From a68180b179ae57f16ff0a8d74b72b43b501d36c6 Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Tue, 2 Jun 2020 01:49:37 -0700 Subject: [PATCH] [HUDI-988] Fix Unit Test Flakiness : Ensure all instantiations of HoodieWriteClient is closed properly. Fix bug in TestRollbacks. Make CLI unit tests for Hudi CLI check skip redering strings --- .../commands/TestArchivedCommitsCommand.java | 12 ++++-- .../hudi/cli/commands/TestCleansCommand.java | 8 +++- .../commands/TestFileSystemViewCommand.java | 12 ++++-- .../commands/TestHoodieLogFileCommand.java | 15 ++++--- .../hudi/cli/commands/TestRepairsCommand.java | 19 +++++---- .../cli/commands/TestRollbacksCommand.java | 14 ++++--- .../cli/commands/TestSavepointsCommand.java | 5 ++- .../cli/commands/TestSparkEnvCommand.java | 8 +++- .../hudi/cli/commands/TestStatsCommand.java | 10 +++-- .../AbstractShellIntegrationTest.java | 9 ++++ .../org/apache/hudi/client/TestMultiFS.java | 4 -- .../client/TestUpdateSchemaEvolution.java | 4 +- .../apache/hudi/index/TestHoodieIndex.java | 7 +--- .../index/bloom/TestHoodieBloomIndex.java | 2 +- .../bloom/TestHoodieGlobalBloomIndex.java | 2 +- .../hudi/index/hbase/TestHBaseIndex.java | 21 ++++------ .../hbase/TestHBaseQPSResourceAllocator.java | 2 +- .../io/TestHoodieKeyLocationFetchHandle.java | 2 +- .../apache/hudi/io/TestHoodieMergeHandle.java | 11 ++--- .../table/TestHoodieMergeOnReadTable.java | 41 +++++++++---------- .../commit/TestCopyOnWriteActionExecutor.java | 4 +- .../action/commit/TestUpsertPartitioner.java | 2 +- .../action/compact/TestHoodieCompactor.java | 9 ++-- .../hudi/testutils/HoodieClientTestBase.java | 19 --------- .../testutils/HoodieClientTestHarness.java | 37 ++++++++++++++++- pom.xml | 2 +- 26 files changed, 160 insertions(+), 121 deletions(-) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java index d25943dcb..ef303adbb 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java @@ -146,7 +146,9 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest { String expectedResult = HoodiePrintHelper.print( header, new HashMap<>(), "", false, -1, false, rows); - assertEquals(expectedResult, cr.getResult().toString()); + expectedResult = removeNonWordAndStripSpace(expectedResult); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expectedResult, got); } /** @@ -169,7 +171,9 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest { } rows.add(new Comparable[] {"103", "commit"}); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, 10, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); // Test with Metadata and no limit cr = getShell().executeCommand("show archived commits --skipMetadata false --limit -1"); @@ -189,6 +193,8 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest { } header = header.addTableHeaderField("CommitDetails"); expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java index 849f05300..69aa5b37a 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java @@ -128,7 +128,9 @@ public class TestCleansCommand extends AbstractShellIntegrationTest { rows.add(new Comparable[] {clean.getTimestamp(), "102", "0", getLatestCleanTimeTakenInMillis().toString()}); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -168,7 +170,9 @@ public class TestCleansCommand extends AbstractShellIntegrationTest { HoodieCleaningPolicy.KEEP_LATEST_COMMITS, "0", "0"}); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java index aec393704..f92caea0c 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java @@ -147,7 +147,9 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_DELTA_FILE_SIZE) .addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES); String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -188,7 +190,9 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_DATA_FILE_SIZE); String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -262,6 +266,8 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_SCHEDULED) .addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_UNSCHEDULED); String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index e484bef16..0c52220e0 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -135,8 +135,9 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest { rows.add(output); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -151,8 +152,9 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest { List records = SchemaTestUtil.generateTestRecords(0, 10); String[][] rows = records.stream().map(r -> new String[]{r.toString()}).toArray(String[][]::new); String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_RECORDS}, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -216,7 +218,8 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest { assertNotNull(rows); String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_RECORDS}, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java index 6b72e9a8d..1ada2a2ed 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java @@ -96,8 +96,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { .toArray(String[][]::new); String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH, HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -126,8 +127,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { .toArray(String[][]::new); String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH, HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); cr = getShell().executeCommand("repair addpartitionmeta"); @@ -137,7 +139,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { .toArray(String[][]::new); expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH, HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows); - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -170,8 +174,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { .toArray(String[][]::new); String expect = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); - - assertEquals(expect, cr.getResult().toString()); + expect = removeNonWordAndStripSpace(expect); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expect, got); } /** diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index 23fbcf26e..245b76b81 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -88,7 +88,7 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest { HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); - try (HoodieWriteClient client = new HoodieWriteClient(jsc, config, false)) { + try (HoodieWriteClient client = getHoodieWriteClient(config)) { // Rollback inflight commit3 and commit2 client.rollback(commitTime3); client.rollback(commitTime2); @@ -119,7 +119,7 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest { row[1] = c; // expect data row[2] = 3; - row[3] = 0; + row[3] = metadata.getTimeTakenInMillis(); row[4] = 3; rows.add(row); }); @@ -134,8 +134,9 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS) .addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_PARTITIONS); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -176,7 +177,8 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_DELETED_FILE) .addTableHeaderField(HoodieTableHeaderFields.HEADER_SUCCEEDED); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java index 1ec0da4e3..1158ac6d5 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java @@ -75,8 +75,9 @@ public class TestSavepointsCommand extends AbstractShellIntegrationTest { String[][] rows = Arrays.asList("100", "101", "102", "103").stream().sorted(Comparator.reverseOrder()) .map(instant -> new String[]{instant}).toArray(String[][]::new); String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_SAVEPOINT_TIME}, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java index ee6f57045..19fcf2f42 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java @@ -40,7 +40,9 @@ public class TestSparkEnvCommand extends AbstractShellIntegrationTest { // First, be empty CommandResult cr = getShell().executeCommand("show envs all"); String nullResult = HoodiePrintHelper.print(new String[] {"key", "value"}, new String[0][2]); - assertEquals(nullResult, cr.getResult().toString()); + nullResult = removeNonWordAndStripSpace(nullResult); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(nullResult, got); // Set SPARK_HOME cr = getShell().executeCommand("set --conf SPARK_HOME=/usr/etc/spark"); @@ -49,6 +51,8 @@ public class TestSparkEnvCommand extends AbstractShellIntegrationTest { //Get cr = getShell().executeCommand("show env --key SPARK_HOME"); String result = HoodiePrintHelper.print(new String[] {"key", "value"}, new String[][] {new String[] {"SPARK_HOME", "/usr/etc/spark"}}); - assertEquals(result, cr.getResult().toString()); + result = removeNonWordAndStripSpace(result); + got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(result, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java index 015697775..61c2fcb5c 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java @@ -103,8 +103,9 @@ public class TestStatsCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_WRITTEN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_WRITE_AMPLIFICATION_FACTOR); String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows); - - assertEquals(expected, cr.getResult().toString()); + expected = removeNonWordAndStripSpace(expected); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expected, got); } /** @@ -170,7 +171,8 @@ public class TestStatsCommand extends AbstractShellIntegrationTest { .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); String expect = HoodiePrintHelper.print(header, new StatsCommand().getFieldNameToConverterMap(), "", false, -1, false, rows); - - assertEquals(expect, cr.getResult().toString()); + expect = removeNonWordAndStripSpace(expect); + String got = removeNonWordAndStripSpace(cr.getResult().toString()); + assertEquals(expect, got); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java index ef053f0c1..0c98135dc 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java @@ -58,4 +58,13 @@ public abstract class AbstractShellIntegrationTest extends HoodieClientTestHarne protected static JLineShellComponent getShell() { return shell; } + + /** + * Helper to prepare string for matching. + * @param str Input string. + * @return pruned string with non word characters removed. + */ + protected String removeNonWordAndStripSpace(String str) { + return str.replaceAll("[\\s]+", ",").replaceAll("[\\W]+", ","); + } } diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java index 53c615eab..02efe8e51 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java @@ -68,10 +68,6 @@ public class TestMultiFS extends HoodieClientTestHarness { cleanupTestDataGenerator(); } - private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig config) { - return new HoodieWriteClient(jsc, config); - } - protected HoodieWriteConfig getHoodieWriteConfig(String basePath) { return HoodieWriteConfig.newBuilder().withPath(basePath).withEmbeddedTimelineServerEnabled(true) .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2).forTable(tableName) diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java index 7bd9d18ad..d20b9fe61 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java @@ -18,6 +18,7 @@ package org.apache.hudi.client; +import java.io.IOException; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -59,8 +60,9 @@ public class TestUpdateSchemaEvolution extends HoodieClientTestHarness { } @AfterEach - public void tearDown() { + public void tearDown() throws IOException { cleanupSparkContexts(); + cleanupFileSystem(); } @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index 46256b6fc..ea5f85137 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -112,7 +112,8 @@ public class TestHoodieIndex extends HoodieClientTestHarness { public void tearDown() throws IOException { cleanupSparkContexts(); cleanupFileSystem(); - cleanupMetaClient(); + cleanupClients(); + cleanupTestDataGenerator(); } @ParameterizedTest @@ -544,10 +545,6 @@ public class TestHoodieIndex extends HoodieClientTestHarness { .withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE).build()); } - private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { - return new HoodieWriteClient(jsc, cfg, false); - } - private void instantiateIndex() { config = getConfigBuilder() .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType) diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index c5348afc5..bfbfa978e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -99,7 +99,7 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { public void tearDown() throws Exception { cleanupSparkContexts(); cleanupFileSystem(); - cleanupMetaClient(); + cleanupClients(); } private HoodieWriteConfig makeConfig(boolean rangePruning, boolean treeFiltering, boolean bucketizedChecking) { diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index 1dac30cb1..3847047c5 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -82,7 +82,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { @AfterEach public void tearDown() { cleanupSparkContexts(); - cleanupMetaClient(); + cleanupClients(); } @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java index e3b9529b0..6763fffd0 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java @@ -91,6 +91,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { @AfterAll public static void clean() throws Exception { if (utility != null) { + utility.deleteTable(tableName); utility.shutdownMiniCluster(); } } @@ -123,11 +124,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { public void tearDown() throws Exception { cleanupSparkContexts(); cleanupTestDataGenerator(); - cleanupMetaClient(); - } - - private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { - return new HoodieWriteClient(jsc, config); + cleanupClients(); } @Test @@ -140,7 +137,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // Load to memory HoodieWriteConfig config = getConfig(); HBaseIndex index = new HBaseIndex(config); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { metaClient = HoodieTableMetaClient.reload(metaClient); HoodieTable hoodieTable = HoodieTable.create(metaClient, config, hadoopConf); @@ -180,7 +177,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // Load to memory HoodieWriteConfig config = getConfig(); HBaseIndex index = new HBaseIndex(config); - HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); + HoodieWriteClient writeClient = getHoodieWriteClient(config); writeClient.startCommitWithTime(newCommitTime); metaClient = HoodieTableMetaClient.reload(metaClient); HoodieTable hoodieTable = HoodieTable.create(metaClient, config, hadoopConf); @@ -214,7 +211,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // Load to memory HoodieWriteConfig config = getConfig(); HBaseIndex index = new HBaseIndex(config); - HoodieWriteClient writeClient = getWriteClient(config); + HoodieWriteClient writeClient = getHoodieWriteClient(config); String newCommitTime = writeClient.startCommit(); List records = dataGen.generateInserts(newCommitTime, 200); @@ -264,7 +261,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // only for test, set the hbaseConnection to mocked object index.setHbaseConnection(hbaseConnection); - HoodieWriteClient writeClient = getWriteClient(config); + HoodieWriteClient writeClient = getHoodieWriteClient(config); // start a commit and generate test data String newCommitTime = writeClient.startCommit(); @@ -289,7 +286,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { public void testTotalPutsBatching() throws Exception { HoodieWriteConfig config = getConfig(); HBaseIndex index = new HBaseIndex(config); - HoodieWriteClient writeClient = getWriteClient(config); + HoodieWriteClient writeClient = getHoodieWriteClient(config); // start a commit and generate test data String newCommitTime = writeClient.startCommit(); @@ -406,7 +403,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // Load to memory HoodieWriteConfig config = getConfig(2); HBaseIndex index = new HBaseIndex(config); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { metaClient = HoodieTableMetaClient.reload(metaClient); HoodieTable hoodieTable = HoodieTable.create(metaClient, config, hadoopConf); @@ -446,7 +443,7 @@ public class TestHBaseIndex extends HoodieClientTestHarness { // Load to memory HoodieWriteConfig config = getConfig(); HBaseIndex index = new HBaseIndex(config); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { metaClient = HoodieTableMetaClient.reload(metaClient); HoodieTable hoodieTable = HoodieTable.create(metaClient, config, hadoopConf); diff --git a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java index f85a76356..856dc6a80 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java @@ -59,7 +59,7 @@ public class TestHBaseQPSResourceAllocator extends HoodieClientTestHarness { @AfterEach public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupMetaClient(); + cleanupClients(); if (utility != null) { utility.shutdownMiniCluster(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java index cbea30ce3..af0e728b1 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java @@ -84,7 +84,7 @@ public class TestHoodieKeyLocationFetchHandle extends HoodieClientTestHarness { public void tearDown() throws IOException { cleanupSparkContexts(); cleanupFileSystem(); - cleanupMetaClient(); + cleanupClients(); } @Test diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 160f16978..fa6f41a09 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -69,11 +69,8 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness { cleanupFileSystem(); cleanupTestDataGenerator(); cleanupSparkContexts(); - cleanupMetaClient(); - } - - private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { - return new HoodieWriteClient(jsc, config); + cleanupClients(); + cleanupFileSystem(); } @Test @@ -84,7 +81,7 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness { // Build a write config with bulkinsertparallelism set HoodieWriteConfig cfg = getConfigBuilder().build(); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { FileSystem fs = FSUtils.getFs(basePath, hadoopConf); /** @@ -226,7 +223,7 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness { public void testHoodieMergeHandleWriteStatMetrics() throws Exception { // insert 100 records HoodieWriteConfig config = getConfigBuilder().build(); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 7913bafd9..cd3adae19 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -121,16 +121,13 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { cleanupDFS(); cleanupSparkContexts(); cleanupTestDataGenerator(); - } - - private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { - return new HoodieWriteClient(jsc, config); + cleanupClients(); } @Test public void testSimpleInsertAndUpdate() throws Exception { HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts) @@ -178,7 +175,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { String partitionPath = "2020/02/20"; // use only one partition for this test dataGen = new HoodieTestDataGenerator(new String[] { partitionPath }); HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts) @@ -270,7 +267,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @Test public void testMetadataAggregateFromWriteStatus() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false).withWriteStatusClass(MetadataMergeWriteStatus.class).build(); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { String newCommitTime = "001"; List records = dataGen.generateInserts(newCommitTime, 200); @@ -293,7 +290,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @Test public void testSimpleInsertUpdateAndDelete() throws Exception { HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts, written as parquet file) @@ -378,7 +375,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.COPY_ON_WRITE); HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts) @@ -431,7 +428,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { public void testRollbackWithDeltaAndCompactionCommit() throws Exception { HoodieWriteConfig cfg = getConfig(false); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { // Test delta commit rollback /** @@ -474,7 +471,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { */ final String commitTime1 = "002"; // WriteClient with custom config (disable small file handling) - try (HoodieWriteClient secondClient = getWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff());) { + try (HoodieWriteClient secondClient = getHoodieWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff());) { secondClient.startCommitWithTime(commitTime1); List copyOfRecords = new ArrayList<>(records); @@ -504,7 +501,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { * Write 3 (inserts + updates - testing successful delta commit) */ final String commitTime2 = "002"; - try (HoodieWriteClient thirdClient = getWriteClient(cfg);) { + try (HoodieWriteClient thirdClient = getHoodieWriteClient(cfg);) { thirdClient.startCommitWithTime(commitTime2); List copyOfRecords = new ArrayList<>(records); @@ -580,7 +577,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { public void testMultiRollbackWithDeltaAndCompactionCommit() throws Exception { HoodieWriteConfig cfg = getConfig(false); - try (final HoodieWriteClient client = getWriteClient(cfg);) { + try (final HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts) */ @@ -621,7 +618,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { */ newCommitTime = "002"; // WriteClient with custom config (disable small file handling) - HoodieWriteClient nClient = getWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff()); + HoodieWriteClient nClient = getHoodieWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff()); nClient.startCommitWithTime(newCommitTime); List copyOfRecords = new ArrayList<>(records); @@ -744,7 +741,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @Test public void testUpsertPartitioner() throws Exception { HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts, written as parquet file) @@ -823,7 +820,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { public void testLogFileCountsAfterCompaction() throws Exception { // insert 100 records HoodieWriteConfig config = getConfig(true); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); @@ -897,7 +894,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { // insert 100 records // Setting IndexType to be InMemory to simulate Global Index nature HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); @@ -934,7 +931,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { // insert 100 records // Setting IndexType to be InMemory to simulate Global Index nature HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); @@ -1005,7 +1002,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { // insert 100 records // Setting IndexType to be InMemory to simulate Global Index nature HoodieWriteConfig config = getConfigBuilder(false, IndexType.INMEMORY).build(); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); @@ -1057,7 +1054,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { public void testRollingStatsInMetadata() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false, IndexType.INMEMORY).withAutoCommit(false).build(); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath); HoodieTable table = HoodieTable.create(metaClient, cfg, hadoopConf); @@ -1156,7 +1153,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @Test public void testRollingStatsWithSmallFileHandling() throws Exception { HoodieWriteConfig cfg = getConfigBuilder(false, IndexType.INMEMORY).withAutoCommit(false).build(); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath); Map fileIdToInsertsMap = new HashMap<>(); Map fileIdToUpsertsMap = new HashMap<>(); @@ -1290,7 +1287,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @Test public void testHandleUpdateWithMultiplePartitions() throws Exception { HoodieWriteConfig cfg = getConfig(true); - try (HoodieWriteClient client = getWriteClient(cfg);) { + try (HoodieWriteClient client = getHoodieWriteClient(cfg);) { /** * Write 1 (only inserts, written as parquet file) diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java index 513f63ac3..25b64ae29 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java @@ -85,7 +85,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestHarness { @AfterEach public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupMetaClient(); + cleanupClients(); cleanupFileSystem(); cleanupTestDataGenerator(); } @@ -129,7 +129,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestHarness { // Prepare the AvroParquetIO HoodieWriteConfig config = makeHoodieClientConfig(); String firstCommitTime = HoodieTestUtils.makeNewCommitTime(); - HoodieWriteClient writeClient = new HoodieWriteClient(jsc, config); + HoodieWriteClient writeClient = getHoodieWriteClient(config); writeClient.startCommitWithTime(firstCommitTime); metaClient = HoodieTableMetaClient.reload(metaClient); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java index 066c54197..7f15379f8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java @@ -62,7 +62,7 @@ public class TestUpsertPartitioner extends HoodieClientTestHarness { @AfterEach public void tearDown() throws Exception { cleanupSparkContexts(); - cleanupMetaClient(); + cleanupClients(); cleanupFileSystem(); cleanupTestDataGenerator(); } diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java index 7bb1bf912..67860d76a 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java @@ -81,10 +81,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness { cleanupFileSystem(); cleanupTestDataGenerator(); cleanupSparkContexts(); - } - - private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { - return new HoodieWriteClient(jsc, config); + cleanupClients(); } private HoodieWriteConfig getConfig() { @@ -120,7 +117,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness { HoodieWriteConfig config = getConfig(); metaClient = HoodieTableMetaClient.reload(metaClient); HoodieTable table = HoodieTable.create(metaClient, config, hadoopConf); - try (HoodieWriteClient writeClient = getWriteClient(config);) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) { String newCommitTime = writeClient.startCommit(); List records = dataGen.generateInserts(newCommitTime, 100); @@ -137,7 +134,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness { public void testWriteStatusContentsAfterCompaction() throws Exception { // insert 100 records HoodieWriteConfig config = getConfig(); - try (HoodieWriteClient writeClient = getWriteClient(config)) { + try (HoodieWriteClient writeClient = getHoodieWriteClient(config)) { String newCommitTime = "100"; writeClient.startCommitWithTime(newCommitTime); diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java index b418d4f70..14123b264 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java @@ -18,7 +18,6 @@ package org.apache.hudi.testutils; -import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.HoodieCleanStat; @@ -49,7 +48,6 @@ import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.sql.SQLContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -83,23 +81,6 @@ public class HoodieClientTestBase extends HoodieClientTestHarness { cleanupResources(); } - public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { - return getHoodieWriteClient(cfg, false); - } - - public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) { - return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg)); - } - - public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, - HoodieIndex index) { - return new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index); - } - - public HoodieReadClient getHoodieReadClient(String basePath) { - return new HoodieReadClient(jsc, basePath, SQLContext.getOrCreate(jsc.sc())); - } - /** * Get Default HoodieWriteConfig for tests. * diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java index 111294d9e..70bd59197 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java @@ -17,6 +17,8 @@ package org.apache.hudi.testutils; +import org.apache.hudi.client.HoodieReadClient; +import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.client.SparkTaskContextSupplier; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -30,6 +32,8 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.slf4j.Logger; @@ -56,6 +60,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im protected transient ExecutorService executorService; protected transient HoodieTableMetaClient metaClient; private static AtomicInteger instantGen = new AtomicInteger(1); + protected transient HoodieWriteClient client; protected final SparkTaskContextSupplier supplier = new SparkTaskContextSupplier(); @@ -84,7 +89,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im * Cleanups resource group for the subclasses of {@link HoodieClientTestBase}. */ public void cleanupResources() throws IOException { - cleanupMetaClient(); + cleanupClients(); cleanupSparkContexts(); cleanupTestDataGenerator(); cleanupFileSystem(); @@ -182,8 +187,12 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im /** * Cleanups table type. */ - protected void cleanupMetaClient() { + protected void cleanupClients() { metaClient = null; + if (null != client) { + client.close(); + client = null; + } } /** @@ -227,6 +236,9 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im if (hdfsTestService != null) { hdfsTestService.stop(); dfsCluster.shutdown(); + hdfsTestService = null; + dfsCluster = null; + dfs = null; } // Need to closeAll to clear FileSystem.Cache, required because DFS and LocalFS used in the // same JVM @@ -267,4 +279,25 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im } } + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { + return getHoodieWriteClient(cfg, false); + } + + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit) { + return getHoodieWriteClient(cfg, rollbackInflightCommit, HoodieIndex.createIndex(cfg)); + } + + public HoodieReadClient getHoodieReadClient(String basePath) { + return new HoodieReadClient(jsc, basePath, SQLContext.getOrCreate(jsc.sc())); + } + + public HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, + HoodieIndex index) { + if (null != client) { + client.close(); + client = null; + } + client = new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index); + return client; + } } diff --git a/pom.xml b/pom.xml index cc39f4136..e218cff02 100644 --- a/pom.xml +++ b/pom.xml @@ -246,7 +246,7 @@ ${maven-surefire-plugin.version} ${skipUTs} - -Xmx2g + -Xmx4g 120