diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java index 7d5cee693..f69e3d3ec 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java @@ -54,7 +54,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest { private String tablePath; @BeforeEach - public void init() throws IOException { + public void init() throws Exception { initDFS(); jsc.hadoopConfiguration().addResource(dfs.getConf()); HoodieCLI.conf = dfs.getConf(); @@ -156,7 +156,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest { * Test for command: show archived commits. */ @Test - public void testShowCommits() throws IOException { + public void testShowCommits() throws Exception { CommandResult cr = getShell().executeCommand("show archived commits"); assertTrue(cr.isSuccess()); final List rows = new ArrayList<>(); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java index 3da718964..2311aaa22 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java @@ -62,7 +62,7 @@ public class TestCleansCommand extends AbstractShellIntegrationTest { private URL propsFilePath; @BeforeEach - public void init() throws IOException { + public void init() throws Exception { HoodieCLI.conf = jsc.hadoopConfiguration(); String tableName = "test_table"; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java index 44e2b8097..c2509b12d 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java @@ -76,17 +76,19 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); } - private LinkedHashMap generateData() { + private LinkedHashMap generateData() throws Exception { // generate data and metadata LinkedHashMap data = new LinkedHashMap<>(); data.put("102", new Integer[] {15, 10}); data.put("101", new Integer[] {20, 10}); data.put("100", new Integer[] {15, 15}); - data.forEach((key, value) -> { + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + Integer[] value = entry.getValue(); HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(), Option.of(value[0]), Option.of(value[1])); - }); + } metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(), @@ -138,7 +140,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commits show' command. */ @Test - public void testShowCommits() throws IOException { + public void testShowCommits() throws Exception { Map data = generateData(); CommandResult cr = getShell().executeCommand("commits show"); @@ -154,7 +156,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commits showarchived' command. */ @Test - public void testShowArchivedCommits() throws IOException { + public void testShowArchivedCommits() throws Exception { // Generate archive HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath) .withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) @@ -168,10 +170,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { data.put("102", new Integer[] {25, 45}); data.put("101", new Integer[] {35, 15}); - data.forEach((key, value) -> { + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + Integer[] value = entry.getValue(); HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(), Option.of(value[0]), Option.of(value[1])); - }); + } // archive metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); @@ -198,7 +202,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commit showpartitions' command. */ @Test - public void testShowCommitPartitions() { + public void testShowCommitPartitions() throws Exception { Map data = generateData(); String commitInstant = "101"; @@ -235,7 +239,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commit showfiles' command. */ @Test - public void testShowCommitFiles() { + public void testShowCommitFiles() throws Exception { Map data = generateData(); String commitInstant = "101"; @@ -270,7 +274,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commits compare' command. */ @Test - public void testCompareCommits() throws IOException { + public void testCompareCommits() throws Exception { Map data = generateData(); String tableName2 = "test_table2"; @@ -278,10 +282,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType()); data.remove("102"); - data.forEach((key, value) -> { + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + Integer[] value = entry.getValue(); HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(), Option.of(value[0]), Option.of(value[1])); - }); + } CommandResult cr = getShell().executeCommand(String.format("commits compare --path %s", tablePath2)); assertTrue(cr.isSuccess()); @@ -298,7 +304,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { * Test case of 'commits sync' command. */ @Test - public void testSyncCommits() throws IOException { + public void testSyncCommits() throws Exception { Map data = generateData(); String tableName2 = "test_table2"; @@ -306,10 +312,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest { HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType(), tableName2); data.remove("102"); - data.forEach((key, value) -> { + for (Map.Entry entry : data.entrySet()) { + String key = entry.getKey(); + Integer[] value = entry.getValue(); HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(), Option.of(value[0]), Option.of(value[1])); - }); + } CommandResult cr = getShell().executeCommand(String.format("commits sync --path %s", tablePath2)); assertTrue(cr.isSuccess()); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index e7a460301..10197a325 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -26,12 +26,12 @@ import org.apache.hudi.cli.TableHeader; import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; -import org.apache.hudi.common.testutils.HoodieTestDataGenerator; -import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; @@ -41,14 +41,18 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -59,39 +63,37 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class TestRollbacksCommand extends AbstractShellIntegrationTest { @BeforeEach - public void init() throws IOException { + public void init() throws Exception { String tableName = "test_table"; - String tablePath = basePath + File.separator + tableName; + String tablePath = Paths.get(basePath, tableName).toString(); new TableCommand().createTable( tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(), "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); - + metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); //Create some commits files and parquet files - String commitTime1 = "100"; - String commitTime2 = "101"; - String commitTime3 = "102"; - HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath); - - // two commit files - HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2); - // one .inflight commit file - HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime3); - - // generate commit files for commits - for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) { - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3"); - } - + Map partitionAndFileId = new HashMap() { + { + put(DEFAULT_FIRST_PARTITION_PATH, "file-1"); + put(DEFAULT_SECOND_PARTITION_PATH, "file-2"); + put(DEFAULT_THIRD_PARTITION_PATH, "file-3"); + } + }; + HoodieTestTable.of(metaClient) + .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS) + .addCommit("100") + .withBaseFilesInPartitions(partitionAndFileId) + .addCommit("101") + .withBaseFilesInPartitions(partitionAndFileId) + .addInflightCommit("102") + .withBaseFilesInPartitions(partitionAndFileId); // generate two rollback HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); try (HoodieWriteClient client = getHoodieWriteClient(config)) { // Rollback inflight commit3 and commit2 - client.rollback(commitTime3); - client.rollback(commitTime2); + client.rollback("102"); + client.rollback("101"); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java index 35e5b47da..cd4f19602 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java @@ -73,17 +73,19 @@ public class TestStatsCommand extends AbstractShellIntegrationTest { * Test case for command 'stats wa'. */ @Test - public void testWriteAmplificationStats() { + public void testWriteAmplificationStats() throws Exception { // generate data and metadata Map data = new LinkedHashMap<>(); data.put("100", new Integer[] {15, 10}); data.put("101", new Integer[] {20, 10}); data.put("102", new Integer[] {15, 15}); - data.forEach((key, value) -> { - HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(), - Option.of(value[0]), Option.of(value[1])); - }); + for (Map.Entry entry : data.entrySet()) { + String k = entry.getKey(); + Integer[] v = entry.getValue(); + HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, k, jsc.hadoopConfiguration(), + Option.of(v[0]), Option.of(v[1])); + } CommandResult cr = getShell().executeCommand("stats wa"); assertTrue(cr.isSuccess()); @@ -93,7 +95,7 @@ public class TestStatsCommand extends AbstractShellIntegrationTest { DecimalFormat df = new DecimalFormat("#.00"); data.forEach((key, value) -> { // there are two partitions, so need to *2 - rows.add(new Comparable[]{key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])}); + rows.add(new Comparable[] {key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])}); }); int totalWrite = data.values().stream().map(integers -> integers[0] * 2).mapToInt(s -> s).sum(); int totalUpdate = data.values().stream().map(integers -> integers[1] * 2).mapToInt(s -> s).sum(); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java index 2334f12b4..b9aa3f731 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java @@ -27,8 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.testutils.FileCreateUtils; -import org.apache.hudi.common.testutils.HoodieTestDataGenerator; -import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -36,11 +35,14 @@ import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.File; import java.io.IOException; -import java.util.Arrays; +import java.nio.file.Paths; import java.util.Properties; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -51,45 +53,31 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest { private String tablePath; @BeforeEach - public void init() throws IOException { + public void init() throws Exception { String tableName = "test_table"; - tablePath = basePath + File.separator + tableName; + tablePath = Paths.get(basePath, tableName).toString(); new TableCommand().createTable( tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); - + metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); //Create some commits files and parquet files - String commitTime1 = "100"; - String commitTime2 = "101"; - HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath); - - // one commit file - HoodieTestUtils.createCommitFiles(tablePath, commitTime1); - // one .inflight commit file - HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime2); - - // generate commit files for commit 100 - for (String commitTime : Arrays.asList(commitTime1)) { - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3"); - } - - // generate commit and marker files for inflight commit 101 - for (String commitTime : Arrays.asList(commitTime2)) { - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1"); - FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1", IOType.MERGE); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2"); - FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2", IOType.MERGE); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3"); - FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3", IOType.MERGE); - } + HoodieTestTable.of(metaClient) + .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS) + .addCommit("100") + .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1") + .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2") + .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3") + .addInflightCommit("101") + .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1") + .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2") + .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3") + .withMarkerFile(DEFAULT_FIRST_PARTITION_PATH, "file-1", IOType.MERGE) + .withMarkerFile(DEFAULT_SECOND_PARTITION_PATH, "file-2", IOType.MERGE) + .withMarkerFile(DEFAULT_THIRD_PARTITION_PATH, "file-3", IOType.MERGE); } @Test public void testDowngradeCommand() throws Exception { - metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); - // update hoodie.table.version to 1 metaClient.getTableConfig().setTableVersion(HoodieTableVersion.ONE); try (FSDataOutputStream os = metaClient.getFs().create(new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE), true)) { @@ -98,7 +86,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest { metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); // verify marker files for inflight commit exists - for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) { + for (String partitionPath : DEFAULT_PARTITION_PATHS) { assertEquals(1, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE)); } @@ -112,7 +100,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest { assertTableVersionFromPropertyFile(); // verify marker files are non existant - for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) { + for (String partitionPath : DEFAULT_PARTITION_PATHS) { assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE)); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java index aa87cdd1a..adc61457a 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java @@ -26,17 +26,21 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; -import org.apache.hudi.common.testutils.HoodieTestDataGenerator; -import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestTable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; -import java.util.Arrays; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -48,43 +52,45 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ public class ITTestCommitsCommand extends AbstractShellIntegrationTest { - private String tablePath; - @BeforeEach public void init() throws IOException { - String tableName = "test_table"; - tablePath = basePath + File.separator + tableName; + String tableName = "test_table_" + ITTestCommitsCommand.class.getName(); + String tablePath = Paths.get(basePath, tableName).toString(); HoodieCLI.conf = jsc.hadoopConfiguration(); // Create table and connect new TableCommand().createTable( tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); + metaClient.setBasePath(tablePath); + metaClient = HoodieTableMetaClient.reload(metaClient); } /** * Test case of 'commit rollback' command. */ @Test - public void testRollbackCommit() throws IOException { + public void testRollbackCommit() throws Exception { //Create some commits files and parquet files - String commitTime1 = "100"; - String commitTime2 = "101"; - String commitTime3 = "102"; - HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath); - - // three commit files - HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2, commitTime3); - - // generate commit files for commits - for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) { - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2"); - HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3"); - } + Map partitionAndFileId = new HashMap() { + { + put(DEFAULT_FIRST_PARTITION_PATH, "file-1"); + put(DEFAULT_SECOND_PARTITION_PATH, "file-2"); + put(DEFAULT_THIRD_PARTITION_PATH, "file-3"); + } + }; + final String rollbackCommit = "102"; + HoodieTestTable.of(metaClient) + .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS) + .addCommit("100") + .withBaseFilesInPartitions(partitionAndFileId) + .addCommit("101") + .withBaseFilesInPartitions(partitionAndFileId) + .addCommit(rollbackCommit) + .withBaseFilesInPartitions(partitionAndFileId); CommandResult cr = getShell().executeCommand(String.format("commit rollback --commit %s --sparkMaster %s --sparkMemory %s", - commitTime3, "local", "4G")); + rollbackCommit, "local", "4G")); assertTrue(cr.isSuccess()); metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java index b86e74838..f4d8019be 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java @@ -23,18 +23,14 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.FileCreateUtils; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; -import org.apache.hudi.common.testutils.HoodieTestUtils; -import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; -import org.apache.hudi.exception.HoodieIOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; @@ -42,6 +38,9 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import static org.apache.hudi.common.testutils.FileCreateUtils.baseFileName; +import static org.apache.hudi.common.util.CollectionUtils.createImmutableList; + /** * Class to be used in tests to keep generating test inserts and updates against a corpus. */ @@ -62,67 +61,53 @@ public class HoodieTestCommitMetadataGenerator extends HoodieTestDataGenerator { /** * Create a commit file with default CommitMetadata. */ - public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) { + public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) throws Exception { createCommitFileWithMetadata(basePath, commitTime, configuration, Option.empty(), Option.empty()); } public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration, - Option writes, Option updates) { + Option writes, Option updates) throws Exception { createCommitFileWithMetadata(basePath, commitTime, configuration, UUID.randomUUID().toString(), UUID.randomUUID().toString(), writes, updates); } public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration, - String fileId1, String fileId2, Option writes, Option updates) { - Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime), - HoodieTimeline.makeRequestedCommitFileName(commitTime)) - .forEach(f -> { - Path commitFile = new Path( - basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f); - FSDataOutputStream os = null; - try { - FileSystem fs = FSUtils.getFs(basePath, configuration); - os = fs.create(commitFile, true); - // Generate commitMetadata - HoodieCommitMetadata commitMetadata = - generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates); - // Write empty commit metadata - os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8))); - } catch (IOException ioe) { - throw new HoodieIOException(ioe.getMessage(), ioe); - } finally { - if (null != os) { - try { - os.close(); - } catch (IOException e) { - throw new HoodieIOException(e.getMessage(), e); - } - } - } - }); + String fileId1, String fileId2, Option writes, Option updates) throws Exception { + List commitFileNames = Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime), + HoodieTimeline.makeRequestedCommitFileName(commitTime)); + for (String name : commitFileNames) { + Path commitFilePath = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + name); + try (FSDataOutputStream os = FSUtils.getFs(basePath, configuration).create(commitFilePath, true)) { + // Generate commitMetadata + HoodieCommitMetadata commitMetadata = + generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates); + // Write empty commit metadata + os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8))); + } + } } /** * Generate commitMetadata in path. */ - public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws IOException { + public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws Exception { return generateCommitMetadata(basePath, commitTime, Option.empty(), Option.empty()); } public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime, - Option writes, Option updates) throws IOException { + Option writes, Option updates) throws Exception { return generateCommitMetadata(basePath, commitTime, UUID.randomUUID().toString(), UUID.randomUUID().toString(), writes, updates); } public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime, String fileId1, - String fileId2, Option writes, Option updates) throws IOException { - String file1P0C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1); - String file1P1C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2); + String fileId2, Option writes, Option updates) throws Exception { + FileCreateUtils.createBaseFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1); + FileCreateUtils.createBaseFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2); return generateCommitMetadata(new HashMap>() { { - put(DEFAULT_FIRST_PARTITION_PATH, CollectionUtils.createImmutableList(file1P0C0)); - put(DEFAULT_SECOND_PARTITION_PATH, CollectionUtils.createImmutableList(file1P1C0)); + put(DEFAULT_FIRST_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_FIRST_PARTITION_PATH, fileId1))); + put(DEFAULT_SECOND_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_SECOND_PARTITION_PATH, fileId2))); } }, writes, updates); } diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java index ba4ffb473..c9a03c74d 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java @@ -33,7 +33,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; -import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; @@ -53,20 +53,22 @@ import org.apache.hudi.table.MarkerFiles; import org.apache.hudi.table.action.commit.WriteHelper; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; +import org.apache.hudi.testutils.HoodieWriteableTestTable; import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; -import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -102,6 +104,12 @@ import static org.mockito.Mockito.when; public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { private static final Logger LOG = LogManager.getLogger(TestHoodieClientOnCopyOnWriteStorage.class); + private HoodieTestTable testTable; + + @BeforeEach + public void setUpTestTable() { + testTable = HoodieWriteableTestTable.of(metaClient); + } /** * Test Auto Commit behavior for HoodieWriteClient insert API. @@ -170,10 +178,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { JavaRDD result = insertFirstBatch(cfg, client, newCommitTime, prevCommitTime, numRecords, writeFn, isPrepped, false, numRecords); - assertFalse(HoodieTestUtils.doesCommitExist(basePath, newCommitTime), + assertFalse(testTable.commitExists(newCommitTime), "If Autocommit is false, then commit should not be made automatically"); assertTrue(client.commit(newCommitTime, result), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, newCommitTime), + assertTrue(testTable.commitExists(newCommitTime), "After explicit commit, commit file should be created"); } } @@ -985,7 +993,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { JavaRDD result = client.bulkInsert(writeRecords, instantTime); assertTrue(client.commit(instantTime, result), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created"); // Get parquet file paths from commit metadata @@ -998,16 +1006,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { Collection commitPathNames = commitMetadata.getFileIdAndFullPaths(basePath).values(); // Read from commit file - String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime); - FileInputStream inputStream = new FileInputStream(filename); - String everything = FileIOUtils.readAsUTFString(inputStream); - HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class); - HashMap paths = metadata.getFileIdAndFullPaths(basePath); - inputStream.close(); - - // Compare values in both to make sure they are equal. - for (String pathName : paths.values()) { - assertTrue(commitPathNames.contains(pathName)); + try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime))) { + String everything = FileIOUtils.readAsUTFString(inputStream); + HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class); + HashMap paths = metadata.getFileIdAndFullPaths(basePath); + // Compare values in both to make sure they are equal. + for (String pathName : paths.values()) { + assertTrue(commitPathNames.contains(pathName)); + } } } } @@ -1017,65 +1023,61 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { */ @Test public void testMetadataStatsOnCommit() throws Exception { - HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build(); HoodieWriteClient client = getHoodieWriteClient(cfg); - HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath); - String instantTime = "000"; - client.startCommitWithTime(instantTime); + String instantTime0 = "000"; + client.startCommitWithTime(instantTime0); - List records = dataGen.generateInserts(instantTime, 200); - JavaRDD writeRecords = jsc.parallelize(records, 1); + List records0 = dataGen.generateInserts(instantTime0, 200); + JavaRDD writeRecords0 = jsc.parallelize(records0, 1); + JavaRDD result0 = client.bulkInsert(writeRecords0, instantTime0); - JavaRDD result = client.bulkInsert(writeRecords, instantTime); - - assertTrue(client.commit(instantTime, result), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(client.commit(instantTime0, result0), "Commit should succeed"); + assertTrue(testTable.commitExists(instantTime0), "After explicit commit, commit file should be created"); // Read from commit file - String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime); - FileInputStream inputStream = new FileInputStream(filename); - String everything = FileIOUtils.readAsUTFString(inputStream); - HoodieCommitMetadata metadata = - HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class); - int inserts = 0; - for (Map.Entry> pstat : metadata.getPartitionToWriteStats().entrySet()) { - for (HoodieWriteStat stat : pstat.getValue()) { - inserts += stat.getNumInserts(); + try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime0))) { + String everything = FileIOUtils.readAsUTFString(inputStream); + HoodieCommitMetadata metadata = + HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class); + int inserts = 0; + for (Map.Entry> pstat : metadata.getPartitionToWriteStats().entrySet()) { + for (HoodieWriteStat stat : pstat.getValue()) { + inserts += stat.getNumInserts(); + } } + assertEquals(200, inserts); } - assertEquals(200, inserts); // Update + Inserts such that they just expand file1 - instantTime = "001"; - client.startCommitWithTime(instantTime); + String instantTime1 = "001"; + client.startCommitWithTime(instantTime1); - records = dataGen.generateUpdates(instantTime, records); - writeRecords = jsc.parallelize(records, 1); - result = client.upsert(writeRecords, instantTime); + List records1 = dataGen.generateUpdates(instantTime1, records0); + JavaRDD writeRecords1 = jsc.parallelize(records1, 1); + JavaRDD result1 = client.upsert(writeRecords1, instantTime1); - assertTrue(client.commit(instantTime, result), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(client.commit(instantTime1, result1), "Commit should succeed"); + assertTrue(testTable.commitExists(instantTime1), "After explicit commit, commit file should be created"); // Read from commit file - filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime); - inputStream = new FileInputStream(filename); - everything = FileIOUtils.readAsUTFString(inputStream); - metadata = HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class); - inserts = 0; - int upserts = 0; - for (Map.Entry> pstat : metadata.getPartitionToWriteStats().entrySet()) { - for (HoodieWriteStat stat : pstat.getValue()) { - inserts += stat.getNumInserts(); - upserts += stat.getNumUpdateWrites(); + try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime1))) { + String everything = FileIOUtils.readAsUTFString(inputStream); + HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class); + int inserts = 0; + int upserts = 0; + for (Map.Entry> pstat : metadata.getPartitionToWriteStats().entrySet()) { + for (HoodieWriteStat stat : pstat.getValue()) { + inserts += stat.getNumInserts(); + upserts += stat.getNumUpdateWrites(); + } } + assertEquals(0, inserts); + assertEquals(200, upserts); } - assertEquals(0, inserts); - assertEquals(200, upserts); - } /** @@ -1095,13 +1097,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { metaClient.getFs().delete(result.getKey(), false); if (!enableOptimisticConsistencyGuard) { assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created"); // Marker directory must be removed assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime)))); } else { // with optimistic, first client.commit should have succeeded. - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(testTable.commitExists(instantTime), "After explicit commit, commit file should be created"); // Marker directory must be removed assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime)))); @@ -1124,13 +1126,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { if (!enableOptimisticConsistencyGuard) { // Rollback of this commit should succeed with FailSafeCG client.rollback(instantTime); - assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present"); // Marker directory must be removed after rollback assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime)))); } else { // if optimistic CG is enabled, commit should have succeeded. - assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertTrue(testTable.commitExists(instantTime), "With optimistic CG, first commit should succeed. commit file should be present"); // Marker directory must be removed after rollback assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime)))); @@ -1145,7 +1147,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { } else { // rollback of a completed commit should succeed if using list based rollback client.rollback(instantTime); - assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime), + assertFalse(testTable.commitExists(instantTime), "After explicit rollback, commit file should not be present"); } } @@ -1216,7 +1218,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { JavaRDD writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1); JavaRDD result = client.bulkInsert(writeRecords, firstInstantTime); assertTrue(client.commit(firstInstantTime, result), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime), + assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created"); // Check the entire dataset has all records still @@ -1235,7 +1237,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { JavaRDD inserts = client.bulkInsert(insertRecords, nextInstantTime); JavaRDD upserts = client.upsert(updateRecords, nextInstantTime); assertTrue(client.commit(nextInstantTime, inserts.union(upserts)), "Commit should succeed"); - assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime), + assertTrue(testTable.commitExists(firstInstantTime), "After explicit commit, commit file should be created"); int totalRecords = 2 * numRecords; assertEquals(totalRecords, HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count(), diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index 7f00f5a14..a1dbe086e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -172,19 +172,6 @@ public class HoodieTestUtils { } } - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static void createInflightCommitFiles(String basePath, String... instantTimes) throws IOException { - - for (String instantTime : instantTimes) { - new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" - + HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile(); - new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeInflightCommitFileName( - instantTime)).createNewFile(); - } - } - public static void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... instantTimes) { for (String instantTime : instantTimes) { Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(instantTime), @@ -257,22 +244,6 @@ public class HoodieTestUtils { TimelineMetadataUtils.serializeCompactionPlan(plan)); } - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static String getCommitFilePath(String basePath, String instantTime) { - return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION; - } - - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static boolean doesCommitExist(String basePath, String instantTime) { - return new File( - basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION) - .exists(); - } - public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath, String instantTime, Configuration configuration) throws IOException {