From c5e10d668f9366f29bdf7721f7efe4140782527b Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Sun, 11 Oct 2020 23:39:10 -0700 Subject: [PATCH] [HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2167) Remove APIs in `HoodieTestUtils` - `createCommitFiles` - `createDataFile` - `createNewLogFile` - `createCompactionRequest` Migrated usages in `TestCleaner#testPendingCompactions`. Also improved some API names in `HoodieTestTable`. --- .../hudi/cli/integ/ITTestRepairsCommand.java | 2 +- .../org/apache/hudi/table/TestCleaner.java | 150 +++++++++--------- .../TestMarkerBasedRollbackStrategy.java | 8 +- .../common/testutils/HoodieTestTable.java | 31 +++- .../common/testutils/HoodieTestUtils.java | 75 --------- .../hadoop/TestHoodieROTablePathFilter.java | 2 +- 6 files changed, 112 insertions(+), 156 deletions(-) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java index f277e33b6..133dcb057 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java @@ -87,7 +87,7 @@ public class ITTestRepairsCommand extends AbstractShellIntegrationTest { testTable.addCommit("20160401010101") .withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "1", hoodieRecords1) .withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "2", hoodieRecords2) - .withLogFile(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH); + .getFileIdWithLogFile(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH); testTable.withInserts(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "4", hoodieRecords1) .withInserts(HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "6", hoodieRecords1); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java index 152a9814a..00f1ea00e 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -51,7 +51,6 @@ import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataMigra import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanMigrator; import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanV1MigrationHandler; import org.apache.hudi.common.table.view.TableFileSystemView; -import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CleanerUtils; @@ -155,7 +154,7 @@ public class TestCleaner extends HoodieClientTestBase { assertTrue(table.getCompletedCleanTimeline().empty()); HoodieIndex index = SparkHoodieIndex.createIndex(cfg); - List taggedRecords = ((JavaRDD)index.tagLocation(jsc.parallelize(records, 1), context, table)).collect(); + List taggedRecords = ((JavaRDD) index.tagLocation(jsc.parallelize(records, 1), context, table)).collect(); checkTaggedRecords(taggedRecords, newCommitTime); } @@ -550,7 +549,7 @@ public class TestCleaner extends HoodieClientTestBase { Map partitionAndFileId002 = testTable.addCommit("00000000000002") .withBaseFilesInPartition(p0, file1P0C0) .withBaseFilesInPartition(p1, file1P1C0) - .withBaseFilesInPartitions(p0, p1); + .getFileIdsWithBaseFilesInPartitions(p0, p1); List hoodieCleanStatsTwo = runCleaner(config, 1); // enableBootstrapSourceClean would delete the bootstrap base file as the same time @@ -592,7 +591,7 @@ public class TestCleaner extends HoodieClientTestBase { // make next commit, with 2 updates to existing files, and 1 insert String file3P0C2 = testTable.addCommit("00000000000003") .withBaseFilesInPartition(p0, file1P0C0, file2P0C1) - .withBaseFilesInPartitions(p0).get(p0); + .getFileIdsWithBaseFilesInPartitions(p0).get(p0); List hoodieCleanStatsThree = runCleaner(config, 3); assertEquals(2, getCleanStat(hoodieCleanStatsThree, p0) @@ -625,7 +624,7 @@ public class TestCleaner extends HoodieClientTestBase { String p0 = "2020/01/01"; // Make 3 files, one base file and 2 log files associated with base file - String file1P0 = testTable.addDeltaCommit("000").withBaseFilesInPartitions(p0).get(p0); + String file1P0 = testTable.addDeltaCommit("000").getFileIdsWithBaseFilesInPartitions(p0).get(p0); testTable.forDeltaCommit("000") .withLogFile(p0, file1P0, 1) .withLogFile(p0, file1P0, 2); @@ -865,7 +864,7 @@ public class TestCleaner extends HoodieClientTestBase { assertTrue(testTable.baseFileExists(p1, "00000000000001", file1P1C0)); // make next commit, with 1 insert & 1 update per partition - Map partitionAndFileId002 = testTable.addInflightCommit("00000000000002").withBaseFilesInPartitions(p0, p1); + Map partitionAndFileId002 = testTable.addInflightCommit("00000000000002").getFileIdsWithBaseFilesInPartitions(p0, p1); String file2P0C1 = partitionAndFileId002.get(p0); String file2P1C1 = partitionAndFileId002.get(p1); testTable.forCommit("00000000000002").withBaseFilesInPartition(p0, file1P0C0).withBaseFilesInPartition(p1, file1P1C0); @@ -889,7 +888,7 @@ public class TestCleaner extends HoodieClientTestBase { String file3P0C2 = testTable.addInflightCommit("00000000000003") .withBaseFilesInPartition(p0, file1P0C0) .withBaseFilesInPartition(p0, file2P0C1) - .withBaseFilesInPartitions(p0).get(p0); + .getFileIdsWithBaseFilesInPartitions(p0).get(p0); commitMetadata = generateCommitMetadata(CollectionUtils .createImmutableMap(p0, CollectionUtils.createImmutableList(file1P0C0, file2P0C1, file3P0C2))); @@ -906,7 +905,7 @@ public class TestCleaner extends HoodieClientTestBase { String file4P0C3 = testTable.addInflightCommit("00000000000004") .withBaseFilesInPartition(p0, file1P0C0) .withBaseFilesInPartition(p0, file2P0C1) - .withBaseFilesInPartitions(p0).get(p0); + .getFileIdsWithBaseFilesInPartitions(p0).get(p0); commitMetadata = generateCommitMetadata(CollectionUtils.createImmutableMap( p0, CollectionUtils.createImmutableList(file1P0C0, file2P0C1, file4P0C3))); metaClient.getActiveTimeline().saveAsComplete( @@ -1021,7 +1020,7 @@ public class TestCleaner extends HoodieClientTestBase { * Test Keep Latest Commits when there are pending compactions. */ @Test - public void testKeepLatestCommitsWithPendingCompactions() throws IOException { + public void testKeepLatestCommitsWithPendingCompactions() throws Exception { HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withAssumeDatePartitioning(true) .withCompactionConfig(HoodieCompactionConfig.newBuilder() .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).retainCommits(2).build()) @@ -1043,7 +1042,7 @@ public class TestCleaner extends HoodieClientTestBase { */ @ParameterizedTest @ValueSource(booleans = {false, true}) - public void testKeepLatestVersionsWithPendingCompactions(boolean retryFailure) throws IOException { + public void testKeepLatestVersionsWithPendingCompactions(boolean retryFailure) throws Exception { HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withAssumeDatePartitioning(true) .withCompactionConfig(HoodieCompactionConfig.newBuilder() @@ -1098,73 +1097,82 @@ public class TestCleaner extends HoodieClientTestBase { * @param expNumFilesDeleted Number of files deleted */ private void testPendingCompactions(HoodieWriteConfig config, int expNumFilesDeleted, - int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws IOException { + int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws Exception { HoodieTableMetaClient metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); - String[] instants = new String[] {"000", "001", "003", "005", "007", "009", "011", "013"}; - String[] compactionInstants = new String[] {"002", "004", "006", "008", "010"}; - Map expFileIdToPendingCompaction = new HashMap<>(); - Map fileIdToLatestInstantBeforeCompaction = new HashMap<>(); - Map> compactionInstantsToFileSlices = new HashMap<>(); - - for (String instant : instants) { - HoodieTestUtils.createCommitFiles(basePath, instant); - } + final String partition = "2016/03/15"; + Map expFileIdToPendingCompaction = new HashMap() { + { + put("fileId2", "004"); + put("fileId3", "006"); + put("fileId4", "008"); + put("fileId5", "010"); + } + }; + Map fileIdToLatestInstantBeforeCompaction = new HashMap() { + { + put("fileId1", "000"); + put("fileId2", "000"); + put("fileId3", "001"); + put("fileId4", "003"); + put("fileId5", "005"); + put("fileId6", "009"); + put("fileId7", "011"); + } + }; // Generate 7 file-groups. First one has only one slice and no pending compaction. File Slices (2 - 5) has // multiple versions with pending compaction. File Slices (6 - 7) have multiple file-slices but not under // compactions // FileIds 2-5 will be under compaction - int maxNumFileIds = 7; - String[] fileIds = new String[] {"fileId1", "fileId2", "fileId3", "fileId4", "fileId5", "fileId6", "fileId7"}; - int maxNumFileIdsForCompaction = 4; - for (int i = 0; i < maxNumFileIds; i++) { - final String fileId = HoodieTestUtils.createDataFile(basePath, - HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], fileIds[i]); - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], - fileId, Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], - fileId, Option.of(2)); - fileIdToLatestInstantBeforeCompaction.put(fileId, instants[0]); - for (int j = 1; j <= i; j++) { - if (j == i && j <= maxNumFileIdsForCompaction) { - expFileIdToPendingCompaction.put(fileId, compactionInstants[j]); - metaClient = HoodieTableMetaClient.reload(metaClient); - HoodieTable table = HoodieSparkTable.create(config, context, metaClient); - FileSlice slice = - table.getSliceView().getLatestFileSlices(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH) - .filter(fs -> fs.getFileId().equals(fileId)).findFirst().get(); - List slices = new ArrayList<>(); - if (compactionInstantsToFileSlices.containsKey(compactionInstants[j])) { - slices = compactionInstantsToFileSlices.get(compactionInstants[j]); - } - slices.add(slice); - compactionInstantsToFileSlices.put(compactionInstants[j], slices); - // Add log-files to simulate delta-commits after pending compaction - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, - compactionInstants[j], fileId, Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, - compactionInstants[j], fileId, Option.of(2)); - } else { - HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j], - fileId); - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, - instants[j], fileId, Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, - instants[j], fileId, Option.of(2)); - fileIdToLatestInstantBeforeCompaction.put(fileId, instants[j]); - } - } - } - - // Setup pending compaction plans - for (String instant : compactionInstants) { - List fileSliceList = compactionInstantsToFileSlices.get(instant); - if (null != fileSliceList) { - HoodieTestUtils.createCompactionRequest(metaClient, instant, fileSliceList.stream() - .map(fs -> Pair.of(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fs)).collect(Collectors.toList())); - } - } + HoodieTestTable.of(metaClient) + .addCommit("000") + .withBaseFilesInPartition(partition, "fileId1", "fileId2", "fileId3", "fileId4", "fileId5", "fileId6", "fileId7") + .withLogFile(partition, "fileId1", 1, 2) + .withLogFile(partition, "fileId2", 1, 2) + .withLogFile(partition, "fileId3", 1, 2) + .withLogFile(partition, "fileId4", 1, 2) + .withLogFile(partition, "fileId5", 1, 2) + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addCommit("001") + .withBaseFilesInPartition(partition, "fileId3", "fileId4", "fileId5", "fileId6", "fileId7") + .withLogFile(partition, "fileId3", 1, 2) + .withLogFile(partition, "fileId4", 1, 2) + .withLogFile(partition, "fileId5", 1, 2) + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addCommit("003") + .withBaseFilesInPartition(partition, "fileId4", "fileId5", "fileId6", "fileId7") + .withLogFile(partition, "fileId4", 1, 2) + .withLogFile(partition, "fileId5", 1, 2) + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addRequestedCompaction("004", new FileSlice(partition, "000", "fileId2")) + .withLogFile(partition, "fileId2", 1, 2) + .addCommit("005") + .withBaseFilesInPartition(partition, "fileId5", "fileId6", "fileId7") + .withLogFile(partition, "fileId5", 1, 2) + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addRequestedCompaction("006", new FileSlice(partition, "001", "fileId3")) + .withLogFile(partition, "fileId3", 1, 2) + .addCommit("007") + .withBaseFilesInPartition(partition, "fileId6", "fileId7") + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addRequestedCompaction("008", new FileSlice(partition, "003", "fileId4")) + .withLogFile(partition, "fileId4", 1, 2) + .addCommit("009") + .withBaseFilesInPartition(partition, "fileId6", "fileId7") + .withLogFile(partition, "fileId6", 1, 2) + .withLogFile(partition, "fileId7", 1, 2) + .addRequestedCompaction("010", new FileSlice(partition, "005", "fileId5")) + .withLogFile(partition, "fileId5", 1, 2) + .addCommit("011") + .withBaseFilesInPartition(partition, "fileId7") + .withLogFile(partition, "fileId7", 1, 2) + .addCommit("013"); // Clean now metaClient = HoodieTableMetaClient.reload(metaClient); @@ -1177,7 +1185,7 @@ public class TestCleaner extends HoodieClientTestBase { expFileIdToPendingCompaction.forEach((fileId, value) -> { String baseInstantForCompaction = fileIdToLatestInstantBeforeCompaction.get(fileId); Option fileSliceForCompaction = Option.fromJavaOptional(hoodieTable.getSliceView() - .getLatestFileSlicesBeforeOrOn(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, baseInstantForCompaction, + .getLatestFileSlicesBeforeOrOn(partition, baseInstantForCompaction, true) .filter(fs -> fs.getFileId().equals(fileId)).findFirst()); assertTrue(fileSliceForCompaction.isPresent(), "Base Instant for Compaction must be preserved"); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java index 191e90fa7..7acff79ef 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java @@ -59,10 +59,10 @@ public class TestMarkerBasedRollbackStrategy extends HoodieClientTestBase { // given: wrote some base files and corresponding markers HoodieTestTable testTable = HoodieTestTable.of(metaClient); String f0 = testTable.addRequestedCommit("000") - .withBaseFilesInPartitions("partA").get("partA"); + .getFileIdsWithBaseFilesInPartitions("partA").get("partA"); String f1 = testTable.addCommit("001") .withBaseFilesInPartition("partA", f0) - .withBaseFilesInPartitions("partB").get("partB"); + .getFileIdsWithBaseFilesInPartitions("partB").get("partB"); String f2 = "f2"; testTable.forCommit("001") .withMarkerFile("partA", f0, IOType.MERGE) @@ -90,10 +90,10 @@ public class TestMarkerBasedRollbackStrategy extends HoodieClientTestBase { // given: wrote some base + log files and corresponding markers HoodieTestTable testTable = HoodieTestTable.of(metaClient); String f2 = testTable.addRequestedDeltaCommit("000") - .withBaseFilesInPartitions("partA").get("partA"); + .getFileIdsWithBaseFilesInPartitions("partA").get("partA"); String f1 = testTable.addDeltaCommit("001") .withLogFile("partA", f2) - .withBaseFilesInPartitions("partB").get("partB"); + .getFileIdsWithBaseFilesInPartitions("partB").get("partB"); String f3 = "f3"; String f4 = "f4"; testTable.forDeltaCommit("001") diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index 9cacf1faa..3663917a5 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -21,12 +21,19 @@ package org.apache.hudi.common.testutils; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanerPlan; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; import org.apache.hudi.common.model.IOType; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.common.util.collection.Pair; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -189,6 +196,20 @@ public class HoodieTestTable { return this; } + public HoodieTestTable addRequestedCompaction(String instantTime, HoodieCompactionPlan compactionPlan) throws IOException { + HoodieInstant compactionInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instantTime); + metaClient.getActiveTimeline().saveToCompactionRequested(compactionInstant, + TimelineMetadataUtils.serializeCompactionPlan(compactionPlan)); + return addRequestedCompaction(instantTime); + } + + public HoodieTestTable addRequestedCompaction(String instantTime, FileSlice... fileSlices) throws IOException { + HoodieCompactionPlan plan = CompactionUtils + .buildFromFileSlices(Arrays.stream(fileSlices).map(fs -> Pair.of(fs.getPartitionPath(), fs)) + .collect(Collectors.toList()), Option.empty(), Option.empty()); + return addRequestedCompaction(instantTime, plan); + } + public HoodieTestTable addCompaction(String instantTime) throws IOException { createRequestedCompaction(basePath, instantTime); createInflightCompaction(basePath, instantTime); @@ -245,7 +266,7 @@ public class HoodieTestTable { * * @return A {@link Map} of partition and its newly inserted file's id. */ - public Map withBaseFilesInPartitions(String... partitions) throws Exception { + public Map getFileIdsWithBaseFilesInPartitions(String... partitions) throws Exception { Map partitionFileIdMap = new HashMap<>(); for (String p : partitions) { String fileId = UUID.randomUUID().toString(); @@ -277,7 +298,7 @@ public class HoodieTestTable { return this; } - public String withLogFile(String partitionPath) throws Exception { + public String getFileIdWithLogFile(String partitionPath) throws Exception { String fileId = UUID.randomUUID().toString(); withLogFile(partitionPath, fileId); return fileId; @@ -287,8 +308,10 @@ public class HoodieTestTable { return withLogFile(partitionPath, fileId, 0); } - public HoodieTestTable withLogFile(String partitionPath, String fileId, int version) throws Exception { - FileCreateUtils.createLogFile(basePath, partitionPath, currentInstantTime, fileId, version); + public HoodieTestTable withLogFile(String partitionPath, String fileId, int... versions) throws Exception { + for (int version : versions) { + FileCreateUtils.createLogFile(basePath, partitionPath, currentInstantTime, fileId, version); + } return this; } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index 1f86a5925..d94f41f91 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -18,9 +18,6 @@ package org.apache.hudi.common.testutils; -import org.apache.hudi.avro.model.HoodieCompactionPlan; -import org.apache.hudi.common.fs.FSUtils; -import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieTableType; @@ -28,27 +25,15 @@ import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.log.HoodieLogFormat; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.table.timeline.HoodieInstant.State; -import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; -import org.apache.hudi.common.util.CompactionUtils; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.Pair; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.JavaSerializer; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -122,66 +107,6 @@ public class HoodieTestUtils { return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties); } - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static void createCommitFiles(String basePath, String... instantTimes) throws IOException { - for (String instantTime : instantTimes) { - new File( - basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" - + HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile(); - new File( - basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" - + HoodieTimeline.makeInflightCommitFileName(instantTime)).createNewFile(); - new File( - basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCommitFileName(instantTime)) - .createNewFile(); - } - } - - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static String createDataFile(String basePath, String partitionPath, String instantTime, String fileID) - throws IOException { - String folderPath = basePath + "/" + partitionPath + "/"; - new File(folderPath).mkdirs(); - new File(folderPath + FSUtils.makeDataFileName(instantTime, DEFAULT_WRITE_TOKEN, fileID)).createNewFile(); - return fileID; - } - - /** - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static String createNewLogFile(FileSystem fs, String basePath, String partitionPath, String instantTime, - String fileID, Option version) throws IOException { - String folderPath = basePath + "/" + partitionPath + "/"; - boolean makeDir = fs.mkdirs(new Path(folderPath)); - if (!makeDir) { - throw new IOException("cannot create directory for path " + folderPath); - } - boolean createFile = fs.createNewFile(new Path(folderPath + FSUtils.makeLogFileName(fileID, ".log", instantTime, - version.orElse(DEFAULT_LOG_VERSION), HoodieLogFormat.UNKNOWN_WRITE_TOKEN))); - if (!createFile) { - throw new IOException( - StringUtils.format("cannot create data file for commit %s and fileId %s", instantTime, fileID)); - } - return fileID; - } - - /** - * TODO: incorporate into {@link HoodieTestTable}. - * - * @deprecated Use {@link HoodieTestTable} instead. - */ - public static void createCompactionRequest(HoodieTableMetaClient metaClient, String instant, - List> fileSliceList) throws IOException { - HoodieCompactionPlan plan = CompactionUtils.buildFromFileSlices(fileSliceList, Option.empty(), Option.empty()); - HoodieInstant compactionInstant = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instant); - metaClient.getActiveTimeline().saveToCompactionRequested(compactionInstant, - TimelineMetadataUtils.serializeCompactionPlan(plan)); - } - public static T serializeDeserialize(T object, Class clazz) { // Using Kyro as the default serializer in Spark Jobs Kryo kryo = new Kryo(); diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java index 5a6070fde..ba88df3e4 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java @@ -88,7 +88,7 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness { public void testPartitionPathsAsNonHoodiePaths() throws Exception { final String p1 = "2017/01/01"; final String p2 = "2017/01/02"; - testTable.addCommit("001").withBaseFilesInPartitions(p1, p2); + testTable.addCommit("001").getFileIdsWithBaseFilesInPartitions(p1, p2); Path partitionPath1 = testTable.getPartitionPath(p1).getParent(); Path partitionPath2 = testTable.getPartitionPath(p2).getParent(); assertTrue(pathFilter.accept(partitionPath1), "Directories should be accepted");