lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-402]: code clean up in test cases

Browse Source
This commit is contained in:
Pratyaksh Sharma
2019-12-29 15:00:06 +05:30
committed by vinoth chandar
parent 98c0d8cf60
commit dde21e7315
62 changed files with 320 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-client/src/test/java/org/apache/hudi/HoodieClientTestHarness.java
View File

@@ -190,18 +190,16 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im
/** /**
* Initializes a test data generator which used to generate test datas. * Initializes a test data generator which used to generate test datas.
* *
* @throws IOException
*/ */
protected void initTestDataGenerator() throws IOException { protected void initTestDataGenerator() {
dataGen = new HoodieTestDataGenerator(); dataGen = new HoodieTestDataGenerator();
} }
/** /**
* Cleanups test data generator. * Cleanups test data generator.
* *
* @throws IOException
*/ */
protected void cleanupTestDataGenerator() throws IOException { protected void cleanupTestDataGenerator() {
dataGen = null; dataGen = null;
} }

49
hudi-client/src/test/java/org/apache/hudi/TestCleaner.java
View File

@@ -365,7 +365,7 @@ public class TestCleaner extends TestHoodieClientBase {
insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn); insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn);
// Keep doing some writes and clean inline. Make sure we have expected number of files remaining. // Keep doing some writes and clean inline. Make sure we have expected number of files remaining.
HoodieTestUtils.monotonicIncreasingCommitTimestamps(8, 1).stream().forEach(newCommitTime -> { HoodieTestUtils.monotonicIncreasingCommitTimestamps(8, 1).forEach(newCommitTime -> {
try { try {
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
List<HoodieRecord> records = recordUpsertGenWrappedFunction.apply(newCommitTime, 100); List<HoodieRecord> records = recordUpsertGenWrappedFunction.apply(newCommitTime, 100);
@@ -441,7 +441,7 @@ public class TestCleaner extends TestHoodieClientBase {
Assert.assertEquals(new Integer(0), cleanMetadata2.getTotalFilesDeleted()); Assert.assertEquals(new Integer(0), cleanMetadata2.getTotalFilesDeleted());
Assert.assertEquals(cleanMetadata1.getPartitionMetadata().keySet(), Assert.assertEquals(cleanMetadata1.getPartitionMetadata().keySet(),
cleanMetadata2.getPartitionMetadata().keySet()); cleanMetadata2.getPartitionMetadata().keySet());
cleanMetadata1.getPartitionMetadata().keySet().stream().forEach(k -> { cleanMetadata1.getPartitionMetadata().keySet().forEach(k -> {
HoodieCleanPartitionMetadata p1 = cleanMetadata1.getPartitionMetadata().get(k); HoodieCleanPartitionMetadata p1 = cleanMetadata1.getPartitionMetadata().get(k);
HoodieCleanPartitionMetadata p2 = cleanMetadata2.getPartitionMetadata().get(k); HoodieCleanPartitionMetadata p2 = cleanMetadata2.getPartitionMetadata().get(k);
Assert.assertEquals(p1.getDeletePathPatterns(), p2.getDeletePathPatterns()); Assert.assertEquals(p1.getDeletePathPatterns(), p2.getDeletePathPatterns());
@@ -450,7 +450,8 @@ public class TestCleaner extends TestHoodieClientBase {
Assert.assertEquals(k, p1.getPartitionPath()); Assert.assertEquals(k, p1.getPartitionPath());
}); });
} }
List<HoodieCleanStat> stats = cleanMetadata1.getPartitionMetadata().values().stream()
return cleanMetadata1.getPartitionMetadata().values().stream()
.map(x -> new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath()) .map(x -> new HoodieCleanStat.Builder().withPartitionPath(x.getPartitionPath())
.withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles()) .withFailedDeletes(x.getFailedDeleteFiles()).withSuccessfulDeletes(x.getSuccessDeleteFiles())
.withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns()) .withPolicy(HoodieCleaningPolicy.valueOf(x.getPolicy())).withDeletePathPattern(x.getDeletePathPatterns())
@@ -459,8 +460,6 @@ public class TestCleaner extends TestHoodieClientBase {
: null)) : null))
.build()) .build())
.collect(Collectors.toList()); .collect(Collectors.toList());
return stats;
} }
/** /**
@@ -482,7 +481,6 @@ public class TestCleaner extends TestHoodieClientBase {
String file1P1C0 = String file1P1C0 =
HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000"); HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config); List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config);
assertEquals("Must not clean any files", 0, assertEquals("Must not clean any files", 0,
@@ -499,7 +497,6 @@ public class TestCleaner extends TestHoodieClientBase {
// make next commit, with 1 insert & 1 update per partition // make next commit, with 1 insert & 1 update per partition
HoodieTestUtils.createCommitFiles(basePath, "001"); HoodieTestUtils.createCommitFiles(basePath, "001");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc);
String file2P0C1 = String file2P0C1 =
HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001"); // insert HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001"); // insert
@@ -527,7 +524,6 @@ public class TestCleaner extends TestHoodieClientBase {
// make next commit, with 2 updates to existing files, and 1 insert // make next commit, with 2 updates to existing files, and 1 insert
HoodieTestUtils.createCommitFiles(basePath, "002"); HoodieTestUtils.createCommitFiles(basePath, "002");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update
HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update
@@ -567,7 +563,6 @@ public class TestCleaner extends TestHoodieClientBase {
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(1).build()) .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(1).build())
.build(); .build();
HoodieTableMetaClient metaClient =
HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ);
// Make 3 files, one base file and 2 log files associated with base file // Make 3 files, one base file and 2 log files associated with base file
@@ -575,23 +570,18 @@ public class TestCleaner extends TestHoodieClientBase {
HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000"); HoodieTestUtils.createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000");
String file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath, String file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.empty()); HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.empty());
String file2P0L1 = HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.of(2)); HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.of(2));
// make 1 compaction commit // make 1 compaction commit
HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "000"); HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "000");
// Make 4 files, one base file and 3 log files associated with base file // Make 4 files, one base file and 3 log files associated with base file
HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0); HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0);
file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
"001", file1P0, Option.empty());
file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
"001", file1P0, Option.of(2));
file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, file2P0L0 = HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
"001", file1P0, Option.of(3)); "001", file1P0, Option.of(3));
// make 1 compaction commit // make 1 compaction commit
HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "001"); HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "001");
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
List<HoodieCleanStat> hoodieCleanStats = runCleaner(config); List<HoodieCleanStat> hoodieCleanStats = runCleaner(config);
assertEquals("Must clean three files, one parquet and 2 log files", 3, assertEquals("Must clean three files, one parquet and 2 log files", 3,
getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles()
@@ -679,14 +669,14 @@ public class TestCleaner extends TestHoodieClientBase {
Assert.assertEquals(map1.keySet(), map2.keySet()); Assert.assertEquals(map1.keySet(), map2.keySet());
List<String> partitions1 = map1.values().stream().map(m -> m.getPartitionPath()).collect( List<String> partitions1 = map1.values().stream().map(HoodieCleanPartitionMetadata::getPartitionPath).collect(
Collectors.toList()); Collectors.toList());
List<String> partitions2 = map2.values().stream().map(m -> m.getPartitionPath()).collect( List<String> partitions2 = map2.values().stream().map(HoodieCleanPartitionMetadata::getPartitionPath).collect(
Collectors.toList()); Collectors.toList());
Assert.assertEquals(partitions1, partitions2); Assert.assertEquals(partitions1, partitions2);
List<String> policies1 = map1.values().stream().map(m -> m.getPolicy()).collect(Collectors.toList()); List<String> policies1 = map1.values().stream().map(HoodieCleanPartitionMetadata::getPolicy).collect(Collectors.toList());
List<String> policies2 = map2.values().stream().map(m -> m.getPolicy()).collect(Collectors.toList()); List<String> policies2 = map2.values().stream().map(HoodieCleanPartitionMetadata::getPolicy).collect(Collectors.toList());
Assert.assertEquals(policies1, policies2); Assert.assertEquals(policies1, policies2);
} }
@@ -758,7 +748,6 @@ public class TestCleaner extends TestHoodieClientBase {
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8))); Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config, simulateFailureRetry); List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config, simulateFailureRetry);
assertEquals("Must not clean any files", 0, assertEquals("Must not clean any files", 0,
@@ -775,7 +764,6 @@ public class TestCleaner extends TestHoodieClientBase {
// make next commit, with 1 insert & 1 update per partition // make next commit, with 1 insert & 1 update per partition
HoodieTestUtils.createInflightCommitFiles(basePath, "001"); HoodieTestUtils.createInflightCommitFiles(basePath, "001");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc);
String file2P0C1 = String file2P0C1 =
HoodieTestUtils HoodieTestUtils
@@ -815,7 +803,6 @@ public class TestCleaner extends TestHoodieClientBase {
// make next commit, with 2 updates to existing files, and 1 insert // make next commit, with 2 updates to existing files, and 1 insert
HoodieTestUtils.createInflightCommitFiles(basePath, "002"); HoodieTestUtils.createInflightCommitFiles(basePath, "002");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieTestUtils HoodieTestUtils
.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update
@@ -843,7 +830,6 @@ public class TestCleaner extends TestHoodieClientBase {
// make next commit, with 2 updates to existing files, and 1 insert // make next commit, with 2 updates to existing files, and 1 insert
HoodieTestUtils.createInflightCommitFiles(basePath, "003"); HoodieTestUtils.createInflightCommitFiles(basePath, "003");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieTestUtils HoodieTestUtils
.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file1P0C0); // update .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file1P0C0); // update
@@ -938,7 +924,6 @@ public class TestCleaner extends TestHoodieClientBase {
HoodieTestUtils.createCommitFiles(basePath, "000"); HoodieTestUtils.createCommitFiles(basePath, "000");
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config); List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config);
assertTrue("HoodieCleanStats should be empty for a table with empty partitionPaths", hoodieCleanStatsOne.isEmpty()); assertTrue("HoodieCleanStats should be empty for a table with empty partitionPaths", hoodieCleanStatsOne.isEmpty());
@@ -1078,15 +1063,13 @@ public class TestCleaner extends TestHoodieClientBase {
// Clean now // Clean now
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
List<HoodieCleanStat> hoodieCleanStats = runCleaner(config, retryFailure); List<HoodieCleanStat> hoodieCleanStats = runCleaner(config, retryFailure);
// Test for safety // Test for safety
final HoodieTableMetaClient newMetaClient = HoodieTableMetaClient.reload(metaClient); final HoodieTableMetaClient newMetaClient = HoodieTableMetaClient.reload(metaClient);
final HoodieTable hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc); final HoodieTable hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc);
expFileIdToPendingCompaction.entrySet().stream().forEach(entry -> { expFileIdToPendingCompaction.forEach((fileId, value) -> {
String fileId = entry.getKey();
String baseInstantForCompaction = fileIdToLatestInstantBeforeCompaction.get(fileId); String baseInstantForCompaction = fileIdToLatestInstantBeforeCompaction.get(fileId);
Option<FileSlice> fileSliceForCompaction = Option.fromJavaOptional(hoodieTable.getRTFileSystemView() Option<FileSlice> fileSliceForCompaction = Option.fromJavaOptional(hoodieTable.getRTFileSystemView()
.getLatestFileSlicesBeforeOrOn(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, baseInstantForCompaction, .getLatestFileSlicesBeforeOrOn(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, baseInstantForCompaction,
@@ -1112,7 +1095,7 @@ public class TestCleaner extends TestHoodieClientBase {
}); });
}).filter(x -> x).count(); }).filter(x -> x).count();
long numDeleted = long numDeleted =
hoodieCleanStats.stream().flatMap(cleanStat -> cleanStat.getDeletePathPatterns().stream()).count(); hoodieCleanStats.stream().mapToLong(cleanStat -> cleanStat.getDeletePathPatterns().size()).sum();
// Tighter check for regression // Tighter check for regression
Assert.assertEquals("Correct number of files deleted", expNumFilesDeleted, numDeleted); Assert.assertEquals("Correct number of files deleted", expNumFilesDeleted, numDeleted);
Assert.assertEquals("Correct number of files under compaction deleted", expNumFilesUnderCompactionDeleted, Assert.assertEquals("Correct number of files under compaction deleted", expNumFilesUnderCompactionDeleted,
@@ -1170,15 +1153,13 @@ public class TestCleaner extends TestHoodieClientBase {
private static HoodieCommitMetadata generateCommitMetadata(Map<String, List<String>> partitionToFilePaths) { private static HoodieCommitMetadata generateCommitMetadata(Map<String, List<String>> partitionToFilePaths) {
HoodieCommitMetadata metadata = new HoodieCommitMetadata(); HoodieCommitMetadata metadata = new HoodieCommitMetadata();
partitionToFilePaths.entrySet().forEach(e -> { partitionToFilePaths.forEach((key, value) -> value.forEach(f -> {
e.getValue().forEach(f -> {
HoodieWriteStat writeStat = new HoodieWriteStat(); HoodieWriteStat writeStat = new HoodieWriteStat();
writeStat.setPartitionPath(e.getKey()); writeStat.setPartitionPath(key);
writeStat.setPath(f); writeStat.setPath(f);
writeStat.setFileId(f); writeStat.setFileId(f);
metadata.addWriteStat(e.getKey(), writeStat); metadata.addWriteStat(key, writeStat);
}); }));
});
return metadata; return metadata;
} }
} }

6
hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java
View File

@@ -127,9 +127,7 @@ public class TestClientRollback extends TestHoodieClientBase {
table = HoodieTable.getHoodieTable(metaClient, getConfig(), jsc); table = HoodieTable.getHoodieTable(metaClient, getConfig(), jsc);
final ReadOptimizedView view2 = table.getROFileSystemView(); final ReadOptimizedView view2 = table.getROFileSystemView();
dataFiles = partitionPaths.stream().flatMap(s -> { dataFiles = partitionPaths.stream().flatMap(s -> view2.getAllDataFiles(s).filter(f -> f.getCommitTime().equals("004"))).collect(Collectors.toList());
return view2.getAllDataFiles(s).filter(f -> f.getCommitTime().equals("004"));
}).collect(Collectors.toList());
assertEquals("The data files for commit 004 should be present", 3, dataFiles.size()); assertEquals("The data files for commit 004 should be present", 3, dataFiles.size());
// rolling back to a non existent savepoint must not succeed // rolling back to a non existent savepoint must not succeed
@@ -205,7 +203,7 @@ public class TestClientRollback extends TestHoodieClientBase {
// Rollback commit 1 (this should fail, since commit2 is still around) // Rollback commit 1 (this should fail, since commit2 is still around)
try { try {
client.rollback(commitTime1); client.rollback(commitTime1);
assertTrue("Should have thrown an exception ", false); fail("Should have thrown an exception ");
} catch (HoodieRollbackException hrbe) { } catch (HoodieRollbackException hrbe) {
// should get here // should get here
} }

41
hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java
View File

@@ -20,6 +20,8 @@ package org.apache.hudi;
import org.apache.hudi.CompactionAdminClient.ValidationOpResult; import org.apache.hudi.CompactionAdminClient.ValidationOpResult;
import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -31,6 +33,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.func.OperationResult;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.junit.After; import org.junit.After;
@@ -39,12 +42,12 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ;
@@ -91,7 +94,7 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
CompactionTestUtils.setupAndValidateCompactionOperations(metaClient, false, numEntriesPerInstant, CompactionTestUtils.setupAndValidateCompactionOperations(metaClient, false, numEntriesPerInstant,
numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant); numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant);
Map<String, CompactionOperation> instantsWithOp = Map<String, CompactionOperation> instantsWithOp =
Arrays.asList("001", "003", "005", "007").stream().map(instant -> { Stream.of("001", "003", "005", "007").map(instant -> {
try { try {
return Pair.of(instant, CompactionUtils.getCompactionPlan(metaClient, instant)); return Pair.of(instant, CompactionUtils.getCompactionPlan(metaClient, instant));
} catch (IOException ioe) { } catch (IOException ioe) {
@@ -132,7 +135,7 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true);
List<ValidationOpResult> result = client.validateCompactionPlan(metaClient, compactionInstant, 1); List<ValidationOpResult> result = client.validateCompactionPlan(metaClient, compactionInstant, 1);
if (expNumRepairs > 0) { if (expNumRepairs > 0) {
Assert.assertTrue("Expect some failures in validation", result.stream().filter(r -> !r.isSuccess()).count() > 0); Assert.assertTrue("Expect some failures in validation", result.stream().anyMatch(r -> !r.isSuccess()));
} }
// Now repair // Now repair
List<Pair<HoodieLogFile, HoodieLogFile>> undoFiles = List<Pair<HoodieLogFile, HoodieLogFile>> undoFiles =
@@ -154,15 +157,13 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
} else { } else {
Assert.assertTrue("Rename Files must be empty", renameFiles.isEmpty()); Assert.assertTrue("Rename Files must be empty", renameFiles.isEmpty());
} }
expRenameFiles.entrySet().stream().forEach(r -> { expRenameFiles.forEach((key, value) -> LOG.info("Key :" + key + " renamed to " + value + " rolled back to "
LOG.info("Key :" + r.getKey() + " renamed to " + r.getValue() + " rolled back to " + renameFilesFromUndo.get(key)));
+ renameFilesFromUndo.get(r.getKey()));
});
Assert.assertEquals("Undo must completely rollback renames", expRenameFiles, renameFilesFromUndo); Assert.assertEquals("Undo must completely rollback renames", expRenameFiles, renameFilesFromUndo);
// Now expect validation to succeed // Now expect validation to succeed
result = client.validateCompactionPlan(metaClient, compactionInstant, 1); result = client.validateCompactionPlan(metaClient, compactionInstant, 1);
Assert.assertTrue("Expect no failures in validation", result.stream().filter(r -> !r.isSuccess()).count() == 0); Assert.assertTrue("Expect no failures in validation", result.stream().allMatch(OperationResult::isSuccess));
Assert.assertEquals("Expected Num Repairs", expNumRepairs, undoFiles.size()); Assert.assertEquals("Expected Num Repairs", expNumRepairs, undoFiles.size());
} }
@@ -176,7 +177,7 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
// Ensure compaction-plan is good to begin with // Ensure compaction-plan is good to begin with
List<ValidationOpResult> validationResults = client.validateCompactionPlan(metaClient, compactionInstant, 1); List<ValidationOpResult> validationResults = client.validateCompactionPlan(metaClient, compactionInstant, 1);
Assert.assertFalse("Some validations failed", Assert.assertFalse("Some validations failed",
validationResults.stream().filter(v -> !v.isSuccess()).findAny().isPresent()); validationResults.stream().anyMatch(v -> !v.isSuccess()));
} }
private void validateRenameFiles(List<Pair<HoodieLogFile, HoodieLogFile>> renameFiles, String ingestionInstant, private void validateRenameFiles(List<Pair<HoodieLogFile, HoodieLogFile>> renameFiles, String ingestionInstant,
@@ -185,14 +186,14 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
Set<HoodieLogFile> uniqNewLogFiles = new HashSet<>(); Set<HoodieLogFile> uniqNewLogFiles = new HashSet<>();
Set<HoodieLogFile> uniqOldLogFiles = new HashSet<>(); Set<HoodieLogFile> uniqOldLogFiles = new HashSet<>();
renameFiles.stream().forEach(lfPair -> { renameFiles.forEach(lfPair -> {
Assert.assertFalse("Old Log File Names do not collide", uniqOldLogFiles.contains(lfPair.getKey())); Assert.assertFalse("Old Log File Names do not collide", uniqOldLogFiles.contains(lfPair.getKey()));
Assert.assertFalse("New Log File Names do not collide", uniqNewLogFiles.contains(lfPair.getValue())); Assert.assertFalse("New Log File Names do not collide", uniqNewLogFiles.contains(lfPair.getValue()));
uniqOldLogFiles.add(lfPair.getKey()); uniqOldLogFiles.add(lfPair.getKey());
uniqNewLogFiles.add(lfPair.getValue()); uniqNewLogFiles.add(lfPair.getValue());
}); });
renameFiles.stream().forEach(lfPair -> { renameFiles.forEach(lfPair -> {
HoodieLogFile oldLogFile = lfPair.getLeft(); HoodieLogFile oldLogFile = lfPair.getLeft();
HoodieLogFile newLogFile = lfPair.getValue(); HoodieLogFile newLogFile = lfPair.getValue();
Assert.assertEquals("Base Commit time is expected", ingestionInstant, newLogFile.getBaseCommitTime()); Assert.assertEquals("Base Commit time is expected", ingestionInstant, newLogFile.getBaseCommitTime());
@@ -234,18 +235,18 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
// Log files belonging to file-slices created because of compaction request must be renamed // Log files belonging to file-slices created because of compaction request must be renamed
Set<HoodieLogFile> gotLogFilesToBeRenamed = renameFiles.stream().map(p -> p.getLeft()).collect(Collectors.toSet()); Set<HoodieLogFile> gotLogFilesToBeRenamed = renameFiles.stream().map(Pair::getLeft).collect(Collectors.toSet());
final HoodieTableFileSystemView fsView = final HoodieTableFileSystemView fsView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline()); new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
Set<HoodieLogFile> expLogFilesToBeRenamed = fsView.getLatestFileSlices(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]) Set<HoodieLogFile> expLogFilesToBeRenamed = fsView.getLatestFileSlices(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0])
.filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)).flatMap(fs -> fs.getLogFiles()) .filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)).flatMap(FileSlice::getLogFiles)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
Assert.assertEquals("Log files belonging to file-slices created because of compaction request must be renamed", Assert.assertEquals("Log files belonging to file-slices created because of compaction request must be renamed",
expLogFilesToBeRenamed, gotLogFilesToBeRenamed); expLogFilesToBeRenamed, gotLogFilesToBeRenamed);
if (skipUnSchedule) { if (skipUnSchedule) {
// Do the renaming only but do not touch the compaction plan - Needed for repair tests // Do the renaming only but do not touch the compaction plan - Needed for repair tests
renameFiles.stream().forEach(lfPair -> { renameFiles.forEach(lfPair -> {
try { try {
client.renameLogFile(metaClient, lfPair.getLeft(), lfPair.getRight()); client.renameLogFile(metaClient, lfPair.getLeft(), lfPair.getRight());
} catch (IOException e) { } catch (IOException e) {
@@ -273,12 +274,12 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
newFsView.getLatestFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant, true) newFsView.getLatestFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant, true)
.filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)).forEach(fs -> { .filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)).forEach(fs -> {
Assert.assertFalse("No Data file must be present", fs.getDataFile().isPresent()); Assert.assertFalse("No Data file must be present", fs.getDataFile().isPresent());
Assert.assertTrue("No Log Files", fs.getLogFiles().count() == 0); Assert.assertEquals("No Log Files", 0, fs.getLogFiles().count());
}); });
// Ensure same number of log-files before and after renaming per fileId // Ensure same number of log-files before and after renaming per fileId
Map<String, Long> fileIdToCountsAfterRenaming = Map<String, Long> fileIdToCountsAfterRenaming =
newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(fg -> fg.getAllFileSlices()) newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(HoodieFileGroup::getAllFileSlices)
.filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant)) .filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant))
.map(fs -> Pair.of(fs.getFileId(), fs.getLogFiles().count())) .map(fs -> Pair.of(fs.getFileId(), fs.getLogFiles().count()))
.collect(Collectors.toMap(Pair::getKey, Pair::getValue)); .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
@@ -305,12 +306,12 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
// Log files belonging to file-slices created because of compaction request must be renamed // Log files belonging to file-slices created because of compaction request must be renamed
Set<HoodieLogFile> gotLogFilesToBeRenamed = renameFiles.stream().map(p -> p.getLeft()).collect(Collectors.toSet()); Set<HoodieLogFile> gotLogFilesToBeRenamed = renameFiles.stream().map(Pair::getLeft).collect(Collectors.toSet());
final HoodieTableFileSystemView fsView = final HoodieTableFileSystemView fsView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline()); new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
Set<HoodieLogFile> expLogFilesToBeRenamed = fsView.getLatestFileSlices(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]) Set<HoodieLogFile> expLogFilesToBeRenamed = fsView.getLatestFileSlices(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0])
.filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)) .filter(fs -> fs.getBaseInstantTime().equals(compactionInstant))
.filter(fs -> fs.getFileId().equals(op.getFileId())).flatMap(fs -> fs.getLogFiles()) .filter(fs -> fs.getFileId().equals(op.getFileId())).flatMap(FileSlice::getLogFiles)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
Assert.assertEquals("Log files belonging to file-slices created because of compaction request must be renamed", Assert.assertEquals("Log files belonging to file-slices created because of compaction request must be renamed",
expLogFilesToBeRenamed, gotLogFilesToBeRenamed); expLogFilesToBeRenamed, gotLogFilesToBeRenamed);
@@ -334,12 +335,12 @@ public class TestCompactionAdminClient extends TestHoodieClientBase {
.filter(fs -> fs.getBaseInstantTime().equals(compactionInstant)) .filter(fs -> fs.getBaseInstantTime().equals(compactionInstant))
.filter(fs -> fs.getFileId().equals(op.getFileId())).forEach(fs -> { .filter(fs -> fs.getFileId().equals(op.getFileId())).forEach(fs -> {
Assert.assertFalse("No Data file must be present", fs.getDataFile().isPresent()); Assert.assertFalse("No Data file must be present", fs.getDataFile().isPresent());
Assert.assertTrue("No Log Files", fs.getLogFiles().count() == 0); Assert.assertEquals("No Log Files", 0, fs.getLogFiles().count());
}); });
// Ensure same number of log-files before and after renaming per fileId // Ensure same number of log-files before and after renaming per fileId
Map<String, Long> fileIdToCountsAfterRenaming = Map<String, Long> fileIdToCountsAfterRenaming =
newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(fg -> fg.getAllFileSlices()) newFsView.getAllFileGroups(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0]).flatMap(HoodieFileGroup::getAllFileSlices)
.filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant)) .filter(fs -> fs.getBaseInstantTime().equals(ingestionInstant))
.filter(fs -> fs.getFileId().equals(op.getFileId())) .filter(fs -> fs.getFileId().equals(op.getFileId()))
.map(fs -> Pair.of(fs.getFileId(), fs.getLogFiles().count())) .map(fs -> Pair.of(fs.getFileId(), fs.getLogFiles().count()))

55
hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java
View File

@@ -26,7 +26,6 @@ import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.SyncableFileSystemView; import org.apache.hudi.common.table.SyncableFileSystemView;
@@ -54,12 +53,12 @@ import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.function.Function;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
@@ -217,8 +216,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
if (!partitionToKeys.containsKey(partitionPath)) { if (!partitionToKeys.containsKey(partitionPath)) {
partitionToKeys.put(partitionPath, new HashSet<>()); partitionToKeys.put(partitionPath, new HashSet<>());
} }
assertTrue("key " + key + " is duplicate within partition " + partitionPath, assertFalse("key " + key + " is duplicate within partition " + partitionPath, partitionToKeys.get(partitionPath).contains(key));
!partitionToKeys.get(partitionPath).contains(key));
partitionToKeys.get(partitionPath).add(key); partitionToKeys.get(partitionPath).add(key);
} }
} }
@@ -253,11 +251,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
* @param keyGenFunction Keys Generation function * @param keyGenFunction Keys Generation function
* @return Wrapped function * @return Wrapped function
*/ */
private Function2<List<HoodieKey>, String, Integer> wrapDeleteKeysGenFunctionForPreppedCalls( private Function<Integer, List<HoodieKey>> wrapDeleteKeysGenFunctionForPreppedCalls(
final HoodieWriteConfig writeConfig, final Function2<List<HoodieKey>, String, Integer> keyGenFunction) { final HoodieWriteConfig writeConfig, final Function<Integer, List<HoodieKey>> keyGenFunction) {
return (commit, numRecords) -> { return (numRecords) -> {
final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc); final HoodieIndex index = HoodieIndex.createIndex(writeConfig, jsc);
List<HoodieKey> records = keyGenFunction.apply(commit, numRecords); List<HoodieKey> records = keyGenFunction.apply(numRecords);
final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true); final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, writeConfig, jsc); HoodieTable table = HoodieTable.getHoodieTable(metaClient, writeConfig, jsc);
JavaRDD<HoodieRecord> recordsToDelete = jsc.parallelize(records, 1) JavaRDD<HoodieRecord> recordsToDelete = jsc.parallelize(records, 1)
@@ -292,8 +290,8 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
* @param wrapped Actual Records Generation function * @param wrapped Actual Records Generation function
* @return Wrapped Function * @return Wrapped Function
*/ */
Function2<List<HoodieKey>, String, Integer> generateWrapDeleteKeysFn(boolean isPreppedAPI, Function<Integer, List<HoodieKey>> generateWrapDeleteKeysFn(boolean isPreppedAPI,
HoodieWriteConfig writeConfig, Function2<List<HoodieKey>, String, Integer> wrapped) { HoodieWriteConfig writeConfig, Function<Integer, List<HoodieKey>> wrapped) {
if (isPreppedAPI) { if (isPreppedAPI) {
return wrapDeleteKeysGenFunctionForPreppedCalls(writeConfig, wrapped); return wrapDeleteKeysGenFunctionForPreppedCalls(writeConfig, wrapped);
} else { } else {
@@ -381,7 +379,7 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
int numRecordsInThisCommit, int numRecordsInThisCommit,
Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieKey>, String> deleteFn, boolean isPreppedAPI, Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieKey>, String> deleteFn, boolean isPreppedAPI,
boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception {
final Function2<List<HoodieKey>, String, Integer> keyGenFunction = final Function<Integer, List<HoodieKey>> keyGenFunction =
generateWrapDeleteKeysFn(isPreppedAPI, writeConfig, dataGen::generateUniqueDeletes); generateWrapDeleteKeysFn(isPreppedAPI, writeConfig, dataGen::generateUniqueDeletes);
return deleteBatch(client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit, return deleteBatch(client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit,
@@ -476,14 +474,14 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
*/ */
JavaRDD<WriteStatus> deleteBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime, JavaRDD<WriteStatus> deleteBatch(HoodieWriteClient client, String newCommitTime, String prevCommitTime,
String initCommitTime, int numRecordsInThisCommit, String initCommitTime, int numRecordsInThisCommit,
Function2<List<HoodieKey>, String, Integer> keyGenFunction, Function<Integer, List<HoodieKey>> keyGenFunction,
Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieKey>, String> deleteFn, Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieKey>, String> deleteFn,
boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception { boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords) throws Exception {
// Delete 1 (only deletes) // Delete 1 (only deletes)
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
List<HoodieKey> keysToDelete = keyGenFunction.apply(newCommitTime, numRecordsInThisCommit); List<HoodieKey> keysToDelete = keyGenFunction.apply(numRecordsInThisCommit);
JavaRDD<HoodieKey> deleteRecords = jsc.parallelize(keysToDelete, 1); JavaRDD<HoodieKey> deleteRecords = jsc.parallelize(keysToDelete, 1);
JavaRDD<WriteStatus> result = deleteFn.apply(client, deleteRecords, newCommitTime); JavaRDD<WriteStatus> result = deleteFn.apply(client, deleteRecords, newCommitTime);
@@ -533,37 +531,6 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
return hoodieCleanStatsTwo.stream().filter(e -> e.getPartitionPath().equals(partitionPath)).findFirst().orElse(null); return hoodieCleanStatsTwo.stream().filter(e -> e.getPartitionPath().equals(partitionPath)).findFirst().orElse(null);
} }
/**
* Utility to simulate commit touching files in a partition.
*
* @param files List of file-Ids to be touched
* @param partitionPath Partition
* @param commitTime Commit Timestamp
* @throws IOException in case of error
*/
void updateAllFilesInPartition(List<String> files, String partitionPath, String commitTime) throws IOException {
for (String fileId : files) {
HoodieTestUtils.createDataFile(basePath, partitionPath, commitTime, fileId);
}
}
/**
* Helper methods to create new data files in a partition.
*
* @param partitionPath Partition
* @param commitTime Commit Timestamp
* @param numFiles Number of files to be added
* @return Created files
* @throws IOException in case of error
*/
List<String> createFilesInPartition(String partitionPath, String commitTime, int numFiles) throws IOException {
List<String> files = new ArrayList<>();
for (int i = 0; i < numFiles; i++) {
files.add(HoodieTestUtils.createNewDataFile(basePath, partitionPath, commitTime));
}
return files;
}
// Functional Interfaces for passing lambda and Hoodie Write API contexts // Functional Interfaces for passing lambda and Hoodie Write API contexts
@FunctionalInterface @FunctionalInterface

8
hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -237,7 +237,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
* @return Hoodie Write Client * @return Hoodie Write Client
* @throws Exception in case of error * @throws Exception in case of error
*/ */
private HoodieWriteClient getWriteClientWithDummyIndex(final boolean isGlobal) throws Exception { private HoodieWriteClient getWriteClientWithDummyIndex(final boolean isGlobal) {
HoodieIndex index = mock(HoodieIndex.class); HoodieIndex index = mock(HoodieIndex.class);
when(index.isGlobal()).thenReturn(isGlobal); when(index.isGlobal()).thenReturn(isGlobal);
return getHoodieWriteClient(getConfigBuilder().build(), false, index); return getHoodieWriteClient(getConfigBuilder().build(), false, index);
@@ -637,7 +637,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
public void testDeletesWithDeleteApi() throws Exception { public void testDeletesWithDeleteApi() throws Exception {
final String testPartitionPath = "2016/09/26"; final String testPartitionPath = "2016/09/26";
final int insertSplitLimit = 100; final int insertSplitLimit = 100;
List<String> keysSoFar = new ArrayList<>();
// setup the small file handling params // setup the small file handling params
HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit); // hold upto 200 records max HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit); // hold upto 200 records max
dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath}); dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath});
@@ -649,7 +648,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
client.startCommitWithTime(commitTime1); client.startCommitWithTime(commitTime1);
List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, insertSplitLimit); // this writes ~500kb List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, insertSplitLimit); // this writes ~500kb
Set<String> keys1 = HoodieClientTestUtils.getRecordKeys(inserts1); Set<String> keys1 = HoodieClientTestUtils.getRecordKeys(inserts1);
keysSoFar.addAll(keys1); List<String> keysSoFar = new ArrayList<>(keys1);
JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1); JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1).collect(); List<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1).collect();
@@ -763,7 +762,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
* Test delete with delete api. * Test delete with delete api.
*/ */
@Test @Test
public void testDeletesWithoutInserts() throws Exception { public void testDeletesWithoutInserts() {
final String testPartitionPath = "2016/09/26"; final String testPartitionPath = "2016/09/26";
final int insertSplitLimit = 100; final int insertSplitLimit = 100;
// setup the small file handling params // setup the small file handling params
@@ -844,7 +843,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build(); HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build();
HoodieWriteClient client = getHoodieWriteClient(cfg); HoodieWriteClient client = getHoodieWriteClient(cfg);
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
String commitTime = "000"; String commitTime = "000";
client.startCommitWithTime(commitTime); client.startCommitWithTime(commitTime);

7
hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java
View File

@@ -23,7 +23,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.util.Arrays; import java.util.Arrays;
@@ -31,7 +30,7 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals;
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
/** /**
@@ -93,7 +92,7 @@ public class TestHoodieReadClient extends TestHoodieClientBase {
JavaRDD<HoodieRecord> filteredRDD = readClient.filterExists(recordsRDD); JavaRDD<HoodieRecord> filteredRDD = readClient.filterExists(recordsRDD);
// Should not find any files // Should not find any files
assertTrue(filteredRDD.collect().size() == 100); assertEquals(100, filteredRDD.collect().size());
JavaRDD<HoodieRecord> smallRecordsRDD = jsc.parallelize(records.subList(0, 75), 1); JavaRDD<HoodieRecord> smallRecordsRDD = jsc.parallelize(records.subList(0, 75), 1);
// We create three parquet file, each having one record. (3 different partitions) // We create three parquet file, each having one record. (3 different partitions)
@@ -105,7 +104,7 @@ public class TestHoodieReadClient extends TestHoodieClientBase {
filteredRDD = anotherReadClient.filterExists(recordsRDD); filteredRDD = anotherReadClient.filterExists(recordsRDD);
List<HoodieRecord> result = filteredRDD.collect(); List<HoodieRecord> result = filteredRDD.collect();
// Check results // Check results
Assert.assertEquals(25, result.size()); assertEquals(25, result.size());
} }
} }
} }

4
hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java
View File

@@ -23,14 +23,12 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import java.io.IOException;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
public class TestWriteStatus { public class TestWriteStatus {
@Test @Test
public void testFailureFraction() throws IOException { public void testFailureFraction() {
WriteStatus status = new WriteStatus(true, 0.1); WriteStatus status = new WriteStatus(true, 0.1);
Throwable t = new Exception("some error in writing"); Throwable t = new Exception("some error in writing");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {

2
hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java
View File

@@ -164,7 +164,7 @@ public class HoodieClientTestUtils {
String commitTime) { String commitTime) {
HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime); HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTime);
if (!commitTimeline.containsInstant(commitInstant)) { if (!commitTimeline.containsInstant(commitInstant)) {
new HoodieException("No commit exists at " + commitTime); throw new HoodieException("No commit exists at " + commitTime);
} }
try { try {
HashMap<String, String> paths = HashMap<String, String> paths =

3
hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java
View File

@@ -45,8 +45,7 @@ import java.util.stream.Collectors;
*/ */
public class HoodieMergeOnReadTestUtils { public class HoodieMergeOnReadTestUtils {
public static List<GenericRecord> getRecordsUsingInputFormat(List<String> inputPaths, String basePath) public static List<GenericRecord> getRecordsUsingInputFormat(List<String> inputPaths, String basePath) {
throws IOException {
JobConf jobConf = new JobConf(); JobConf jobConf = new JobConf();
Schema schema = HoodieAvroUtils.addMetadataFields( Schema schema = HoodieAvroUtils.addMetadataFields(
new Schema.Parser().parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)); new Schema.Parser().parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA));

22
hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
View File

@@ -24,7 +24,6 @@ import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -121,7 +120,7 @@ public class HoodieTestDataGenerator {
/** /**
* Generates a new avro record of the above schema format, retaining the key if optionally provided. * Generates a new avro record of the above schema format, retaining the key if optionally provided.
*/ */
public static HoodieAvroPayload generateAvroPayload(HoodieKey key, String commitTime) throws IOException { public static HoodieAvroPayload generateAvroPayload(HoodieKey key, String commitTime) {
GenericRecord rec = generateGenericRecord(key.getRecordKey(), "rider-" + commitTime, "driver-" + commitTime, 0.0); GenericRecord rec = generateGenericRecord(key.getRecordKey(), "rider-" + commitTime, "driver-" + commitTime, 0.0);
return new HoodieAvroPayload(Option.of(rec)); return new HoodieAvroPayload(Option.of(rec));
} }
@@ -141,10 +140,6 @@ public class HoodieTestDataGenerator {
return rec; return rec;
} }
public static void createCommitFile(String basePath, String commitTime) throws IOException {
createCommitFile(basePath, commitTime, HoodieTestUtils.getDefaultHadoopConf());
}
public static void createCommitFile(String basePath, String commitTime, Configuration configuration) { public static void createCommitFile(String basePath, String commitTime, Configuration configuration) {
Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime), Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
HoodieTimeline.makeRequestedCommitFileName(commitTime)).forEach(f -> { HoodieTimeline.makeRequestedCommitFileName(commitTime)).forEach(f -> {
@@ -213,7 +208,7 @@ public class HoodieTestDataGenerator {
/** /**
* Generates new inserts, uniformly across the partition paths above. It also updates the list of existing keys. * Generates new inserts, uniformly across the partition paths above. It also updates the list of existing keys.
*/ */
public List<HoodieRecord> generateInserts(String commitTime, Integer n) throws IOException { public List<HoodieRecord> generateInserts(String commitTime, Integer n) {
return generateInsertsStream(commitTime, n).collect(Collectors.toList()); return generateInsertsStream(commitTime, n).collect(Collectors.toList());
} }
@@ -249,7 +244,7 @@ public class HoodieTestDataGenerator {
return copy; return copy;
} }
public List<HoodieRecord> generateInsertsWithHoodieAvroPayload(String commitTime, int limit) throws IOException { public List<HoodieRecord> generateInsertsWithHoodieAvroPayload(String commitTime, int limit) {
List<HoodieRecord> inserts = new ArrayList<>(); List<HoodieRecord> inserts = new ArrayList<>();
int currSize = getNumExistingKeys(); int currSize = getNumExistingKeys();
for (int i = 0; i < limit; i++) { for (int i = 0; i < limit; i++) {
@@ -267,8 +262,7 @@ public class HoodieTestDataGenerator {
return inserts; return inserts;
} }
public List<HoodieRecord> generateUpdatesWithHoodieAvroPayload(String commitTime, List<HoodieRecord> baseRecords) public List<HoodieRecord> generateUpdatesWithHoodieAvroPayload(String commitTime, List<HoodieRecord> baseRecords) {
throws IOException {
List<HoodieRecord> updates = new ArrayList<>(); List<HoodieRecord> updates = new ArrayList<>();
for (HoodieRecord baseRecord : baseRecords) { for (HoodieRecord baseRecord : baseRecords) {
HoodieRecord record = new HoodieRecord(baseRecord.getKey(), generateAvroPayload(baseRecord.getKey(), commitTime)); HoodieRecord record = new HoodieRecord(baseRecord.getKey(), generateAvroPayload(baseRecord.getKey(), commitTime));
@@ -365,12 +359,11 @@ public class HoodieTestDataGenerator {
/** /**
* Generates deduped delete of keys previously inserted, randomly distributed across the keys above. * Generates deduped delete of keys previously inserted, randomly distributed across the keys above.
* *
* @param commitTime Commit Timestamp
* @param n Number of unique records * @param n Number of unique records
* @return list of hoodie record updates * @return list of hoodie record updates
*/ */
public List<HoodieKey> generateUniqueDeletes(String commitTime, Integer n) { public List<HoodieKey> generateUniqueDeletes(Integer n) {
return generateUniqueDeleteStream(commitTime, n).collect(Collectors.toList()); return generateUniqueDeleteStream(n).collect(Collectors.toList());
} }
/** /**
@@ -407,11 +400,10 @@ public class HoodieTestDataGenerator {
/** /**
* Generates deduped delete of keys previously inserted, randomly distributed across the keys above. * Generates deduped delete of keys previously inserted, randomly distributed across the keys above.
* *
* @param commitTime Commit Timestamp
* @param n Number of unique records * @param n Number of unique records
* @return stream of hoodie record updates * @return stream of hoodie record updates
*/ */
public Stream<HoodieKey> generateUniqueDeleteStream(String commitTime, Integer n) { public Stream<HoodieKey> generateUniqueDeleteStream(Integer n) {
final Set<KeyPartition> used = new HashSet<>(); final Set<KeyPartition> used = new HashSet<>();
if (n > numExistingKeys) { if (n > numExistingKeys) {

2
hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java
View File

@@ -56,7 +56,7 @@ public class TestBoundedInMemoryExecutor extends HoodieClientTestHarness {
} }
@Test @Test
public void testExecutor() throws Exception { public void testExecutor() {
final List<HoodieRecord> hoodieRecords = dataGen.generateInserts(commitTime, 100); final List<HoodieRecord> hoodieRecords = dataGen.generateInserts(commitTime, 100);

2
hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java
View File

@@ -221,7 +221,7 @@ public class TestBoundedInMemoryQueue extends HoodieClientTestHarness {
new BoundedInMemoryQueue(memoryLimitInBytes, getTransformFunction(HoodieTestDataGenerator.avroSchema)); new BoundedInMemoryQueue(memoryLimitInBytes, getTransformFunction(HoodieTestDataGenerator.avroSchema));
// Produce // Produce
Future<Boolean> resFuture = executorService.submit(() -> { executorService.submit(() -> {
new IteratorBasedQueueProducer<>(hoodieRecords.iterator()).produce(queue); new IteratorBasedQueueProducer<>(hoodieRecords.iterator()).produce(queue);
return true; return true;
}); });

2
hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java
View File

@@ -35,7 +35,6 @@ import org.apache.hudi.io.HoodieCreateHandle;
import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.hudi.table.HoodieCopyOnWriteTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@@ -102,7 +101,6 @@ public class TestUpdateMapFunction extends HoodieClientTestHarness {
// Now try an update with an evolved schema // Now try an update with an evolved schema
// Evolved schema does not have guarantee on preserving the original field ordering // Evolved schema does not have guarantee on preserving the original field ordering
final HoodieWriteConfig config2 = makeHoodieClientConfig("/exampleEvolvedSchema.txt"); final HoodieWriteConfig config2 = makeHoodieClientConfig("/exampleEvolvedSchema.txt");
final Schema schema = new Schema.Parser().parse(config2.getSchema());
final WriteStatus insertResult = statuses.get(0); final WriteStatus insertResult = statuses.get(0);
String fileId = insertResult.getFileId(); String fileId = insertResult.getFileId();

28
hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java
View File

@@ -62,6 +62,7 @@ import java.util.List;
import scala.Tuple2; import scala.Tuple2;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyObject;
@@ -80,7 +81,7 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
private static Configuration hbaseConfig; private static Configuration hbaseConfig;
private static String tableName = "test_table"; private static String tableName = "test_table";
public TestHbaseIndex() throws Exception {} public TestHbaseIndex() {}
@AfterClass @AfterClass
public static void clean() throws Exception { public static void clean() throws Exception {
@@ -155,10 +156,10 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc); hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc);
javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable); javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable);
assertTrue(javaRDD.filter(record -> record.isCurrentLocationKnown()).collect().size() == 200); assertEquals(200, javaRDD.filter(record -> record.isCurrentLocationKnown()).collect().size());
assertTrue(javaRDD.map(record -> record.getKey().getRecordKey()).distinct().count() == 200); assertEquals(200, javaRDD.map(record -> record.getKey().getRecordKey()).distinct().count());
assertTrue(javaRDD.filter(record -> (record.getCurrentLocation() != null assertEquals(200, javaRDD.filter(record -> (record.getCurrentLocation() != null
&& record.getCurrentLocation().getInstantTime().equals(newCommitTime))).distinct().count() == 200); && record.getCurrentLocation().getInstantTime().equals(newCommitTime))).distinct().count());
} }
} }
@@ -194,10 +195,10 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc); hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc);
JavaRDD<HoodieRecord> javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable); JavaRDD<HoodieRecord> javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable);
assertTrue(javaRDD.filter(record -> record.isCurrentLocationKnown()).collect().size() == 10); assertEquals(10, javaRDD.filter(HoodieRecord::isCurrentLocationKnown).collect().size());
assertTrue(javaRDD.map(record -> record.getKey().getRecordKey()).distinct().count() == 10); assertEquals(10, javaRDD.map(record -> record.getKey().getRecordKey()).distinct().count());
assertTrue(javaRDD.filter(record -> (record.getCurrentLocation() != null assertEquals(10, javaRDD.filter(record -> (record.getCurrentLocation() != null
&& record.getCurrentLocation().getInstantTime().equals(newCommitTime))).distinct().count() == 10); && record.getCurrentLocation().getInstantTime().equals(newCommitTime))).distinct().count());
} }
@Test @Test
@@ -211,7 +212,6 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200);
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1); JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc);
// Insert 200 records // Insert 200 records
JavaRDD<WriteStatus> writeStatues = writeClient.upsert(writeRecords, newCommitTime); JavaRDD<WriteStatus> writeStatues = writeClient.upsert(writeRecords, newCommitTime);
@@ -219,13 +219,13 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
// commit this upsert // commit this upsert
writeClient.commit(newCommitTime, writeStatues); writeClient.commit(newCommitTime, writeStatues);
hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieTable hoodieTable = HoodieTable.getHoodieTable(metaClient, config, jsc);
// Now tagLocation for these records, hbaseIndex should tag them // Now tagLocation for these records, hbaseIndex should tag them
JavaRDD<HoodieRecord> javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable); JavaRDD<HoodieRecord> javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable);
assert (javaRDD.filter(record -> record.isCurrentLocationKnown()).collect().size() == 200); assert (javaRDD.filter(HoodieRecord::isCurrentLocationKnown).collect().size() == 200);
// check tagged records are tagged with correct fileIds // check tagged records are tagged with correct fileIds
List<String> fileIds = writeStatues.map(status -> status.getFileId()).collect(); List<String> fileIds = writeStatues.map(WriteStatus::getFileId).collect();
assert (javaRDD.filter(record -> record.getCurrentLocation().getFileId() == null).collect().size() == 0); assert (javaRDD.filter(record -> record.getCurrentLocation().getFileId() == null).collect().size() == 0);
List<String> taggedFileIds = javaRDD.map(record -> record.getCurrentLocation().getFileId()).distinct().collect(); List<String> taggedFileIds = javaRDD.map(record -> record.getCurrentLocation().getFileId()).distinct().collect();
@@ -238,7 +238,7 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
// Now tagLocation for these records, hbaseIndex should not tag them since it was a rolled // Now tagLocation for these records, hbaseIndex should not tag them since it was a rolled
// back commit // back commit
javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable); javaRDD = index.tagLocation(writeRecords, jsc, hoodieTable);
assert (javaRDD.filter(record -> record.isCurrentLocationKnown()).collect().size() == 0); assert (javaRDD.filter(HoodieRecord::isCurrentLocationKnown).collect().size() == 0);
assert (javaRDD.filter(record -> record.getCurrentLocation() != null).collect().size() == 0); assert (javaRDD.filter(record -> record.getCurrentLocation() != null).collect().size() == 0);
} }

2
hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java
View File

@@ -49,7 +49,7 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
} }
@Test @Test
public void testCreateIndex() throws Exception { public void testCreateIndex() {
HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder(); HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder(); HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
// Different types // Different types

5
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
View File

@@ -110,12 +110,11 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
} }
private HoodieWriteConfig makeConfig() { private HoodieWriteConfig makeConfig() {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath) return HoodieWriteConfig.newBuilder().withPath(basePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().bloomIndexPruneByRanges(rangePruning) .withIndexConfig(HoodieIndexConfig.newBuilder().bloomIndexPruneByRanges(rangePruning)
.bloomIndexTreebasedFilter(treeFiltering).bloomIndexBucketizedChecking(bucketizedChecking) .bloomIndexTreebasedFilter(treeFiltering).bloomIndexBucketizedChecking(bucketizedChecking)
.bloomIndexKeysPerBucket(2).build()) .bloomIndexKeysPerBucket(2).build())
.build(); .build();
return config;
} }
@Test @Test
@@ -278,7 +277,7 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
} }
@Test @Test
public void testTagLocationWithEmptyRDD() throws Exception { public void testTagLocationWithEmptyRDD() {
// We have some records to be tagged (two different partitions) // We have some records to be tagged (two different partitions)
JavaRDD<HoodieRecord> recordRDD = jsc.emptyRDD(); JavaRDD<HoodieRecord> recordRDD = jsc.emptyRDD();
// Also create the metadata and config // Also create the metadata and config

2
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
View File

@@ -61,7 +61,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness {
private String schemaStr; private String schemaStr;
private Schema schema; private Schema schema;
public TestHoodieGlobalBloomIndex() throws Exception { public TestHoodieGlobalBloomIndex() {
} }
@Before @Before

3
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java
View File

@@ -28,7 +28,6 @@ import java.util.Random;
import java.util.UUID; import java.util.UUID;
import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
/** /**
* Tests {@link KeyRangeLookupTree}. * Tests {@link KeyRangeLookupTree}.
@@ -152,7 +151,7 @@ public class TestKeyRangeLookupTree {
if (!expectedMatches.containsKey(iStr)) { if (!expectedMatches.containsKey(iStr)) {
assertEquals(Collections.EMPTY_SET, keyRangeLookupTree.getMatchingIndexFiles(iStr)); assertEquals(Collections.EMPTY_SET, keyRangeLookupTree.getMatchingIndexFiles(iStr));
} else { } else {
assertTrue(expectedMatches.get(iStr).equals(keyRangeLookupTree.getMatchingIndexFiles(iStr))); assertEquals(expectedMatches.get(iStr), keyRangeLookupTree.getMatchingIndexFiles(iStr));
} }
} }
} }

2
hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java
View File

@@ -147,7 +147,7 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness {
HoodieTestUtils.createCleanFiles(metaClient, basePath, "103", dfs.getConf()); HoodieTestUtils.createCleanFiles(metaClient, basePath, "103", dfs.getConf());
HoodieTestUtils.createCleanFiles(metaClient, basePath, "104", dfs.getConf()); HoodieTestUtils.createCleanFiles(metaClient, basePath, "104", dfs.getConf());
HoodieTestUtils.createCleanFiles(metaClient, basePath, "105", dfs.getConf()); HoodieTestUtils.createCleanFiles(metaClient, basePath, "105", dfs.getConf());
HoodieTestUtils.createPendingCleanFiles(metaClient, dfs.getConf(), "106", "107"); HoodieTestUtils.createPendingCleanFiles(metaClient, "106", "107");
// reload the timeline and get all the commmits before archive // reload the timeline and get all the commmits before archive
timeline = metaClient.getActiveTimeline().reload().getAllCommitsTimeline().filterCompletedInstants(); timeline = metaClient.getActiveTimeline().reload().getAllCommitsTimeline().filterCompletedInstants();

2
hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java
View File

@@ -34,7 +34,6 @@ import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@@ -250,7 +249,6 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness {
// Update all the 100 records // Update all the 100 records
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
newCommitTime = "101"; newCommitTime = "101";
writeClient.startCommitWithTime(newCommitTime); writeClient.startCommitWithTime(newCommitTime);

8
hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java
View File

@@ -87,7 +87,7 @@ public class TestHoodieCompactionStrategy {
assertEquals("BoundedIOCompaction should have resulted in 2 compactions being chosen", 2, returned.size()); assertEquals("BoundedIOCompaction should have resulted in 2 compactions being chosen", 2, returned.size());
// Total size of all the log files // Total size of all the log files
Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB)) Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB))
.map(s -> s.longValue()).reduce((size1, size2) -> size1 + size2).orElse(0L); .map(Double::longValue).reduce(Long::sum).orElse(0L);
assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 610, assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 610,
(long) returnedSize); (long) returnedSize);
} }
@@ -111,7 +111,7 @@ public class TestHoodieCompactionStrategy {
assertEquals("LogFileSizeBasedCompactionStrategy should have resulted in 1 compaction", 1, returned.size()); assertEquals("LogFileSizeBasedCompactionStrategy should have resulted in 1 compaction", 1, returned.size());
// Total size of all the log files // Total size of all the log files
Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB)) Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB))
.map(s -> s.longValue()).reduce((size1, size2) -> size1 + size2).orElse(0L); .map(Double::longValue).reduce(Long::sum).orElse(0L);
assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 1204, assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 1204,
(long) returnedSize); (long) returnedSize);
} }
@@ -227,8 +227,8 @@ public class TestHoodieCompactionStrategy {
private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config, private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,
Map<Long, List<Long>> sizesMap) { Map<Long, List<Long>> sizesMap) {
Map<Long, String> keyToPartitionMap = sizesMap.entrySet().stream() Map<Long, String> keyToPartitionMap = sizesMap.keySet().stream()
.map(e -> Pair.of(e.getKey(), partitionPaths[new Random().nextInt(partitionPaths.length - 1)])) .map(e -> Pair.of(e, partitionPaths[new Random().nextInt(partitionPaths.length - 1)]))
.collect(Collectors.toMap(Pair::getKey, Pair::getValue)); .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
return createCompactionOperations(config, sizesMap, keyToPartitionMap); return createCompactionOperations(config, sizesMap, keyToPartitionMap);
} }

6
hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
View File

@@ -24,7 +24,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.Test; import org.junit.Test;
import static org.apache.hudi.metrics.Metrics.registerGauge; import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -46,7 +46,7 @@ public class TestHoodieJmxMetrics extends TestHoodieMetrics {
@Test @Test
public void testRegisterGauge() { public void testRegisterGauge() {
registerGauge("jmx_metric", 123L); registerGauge("jmx_metric", 123L);
assertTrue(Metrics.getInstance().getRegistry().getGauges() assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
.get("jmx_metric").getValue().toString().equals("123")); .get("jmx_metric").getValue().toString());
} }
} }

4
hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
View File

@@ -24,7 +24,7 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import static org.apache.hudi.metrics.Metrics.registerGauge; import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -41,6 +41,6 @@ public class TestHoodieMetrics {
@Test @Test
public void testRegisterGauge() { public void testRegisterGauge() {
registerGauge("metric1", 123L); registerGauge("metric1", 123L);
assertTrue(Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString().equals("123")); assertEquals("123", Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString());
} }
} }

9
hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
View File

@@ -64,6 +64,7 @@ import java.util.UUID;
import scala.Tuple2; import scala.Tuple2;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -224,7 +225,7 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
} }
} }
} }
assertTrue(updatedParquetFile != null); assertNotNull(updatedParquetFile);
// Check whether the record has been updated // Check whether the record has been updated
Path updatedParquetFilePath = new Path(updatedParquetFile.getAbsolutePath()); Path updatedParquetFilePath = new Path(updatedParquetFile.getAbsolutePath());
BloomFilter updatedFilter = BloomFilter updatedFilter =
@@ -240,16 +241,16 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
ParquetReader updatedReader = ParquetReader.builder(new AvroReadSupport<>(), updatedParquetFilePath).build(); ParquetReader updatedReader = ParquetReader.builder(new AvroReadSupport<>(), updatedParquetFilePath).build();
index = 0; index = 0;
while ((newRecord = (GenericRecord) updatedReader.read()) != null) { while ((newRecord = (GenericRecord) updatedReader.read()) != null) {
assertTrue(newRecord.get("_row_key").toString().equals(records.get(index).getRecordKey())); assertEquals(newRecord.get("_row_key").toString(), records.get(index).getRecordKey());
if (index == 0) { if (index == 0) {
assertTrue(newRecord.get("number").toString().equals("15")); assertEquals("15", newRecord.get("number").toString());
} }
index++; index++;
} }
updatedReader.close(); updatedReader.close();
// Also check the numRecordsWritten // Also check the numRecordsWritten
WriteStatus writeStatus = statuses.get(0); WriteStatus writeStatus = statuses.get(0);
assertTrue("Should be only one file generated", statuses.size() == 1); assertEquals("Should be only one file generated", 1, statuses.size());
assertEquals(4, writeStatus.getStat().getNumWrites());// 3 rewritten records + 1 new record assertEquals(4, writeStatus.getStat().getNumWrites());// 3 rewritten records + 1 new record
} }

133
hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java
View File

@@ -241,7 +241,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
ReadOptimizedView roView = ReadOptimizedView roView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles);
Stream<HoodieDataFile> dataFilesToRead = roView.getLatestDataFiles(); Stream<HoodieDataFile> dataFilesToRead = roView.getLatestDataFiles();
assertTrue(!dataFilesToRead.findAny().isPresent()); assertFalse(dataFilesToRead.findAny().isPresent());
roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles); roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles);
dataFilesToRead = roView.getLatestDataFiles(); dataFilesToRead = roView.getLatestDataFiles();
@@ -284,7 +284,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
dataFilesToRead = roView.getLatestDataFiles(); dataFilesToRead = roView.getLatestDataFiles();
assertTrue(dataFilesToRead.findAny().isPresent()); assertTrue(dataFilesToRead.findAny().isPresent());
List<String> dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); List<String> dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
// Wrote 20 records and deleted 20 records, so remaining 20-20 = 0 // Wrote 20 records and deleted 20 records, so remaining 20-20 = 0
assertEquals("Must contain 0 records", 0, recordsRead.size()); assertEquals("Must contain 0 records", 0, recordsRead.size());
@@ -343,13 +343,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles); new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles);
final String absentCommit = newCommitTime; final String absentCommit = newCommitTime;
assertFalse(roView.getLatestDataFiles().filter(file -> { assertFalse(roView.getLatestDataFiles().anyMatch(file -> absentCommit.equals(file.getCommitTime())));
if (absentCommit.equals(file.getCommitTime())) {
return true;
} else {
return false;
}
}).findAny().isPresent());
} }
} }
@@ -407,7 +401,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
copyOfRecords = dataGen.generateUpdates(commitTime1, copyOfRecords); copyOfRecords = dataGen.generateUpdates(commitTime1, copyOfRecords);
copyOfRecords.addAll(dataGen.generateInserts(commitTime1, 200)); copyOfRecords.addAll(dataGen.generateInserts(commitTime1, 200));
List<String> dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); List<String> dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
assertEquals(recordsRead.size(), 200); assertEquals(recordsRead.size(), 200);
@@ -419,9 +413,9 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
secondClient.rollback(commitTime1); secondClient.rollback(commitTime1);
allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath());
// After rollback, there should be no parquet file with the failed commit time // After rollback, there should be no parquet file with the failed commit time
Assert.assertEquals(Arrays.asList(allFiles).stream() Assert.assertEquals(Arrays.stream(allFiles)
.filter(file -> file.getPath().getName().contains(commitTime1)).collect(Collectors.toList()).size(), 0); .filter(file -> file.getPath().getName().contains(commitTime1)).count(), 0);
dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
assertEquals(recordsRead.size(), 200); assertEquals(recordsRead.size(), 200);
} }
@@ -437,7 +431,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
copyOfRecords = dataGen.generateUpdates(commitTime2, copyOfRecords); copyOfRecords = dataGen.generateUpdates(commitTime2, copyOfRecords);
copyOfRecords.addAll(dataGen.generateInserts(commitTime2, 200)); copyOfRecords.addAll(dataGen.generateInserts(commitTime2, 200));
List<String> dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); List<String> dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
assertEquals(recordsRead.size(), 200); assertEquals(recordsRead.size(), 200);
@@ -452,13 +446,13 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
thirdClient.rollback(commitTime2); thirdClient.rollback(commitTime2);
allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath());
// After rollback, there should be no parquet file with the failed commit time // After rollback, there should be no parquet file with the failed commit time
Assert.assertEquals(Arrays.asList(allFiles).stream() Assert.assertEquals(Arrays.stream(allFiles)
.filter(file -> file.getPath().getName().contains(commitTime2)).collect(Collectors.toList()).size(), 0); .filter(file -> file.getPath().getName().contains(commitTime2)).count(), 0);
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc); hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles); roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles);
dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
// check that the number of records read is still correct after rollback operation // check that the number of records read is still correct after rollback operation
assertEquals(recordsRead.size(), 200); assertEquals(recordsRead.size(), 200);
@@ -470,8 +464,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
newCommitTime = "003"; newCommitTime = "003";
thirdClient.startCommitWithTime(newCommitTime); thirdClient.startCommitWithTime(newCommitTime);
records = dataGen.generateUpdates(newCommitTime, records);
writeStatusJavaRDD = thirdClient.upsert(writeRecords, newCommitTime); writeStatusJavaRDD = thirdClient.upsert(writeRecords, newCommitTime);
thirdClient.commit(newCommitTime, writeStatusJavaRDD); thirdClient.commit(newCommitTime, writeStatusJavaRDD);
statuses = writeStatusJavaRDD.collect(); statuses = writeStatusJavaRDD.collect();
@@ -486,35 +478,20 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles); roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles);
List<HoodieDataFile> dataFiles2 = roView.getLatestDataFiles().collect(Collectors.toList());
final String compactedCommitTime = final String compactedCommitTime =
metaClient.getActiveTimeline().reload().getCommitsTimeline().lastInstant().get().getTimestamp(); metaClient.getActiveTimeline().reload().getCommitsTimeline().lastInstant().get().getTimestamp();
assertTrue(roView.getLatestDataFiles().filter(file -> { assertTrue(roView.getLatestDataFiles().anyMatch(file -> compactedCommitTime.equals(file.getCommitTime())));
if (compactedCommitTime.equals(file.getCommitTime())) {
return true;
} else {
return false;
}
}).findAny().isPresent());
thirdClient.rollback(compactedCommitTime); thirdClient.rollback(compactedCommitTime);
allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
hoodieTable = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles); roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline(), allFiles);
assertFalse(roView.getLatestDataFiles().filter(file -> { assertFalse(roView.getLatestDataFiles().anyMatch(file -> compactedCommitTime.equals(file.getCommitTime())));
if (compactedCommitTime.equals(file.getCommitTime())) {
return true;
} else {
return false;
}
}).findAny().isPresent());
} }
} }
} }
@@ -524,12 +501,10 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
HoodieWriteConfig cfg = getConfig(false); HoodieWriteConfig cfg = getConfig(false);
try (final HoodieWriteClient client = getWriteClient(cfg);) { try (final HoodieWriteClient client = getWriteClient(cfg);) {
List<String> allCommits = new ArrayList<>();
/** /**
* Write 1 (only inserts) * Write 1 (only inserts)
*/ */
String newCommitTime = "001"; String newCommitTime = "001";
allCommits.add(newCommitTime);
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200);
@@ -554,7 +529,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
ReadOptimizedView roView = ReadOptimizedView roView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles);
Stream<HoodieDataFile> dataFilesToRead = roView.getLatestDataFiles(); Stream<HoodieDataFile> dataFilesToRead = roView.getLatestDataFiles();
assertTrue(!dataFilesToRead.findAny().isPresent()); assertFalse(dataFilesToRead.findAny().isPresent());
roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles); roView = new HoodieTableFileSystemView(metaClient, hoodieTable.getCompletedCommitsTimeline(), allFiles);
dataFilesToRead = roView.getLatestDataFiles(); dataFilesToRead = roView.getLatestDataFiles();
@@ -565,7 +540,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
* Write 2 (inserts + updates) * Write 2 (inserts + updates)
*/ */
newCommitTime = "002"; newCommitTime = "002";
allCommits.add(newCommitTime);
// WriteClient with custom config (disable small file handling) // WriteClient with custom config (disable small file handling)
HoodieWriteClient nClient = getWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff()); HoodieWriteClient nClient = getWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff());
nClient.startCommitWithTime(newCommitTime); nClient.startCommitWithTime(newCommitTime);
@@ -589,7 +563,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
* Write 3 (inserts + updates) * Write 3 (inserts + updates)
*/ */
newCommitTime = "003"; newCommitTime = "003";
allCommits.add(newCommitTime);
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
List<HoodieRecord> newInserts = dataGen.generateInserts(newCommitTime, 100); List<HoodieRecord> newInserts = dataGen.generateInserts(newCommitTime, 100);
@@ -606,7 +579,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
String compactionInstantTime = "004"; String compactionInstantTime = "004";
allCommits.add(compactionInstantTime);
client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty()); client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty());
// Compaction commit // Compaction commit
@@ -614,7 +586,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
* Write 4 (updates) * Write 4 (updates)
*/ */
newCommitTime = "005"; newCommitTime = "005";
allCommits.add(newCommitTime);
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
records = dataGen.generateUpdates(newCommitTime, records); records = dataGen.generateUpdates(newCommitTime, records);
@@ -629,7 +600,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
compactionInstantTime = "006"; compactionInstantTime = "006";
allCommits.add(compactionInstantTime);
client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty()); client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty());
JavaRDD<WriteStatus> ws = client.compact(compactionInstantTime); JavaRDD<WriteStatus> ws = client.compact(compactionInstantTime);
client.commitCompaction(compactionInstantTime, ws, Option.empty()); client.commitCompaction(compactionInstantTime, ws, Option.empty());
@@ -641,19 +611,12 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
final String compactedCommitTime = final String compactedCommitTime =
metaClient.getActiveTimeline().reload().getCommitsTimeline().lastInstant().get().getTimestamp(); metaClient.getActiveTimeline().reload().getCommitsTimeline().lastInstant().get().getTimestamp();
assertTrue(roView.getLatestDataFiles().filter(file -> { assertTrue(roView.getLatestDataFiles().anyMatch(file -> compactedCommitTime.equals(file.getCommitTime())));
if (compactedCommitTime.equals(file.getCommitTime())) {
return true;
} else {
return false;
}
}).findAny().isPresent());
/** /**
* Write 5 (updates) * Write 5 (updates)
*/ */
newCommitTime = "007"; newCommitTime = "007";
allCommits.add(newCommitTime);
client.startCommitWithTime(newCommitTime); client.startCommitWithTime(newCommitTime);
copyOfRecords = new ArrayList<>(records); copyOfRecords = new ArrayList<>(records);
copyOfRecords = dataGen.generateUpdates(newCommitTime, copyOfRecords); copyOfRecords = dataGen.generateUpdates(newCommitTime, copyOfRecords);
@@ -673,7 +636,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
roView = roView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles);
dataFilesToRead = roView.getLatestDataFiles(); dataFilesToRead = roView.getLatestDataFiles();
assertTrue(!dataFilesToRead.findAny().isPresent()); assertFalse(dataFilesToRead.findAny().isPresent());
RealtimeView rtView = RealtimeView rtView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles);
List<HoodieFileGroup> fileGroups = List<HoodieFileGroup> fileGroups =
@@ -681,9 +644,9 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
assertTrue(fileGroups.isEmpty()); assertTrue(fileGroups.isEmpty());
// make sure there are no log files remaining // make sure there are no log files remaining
assertTrue(((HoodieTableFileSystemView) rtView).getAllFileGroups() assertEquals(0L, ((HoodieTableFileSystemView) rtView).getAllFileGroups()
.filter(fileGroup -> fileGroup.getAllRawFileSlices().filter(f -> f.getLogFiles().count() == 0).count() == 0) .filter(fileGroup -> fileGroup.getAllRawFileSlices().noneMatch(f -> f.getLogFiles().count() == 0))
.count() == 0L); .count());
} }
} }
@@ -691,10 +654,10 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
protected HoodieWriteConfig getHoodieWriteConfigWithSmallFileHandlingOff() { protected HoodieWriteConfig getHoodieWriteConfigWithSmallFileHandlingOff() {
return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
.withAutoCommit(false).withAssumeDatePartitioning(true) .withAutoCommit(false).withAssumeDatePartitioning(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1 * 1024) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024)
.withInlineCompaction(false).withMaxNumDeltaCommitsBeforeCompaction(1).build()) .withInlineCompaction(false).withMaxNumDeltaCommitsBeforeCompaction(1).build())
.withEmbeddedTimelineServerEnabled(true) .withEmbeddedTimelineServerEnabled(true)
.withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1 * 1024).build()).forTable("test-trip-table") .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024).build()).forTable("test-trip-table")
.build(); .build();
} }
@@ -767,10 +730,9 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
Map<String, Long> parquetFileIdToNewSize = Map<String, Long> parquetFileIdToNewSize =
newDataFilesList.stream().collect(Collectors.toMap(HoodieDataFile::getFileId, HoodieDataFile::getFileSize)); newDataFilesList.stream().collect(Collectors.toMap(HoodieDataFile::getFileId, HoodieDataFile::getFileSize));
assertTrue(parquetFileIdToNewSize.entrySet().stream() assertTrue(parquetFileIdToNewSize.entrySet().stream().anyMatch(entry -> parquetFileIdToSize.get(entry.getKey()) < entry.getValue()));
.filter(entry -> parquetFileIdToSize.get(entry.getKey()) < entry.getValue()).count() > 0);
List<String> dataFiles = roView.getLatestDataFiles().map(hf -> hf.getPath()).collect(Collectors.toList()); List<String> dataFiles = roView.getLatestDataFiles().map(HoodieDataFile::getPath).collect(Collectors.toList());
List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath); List<GenericRecord> recordsRead = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(dataFiles, basePath);
// Wrote 20 records in 2 batches // Wrote 20 records in 2 batches
assertEquals("Must contain 40 records", 40, recordsRead.size()); assertEquals("Must contain 40 records", 40, recordsRead.size());
@@ -787,13 +749,11 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1); JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1);
List<WriteStatus> statuses = writeClient.insert(recordsRDD, newCommitTime).collect(); writeClient.insert(recordsRDD, newCommitTime).collect();
// Update all the 100 records // Update all the 100 records
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieTimeline timeline2 = metaClient.getActiveTimeline();
newCommitTime = "101"; newCommitTime = "101";
writeClient.startCommitWithTime(newCommitTime); writeClient.startCommitWithTime(newCommitTime);
@@ -808,7 +768,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
// Verify that all data file has one log file // Verify that all data file has one log file
metaClient = HoodieTableMetaClient.reload(metaClient); metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieTable.getHoodieTable(metaClient, config, jsc); HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
// In writeRecordsToLogFiles, no commit files are getting added, so resetting file-system view state // In writeRecordsToLogFiles, no commit files are getting added, so resetting file-system view state
((SyncableFileSystemView) (table.getRTFileSystemView())).reset(); ((SyncableFileSystemView) (table.getRTFileSystemView())).reset();
@@ -842,13 +802,10 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
List<FileSlice> groupedLogFiles = List<FileSlice> groupedLogFiles =
table.getRTFileSystemView().getLatestFileSlices(partitionPath).collect(Collectors.toList()); table.getRTFileSystemView().getLatestFileSlices(partitionPath).collect(Collectors.toList());
for (FileSlice slice : groupedLogFiles) { for (FileSlice slice : groupedLogFiles) {
assertTrue("After compaction there should be no log files visiable on a Realtime view", assertEquals("After compaction there should be no log files visiable on a Realtime view", 0, slice.getLogFiles().count());
slice.getLogFiles().collect(Collectors.toList()).isEmpty());
} }
List<WriteStatus> writeStatuses = result.collect(); List<WriteStatus> writeStatuses = result.collect();
assertTrue(writeStatuses.stream() assertTrue(writeStatuses.stream().anyMatch(writeStatus -> writeStatus.getStat().getPartitionPath().contentEquals(partitionPath)));
.filter(writeStatus -> writeStatus.getStat().getPartitionPath().contentEquals(partitionPath))
.count() > 0);
} }
} }
} }
@@ -874,10 +831,9 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
long numLogFiles = 0; long numLogFiles = 0;
for (String partitionPath : dataGen.getPartitionPaths()) { for (String partitionPath : dataGen.getPartitionPaths()) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath) assertEquals(0, tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); .filter(fileSlice -> fileSlice.getDataFile().isPresent()).count());
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath) Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).anyMatch(fileSlice -> fileSlice.getLogFiles().count() > 0));
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0);
numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath) numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count(); .filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();
} }
@@ -886,7 +842,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
// Do a compaction // Do a compaction
String commitTime = writeClient.scheduleCompaction(Option.empty()).get().toString(); String commitTime = writeClient.scheduleCompaction(Option.empty()).get().toString();
statuses = writeClient.compact(commitTime); statuses = writeClient.compact(commitTime);
Assert.assertTrue(statuses.map(status -> status.getStat().getPath().contains("parquet")).count() == numLogFiles); assertEquals(statuses.map(status -> status.getStat().getPath().contains("parquet")).count(), numLogFiles);
Assert.assertEquals(statuses.count(), numLogFiles); Assert.assertEquals(statuses.count(), numLogFiles);
writeClient.commitCompaction(commitTime, statuses, Option.empty()); writeClient.commitCompaction(commitTime, statuses, Option.empty());
} }
@@ -911,7 +867,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
Assert.assertEquals( Assert.assertEquals(
writeStatuses.stream().filter(writeStatus -> !writeStatus.getStat().getPath().contains("log")).count(), 0); writeStatuses.stream().filter(writeStatus -> !writeStatus.getStat().getPath().contains("log")).count(), 0);
Assert.assertTrue( Assert.assertTrue(
writeStatuses.stream().filter(writeStatus -> writeStatus.getStat().getPath().contains("log")).count() > 0); writeStatuses.stream().anyMatch(writeStatus -> writeStatus.getStat().getPath().contains("log")));
// rollback a failed commit // rollback a failed commit
boolean rollback = writeClient.rollback(newCommitTime); boolean rollback = writeClient.rollback(newCommitTime);
@@ -951,14 +907,12 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
long numLogFiles = 0; long numLogFiles = 0;
for (String partitionPath : dataGen.getPartitionPaths()) { for (String partitionPath : dataGen.getPartitionPaths()) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath) Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).noneMatch(fileSlice -> fileSlice.getDataFile().isPresent()));
.filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).noneMatch(fileSlice -> fileSlice.getLogFiles().count() > 0));
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() == 0);
numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath) numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count(); .filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();
} }
Assert.assertTrue(numLogFiles == 0); assertEquals(0, numLogFiles);
metaClient.getFs().copyFromLocalFile(new Path(file.getAbsolutePath()), metaClient.getFs().copyFromLocalFile(new Path(file.getAbsolutePath()),
new Path(metaClient.getMetaPath(), fileName)); new Path(metaClient.getMetaPath(), fileName));
Thread.sleep(1000); Thread.sleep(1000);
@@ -990,10 +944,8 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
long numLogFiles = 0; long numLogFiles = 0;
for (String partitionPath : dataGen.getPartitionPaths()) { for (String partitionPath : dataGen.getPartitionPaths()) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath) Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).noneMatch(fileSlice -> fileSlice.getDataFile().isPresent()));
.filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).anyMatch(fileSlice -> fileSlice.getLogFiles().count() > 0));
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0);
numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath) numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count(); .filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();
} }
@@ -1003,7 +955,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
newCommitTime = writeClient.scheduleCompaction(Option.empty()).get().toString(); newCommitTime = writeClient.scheduleCompaction(Option.empty()).get().toString();
statuses = writeClient.compact(newCommitTime); statuses = writeClient.compact(newCommitTime);
// Ensure all log files have been compacted into parquet files // Ensure all log files have been compacted into parquet files
Assert.assertTrue(statuses.map(status -> status.getStat().getPath().contains("parquet")).count() == numLogFiles); assertEquals(statuses.map(status -> status.getStat().getPath().contains("parquet")).count(), numLogFiles);
Assert.assertEquals(statuses.count(), numLogFiles); Assert.assertEquals(statuses.count(), numLogFiles);
writeClient.commitCompaction(newCommitTime, statuses, Option.empty()); writeClient.commitCompaction(newCommitTime, statuses, Option.empty());
// Trigger a rollback of compaction // Trigger a rollback of compaction
@@ -1014,10 +966,8 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
Option<HoodieInstant> lastInstant = ((SyncableFileSystemView) tableRTFileSystemView).getLastInstant(); Option<HoodieInstant> lastInstant = ((SyncableFileSystemView) tableRTFileSystemView).getLastInstant();
System.out.println("Last Instant =" + lastInstant); System.out.println("Last Instant =" + lastInstant);
for (String partitionPath : dataGen.getPartitionPaths()) { for (String partitionPath : dataGen.getPartitionPaths()) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath) Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).noneMatch(fileSlice -> fileSlice.getDataFile().isPresent()));
.filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).anyMatch(fileSlice -> fileSlice.getLogFiles().count() > 0));
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath)
.filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0);
} }
} }
} }
@@ -1134,7 +1084,6 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
Map<String, Long> fileIdToInsertsMap = new HashMap<>(); Map<String, Long> fileIdToInsertsMap = new HashMap<>();
Map<String, Long> fileIdToUpsertsMap = new HashMap<>(); Map<String, Long> fileIdToUpsertsMap = new HashMap<>();
HoodieTable table = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
String commitTime = "000"; String commitTime = "000";
client.startCommitWithTime(commitTime); client.startCommitWithTime(commitTime);
@@ -1146,7 +1095,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
assertTrue("Commit should succeed", client.commit(commitTime, statuses)); assertTrue("Commit should succeed", client.commit(commitTime, statuses));
// Read from commit file // Read from commit file
table = HoodieTable.getHoodieTable(metaClient, cfg, jsc); HoodieTable table = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes( HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
table.getActiveTimeline() table.getActiveTimeline()
.getInstantDetails(table.getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), .getInstantDetails(table.getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(),

7
hudi-common/src/test/java/org/apache/hudi/common/bloom/filter/TestBloomFilter.java
View File

@@ -24,7 +24,6 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized; import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters; import org.junit.runners.Parameterized.Parameters;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@@ -55,7 +54,7 @@ public class TestBloomFilter {
@Test @Test
public void testAddKey() { public void testAddKey() {
List<String> inputs = new ArrayList<>(); List<String> inputs;
int[] sizes = {100, 1000, 10000}; int[] sizes = {100, 1000, 10000};
for (int size : sizes) { for (int size : sizes) {
inputs = new ArrayList<>(); inputs = new ArrayList<>();
@@ -78,9 +77,9 @@ public class TestBloomFilter {
} }
@Test @Test
public void testSerialize() throws IOException, ClassNotFoundException { public void testSerialize() {
List<String> inputs = new ArrayList<>(); List<String> inputs;
int[] sizes = {100, 1000, 10000}; int[] sizes = {100, 1000, 10000};
for (int size : sizes) { for (int size : sizes) {
inputs = new ArrayList<>(); inputs = new ArrayList<>();

23
hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
View File

@@ -79,7 +79,7 @@ public class HdfsTestService {
// Configure and start the HDFS cluster // Configure and start the HDFS cluster
// boolean format = shouldFormatDFSCluster(localDFSLocation, clean); // boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
hadoopConf = configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort, namenodeHttpPort, hadoopConf = configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
datanodePort, datanodeIpcPort, datanodeHttpPort); datanodePort, datanodeIpcPort, datanodeHttpPort);
miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true) miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true)
.checkDataNodeHostConfig(true).build(); .checkDataNodeHostConfig(true).build();
@@ -87,7 +87,7 @@ public class HdfsTestService {
return miniDfsCluster; return miniDfsCluster;
} }
public void stop() throws IOException { public void stop() {
LOG.info("HDFS Minicluster service being shut down."); LOG.info("HDFS Minicluster service being shut down.");
miniDfsCluster.shutdown(); miniDfsCluster.shutdown();
miniDfsCluster = null; miniDfsCluster = null;
@@ -104,23 +104,6 @@ public class HdfsTestService {
return baseFsLocation + Path.SEPARATOR + "dfs"; return baseFsLocation + Path.SEPARATOR + "dfs";
} }
/**
* Returns true if we should format the DFS Cluster. We'll format if clean is true, or if the dfsFsLocation does not
* exist.
*
* @param localDFSLocation The location on the local FS to hold the HDFS metadata and block data
* @param clean Specifies if we want to start a clean cluster
* @return Returns true if we should format a DFSCluster, otherwise false
*/
private static boolean shouldFormatDFSCluster(String localDFSLocation, boolean clean) {
boolean format = true;
File f = new File(localDFSLocation);
if (f.exists() && f.isDirectory() && !clean) {
format = false;
}
return format;
}
/** /**
* Configure the DFS Cluster before launching it. * Configure the DFS Cluster before launching it.
* *
@@ -130,7 +113,7 @@ public class HdfsTestService {
* @return The updated Configuration object. * @return The updated Configuration object.
*/ */
private static Configuration configureDFSCluster(Configuration config, String localDFSLocation, String bindIP, private static Configuration configureDFSCluster(Configuration config, String localDFSLocation, String bindIP,
int namenodeRpcPort, int namenodeHttpPort, int datanodePort, int datanodeIpcPort, int datanodeHttpPort) { int namenodeRpcPort, int datanodePort, int datanodeIpcPort, int datanodeHttpPort) {
LOG.info("HDFS force binding to ip: " + bindIP); LOG.info("HDFS force binding to ip: " + bindIP);
config.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + bindIP + ":" + namenodeRpcPort); config.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + bindIP + ":" + namenodeRpcPort);

38
hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
View File

@@ -139,15 +139,7 @@ public class HoodieTestUtils {
} }
} }
public static final void createDeltaCommitFiles(String basePath, String... commitTimes) throws IOException { public static final void createMetadataFolder(String basePath) {
for (String commitTime : commitTimes) {
new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeDeltaFileName(commitTime))
.createNewFile();
}
}
public static final void createMetadataFolder(String basePath) throws IOException {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME).mkdirs(); new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME).mkdirs();
} }
@@ -161,8 +153,7 @@ public class HoodieTestUtils {
} }
} }
public static final void createPendingCleanFiles(HoodieTableMetaClient metaClient, Configuration configuration, public static final void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... commitTimes) {
String... commitTimes) throws IOException {
for (String commitTime : commitTimes) { for (String commitTime : commitTimes) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(commitTime), Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(commitTime),
HoodieTimeline.makeInflightCleanerFileName(commitTime)).forEach(f -> { HoodieTimeline.makeInflightCleanerFileName(commitTime)).forEach(f -> {
@@ -303,7 +294,7 @@ public class HoodieTestUtils {
public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath, public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath,
String commitTime, Configuration configuration) String commitTime, Configuration configuration)
throws IOException { throws IOException {
createPendingCleanFiles(metaClient, configuration, commitTime); createPendingCleanFiles(metaClient, commitTime);
Path commitFile = new Path( Path commitFile = new Path(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCleanerFileName(commitTime)); basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCleanerFileName(commitTime));
FileSystem fs = FSUtils.getFs(basePath, configuration); FileSystem fs = FSUtils.getFs(basePath, configuration);
@@ -323,19 +314,6 @@ public class HoodieTestUtils {
} }
} }
public static void createCleanFiles(HoodieTableMetaClient metaClient,
String basePath, String commitTime) throws IOException {
createCleanFiles(metaClient, basePath, commitTime, HoodieTestUtils.getDefaultHadoopConf());
}
public static String makeTestFileName(String instant) {
return instant + TEST_EXTENSION;
}
public static String makeCommitFileName(String instant) {
return instant + ".commit";
}
public static void assertStreamEquals(String message, Stream<?> expected, Stream<?> actual) { public static void assertStreamEquals(String message, Stream<?> expected, Stream<?> actual) {
Iterator<?> iter1 = expected.iterator(); Iterator<?> iter1 = expected.iterator();
Iterator<?> iter2 = actual.iterator(); Iterator<?> iter2 = actual.iterator();
@@ -345,8 +323,7 @@ public class HoodieTestUtils {
assert !iter1.hasNext() && !iter2.hasNext(); assert !iter1.hasNext() && !iter2.hasNext();
} }
public static <T extends Serializable> T serializeDeserialize(T object, Class<T> clazz) public static <T extends Serializable> T serializeDeserialize(T object, Class<T> clazz) {
throws IOException, ClassNotFoundException {
// Using Kyro as the default serializer in Spark Jobs // Using Kyro as the default serializer in Spark Jobs
Kryo kryo = new Kryo(); Kryo kryo = new Kryo();
kryo.register(HoodieTableMetaClient.class, new JavaSerializer()); kryo.register(HoodieTableMetaClient.class, new JavaSerializer());
@@ -367,9 +344,8 @@ public class HoodieTestUtils {
Map<HoodieRecordLocation, List<HoodieRecord>> groupedUpdated = Map<HoodieRecordLocation, List<HoodieRecord>> groupedUpdated =
updatedRecords.stream().collect(Collectors.groupingBy(HoodieRecord::getCurrentLocation)); updatedRecords.stream().collect(Collectors.groupingBy(HoodieRecord::getCurrentLocation));
groupedUpdated.entrySet().forEach(s -> { groupedUpdated.forEach((location, value) -> {
HoodieRecordLocation location = s.getKey(); String partitionPath = value.get(0).getPartitionPath();
String partitionPath = s.getValue().get(0).getPartitionPath();
Writer logWriter; Writer logWriter;
try { try {
@@ -380,7 +356,7 @@ public class HoodieTestUtils {
Map<HoodieLogBlock.HeaderMetadataType, String> header = Maps.newHashMap(); Map<HoodieLogBlock.HeaderMetadataType, String> header = Maps.newHashMap();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, location.getInstantTime()); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, location.getInstantTime());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
logWriter.appendBlock(new HoodieAvroDataBlock(s.getValue().stream().map(r -> { logWriter.appendBlock(new HoodieAvroDataBlock(value.stream().map(r -> {
try { try {
GenericRecord val = (GenericRecord) r.getData().getInsertValue(schema).get(); GenericRecord val = (GenericRecord) r.getData().getInsertValue(schema).get();
HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), ""); HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), "");

4
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java
View File

@@ -33,7 +33,7 @@ public class TestHoodieCommitMetadata {
List<HoodieWriteStat> fakeHoodieWriteStats = HoodieTestUtils.generateFakeHoodieWriteStat(100); List<HoodieWriteStat> fakeHoodieWriteStats = HoodieTestUtils.generateFakeHoodieWriteStat(100);
HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
fakeHoodieWriteStats.stream().forEach(stat -> commitMetadata.addWriteStat(stat.getPartitionPath(), stat)); fakeHoodieWriteStats.forEach(stat -> commitMetadata.addWriteStat(stat.getPartitionPath(), stat));
Assert.assertTrue(commitMetadata.getTotalCreateTime() > 0); Assert.assertTrue(commitMetadata.getTotalCreateTime() > 0);
Assert.assertTrue(commitMetadata.getTotalUpsertTime() > 0); Assert.assertTrue(commitMetadata.getTotalUpsertTime() > 0);
Assert.assertTrue(commitMetadata.getTotalScanTime() > 0); Assert.assertTrue(commitMetadata.getTotalScanTime() > 0);
@@ -43,7 +43,7 @@ public class TestHoodieCommitMetadata {
HoodieCommitMetadata metadata = HoodieCommitMetadata metadata =
HoodieCommitMetadata.fromJsonString(serializedCommitMetadata, HoodieCommitMetadata.class); HoodieCommitMetadata.fromJsonString(serializedCommitMetadata, HoodieCommitMetadata.class);
// Make sure timing metrics are not written to instant file // Make sure timing metrics are not written to instant file
Assert.assertTrue(metadata.getTotalScanTime() == 0); Assert.assertEquals(0, (long) metadata.getTotalScanTime());
Assert.assertTrue(metadata.getTotalLogFilesCompacted() > 0); Assert.assertTrue(metadata.getTotalLogFilesCompacted() > 0);
} }
} }

2
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java
View File

@@ -18,7 +18,6 @@
package org.apache.hudi.common.model; package org.apache.hudi.common.model;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.FSUtils;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@@ -46,7 +45,6 @@ public class TestHoodieWriteStat {
Path basePath = new Path(basePathString); Path basePath = new Path(basePathString);
Path partitionPath = new Path(basePath, partitionPathString); Path partitionPath = new Path(basePath, partitionPathString);
Path tempPath = new Path(basePath, HoodieTableMetaClient.TEMPFOLDER_NAME);
Path finalizeFilePath = new Path(partitionPath, FSUtils.makeDataFileName(commitTime, writeToken, fileName)); Path finalizeFilePath = new Path(partitionPath, FSUtils.makeDataFileName(commitTime, writeToken, fileName));
HoodieWriteStat writeStat = new HoodieWriteStat(); HoodieWriteStat writeStat = new HoodieWriteStat();

4
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
View File

@@ -61,7 +61,7 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
} }
@Test @Test
public void checkSerDe() throws IOException, ClassNotFoundException { public void checkSerDe() {
// check if this object is serialized and de-serialized, we are able to read from the file system // check if this object is serialized and de-serialized, we are able to read from the file system
HoodieTableMetaClient deseralizedMetaClient = HoodieTableMetaClient deseralizedMetaClient =
HoodieTestUtils.serializeDeserialize(metaClient, HoodieTableMetaClient.class); HoodieTestUtils.serializeDeserialize(metaClient, HoodieTableMetaClient.class);
@@ -78,7 +78,7 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
} }
@Test @Test
public void checkCommitTimeline() throws IOException { public void checkCommitTimeline() {
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline(); HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
assertTrue("Should be empty commit timeline", activeCommitTimeline.empty()); assertTrue("Should be empty commit timeline", activeCommitTimeline.empty());

29
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
View File

@@ -30,7 +30,6 @@ import org.apache.hudi.common.table.log.HoodieLogFormat.Reader;
import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer;
import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
import org.apache.hudi.common.table.log.block.HoodieCommandBlock; import org.apache.hudi.common.table.log.block.HoodieCommandBlock;
import org.apache.hudi.common.table.log.block.HoodieCorruptBlock;
import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
@@ -471,7 +470,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
} }
assertEquals("Scanner records count should be the same as appended records", scannedRecords.size(), assertEquals("Scanner records count should be the same as appended records", scannedRecords.size(),
allRecords.stream().flatMap(records -> records.stream()).collect(Collectors.toList()).size()); allRecords.stream().mapToLong(Collection::size).sum());
} }
@@ -511,8 +510,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertTrue("We should have corrupted block next", reader.hasNext()); assertTrue("We should have corrupted block next", reader.hasNext());
HoodieLogBlock block = reader.next(); HoodieLogBlock block = reader.next();
assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType()); assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType());
HoodieCorruptBlock corruptBlock = (HoodieCorruptBlock) block;
// assertEquals("", "something-random", new String(corruptBlock.getCorruptedBytes()));
assertFalse("There should be no more block left", reader.hasNext()); assertFalse("There should be no more block left", reader.hasNext());
reader.close(); reader.close();
@@ -551,8 +548,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertTrue("We should get the 2nd corrupted block next", reader.hasNext()); assertTrue("We should get the 2nd corrupted block next", reader.hasNext());
block = reader.next(); block = reader.next();
assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType()); assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType());
corruptBlock = (HoodieCorruptBlock) block;
// assertEquals("", "something-else-random", new String(corruptBlock.getCorruptedBytes()));
assertTrue("We should get the last block next", reader.hasNext()); assertTrue("We should get the last block next", reader.hasNext());
reader.next(); reader.next();
assertFalse("We should have no more blocks left", reader.hasNext()); assertFalse("We should have no more blocks left", reader.hasNext());
@@ -817,7 +812,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE, header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE,
String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal())); String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal()));
HoodieCommandBlock commandBlock = new HoodieCommandBlock(header); HoodieCommandBlock commandBlock = new HoodieCommandBlock(header);
writer = writer.appendBlock(commandBlock); writer.appendBlock(commandBlock);
readKeys.clear(); readKeys.clear();
scanner = new HoodieMergedLogRecordScanner(fs, basePath, allLogFiles, schema, "101", 10240L, readBlocksLazily, scanner = new HoodieMergedLogRecordScanner(fs, basePath, allLogFiles, schema, "101", 10240L, readBlocksLazily,
@@ -855,10 +850,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
dataBlock = new HoodieAvroDataBlock(records2, header); dataBlock = new HoodieAvroDataBlock(records2, header);
writer = writer.appendBlock(dataBlock); writer = writer.appendBlock(dataBlock);
List<String> originalKeys =
copyOfRecords1.stream().map(s -> ((GenericRecord) s).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString())
.collect(Collectors.toList());
// Delete 50 keys // Delete 50 keys
// Delete 50 keys // Delete 50 keys
List<HoodieKey> deletedKeys = copyOfRecords1.stream() List<HoodieKey> deletedKeys = copyOfRecords1.stream()
@@ -881,7 +872,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// it's okay // it's okay
} }
// Attempt 2 : Write another rollback blocks for a failed write // Attempt 2 : Write another rollback blocks for a failed write
writer = writer.appendBlock(commandBlock); writer.appendBlock(commandBlock);
List<String> allLogFiles = List<String> allLogFiles =
FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100") FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100")
@@ -937,7 +928,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal())); String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal()));
HoodieCommandBlock commandBlock = new HoodieCommandBlock(header); HoodieCommandBlock commandBlock = new HoodieCommandBlock(header);
writer = writer.appendBlock(commandBlock); writer = writer.appendBlock(commandBlock);
writer = writer.appendBlock(commandBlock); writer.appendBlock(commandBlock);
List<String> allLogFiles = List<String> allLogFiles =
FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100") FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100")
@@ -970,7 +961,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE, header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE,
String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal())); String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal()));
HoodieCommandBlock commandBlock = new HoodieCommandBlock(header); HoodieCommandBlock commandBlock = new HoodieCommandBlock(header);
writer = writer.appendBlock(commandBlock); writer.appendBlock(commandBlock);
List<String> allLogFiles = List<String> allLogFiles =
FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100") FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100")
@@ -1008,10 +999,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
writer = writer.appendBlock(dataBlock); writer = writer.appendBlock(dataBlock);
writer = writer.appendBlock(dataBlock); writer = writer.appendBlock(dataBlock);
List<String> originalKeys =
copyOfRecords1.stream().map(s -> ((GenericRecord) s).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString())
.collect(Collectors.toList());
// Delete 50 keys // Delete 50 keys
// Delete 50 keys // Delete 50 keys
List<HoodieKey> deletedKeys = copyOfRecords1.stream() List<HoodieKey> deletedKeys = copyOfRecords1.stream()
@@ -1026,7 +1013,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE, header.put(HoodieLogBlock.HeaderMetadataType.COMMAND_BLOCK_TYPE,
String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal())); String.valueOf(HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK.ordinal()));
HoodieCommandBlock commandBlock = new HoodieCommandBlock(header); HoodieCommandBlock commandBlock = new HoodieCommandBlock(header);
writer = writer.appendBlock(commandBlock); writer.appendBlock(commandBlock);
List<String> allLogFiles = List<String> allLogFiles =
FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100") FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1", HoodieLogFile.DELTA_EXTENSION, "100")
@@ -1275,8 +1262,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION) HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withFileId("test-fileid1").overBaseCommit("100").withFs(fs).build(); .withFileId("test-fileid1").overBaseCommit("100").withFs(fs).build();
List<IndexedRecord> records2 = SchemaTestUtil.generateTestRecords(0, 100); List<IndexedRecord> records2 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
.map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema)).collect(Collectors.toList());
dataBlock = new HoodieAvroDataBlock(records2, header); dataBlock = new HoodieAvroDataBlock(records2, header);
writer = writer.appendBlock(dataBlock); writer = writer.appendBlock(dataBlock);
writer.close(); writer.close();
@@ -1286,8 +1271,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION) HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withFileId("test-fileid1").overBaseCommit("100").withFs(fs).build(); .withFileId("test-fileid1").overBaseCommit("100").withFs(fs).build();
List<IndexedRecord> records3 = SchemaTestUtil.generateTestRecords(0, 100); List<IndexedRecord> records3 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords3 = records3.stream()
.map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema)).collect(Collectors.toList());
dataBlock = new HoodieAvroDataBlock(records3, header); dataBlock = new HoodieAvroDataBlock(records3, header);
writer = writer.appendBlock(dataBlock); writer = writer.appendBlock(dataBlock);
writer.close(); writer.close();

2
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatAppendFailure.java
View File

@@ -137,7 +137,7 @@ public class TestHoodieLogFormatAppendFailure {
.withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits.archive") .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits.archive")
.overBaseCommit("").withFs(fs).build(); .overBaseCommit("").withFs(fs).build();
// The log version should be different for this new writer // The log version should be different for this new writer
Assert.assertFalse(writer.getLogFile().getLogVersion() == logFileVersion); Assert.assertNotEquals(writer.getLogFile().getLogVersion(), logFileVersion);
} }
} }

11
hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java
View File

@@ -22,9 +22,7 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant;
import java.io.IOException;
import java.util.Comparator; import java.util.Comparator;
import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
@@ -33,16 +31,11 @@ import java.util.stream.Stream;
*/ */
public class MockHoodieTimeline extends HoodieActiveTimeline { public class MockHoodieTimeline extends HoodieActiveTimeline {
public MockHoodieTimeline(Stream<String> completed, Stream<String> inflights) throws IOException { public MockHoodieTimeline(Stream<String> completed, Stream<String> inflights) {
super(); super();
this.setInstants(Stream this.setInstants(Stream
.concat(completed.map(s -> new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, s)), .concat(completed.map(s -> new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, s)),
inflights.map(s -> new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, s))) inflights.map(s -> new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, s)))
.sorted(Comparator.comparing(new Function<HoodieInstant, String>() { .sorted(Comparator.comparing(HoodieInstant::getFileName)).collect(Collectors.toList()));
@Override
public String apply(HoodieInstant hoodieInstant) {
return hoodieInstant.getFileName();
}
})).collect(Collectors.toList()));
} }
} }

7
hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
View File

@@ -31,7 +31,6 @@ import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@@ -53,7 +52,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
} }
@Test @Test
public void testLoadingInstantsFromFiles() throws IOException { public void testLoadingInstantsFromFiles() {
HoodieInstant instant1 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "1"); HoodieInstant instant1 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "1");
HoodieInstant instant2 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "3"); HoodieInstant instant2 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "3");
HoodieInstant instant3 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "5"); HoodieInstant instant3 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "5");
@@ -100,7 +99,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
} }
@Test @Test
public void testTimelineOperationsBasic() throws Exception { public void testTimelineOperationsBasic() {
timeline = new HoodieActiveTimeline(metaClient); timeline = new HoodieActiveTimeline(metaClient);
assertTrue(timeline.empty()); assertTrue(timeline.empty());
assertEquals("", 0, timeline.countInstants()); assertEquals("", 0, timeline.countInstants());
@@ -112,7 +111,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
} }
@Test @Test
public void testTimelineOperations() throws Exception { public void testTimelineOperations() {
timeline = new MockHoodieTimeline(Stream.of("01", "03", "05", "07", "09", "11", "13", "15", "17", "19"), timeline = new MockHoodieTimeline(Stream.of("01", "03", "05", "07", "09", "11", "13", "15", "17", "19"),
Stream.of("21", "23")); Stream.of("21", "23"));
HoodieTestUtils.assertStreamEquals("", Stream.of("05", "07", "09", "11"), timeline.getCommitTimeline() HoodieTestUtils.assertStreamEquals("", Stream.of("05", "07", "09", "11"), timeline.getCommitTimeline()

51
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
View File

@@ -294,7 +294,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
assertEquals("Base-Instant must be compaction Instant", compactionRequestedTime, assertEquals("Base-Instant must be compaction Instant", compactionRequestedTime,
slices.get(0).getBaseInstantTime()); slices.get(0).getBaseInstantTime());
assertFalse("Latest File Slice must not have data-file", slices.get(0).getDataFile().isPresent()); assertFalse("Latest File Slice must not have data-file", slices.get(0).getDataFile().isPresent());
assertTrue("Latest File Slice must not have any log-files", slices.get(0).getLogFiles().count() == 0); assertEquals("Latest File Slice must not have any log-files", 0, slices.get(0).getLogFiles().count());
// Fake delta-ingestion after compaction-requested // Fake delta-ingestion after compaction-requested
String deltaInstantTime4 = "5"; String deltaInstantTime4 = "5";
@@ -360,36 +360,28 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals("Expect no data file to be returned", 0, dataFiles.size());
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1));
assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1);
});
} }
dataFiles = roView.getLatestDataFiles(partitionPath).collect(Collectors.toList()); dataFiles = roView.getLatestDataFiles(partitionPath).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals("Expect no data file to be returned", 0, dataFiles.size());
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1));
assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1);
});
} }
dataFiles = roView.getLatestDataFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); dataFiles = roView.getLatestDataFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals("Expect no data file to be returned", 0, dataFiles.size());
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1));
assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1);
});
} }
dataFiles = roView.getLatestDataFilesInRange(allInstantTimes).collect(Collectors.toList()); dataFiles = roView.getLatestDataFilesInRange(allInstantTimes).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals("Expect no data file to be returned", 0, dataFiles.size());
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1));
assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1);
});
} }
/** Inflight/Orphan File-groups needs to be in the view **/ /** Inflight/Orphan File-groups needs to be in the view **/
@@ -517,24 +509,16 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
/** Data Files API tests */ /** Data Files API tests */
dataFiles = roView.getLatestDataFiles().collect(Collectors.toList()); dataFiles = roView.getLatestDataFiles().collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime));
assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime);
});
dataFiles = roView.getLatestDataFiles(partitionPath).collect(Collectors.toList()); dataFiles = roView.getLatestDataFiles(partitionPath).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime));
assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime);
});
dataFiles = roView.getLatestDataFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); dataFiles = roView.getLatestDataFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime));
assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime);
});
dataFiles = roView.getLatestDataFilesInRange(allInstantTimes).collect(Collectors.toList()); dataFiles = roView.getLatestDataFilesInRange(allInstantTimes).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals("Expect only one data-file to be sent", 1, dataFiles.size());
dataFiles.stream().forEach(df -> { dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime));
assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime);
});
assertEquals("Total number of file-slices in partitions matches expected", expTotalFileSlices, assertEquals("Total number of file-slices in partitions matches expected", expTotalFileSlices,
rtView.getAllFileSlices(partitionPath).count()); rtView.getAllFileSlices(partitionPath).count());
@@ -552,7 +536,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String fileId = UUID.randomUUID().toString(); String fileId = UUID.randomUUID().toString();
assertFalse("No commit, should not find any data file", roView.getLatestDataFiles(partitionPath) assertFalse("No commit, should not find any data file", roView.getLatestDataFiles(partitionPath)
.filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().isPresent()); .anyMatch(dfile -> dfile.getFileId().equals(fileId)));
// Only one commit, but is not safe // Only one commit, but is not safe
String commitTime1 = "1"; String commitTime1 = "1";
@@ -560,7 +544,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile(); new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile();
refreshFsView(); refreshFsView();
assertFalse("No commit, should not find any data file", roView.getLatestDataFiles(partitionPath) assertFalse("No commit, should not find any data file", roView.getLatestDataFiles(partitionPath)
.filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().isPresent()); .anyMatch(dfile -> dfile.getFileId().equals(fileId)));
// Make this commit safe // Make this commit safe
HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline(); HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline();
@@ -658,7 +642,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
List<FileSlice> allSlices = rtView.getAllFileSlices("2016/05/01").collect(Collectors.toList()); List<FileSlice> allSlices = rtView.getAllFileSlices("2016/05/01").collect(Collectors.toList());
assertEquals(isLatestFileSliceOnly ? 4 : 8, allSlices.size()); assertEquals(isLatestFileSliceOnly ? 4 : 8, allSlices.size());
Map<String, Long> fileSliceMap = Map<String, Long> fileSliceMap =
allSlices.stream().collect(Collectors.groupingBy(slice -> slice.getFileId(), Collectors.counting())); allSlices.stream().collect(Collectors.groupingBy(FileSlice::getFileId, Collectors.counting()));
assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId1).longValue()); assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId1).longValue());
assertEquals(isLatestFileSliceOnly ? 1 : 3, fileSliceMap.get(fileId2).longValue()); assertEquals(isLatestFileSliceOnly ? 1 : 3, fileSliceMap.get(fileId2).longValue());
assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId3).longValue()); assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId3).longValue());
@@ -677,7 +661,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
filenames = Sets.newHashSet(); filenames = Sets.newHashSet();
List<HoodieLogFile> logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime4, true) List<HoodieLogFile> logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime4, true)
.map(slice -> slice.getLogFiles()).flatMap(logFileList -> logFileList).collect(Collectors.toList()); .map(FileSlice::getLogFiles).flatMap(logFileList -> logFileList).collect(Collectors.toList());
assertEquals(logFilesList.size(), 4); assertEquals(logFilesList.size(), 4);
for (HoodieLogFile logFile : logFilesList) { for (HoodieLogFile logFile : logFilesList) {
filenames.add(logFile.getFileName()); filenames.add(logFile.getFileName());
@@ -709,10 +693,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
} }
logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime3, true) logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime3, true)
.map(slice -> slice.getLogFiles()).flatMap(logFileList -> logFileList).collect(Collectors.toList()); .map(FileSlice::getLogFiles).flatMap(logFileList -> logFileList).collect(Collectors.toList());
assertEquals(logFilesList.size(), 1); assertEquals(logFilesList.size(), 1);
assertTrue(logFilesList.get(0).getFileName() assertEquals(logFilesList.get(0).getFileName(), FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, commitTime3, 0, TEST_WRITE_TOKEN));
.equals(FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, commitTime3, 0, TEST_WRITE_TOKEN)));
} }
@Test @Test
@@ -1091,8 +1074,6 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// Fake delta-ingestion after compaction-requested // Fake delta-ingestion after compaction-requested
String deltaInstantTime4 = "4"; String deltaInstantTime4 = "4";
String deltaInstantTime5 = "6"; String deltaInstantTime5 = "6";
List<String> allInstantTimes = Arrays.asList(instantTime1, deltaInstantTime1, deltaInstantTime2,
compactionRequestedTime, deltaInstantTime4, deltaInstantTime5);
String fileName3 = String fileName3 =
FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, compactionRequestedTime, 0, TEST_WRITE_TOKEN); FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, compactionRequestedTime, 0, TEST_WRITE_TOKEN);
String fileName4 = String fileName4 =
@@ -1119,7 +1100,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
assertEquals("Expect only valid commit", "1", dataFiles.get(0).getCommitTime()); assertEquals("Expect only valid commit", "1", dataFiles.get(0).getCommitTime());
/** Merge API Tests **/ /** Merge API Tests **/
Arrays.asList(partitionPath1, partitionPath2, partitionPath3).stream().forEach(partitionPath -> { Arrays.asList(partitionPath1, partitionPath2, partitionPath3).forEach(partitionPath -> {
List<FileSlice> fileSliceList = List<FileSlice> fileSliceList =
rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
assertEquals("Expect file-slice to be merged", 1, fileSliceList.size()); assertEquals("Expect file-slice to be merged", 1, fileSliceList.size());

67
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
View File

@@ -173,9 +173,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.reloadActiveTimeline(); metaClient.reloadActiveTimeline();
SyncableFileSystemView newView = getFileSystemView(metaClient); SyncableFileSystemView newView = getFileSystemView(metaClient);
for (String partition : partitions) {
newView.getAllFileGroups(partition).count();
}
areViewsConsistent(view, newView, 0L); areViewsConsistent(view, newView, 0L);
@@ -221,7 +218,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
view2.sync(); view2.sync();
SyncableFileSystemView view3 = SyncableFileSystemView view3 =
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
partitions.stream().forEach(p -> view3.getLatestFileSlices(p).count());
view3.sync(); view3.sync();
areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size()); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size());
@@ -234,7 +230,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size()); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size());
SyncableFileSystemView view4 = SyncableFileSystemView view4 =
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
partitions.stream().forEach(p -> view4.getLatestFileSlices(p).count());
view4.sync(); view4.sync();
/** /**
@@ -249,7 +244,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2);
SyncableFileSystemView view5 = SyncableFileSystemView view5 =
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
partitions.stream().forEach(p -> view5.getLatestFileSlices(p).count());
view5.sync(); view5.sync();
/** /**
@@ -269,7 +263,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2); areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size() * 2);
SyncableFileSystemView view6 = SyncableFileSystemView view6 =
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath())); getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
partitions.stream().forEach(p -> view5.getLatestFileSlices(p).count());
view6.sync(); view6.sync();
/** /**
@@ -284,7 +277,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
testMultipleWriteSteps(view2, Arrays.asList("28"), false, "28", 3, testMultipleWriteSteps(view2, Arrays.asList("28"), false, "28", 3,
Arrays.asList(new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "29"))); Arrays.asList(new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "29")));
Arrays.asList(view1, view2, view3, view4, view5, view6).stream().forEach(v -> { Arrays.asList(view1, view2, view3, view4, view5, view6).forEach(v -> {
v.sync(); v.sync();
areViewsConsistent(v, view1, partitions.size() * fileIdsPerPartition.size() * 3); areViewsConsistent(v, view1, partitions.size() * fileIdsPerPartition.size() * 3);
}); });
@@ -318,16 +311,15 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
Map<String, List<String>> deltaInstantMap, Map<String, List<String>> instantsToFiles, Map<String, List<String>> deltaInstantMap, Map<String, List<String>> instantsToFiles,
List<String> cleanedInstants) { List<String> cleanedInstants) {
Assert.assertEquals(newCleanerInstants.size(), cleanedInstants.size()); Assert.assertEquals(newCleanerInstants.size(), cleanedInstants.size());
long initialFileSlices = partitions.stream().mapToLong(p -> view.getAllFileSlices(p).count()).findAny().getAsLong(); long exp = partitions.stream().mapToLong(p1 -> view.getAllFileSlices(p1).count()).findAny().getAsLong();
long exp = initialFileSlices;
LOG.info("Initial File Slices :" + exp); LOG.info("Initial File Slices :" + exp);
for (int idx = 0; idx < newCleanerInstants.size(); idx++) { for (int idx = 0; idx < newCleanerInstants.size(); idx++) {
String instant = cleanedInstants.get(idx); String instant = cleanedInstants.get(idx);
try { try {
List<String> filesToDelete = new ArrayList<>(instantsToFiles.get(instant)); List<String> filesToDelete = new ArrayList<>(instantsToFiles.get(instant));
deltaInstantMap.get(instant).stream().forEach(n -> filesToDelete.addAll(instantsToFiles.get(n))); deltaInstantMap.get(instant).forEach(n -> filesToDelete.addAll(instantsToFiles.get(n)));
performClean(view, instant, filesToDelete, newCleanerInstants.get(idx)); performClean(instant, filesToDelete, newCleanerInstants.get(idx));
exp -= fileIdsPerPartition.size(); exp -= fileIdsPerPartition.size();
final long expTotalFileSlicesPerPartition = exp; final long expTotalFileSlicesPerPartition = exp;
@@ -346,9 +338,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.reloadActiveTimeline(); metaClient.reloadActiveTimeline();
SyncableFileSystemView newView = getFileSystemView(metaClient); SyncableFileSystemView newView = getFileSystemView(metaClient);
for (String partition : partitions) {
newView.getAllFileGroups(partition).count();
}
areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size()); areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size());
} catch (IOException e) { } catch (IOException e) {
throw new HoodieException(e); throw new HoodieException(e);
@@ -368,13 +357,13 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
*/ */
private void testRestore(SyncableFileSystemView view, List<String> newRestoreInstants, boolean isDeltaCommit, private void testRestore(SyncableFileSystemView view, List<String> newRestoreInstants, boolean isDeltaCommit,
Map<String, List<String>> instantsToFiles, List<String> rolledBackInstants, String emptyRestoreInstant, Map<String, List<String>> instantsToFiles, List<String> rolledBackInstants, String emptyRestoreInstant,
boolean isRestore) throws IOException { boolean isRestore) {
Assert.assertEquals(newRestoreInstants.size(), rolledBackInstants.size()); Assert.assertEquals(newRestoreInstants.size(), rolledBackInstants.size());
long initialFileSlices = partitions.stream().mapToLong(p -> view.getAllFileSlices(p).count()).findAny().getAsLong(); long initialFileSlices = partitions.stream().mapToLong(p -> view.getAllFileSlices(p).count()).findAny().getAsLong();
IntStream.range(0, newRestoreInstants.size()).forEach(idx -> { IntStream.range(0, newRestoreInstants.size()).forEach(idx -> {
String instant = rolledBackInstants.get(idx); String instant = rolledBackInstants.get(idx);
try { try {
performRestore(view, instant, instantsToFiles.get(instant), newRestoreInstants.get(idx), isRestore); performRestore(instant, instantsToFiles.get(instant), newRestoreInstants.get(idx), isRestore);
final long expTotalFileSlicesPerPartition = final long expTotalFileSlicesPerPartition =
isDeltaCommit ? initialFileSlices : initialFileSlices - ((idx + 1) * fileIdsPerPartition.size()); isDeltaCommit ? initialFileSlices : initialFileSlices - ((idx + 1) * fileIdsPerPartition.size());
view.sync(); view.sync();
@@ -397,9 +386,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.reloadActiveTimeline(); metaClient.reloadActiveTimeline();
SyncableFileSystemView newView = getFileSystemView(metaClient); SyncableFileSystemView newView = getFileSystemView(metaClient);
for (String partition : partitions) {
newView.getAllFileGroups(partition).count();
}
areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size()); areViewsConsistent(view, newView, expTotalFileSlicesPerPartition * partitions.size());
} catch (IOException e) { } catch (IOException e) {
throw new HoodieException(e); throw new HoodieException(e);
@@ -410,18 +396,16 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
/** /**
* Simulate a Cleaner operation cleaning up an instant. * Simulate a Cleaner operation cleaning up an instant.
* *
* @param view Hoodie View
* @param instant Instant to be cleaner * @param instant Instant to be cleaner
* @param files List of files to be deleted * @param files List of files to be deleted
* @param cleanInstant Cleaner Instant * @param cleanInstant Cleaner Instant
*/ */
private void performClean(SyncableFileSystemView view, String instant, List<String> files, String cleanInstant) private void performClean(String instant, List<String> files, String cleanInstant)
throws IOException { throws IOException {
Map<String, List<String>> partititonToFiles = deleteFiles(files); Map<String, List<String>> partititonToFiles = deleteFiles(files);
List<HoodieCleanStat> cleanStats = partititonToFiles.entrySet().stream().map(e -> { List<HoodieCleanStat> cleanStats = partititonToFiles.entrySet().stream().map(e ->
return new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, e.getKey(), e.getValue(), e.getValue(), new HoodieCleanStat(HoodieCleaningPolicy.KEEP_LATEST_COMMITS, e.getKey(), e.getValue(), e.getValue(),
new ArrayList<>(), Integer.toString(Integer.parseInt(instant) + 1)); new ArrayList<>(), Integer.toString(Integer.parseInt(instant) + 1))).collect(Collectors.toList());
}).collect(Collectors.toList());
HoodieInstant cleanInflightInstant = new HoodieInstant(true, HoodieTimeline.CLEAN_ACTION, cleanInstant); HoodieInstant cleanInflightInstant = new HoodieInstant(true, HoodieTimeline.CLEAN_ACTION, cleanInstant);
metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant); metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant);
@@ -434,17 +418,16 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
/** /**
* Simulate Restore of an instant in timeline and fsview. * Simulate Restore of an instant in timeline and fsview.
* *
* @param view Hoodie View
* @param instant Instant to be rolled-back * @param instant Instant to be rolled-back
* @param files List of files to be deleted as part of rollback * @param files List of files to be deleted as part of rollback
* @param rollbackInstant Restore Instant * @param rollbackInstant Restore Instant
*/ */
private void performRestore(SyncableFileSystemView view, String instant, List<String> files, String rollbackInstant, private void performRestore(String instant, List<String> files, String rollbackInstant,
boolean isRestore) throws IOException { boolean isRestore) throws IOException {
Map<String, List<String>> partititonToFiles = deleteFiles(files); Map<String, List<String>> partititonToFiles = deleteFiles(files);
List<HoodieRollbackStat> rollbackStats = partititonToFiles.entrySet().stream().map(e -> { List<HoodieRollbackStat> rollbackStats = partititonToFiles.entrySet().stream().map(e ->
return new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>()); new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>())
}).collect(Collectors.toList()); ).collect(Collectors.toList());
List<String> rollbacks = new ArrayList<>(); List<String> rollbacks = new ArrayList<>();
rollbacks.add(instant); rollbacks.add(instant);
@@ -491,7 +474,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
for (String f : files) { for (String f : files) {
String fullPath = String.format("%s/%s", metaClient.getBasePath(), f); String fullPath = String.format("%s/%s", metaClient.getBasePath(), f);
new File(fullPath).delete(); new File(fullPath).delete();
String partition = partitions.stream().filter(p -> f.startsWith(p)).findAny().get(); String partition = partitions.stream().filter(f::startsWith).findAny().get();
partititonToFiles.get(partition).add(fullPath); partititonToFiles.get(partition).add(fullPath);
} }
return partititonToFiles; return partititonToFiles;
@@ -515,7 +498,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
AvroUtils.serializeCompactionPlan(plan)); AvroUtils.serializeCompactionPlan(plan));
view.sync(); view.sync();
partitions.stream().forEach(p -> { partitions.forEach(p -> {
view.getLatestFileSlices(p).forEach(fs -> { view.getLatestFileSlices(p).forEach(fs -> {
Assert.assertEquals(instantTime, fs.getBaseInstantTime()); Assert.assertEquals(instantTime, fs.getBaseInstantTime());
Assert.assertEquals(p, fs.getPartitionPath()); Assert.assertEquals(p, fs.getPartitionPath());
@@ -530,7 +513,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.reloadActiveTimeline(); metaClient.reloadActiveTimeline();
SyncableFileSystemView newView = getFileSystemView(metaClient); SyncableFileSystemView newView = getFileSystemView(metaClient);
partitions.forEach(p -> newView.getLatestFileSlices(p).count());
areViewsConsistent(view, newView, initialExpTotalFileSlices + partitions.size() * fileIdsPerPartition.size()); areViewsConsistent(view, newView, initialExpTotalFileSlices + partitions.size() * fileIdsPerPartition.size());
} }
@@ -550,11 +532,9 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
view.sync(); view.sync();
Assert.assertEquals(newLastInstant, view.getLastInstant().get().getTimestamp()); Assert.assertEquals(newLastInstant, view.getLastInstant().get().getTimestamp());
partitions.stream().forEach(p -> { partitions.forEach(p -> view.getLatestFileSlices(p).forEach(fs -> {
view.getLatestFileSlices(p).forEach(fs -> {
Assert.assertEquals(newBaseInstant, fs.getBaseInstantTime()); Assert.assertEquals(newBaseInstant, fs.getBaseInstantTime());
}); }));
});
} }
/** /**
@@ -653,9 +633,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.reloadActiveTimeline(); metaClient.reloadActiveTimeline();
SyncableFileSystemView newView = getFileSystemView(metaClient); SyncableFileSystemView newView = getFileSystemView(metaClient);
for (String partition : partitions) {
newView.getAllFileGroups(partition).count();
}
areViewsConsistent(view, newView, fileIdsPerPartition.size() * partitions.size() * multiple); areViewsConsistent(view, newView, fileIdsPerPartition.size() * partitions.size() * multiple);
instantToFiles.put(instant, filePaths); instantToFiles.put(instant, filePaths);
if (!deltaCommit) { if (!deltaCommit) {
@@ -680,10 +657,10 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
Iterators.elementsEqual(timeline1.getInstants().iterator(), timeline2.getInstants().iterator()); Iterators.elementsEqual(timeline1.getInstants().iterator(), timeline2.getInstants().iterator());
// View Checks // View Checks
Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap1 = partitions.stream().flatMap(p -> view1.getAllFileGroups(p)) Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap1 = partitions.stream().flatMap(view1::getAllFileGroups)
.collect(Collectors.toMap(fg -> fg.getFileGroupId(), fg -> fg)); .collect(Collectors.toMap(HoodieFileGroup::getFileGroupId, fg -> fg));
Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap2 = partitions.stream().flatMap(p -> view2.getAllFileGroups(p)) Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap2 = partitions.stream().flatMap(view2::getAllFileGroups)
.collect(Collectors.toMap(fg -> fg.getFileGroupId(), fg -> fg)); .collect(Collectors.toMap(HoodieFileGroup::getFileGroupId, fg -> fg));
Assert.assertEquals(fileGroupsMap1.keySet(), fileGroupsMap2.keySet()); Assert.assertEquals(fileGroupsMap1.keySet(), fileGroupsMap2.keySet());
long gotSlicesCount = fileGroupsMap1.keySet().stream() long gotSlicesCount = fileGroupsMap1.keySet().stream()
.map(k -> Pair.of(fileGroupsMap1.get(k), fileGroupsMap2.get(k))).mapToLong(e -> { .map(k -> Pair.of(fileGroupsMap1.get(k), fileGroupsMap2.get(k))).mapToLong(e -> {

2
hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java
View File

@@ -128,7 +128,7 @@ public class CompactionTestUtils {
AvroUtils.serializeCompactionPlan(compactionPlan)); AvroUtils.serializeCompactionPlan(compactionPlan));
} }
public static void createDeltaCommit(HoodieTableMetaClient metaClient, String instantTime) throws IOException { public static void createDeltaCommit(HoodieTableMetaClient metaClient, String instantTime) {
HoodieInstant requested = new HoodieInstant(State.REQUESTED, DELTA_COMMIT_ACTION, instantTime); HoodieInstant requested = new HoodieInstant(State.REQUESTED, DELTA_COMMIT_ACTION, instantTime);
metaClient.getActiveTimeline().createNewInstant(requested); metaClient.getActiveTimeline().createNewInstant(requested);
metaClient.getActiveTimeline().transitionRequestedToInflight(requested, Option.empty()); metaClient.getActiveTimeline().transitionRequestedToInflight(requested, Option.empty());

4
hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java
View File

@@ -124,7 +124,7 @@ public class SchemaTestUtil {
} }
public static List<IndexedRecord> updateHoodieTestRecords(List<String> oldRecordKeys, List<IndexedRecord> newRecords, public static List<IndexedRecord> updateHoodieTestRecords(List<String> oldRecordKeys, List<IndexedRecord> newRecords,
String commitTime) throws IOException, URISyntaxException { String commitTime) {
return newRecords.stream().map(p -> { return newRecords.stream().map(p -> {
((GenericRecord) p).put(HoodieRecord.RECORD_KEY_METADATA_FIELD, oldRecordKeys.remove(0)); ((GenericRecord) p).put(HoodieRecord.RECORD_KEY_METADATA_FIELD, oldRecordKeys.remove(0));
@@ -144,7 +144,7 @@ public class SchemaTestUtil {
} }
public static List<HoodieRecord> updateHoodieTestRecordsWithoutHoodieMetadata(List<HoodieRecord> oldRecords, public static List<HoodieRecord> updateHoodieTestRecordsWithoutHoodieMetadata(List<HoodieRecord> oldRecords,
Schema schema, String fieldNameToUpdate, String newValue) throws IOException, URISyntaxException { Schema schema, String fieldNameToUpdate, String newValue) {
return oldRecords.stream().map(r -> { return oldRecords.stream().map(r -> {
try { try {
GenericRecord rec = (GenericRecord) r.getData().getInsertValue(schema).get(); GenericRecord rec = (GenericRecord) r.getData().getInsertValue(schema).get();

2
hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java
View File

@@ -42,7 +42,7 @@ public class SpillableMapTestUtils {
public static List<String> upsertRecords(List<IndexedRecord> iRecords, public static List<String> upsertRecords(List<IndexedRecord> iRecords,
Map<String, HoodieRecord<? extends HoodieRecordPayload>> records) { Map<String, HoodieRecord<? extends HoodieRecordPayload>> records) {
List<String> recordKeys = new ArrayList<>(); List<String> recordKeys = new ArrayList<>();
iRecords.stream().forEach(r -> { iRecords.forEach(r -> {
String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(); String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString(); String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
recordKeys.add(key); recordKeys.add(key);

12
hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
View File

@@ -31,6 +31,8 @@ import java.io.IOException;
import java.io.PrintStream; import java.io.PrintStream;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
/** /**
@@ -85,7 +87,7 @@ public class TestDFSPropertiesConfiguration {
} }
@Test @Test
public void testParsing() throws IOException { public void testParsing() {
DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/t1.props")); DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/t1.props"));
TypedProperties props = cfg.getConfig(); TypedProperties props = cfg.getConfig();
assertEquals(5, props.size()); assertEquals(5, props.size());
@@ -98,19 +100,19 @@ public class TestDFSPropertiesConfiguration {
assertEquals(123, props.getInteger("int.prop")); assertEquals(123, props.getInteger("int.prop"));
assertEquals(113.4, props.getDouble("double.prop"), 0.001); assertEquals(113.4, props.getDouble("double.prop"), 0.001);
assertEquals(true, props.getBoolean("boolean.prop")); assertTrue(props.getBoolean("boolean.prop"));
assertEquals("str", props.getString("string.prop")); assertEquals("str", props.getString("string.prop"));
assertEquals(1354354354, props.getLong("long.prop")); assertEquals(1354354354, props.getLong("long.prop"));
assertEquals(123, props.getInteger("int.prop", 456)); assertEquals(123, props.getInteger("int.prop", 456));
assertEquals(113.4, props.getDouble("double.prop", 223.4), 0.001); assertEquals(113.4, props.getDouble("double.prop", 223.4), 0.001);
assertEquals(true, props.getBoolean("boolean.prop", false)); assertTrue(props.getBoolean("boolean.prop", false));
assertEquals("str", props.getString("string.prop", "default")); assertEquals("str", props.getString("string.prop", "default"));
assertEquals(1354354354, props.getLong("long.prop", 8578494434L)); assertEquals(1354354354, props.getLong("long.prop", 8578494434L));
assertEquals(456, props.getInteger("bad.int.prop", 456)); assertEquals(456, props.getInteger("bad.int.prop", 456));
assertEquals(223.4, props.getDouble("bad.double.prop", 223.4), 0.001); assertEquals(223.4, props.getDouble("bad.double.prop", 223.4), 0.001);
assertEquals(false, props.getBoolean("bad.boolean.prop", false)); assertFalse(props.getBoolean("bad.boolean.prop", false));
assertEquals("default", props.getString("bad.string.prop", "default")); assertEquals("default", props.getString("bad.string.prop", "default"));
assertEquals(8578494434L, props.getLong("bad.long.prop", 8578494434L)); assertEquals(8578494434L, props.getLong("bad.long.prop", 8578494434L));
} }
@@ -122,7 +124,7 @@ public class TestDFSPropertiesConfiguration {
assertEquals(123, props.getInteger("int.prop")); assertEquals(123, props.getInteger("int.prop"));
assertEquals(243.4, props.getDouble("double.prop"), 0.001); assertEquals(243.4, props.getDouble("double.prop"), 0.001);
assertEquals(true, props.getBoolean("boolean.prop")); assertTrue(props.getBoolean("boolean.prop"));
assertEquals("t3.value", props.getString("string.prop")); assertEquals("t3.value", props.getString("string.prop"));
assertEquals(1354354354, props.getLong("long.prop")); assertEquals(1354354354, props.getLong("long.prop"));

27
hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java
View File

@@ -39,11 +39,10 @@ import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/** /**
* Tests file system utils. * Tests file system utils.
@@ -63,18 +62,15 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test @Test
public void testMakeDataFileName() { public void testMakeDataFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
int taskPartitionId = 2;
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
assertTrue(FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName) assertEquals(FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName), fileName + "_" + TEST_WRITE_TOKEN + "_" + commitTime + ".parquet");
.equals(fileName + "_" + TEST_WRITE_TOKEN + "_" + commitTime + ".parquet"));
} }
@Test @Test
public void testMaskFileName() { public void testMaskFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
int taskPartitionId = 2; int taskPartitionId = 2;
assertTrue(FSUtils.maskWithoutFileId(commitTime, taskPartitionId) assertEquals(FSUtils.maskWithoutFileId(commitTime, taskPartitionId), "*_" + taskPartitionId + "_" + commitTime + ".parquet");
.equals("*_" + taskPartitionId + "_" + commitTime + ".parquet"));
} }
@Test @Test
@@ -89,7 +85,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
// All directories including marker dirs. // All directories including marker dirs.
List<String> folders = List<String> folders =
Arrays.asList("2016/04/15", "2016/05/16", ".hoodie/.temp/2/2016/04/15", ".hoodie/.temp/2/2016/05/16"); Arrays.asList("2016/04/15", "2016/05/16", ".hoodie/.temp/2/2016/04/15", ".hoodie/.temp/2/2016/05/16");
folders.stream().forEach(f -> { folders.forEach(f -> {
try { try {
metaClient.getFs().mkdirs(new Path(new Path(basePath), f)); metaClient.getFs().mkdirs(new Path(new Path(basePath), f));
} catch (IOException e) { } catch (IOException e) {
@@ -102,7 +98,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
"2016/05/16/2_1-0-1_20190528120000.parquet", ".hoodie/.temp/2/2016/05/16/2_1-0-1_20190528120000.parquet", "2016/05/16/2_1-0-1_20190528120000.parquet", ".hoodie/.temp/2/2016/05/16/2_1-0-1_20190528120000.parquet",
".hoodie/.temp/2/2016/04/15/1_1-0-1_20190528120000.parquet"); ".hoodie/.temp/2/2016/04/15/1_1-0-1_20190528120000.parquet");
files.stream().forEach(f -> { files.forEach(f -> {
try { try {
metaClient.getFs().create(new Path(new Path(basePath), f)); metaClient.getFs().create(new Path(new Path(basePath), f));
} catch (IOException e) { } catch (IOException e) {
@@ -138,19 +134,17 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test @Test
public void testGetCommitTime() { public void testGetCommitTime() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
int taskPartitionId = 2;
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName); String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName);
assertTrue(FSUtils.getCommitTime(fullFileName).equals(commitTime)); assertEquals(FSUtils.getCommitTime(fullFileName), commitTime);
} }
@Test @Test
public void testGetFileNameWithoutMeta() { public void testGetFileNameWithoutMeta() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
int taskPartitionId = 2;
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName); String fullFileName = FSUtils.makeDataFileName(commitTime, TEST_WRITE_TOKEN, fileName);
assertTrue(FSUtils.getFileId(fullFileName).equals(fileName)); assertEquals(FSUtils.getFileId(fullFileName), fileName);
} }
@Test @Test
@@ -234,7 +228,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
String log1Ver0 = makeOldLogFileName("file1", ".log", "1", 0); String log1Ver0 = makeOldLogFileName("file1", ".log", "1", 0);
String log1Ver1 = makeOldLogFileName("file1", ".log", "1", 1); String log1Ver1 = makeOldLogFileName("file1", ".log", "1", 1);
String log1base2 = makeOldLogFileName("file1", ".log", "2", 0); String log1base2 = makeOldLogFileName("file1", ".log", "2", 0);
List<HoodieLogFile> logFiles = Arrays.asList(log1base2, log1Ver1, log1Ver0).stream().map(f -> new HoodieLogFile(f)) List<HoodieLogFile> logFiles = Stream.of(log1base2, log1Ver1, log1Ver0).map(HoodieLogFile::new)
.collect(Collectors.toList()); .collect(Collectors.toList());
logFiles.sort(HoodieLogFile.getLogFileComparator()); logFiles.sort(HoodieLogFile.getLogFileComparator());
assertEquals(log1Ver0, logFiles.get(0).getFileName()); assertEquals(log1Ver0, logFiles.get(0).getFileName());
@@ -255,8 +249,8 @@ public class TestFSUtils extends HoodieCommonTestHarness {
String log1base2W1 = FSUtils.makeLogFileName("file1", ".log", "2", 0, "1-1-1"); String log1base2W1 = FSUtils.makeLogFileName("file1", ".log", "2", 0, "1-1-1");
List<HoodieLogFile> logFiles = List<HoodieLogFile> logFiles =
Arrays.asList(log1Ver1W1, log1base2W0, log1base2W1, log1Ver1W0, log1Ver0W1, log1Ver0W0).stream() Stream.of(log1Ver1W1, log1base2W0, log1base2W1, log1Ver1W0, log1Ver0W1, log1Ver0W0)
.map(f -> new HoodieLogFile(f)).collect(Collectors.toList()); .map(HoodieLogFile::new).collect(Collectors.toList());
logFiles.sort(HoodieLogFile.getLogFileComparator()); logFiles.sort(HoodieLogFile.getLogFileComparator());
assertEquals(log1Ver0W0, logFiles.get(0).getFileName()); assertEquals(log1Ver0W0, logFiles.get(0).getFileName());
assertEquals(log1Ver0W1, logFiles.get(1).getFileName()); assertEquals(log1Ver0W1, logFiles.get(1).getFileName());
@@ -267,7 +261,6 @@ public class TestFSUtils extends HoodieCommonTestHarness {
} }
public static String makeOldLogFileName(String fileId, String logFileExtension, String baseCommitTime, int version) { public static String makeOldLogFileName(String fileId, String logFileExtension, String baseCommitTime, int version) {
Pattern oldLogFilePattern = Pattern.compile("\\.(.*)_(.*)\\.(.*)\\.([0-9]*)(\\.([0-9]*))");
return "." + String.format("%s_%s%s.%d", fileId, baseCommitTime, logFileExtension, version); return "." + String.format("%s_%s%s.%d", fileId, baseCommitTime, logFileExtension, version);
} }
} }

6
hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
View File

@@ -44,15 +44,15 @@ public class TestHoodieAvroUtils {
continue; continue;
} }
Assert.assertTrue("field name is null", field.name() != null); Assert.assertNotNull("field name is null", field.name());
Map<String, JsonNode> props = field.getJsonProps(); Map<String, JsonNode> props = field.getJsonProps();
Assert.assertTrue("The property is null", props != null); Assert.assertNotNull("The property is null", props);
if (field.name().equals("pii_col")) { if (field.name().equals("pii_col")) {
piiPresent = true; piiPresent = true;
Assert.assertTrue("sensitivity_level is removed in field 'pii_col'", props.containsKey("column_category")); Assert.assertTrue("sensitivity_level is removed in field 'pii_col'", props.containsKey("column_category"));
} else { } else {
Assert.assertTrue("The property shows up but not set", props.size() == 0); Assert.assertEquals("The property shows up but not set", 0, props.size());
} }
} }
Assert.assertTrue("column pii_col doesn't show up", piiPresent); Assert.assertTrue("column pii_col doesn't show up", piiPresent);

18
hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java
View File

@@ -20,7 +20,7 @@ package org.apache.hudi.common.util;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals;
/** /**
* Tests numeric utils. * Tests numeric utils.
@@ -29,14 +29,14 @@ public class TestNumericUtils {
@Test @Test
public void testHumanReadableByteCount() { public void testHumanReadableByteCount() {
assertTrue(NumericUtils.humanReadableByteCount(0).equals("0.0 B")); assertEquals("0.0 B", NumericUtils.humanReadableByteCount(0));
assertTrue(NumericUtils.humanReadableByteCount(27).equals("27.0 B")); assertEquals("27.0 B", NumericUtils.humanReadableByteCount(27));
assertTrue(NumericUtils.humanReadableByteCount(1023).equals("1023.0 B")); assertEquals("1023.0 B", NumericUtils.humanReadableByteCount(1023));
assertTrue(NumericUtils.humanReadableByteCount(1024).equals("1.0 KB")); assertEquals("1.0 KB", NumericUtils.humanReadableByteCount(1024));
assertTrue(NumericUtils.humanReadableByteCount(110592).equals("108.0 KB")); assertEquals("108.0 KB", NumericUtils.humanReadableByteCount(110592));
assertTrue(NumericUtils.humanReadableByteCount(28991029248L).equals("27.0 GB")); assertEquals("27.0 GB", NumericUtils.humanReadableByteCount(28991029248L));
assertTrue(NumericUtils.humanReadableByteCount(1855425871872L).equals("1.7 TB")); assertEquals("1.7 TB", NumericUtils.humanReadableByteCount(1855425871872L));
assertTrue(NumericUtils.humanReadableByteCount(9223372036854775807L).equals("8.0 EB")); assertEquals("8.0 EB", NumericUtils.humanReadableByteCount(9223372036854775807L));
} }
} }

11
hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
View File

@@ -29,7 +29,6 @@ import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.ParquetWriter;
@@ -101,16 +100,6 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
} }
} }
private Configuration getConfiguration() {
if (bloomFilterTypeToTest.equalsIgnoreCase(BloomFilterTypeCode.SIMPLE.name())) {
return HoodieTestUtils.getDefaultHadoopConf();
} else {
org.apache.hadoop.conf.Configuration conf = HoodieTestUtils.getDefaultHadoopConf();
// conf.set();
return conf;
}
}
@Test @Test
public void testFilterParquetRowKeys() throws Exception { public void testFilterParquetRowKeys() throws Exception {
List<String> rowKeys = new ArrayList<>(); List<String> rowKeys = new ArrayList<>();

20
hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java
View File

@@ -59,7 +59,7 @@ public class TestRocksDBManager {
} }
@Test @Test
public void testRocksDBManager() throws Exception { public void testRocksDBManager() {
String prefix1 = "prefix1_"; String prefix1 = "prefix1_";
String prefix2 = "prefix2_"; String prefix2 = "prefix2_";
String prefix3 = "prefix3_"; String prefix3 = "prefix3_";
@@ -77,11 +77,11 @@ public class TestRocksDBManager {
return new Payload(prefix, key, val, family); return new Payload(prefix, key, val, family);
}).collect(Collectors.toList()); }).collect(Collectors.toList());
colFamilies.stream().forEach(family -> dbManager.dropColumnFamily(family)); colFamilies.forEach(family -> dbManager.dropColumnFamily(family));
colFamilies.stream().forEach(family -> dbManager.addColumnFamily(family)); colFamilies.forEach(family -> dbManager.addColumnFamily(family));
Map<String, Map<String, Integer>> countsMap = new HashMap<>(); Map<String, Map<String, Integer>> countsMap = new HashMap<>();
payloads.stream().forEach(payload -> { payloads.forEach(payload -> {
dbManager.put(payload.getFamily(), payload.getKey(), payload); dbManager.put(payload.getFamily(), payload.getKey(), payload);
if (!countsMap.containsKey(payload.family)) { if (!countsMap.containsKey(payload.family)) {
@@ -95,21 +95,21 @@ public class TestRocksDBManager {
c.put(payload.prefix, currCount + 1); c.put(payload.prefix, currCount + 1);
}); });
colFamilies.stream().forEach(family -> { colFamilies.forEach(family -> {
prefixes.stream().forEach(prefix -> { prefixes.forEach(prefix -> {
List<Pair<String, Payload>> gotPayloads = List<Pair<String, Payload>> gotPayloads =
dbManager.<Payload>prefixSearch(family, prefix).collect(Collectors.toList()); dbManager.<Payload>prefixSearch(family, prefix).collect(Collectors.toList());
Integer expCount = countsMap.get(family).get(prefix); Integer expCount = countsMap.get(family).get(prefix);
Assert.assertEquals("Size check for prefix (" + prefix + ") and family (" + family + ")", Assert.assertEquals("Size check for prefix (" + prefix + ") and family (" + family + ")",
expCount == null ? 0L : expCount.longValue(), gotPayloads.size()); expCount == null ? 0L : expCount.longValue(), gotPayloads.size());
gotPayloads.stream().forEach(p -> { gotPayloads.forEach(p -> {
Assert.assertEquals(p.getRight().getFamily(), family); Assert.assertEquals(p.getRight().getFamily(), family);
Assert.assertTrue(p.getRight().getKey().startsWith(prefix)); Assert.assertTrue(p.getRight().getKey().startsWith(prefix));
}); });
}); });
}); });
payloads.stream().forEach(payload -> { payloads.forEach(payload -> {
Payload p = dbManager.get(payload.getFamily(), payload.getKey()); Payload p = dbManager.get(payload.getFamily(), payload.getKey());
Assert.assertEquals("Retrieved correct payload for key :" + payload.getKey(), payload, p); Assert.assertEquals("Retrieved correct payload for key :" + payload.getKey(), payload, p);
@@ -122,8 +122,8 @@ public class TestRocksDBManager {
}); });
// Now do a prefix search // Now do a prefix search
colFamilies.stream().forEach(family -> { colFamilies.forEach(family -> {
prefixes.stream().forEach(prefix -> { prefixes.forEach(prefix -> {
List<Pair<String, Payload>> gotPayloads = List<Pair<String, Payload>> gotPayloads =
dbManager.<Payload>prefixSearch(family, prefix).collect(Collectors.toList()); dbManager.<Payload>prefixSearch(family, prefix).collect(Collectors.toList());
Assert.assertEquals("Size check for prefix (" + prefix + ") and family (" + family + ")", 0, Assert.assertEquals("Size check for prefix (" + prefix + ") and family (" + family + ")", 0,

2
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java
View File

@@ -87,7 +87,7 @@ public class TestDiskBasedMap extends HoodieCommonTestHarness {
List<HoodieRecord> hoodieRecords = SchemaTestUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000); List<HoodieRecord> hoodieRecords = SchemaTestUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000);
Set<String> recordKeys = new HashSet<>(); Set<String> recordKeys = new HashSet<>();
// insert generated records into the map // insert generated records into the map
hoodieRecords.stream().forEach(r -> { hoodieRecords.forEach(r -> {
records.put(r.getRecordKey(), r); records.put(r.getRecordKey(), r);
recordKeys.add(r.getRecordKey()); recordKeys.add(r.getRecordKey());
}); });

9
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
View File

@@ -87,7 +87,6 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
public void testSimpleUpsert() throws IOException, URISyntaxException { public void testSimpleUpsert() throws IOException, URISyntaxException {
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema());
String payloadClazz = HoodieAvroPayload.class.getName();
ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B
@@ -110,7 +109,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
assertTrue(records.getDiskBasedMapNumEntries() > 0); assertTrue(records.getDiskBasedMapNumEntries() > 0);
// iterate over the updated records and compare the value from Map // iterate over the updated records and compare the value from Map
updatedRecords.stream().forEach(record -> { updatedRecords.forEach(record -> {
HoodieRecord rec = records.get(((GenericRecord) record).get(HoodieRecord.RECORD_KEY_METADATA_FIELD)); HoodieRecord rec = records.get(((GenericRecord) record).get(HoodieRecord.RECORD_KEY_METADATA_FIELD));
try { try {
assertEquals(rec.getData().getInsertValue(schema).get(), record); assertEquals(rec.getData().getInsertValue(schema).get(), record);
@@ -196,7 +195,6 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
public void testDataCorrectnessWithUpsertsToDataInMapAndOnDisk() throws IOException, URISyntaxException { public void testDataCorrectnessWithUpsertsToDataInMapAndOnDisk() throws IOException, URISyntaxException {
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema());
String payloadClazz = HoodieAvroPayload.class.getName();
ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B
@@ -248,7 +246,6 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
public void testDataCorrectnessWithoutHoodieMetadata() throws IOException, URISyntaxException { public void testDataCorrectnessWithoutHoodieMetadata() throws IOException, URISyntaxException {
Schema schema = SchemaTestUtil.getSimpleSchema(); Schema schema = SchemaTestUtil.getSimpleSchema();
String payloadClazz = HoodieAvroPayload.class.getName();
ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(schema)); // 16B
@@ -278,7 +275,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
SchemaTestUtil.updateHoodieTestRecordsWithoutHoodieMetadata(recordsToUpdate, schema, fieldName, newValue); SchemaTestUtil.updateHoodieTestRecordsWithoutHoodieMetadata(recordsToUpdate, schema, fieldName, newValue);
// Upsert this updated record // Upsert this updated record
updatedRecords.stream().forEach(r -> { updatedRecords.forEach(r -> {
records.put(r.getRecordKey(), r); records.put(r.getRecordKey(), r);
}); });
GenericRecord gRecord = (GenericRecord) records.get(key).getData().getInsertValue(schema).get(); GenericRecord gRecord = (GenericRecord) records.get(key).getData().getInsertValue(schema).get();
@@ -300,7 +297,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
SchemaTestUtil.updateHoodieTestRecordsWithoutHoodieMetadata(recordsToUpdate, schema, fieldName, newValue); SchemaTestUtil.updateHoodieTestRecordsWithoutHoodieMetadata(recordsToUpdate, schema, fieldName, newValue);
// Upsert this updated record // Upsert this updated record
updatedRecords.stream().forEach(r -> { updatedRecords.forEach(r -> {
records.put(r.getRecordKey(), r); records.put(r.getRecordKey(), r);
}); });
gRecord = (GenericRecord) records.get(key).getData().getInsertValue(schema).get(); gRecord = (GenericRecord) records.get(key).getData().getInsertValue(schema).get();

2
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java
View File

@@ -20,7 +20,6 @@ package org.apache.hudi.hadoop;
import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.common.util.HoodieAvroUtils;
import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SchemaTestUtil;
@@ -159,7 +158,6 @@ public class InputFormatTestUtil {
parquetWriter = new AvroParquetWriter(new Path(dataFile.getAbsolutePath()), schema); parquetWriter = new AvroParquetWriter(new Path(dataFile.getAbsolutePath()), schema);
try { try {
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, numberOfRecords); List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, numberOfRecords);
String commitTime = HoodieActiveTimeline.createNewInstantTime();
Schema hoodieFieldsSchema = HoodieAvroUtils.addMetadataFields(schema); Schema hoodieFieldsSchema = HoodieAvroUtils.addMetadataFields(schema);
for (IndexedRecord record : records) { for (IndexedRecord record : records) {
GenericRecord p = HoodieAvroUtils.rewriteRecord((GenericRecord) record, hoodieFieldsSchema); GenericRecord p = HoodieAvroUtils.rewriteRecord((GenericRecord) record, hoodieFieldsSchema);

9
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java
View File

@@ -26,7 +26,6 @@ import org.apache.hadoop.mapred.RecordReader;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
@@ -61,7 +60,7 @@ public class TestRecordReaderValueIterator {
} }
@Override @Override
public boolean next(IntWritable key, Text value) throws IOException { public boolean next(IntWritable key, Text value) {
if (currIndex >= entries.size()) { if (currIndex >= entries.size()) {
return false; return false;
} }
@@ -82,17 +81,17 @@ public class TestRecordReaderValueIterator {
} }
@Override @Override
public long getPos() throws IOException { public long getPos() {
return currIndex; return currIndex;
} }
@Override @Override
public void close() throws IOException { public void close() {
} }
@Override @Override
public float getProgress() throws IOException { public float getProgress() {
return (currIndex * 1.0F) / entries.size(); return (currIndex * 1.0F) / entries.size();
} }
} }

60
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
View File

@@ -75,6 +75,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@@ -89,7 +90,7 @@ public class TestHoodieRealtimeRecordReader {
@Before @Before
public void setUp() { public void setUp() {
jobConf = new JobConf(); jobConf = new JobConf();
jobConf.set(AbstractRealtimeRecordReader.MAX_DFS_STREAM_BUFFER_SIZE_PROP, String.valueOf(1 * 1024 * 1024)); jobConf.set(AbstractRealtimeRecordReader.MAX_DFS_STREAM_BUFFER_SIZE_PROP, String.valueOf(1024 * 1024));
hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); hadoopConf = HoodieTestUtils.getDefaultHadoopConf();
fs = FSUtils.getFs(basePath.getRoot().getAbsolutePath(), hadoopConf); fs = FSUtils.getFs(basePath.getRoot().getAbsolutePath(), hadoopConf);
} }
@@ -198,7 +199,7 @@ public class TestHoodieRealtimeRecordReader {
FileSlice fileSlice = FileSlice fileSlice =
new FileSlice(partitioned ? FSUtils.getRelativePartitionPath(new Path(basePath.getRoot().getAbsolutePath()), new FileSlice(partitioned ? FSUtils.getRelativePartitionPath(new Path(basePath.getRoot().getAbsolutePath()),
new Path(partitionDir.getAbsolutePath())) : "default", baseInstant, "fileid0"); new Path(partitionDir.getAbsolutePath())) : "default", baseInstant, "fileid0");
logVersionsWithAction.stream().forEach(logVersionWithAction -> { logVersionsWithAction.forEach(logVersionWithAction -> {
try { try {
// update files or generate new log file // update files or generate new log file
int logVersion = logVersionWithAction.getRight(); int logVersion = logVersionWithAction.getRight();
@@ -246,7 +247,7 @@ public class TestHoodieRealtimeRecordReader {
while (recordReader.next(key, value)) { while (recordReader.next(key, value)) {
Writable[] values = value.get(); Writable[] values = value.get();
// check if the record written is with latest commit, here "101" // check if the record written is with latest commit, here "101"
Assert.assertEquals(latestInstant, values[0].toString()); assertEquals(latestInstant, values[0].toString());
key = recordReader.createKey(); key = recordReader.createKey();
value = recordReader.createValue(); value = recordReader.createValue();
} }
@@ -306,17 +307,17 @@ public class TestHoodieRealtimeRecordReader {
int numRecordsAtCommit1 = 0; int numRecordsAtCommit1 = 0;
int numRecordsAtCommit2 = 0; int numRecordsAtCommit2 = 0;
Set<Integer> seenKeys = new HashSet<>(); Set<Integer> seenKeys = new HashSet<>();
Integer lastSeenKeyFromLog = firstBatchLastRecordKey; int lastSeenKeyFromLog = firstBatchLastRecordKey;
while (recordReader.next(key, value)) { while (recordReader.next(key, value)) {
Writable[] values = value.get(); Writable[] values = value.get();
String gotCommit = values[0].toString(); String gotCommit = values[0].toString();
String keyStr = values[2].toString(); String keyStr = values[2].toString();
Integer gotKey = Integer.parseInt(keyStr.substring("key".length())); int gotKey = Integer.parseInt(keyStr.substring("key".length()));
if (gotCommit.equals(newCommitTime)) { if (gotCommit.equals(newCommitTime)) {
numRecordsAtCommit2++; numRecordsAtCommit2++;
Assert.assertTrue(gotKey > firstBatchLastRecordKey); Assert.assertTrue(gotKey > firstBatchLastRecordKey);
Assert.assertTrue(gotKey <= secondBatchLastRecordKey); Assert.assertTrue(gotKey <= secondBatchLastRecordKey);
Assert.assertEquals(gotKey.intValue(), lastSeenKeyFromLog + 1); assertEquals((int) gotKey, lastSeenKeyFromLog + 1);
lastSeenKeyFromLog++; lastSeenKeyFromLog++;
} else { } else {
numRecordsAtCommit1++; numRecordsAtCommit1++;
@@ -329,9 +330,9 @@ public class TestHoodieRealtimeRecordReader {
key = recordReader.createKey(); key = recordReader.createKey();
value = recordReader.createValue(); value = recordReader.createValue();
} }
Assert.assertEquals(numRecords, numRecordsAtCommit1); assertEquals(numRecords, numRecordsAtCommit1);
Assert.assertEquals(numRecords, numRecordsAtCommit2); assertEquals(numRecords, numRecordsAtCommit2);
Assert.assertEquals(2 * numRecords, seenKeys.size()); assertEquals(2 * numRecords, seenKeys.size());
} }
@Test @Test
@@ -390,34 +391,34 @@ public class TestHoodieRealtimeRecordReader {
} }
String recordCommitTimeSuffix = "@" + recordCommitTime; String recordCommitTimeSuffix = "@" + recordCommitTime;
Assert.assertEquals(values[0].toString(), recordCommitTime); assertEquals(values[0].toString(), recordCommitTime);
key = recordReader.createKey(); key = recordReader.createKey();
value = recordReader.createValue(); value = recordReader.createValue();
// Assert type STRING // Assert type STRING
Assert.assertEquals("test value for field: field1", values[5].toString(), "field" + currentRecordNo); assertEquals("test value for field: field1", values[5].toString(), "field" + currentRecordNo);
Assert.assertEquals("test value for field: field2", values[6].toString(), assertEquals("test value for field: field2", values[6].toString(),
"field" + currentRecordNo + recordCommitTimeSuffix); "field" + currentRecordNo + recordCommitTimeSuffix);
Assert.assertEquals("test value for field: name", values[7].toString(), "name" + currentRecordNo); assertEquals("test value for field: name", values[7].toString(), "name" + currentRecordNo);
// Assert type INT // Assert type INT
IntWritable intWritable = (IntWritable) values[8]; IntWritable intWritable = (IntWritable) values[8];
Assert.assertEquals("test value for field: favoriteIntNumber", intWritable.get(), assertEquals("test value for field: favoriteIntNumber", intWritable.get(),
currentRecordNo + recordCommitTime.hashCode()); currentRecordNo + recordCommitTime.hashCode());
// Assert type LONG // Assert type LONG
LongWritable longWritable = (LongWritable) values[9]; LongWritable longWritable = (LongWritable) values[9];
Assert.assertEquals("test value for field: favoriteNumber", longWritable.get(), assertEquals("test value for field: favoriteNumber", longWritable.get(),
currentRecordNo + recordCommitTime.hashCode()); currentRecordNo + recordCommitTime.hashCode());
// Assert type FLOAT // Assert type FLOAT
FloatWritable floatWritable = (FloatWritable) values[10]; FloatWritable floatWritable = (FloatWritable) values[10];
Assert.assertEquals("test value for field: favoriteFloatNumber", floatWritable.get(), assertEquals("test value for field: favoriteFloatNumber", floatWritable.get(),
(float) ((currentRecordNo + recordCommitTime.hashCode()) / 1024.0), 0); (float) ((currentRecordNo + recordCommitTime.hashCode()) / 1024.0), 0);
// Assert type DOUBLE // Assert type DOUBLE
DoubleWritable doubleWritable = (DoubleWritable) values[11]; DoubleWritable doubleWritable = (DoubleWritable) values[11];
Assert.assertEquals("test value for field: favoriteDoubleNumber", doubleWritable.get(), assertEquals("test value for field: favoriteDoubleNumber", doubleWritable.get(),
(currentRecordNo + recordCommitTime.hashCode()) / 1024.0, 0); (currentRecordNo + recordCommitTime.hashCode()) / 1024.0, 0);
// Assert type MAP // Assert type MAP
@@ -425,36 +426,35 @@ public class TestHoodieRealtimeRecordReader {
Writable mapItemValue1 = mapItem.get()[0]; Writable mapItemValue1 = mapItem.get()[0];
Writable mapItemValue2 = mapItem.get()[1]; Writable mapItemValue2 = mapItem.get()[1];
Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get()[0].toString(), assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get()[0].toString(),
"mapItem1"); "mapItem1");
Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get()[0].toString(), assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get()[0].toString(),
"mapItem2"); "mapItem2");
Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get().length, 2); assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get().length, 2);
Assert.assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get().length, 2); assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get().length, 2);
Writable mapItemValue1value = ((ArrayWritable) mapItemValue1).get()[1]; Writable mapItemValue1value = ((ArrayWritable) mapItemValue1).get()[1];
Writable mapItemValue2value = ((ArrayWritable) mapItemValue2).get()[1]; Writable mapItemValue2value = ((ArrayWritable) mapItemValue2).get()[1];
Assert.assertEquals("test value for field: tags[\"mapItem1\"].item1", assertEquals("test value for field: tags[\"mapItem1\"].item1",
((ArrayWritable) mapItemValue1value).get()[0].toString(), "item" + currentRecordNo); ((ArrayWritable) mapItemValue1value).get()[0].toString(), "item" + currentRecordNo);
Assert.assertEquals("test value for field: tags[\"mapItem2\"].item1", assertEquals("test value for field: tags[\"mapItem2\"].item1",
((ArrayWritable) mapItemValue2value).get()[0].toString(), "item2" + currentRecordNo); ((ArrayWritable) mapItemValue2value).get()[0].toString(), "item2" + currentRecordNo);
Assert.assertEquals("test value for field: tags[\"mapItem1\"].item2", assertEquals("test value for field: tags[\"mapItem1\"].item2",
((ArrayWritable) mapItemValue1value).get()[1].toString(), "item" + currentRecordNo + recordCommitTimeSuffix); ((ArrayWritable) mapItemValue1value).get()[1].toString(), "item" + currentRecordNo + recordCommitTimeSuffix);
Assert.assertEquals("test value for field: tags[\"mapItem2\"].item2", assertEquals("test value for field: tags[\"mapItem2\"].item2",
((ArrayWritable) mapItemValue2value).get()[1].toString(), "item2" + currentRecordNo + recordCommitTimeSuffix); ((ArrayWritable) mapItemValue2value).get()[1].toString(), "item2" + currentRecordNo + recordCommitTimeSuffix);
// Assert type RECORD // Assert type RECORD
ArrayWritable recordItem = (ArrayWritable) values[13]; ArrayWritable recordItem = (ArrayWritable) values[13];
Writable[] nestedRecord = recordItem.get(); Writable[] nestedRecord = recordItem.get();
Assert.assertEquals("test value for field: testNestedRecord.isAdmin", ((BooleanWritable) nestedRecord[0]).get(), assertFalse("test value for field: testNestedRecord.isAdmin", ((BooleanWritable) nestedRecord[0]).get());
false); assertEquals("test value for field: testNestedRecord.userId", nestedRecord[1].toString(),
Assert.assertEquals("test value for field: testNestedRecord.userId", nestedRecord[1].toString(),
"UserId" + currentRecordNo + recordCommitTimeSuffix); "UserId" + currentRecordNo + recordCommitTimeSuffix);
// Assert type ARRAY // Assert type ARRAY
ArrayWritable arrayValue = (ArrayWritable) values[14]; ArrayWritable arrayValue = (ArrayWritable) values[14];
Writable[] arrayValues = arrayValue.get(); Writable[] arrayValues = arrayValue.get();
for (int i = 0; i < arrayValues.length; i++) { for (int i = 0; i < arrayValues.length; i++) {
Assert.assertEquals("test value for field: stringArray", "stringArray" + i + recordCommitTimeSuffix, assertEquals("test value for field: stringArray", "stringArray" + i + recordCommitTimeSuffix,
arrayValues[i].toString()); arrayValues[i].toString());
} }
} }
@@ -510,7 +510,7 @@ public class TestHoodieRealtimeRecordReader {
// Try to read all the fields passed by the new schema // Try to read all the fields passed by the new schema
setHiveColumnNameProps(fields, jobConf, true); setHiveColumnNameProps(fields, jobConf, true);
HoodieRealtimeRecordReader recordReader = null; HoodieRealtimeRecordReader recordReader;
try { try {
// validate record reader compaction // validate record reader compaction
recordReader = new HoodieRealtimeRecordReader(split, jobConf, reader); recordReader = new HoodieRealtimeRecordReader(split, jobConf, reader);

2
hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
View File

@@ -69,7 +69,7 @@ public class TestHiveSyncTool {
} }
@After @After
public void teardown() throws IOException, InterruptedException { public void teardown() throws IOException {
TestUtil.clear(); TestUtil.clear();
} }

16
hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java
View File

@@ -88,7 +88,7 @@ public class TestUtil {
static FileSystem fileSystem; static FileSystem fileSystem;
private static Set<String> createdTablesSet = Sets.newHashSet(); private static Set<String> createdTablesSet = Sets.newHashSet();
public static void setUp() throws IOException, InterruptedException, URISyntaxException { public static void setUp() throws IOException, InterruptedException {
if (dfsCluster == null) { if (dfsCluster == null) {
HdfsTestService service = new HdfsTestService(); HdfsTestService service = new HdfsTestService();
dfsCluster = service.start(true); dfsCluster = service.start(true);
@@ -153,7 +153,7 @@ public class TestUtil {
} }
static void createCOWDataset(String commitTime, int numberOfPartitions) static void createCOWDataset(String commitTime, int numberOfPartitions)
throws IOException, InitializationError, URISyntaxException, InterruptedException { throws IOException, InitializationError, URISyntaxException {
Path path = new Path(hiveSyncConfig.basePath); Path path = new Path(hiveSyncConfig.basePath);
FileIOUtils.deleteDirectory(new File(hiveSyncConfig.basePath)); FileIOUtils.deleteDirectory(new File(hiveSyncConfig.basePath));
HoodieTableMetaClient.initTableType(configuration, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE, HoodieTableMetaClient.initTableType(configuration, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE,
@@ -182,7 +182,7 @@ public class TestUtil {
.add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_REALTIME_TABLE); .add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_REALTIME_TABLE);
HoodieCommitMetadata compactionMetadata = new HoodieCommitMetadata(); HoodieCommitMetadata compactionMetadata = new HoodieCommitMetadata();
commitMetadata.getPartitionToWriteStats() commitMetadata.getPartitionToWriteStats()
.forEach((key, value) -> value.stream().forEach(l -> compactionMetadata.addWriteStat(key, l))); .forEach((key, value) -> value.forEach(l -> compactionMetadata.addWriteStat(key, l)));
createCompactionCommitFile(compactionMetadata, commitTime); createCompactionCommitFile(compactionMetadata, commitTime);
// Write a delta commit // Write a delta commit
HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), true); HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), true);
@@ -190,7 +190,7 @@ public class TestUtil {
} }
static void addCOWPartitions(int numberOfPartitions, boolean isParquetSchemaSimple, DateTime startFrom, static void addCOWPartitions(int numberOfPartitions, boolean isParquetSchemaSimple, DateTime startFrom,
String commitTime) throws IOException, URISyntaxException, InterruptedException { String commitTime) throws IOException, URISyntaxException {
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata commitMetadata =
createPartitions(numberOfPartitions, isParquetSchemaSimple, startFrom, commitTime); createPartitions(numberOfPartitions, isParquetSchemaSimple, startFrom, commitTime);
createdTablesSet.add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName); createdTablesSet.add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName);
@@ -207,7 +207,7 @@ public class TestUtil {
.add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_REALTIME_TABLE); .add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_REALTIME_TABLE);
HoodieCommitMetadata compactionMetadata = new HoodieCommitMetadata(); HoodieCommitMetadata compactionMetadata = new HoodieCommitMetadata();
commitMetadata.getPartitionToWriteStats() commitMetadata.getPartitionToWriteStats()
.forEach((key, value) -> value.stream().forEach(l -> compactionMetadata.addWriteStat(key, l))); .forEach((key, value) -> value.forEach(l -> compactionMetadata.addWriteStat(key, l)));
createCompactionCommitFile(compactionMetadata, commitTime); createCompactionCommitFile(compactionMetadata, commitTime);
HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), isLogSchemaSimple); HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), isLogSchemaSimple);
createDeltaCommitFile(deltaMetadata, deltaCommitTime); createDeltaCommitFile(deltaMetadata, deltaCommitTime);
@@ -232,7 +232,7 @@ public class TestUtil {
} }
private static HoodieCommitMetadata createPartitions(int numberOfPartitions, boolean isParquetSchemaSimple, private static HoodieCommitMetadata createPartitions(int numberOfPartitions, boolean isParquetSchemaSimple,
DateTime startFrom, String commitTime) throws IOException, URISyntaxException, InterruptedException { DateTime startFrom, String commitTime) throws IOException, URISyntaxException {
startFrom = startFrom.withTimeAtStartOfDay(); startFrom = startFrom.withTimeAtStartOfDay();
HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
@@ -249,7 +249,7 @@ public class TestUtil {
} }
private static List<HoodieWriteStat> createTestData(Path partPath, boolean isParquetSchemaSimple, String commitTime) private static List<HoodieWriteStat> createTestData(Path partPath, boolean isParquetSchemaSimple, String commitTime)
throws IOException, URISyntaxException, InterruptedException { throws IOException, URISyntaxException {
List<HoodieWriteStat> writeStats = Lists.newArrayList(); List<HoodieWriteStat> writeStats = Lists.newArrayList();
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
// Create 5 files // Create 5 files
@@ -266,7 +266,7 @@ public class TestUtil {
@SuppressWarnings({"unchecked", "deprecation"}) @SuppressWarnings({"unchecked", "deprecation"})
private static void generateParquetData(Path filePath, boolean isParquetSchemaSimple) private static void generateParquetData(Path filePath, boolean isParquetSchemaSimple)
throws IOException, URISyntaxException, InterruptedException { throws IOException, URISyntaxException {
Schema schema = (isParquetSchemaSimple ? SchemaTestUtil.getSimpleSchema() : SchemaTestUtil.getEvolvedSchema()); Schema schema = (isParquetSchemaSimple ? SchemaTestUtil.getSimpleSchema() : SchemaTestUtil.getEvolvedSchema());
org.apache.parquet.schema.MessageType parquetSchema = new AvroSchemaConverter().convert(schema); org.apache.parquet.schema.MessageType parquetSchema = new AvroSchemaConverter().convert(schema);
BloomFilter filter = BloomFilterFactory.createBloomFilter(1000, 0.0001, -1, BloomFilter filter = BloomFilterFactory.createBloomFilter(1000, 0.0001, -1,

25
hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
View File

@@ -121,20 +121,6 @@ public class HiveTestService {
return hiveServer; return hiveServer;
} }
public void stop() throws IOException {
resetSystemProperties();
if (tServer != null) {
tServer.stop();
}
if (hiveServer != null) {
hiveServer.stop();
}
LOG.info("Hive Minicluster service shut down.");
tServer = null;
hiveServer = null;
hadoopConf = null;
}
private HiveConf configureHive(Configuration conf, String localHiveLocation) throws IOException { private HiveConf configureHive(Configuration conf, String localHiveLocation) throws IOException {
conf.set("hive.metastore.local", "false"); conf.set("hive.metastore.local", "false");
conf.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + bindIP + ":" + metastorePort); conf.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + bindIP + ":" + metastorePort);
@@ -196,17 +182,6 @@ public class HiveTestService {
} }
} }
private void resetSystemProperties() {
for (Map.Entry<String, String> entry : sysProps.entrySet()) {
if (entry.getValue() != null) {
System.setProperty(entry.getKey(), entry.getValue());
} else {
System.getProperties().remove(entry.getKey());
}
}
sysProps.clear();
}
private static String getHiveLocation(String baseLocation) { private static String getHiveLocation(String baseLocation) {
return baseLocation + Path.SEPARATOR + "hive"; return baseLocation + Path.SEPARATOR + "hive";
} }

3
hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
View File

@@ -165,8 +165,7 @@ public abstract class ITTestBase {
LOG.error("\n\n ###### Stderr #######\n" + callback.getStderr().toString()); LOG.error("\n\n ###### Stderr #######\n" + callback.getStderr().toString());
if (expectedToSucceed) { if (expectedToSucceed) {
Assert.assertTrue("Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")", Assert.assertEquals("Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")", 0, exitCode);
exitCode == 0);
} else { } else {
Assert.assertTrue("Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")", Assert.assertTrue("Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")",
exitCode != 0); exitCode != 0);

20
hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
View File

@@ -32,7 +32,6 @@ import org.apache.hudi.exception.DatasetNotFoundException;
import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.PartitionValueExtractor;
import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor; import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor;
import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex;
@@ -69,7 +68,7 @@ public class DataSourceUtils {
*/ */
public static String getNestedFieldValAsString(GenericRecord record, String fieldName) { public static String getNestedFieldValAsString(GenericRecord record, String fieldName) {
Object obj = getNestedFieldVal(record, fieldName); Object obj = getNestedFieldVal(record, fieldName);
return (obj == null) ? null : obj.toString(); return obj.toString();
} }
/** /**
@@ -119,17 +118,6 @@ public class DataSourceUtils {
} }
} }
/**
* Create a partition value extractor class via reflection, passing in any configs needed.
*/
public static PartitionValueExtractor createPartitionExtractor(String partitionExtractorClass) {
try {
return (PartitionValueExtractor) ReflectionUtils.loadClass(partitionExtractorClass);
} catch (Throwable e) {
throw new HoodieException("Could not load partition extractor class " + partitionExtractorClass, e);
}
}
/** /**
* Create a payload class via reflection, passing in an ordering/precombine value. * Create a payload class via reflection, passing in an ordering/precombine value.
*/ */
@@ -152,7 +140,7 @@ public class DataSourceUtils {
} }
public static HoodieWriteClient createHoodieClient(JavaSparkContext jssc, String schemaStr, String basePath, public static HoodieWriteClient createHoodieClient(JavaSparkContext jssc, String schemaStr, String basePath,
String tblName, Map<String, String> parameters) throws Exception { String tblName, Map<String, String> parameters) {
// inline compaction is on by default for MOR // inline compaction is on by default for MOR
boolean inlineCompact = parameters.get(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY()) boolean inlineCompact = parameters.get(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY())
@@ -198,7 +186,7 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords, public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) throws Exception { HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) {
HoodieReadClient client = null; HoodieReadClient client = null;
try { try {
client = new HoodieReadClient<>(jssc, writeConfig, timelineService); client = new HoodieReadClient<>(jssc, writeConfig, timelineService);
@@ -217,7 +205,7 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords, public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
Map<String, String> parameters, Option<EmbeddedTimelineService> timelineService) throws Exception { Map<String, String> parameters, Option<EmbeddedTimelineService> timelineService) {
HoodieWriteConfig writeConfig = HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(parameters.get("path")).withProps(parameters).build(); HoodieWriteConfig.newBuilder().withPath(parameters.get("path")).withProps(parameters).build();
return dropDuplicates(jssc, incomingHoodieRecords, writeConfig, timelineService); return dropDuplicates(jssc, incomingHoodieRecords, writeConfig, timelineService);

2
hudi-spark/src/test/java/DataSourceTestUtils.java
View File

@@ -41,7 +41,7 @@ public class DataSourceTestUtils {
} }
public static List<String> convertToStringList(List<HoodieRecord> records) { public static List<String> convertToStringList(List<HoodieRecord> records) {
return records.stream().map(hr -> convertToString(hr)).filter(os -> os.isPresent()).map(os -> os.get()) return records.stream().map(DataSourceTestUtils::convertToString).filter(Option::isPresent).map(Option::get)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

3
hudi-spark/src/test/java/HoodieJavaApp.java
View File

@@ -114,13 +114,12 @@ public class HoodieJavaApp {
} else { } else {
dataGen = new HoodieTestDataGenerator(); dataGen = new HoodieTestDataGenerator();
} }
List<HoodieRecord> recordsSoFar = new ArrayList<>();
/** /**
* Commit with only inserts * Commit with only inserts
*/ */
// Generate some input.. // Generate some input..
recordsSoFar.addAll(dataGen.generateInserts("001"/* ignore */, 100)); List<HoodieRecord> recordsSoFar = new ArrayList<>(dataGen.generateInserts("001"/* ignore */, 100));
List<String> records1 = DataSourceTestUtils.convertToStringList(recordsSoFar); List<String> records1 = DataSourceTestUtils.convertToStringList(recordsSoFar);
Dataset<Row> inputDF1 = spark.read().json(jssc.parallelize(records1, 2)); Dataset<Row> inputDF1 = spark.read().json(jssc.parallelize(records1, 2));

4
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
View File

@@ -291,7 +291,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
} }
@Test @Test
public void testProps() throws IOException { public void testProps() {
TypedProperties props = TypedProperties props =
new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/" + PROPS_FILENAME_TEST_SOURCE)).getConfig(); new DFSPropertiesConfiguration(dfs, new Path(dfsBasePath + "/" + PROPS_FILENAME_TEST_SOURCE)).getConfig();
assertEquals(2, props.getInteger("hoodie.upsert.shuffle.parallelism")); assertEquals(2, props.getInteger("hoodie.upsert.shuffle.parallelism"));
@@ -547,7 +547,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
} }
@Test @Test
public void testDistributedTestDataSource() throws Exception { public void testDistributedTestDataSource() {
TypedProperties props = new TypedProperties(); TypedProperties props = new TypedProperties();
props.setProperty(TestSourceConfig.MAX_UNIQUE_RECORDS_PROP, "1000"); props.setProperty(TestSourceConfig.MAX_UNIQUE_RECORDS_PROP, "1000");
props.setProperty(TestSourceConfig.NUM_SOURCE_PARTITIONS_PROP, "1"); props.setProperty(TestSourceConfig.NUM_SOURCE_PARTITIONS_PROP, "1");

2
hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
View File

@@ -229,7 +229,7 @@ public class UtilitiesTestBase {
} }
} }
public static String[] jsonifyRecords(List<HoodieRecord> records) throws IOException { public static String[] jsonifyRecords(List<HoodieRecord> records) {
return records.stream().map(Helpers::toJsonString).toArray(String[]::new); return records.stream().map(Helpers::toJsonString).toArray(String[]::new);
} }
} }

4
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/DistributedTestDataSource.java
View File

@@ -30,7 +30,6 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import java.util.Iterator;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
@@ -77,8 +76,7 @@ public class DistributedTestDataSource extends AbstractBaseTestSource {
if (!dataGeneratorMap.containsKey(p)) { if (!dataGeneratorMap.containsKey(p)) {
initDataGen(newProps, p); initDataGen(newProps, p);
} }
Iterator<GenericRecord> itr = fetchNextBatch(newProps, perPartitionSourceLimit, commitTime, p).iterator(); return fetchNextBatch(newProps, perPartitionSourceLimit, commitTime, p).iterator();
return itr;
}, true); }, true);
return new InputBatch<>(Option.of(avroRDD), commitTime); return new InputBatch<>(Option.of(avroRDD), commitTime);
} }