lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[minor] following 4270, add unit tests for the keys lost case (#5918)

Browse Source
This commit is contained in:
Danny Chan
2022-06-22 16:56:06 +08:00
committed by GitHub
parent c9590790f8
commit 1dbd9d407a
3 changed files with 45 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
View File

@@ -516,7 +516,7 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
}
executorService.shutdown();
assertTrue(executorService.awaitTermination(10, TimeUnit.SECONDS));
assertTrue(executorService.awaitTermination(60, TimeUnit.SECONDS));
// required to catch exceptions
for (Future f : futures) {
f.get();

25
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java
View File

@@ -38,6 +38,31 @@ public class TestWriteMergeOnRead extends TestWriteCopyOnWrite {
conf.setBoolean(FlinkOptions.COMPACTION_ASYNC_ENABLED, false);
}
@Test
public void testIndexStateBootstrapWithMultiFilesInOneSlice() throws Exception {
// open the function and ingest data
preparePipeline(conf)
.consume(TestData.filterOddRows(TestData.DATA_SET_INSERT))
.assertEmptyDataFiles()
.checkpoint(1)
.assertNextEvent()
.checkpointComplete(1)
.consume(TestData.filterEvenRows(TestData.DATA_SET_INSERT))
.checkpoint(2)
.assertNextEvent()
.checkpointComplete(2)
.checkWrittenData(EXPECTED1, 4)
// write another commit but does not complete it
.consume(TestData.filterEvenRows(TestData.DATA_SET_INSERT))
.checkpoint(3)
.assertNextEvent()
.end();
// reset the config option
conf.setBoolean(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true);
validateIndexLoaded();
}
@Test
public void testIndexStateBootstrapWithCompactionScheduled() throws Exception {
// sets up the delta commits as 1 to generate a new compaction plan.

19
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
View File

@@ -64,6 +64,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -304,6 +305,24 @@ public class TestData {
return inserts;
}
public static List<RowData> filterOddRows(List<RowData> rows) {
return filterRowsByIndexPredicate(rows, i -> i % 2 != 0);
}
public static List<RowData> filterEvenRows(List<RowData> rows) {
return filterRowsByIndexPredicate(rows, i -> i % 2 == 0);
}
private static List<RowData> filterRowsByIndexPredicate(List<RowData> rows, Predicate<Integer> predicate) {
List<RowData> filtered = new ArrayList<>();
for (int i = 0; i < rows.size(); i++) {
if (predicate.test(i)) {
filtered.add(rows.get(i));
}
}
return filtered;
}
private static Integer toIdSafely(Object id) {
if (id == null) {
return -1;