lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2480] FileSlice after pending compaction-requested instant-time… (#3703)

Browse Source 
* [HUDI-2480] FileSlice after pending compaction-requested instant-time is ignored by MOR snapshot reader

* include file slice after a pending compaction for spark reader

Co-authored-by: garyli1019 <yanjia.gary.li@gmail.com>
This commit is contained in:
Danny Chan
2021-11-25 22:30:09 +08:00
committed by GitHub
parent 88067f57a2
commit a2eb2b0b0a
4 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
View File

@@ -463,6 +463,32 @@ public class TestInputFormat {
TestData.assertRowDataEquals(result, TestData.DATA_SET_INSERT);
}
/**
* Test reading file groups with compaction plan scheduled and delta logs.
* File-slice after pending compaction-requested instant-time should also be considered valid.
*/
@Test
void testReadMORWithCompactionPlanScheduled() throws Exception {
Map<String, String> options = new HashMap<>();
// compact for each commit
options.put(FlinkOptions.COMPACTION_DELTA_COMMITS.key(), "1");
options.put(FlinkOptions.COMPACTION_ASYNC_ENABLED.key(), "false");
beforeEach(HoodieTableType.MERGE_ON_READ, options);
// write three commits
for (int i = 0; i < 6; i += 2) {
List<RowData> dataset = TestData.dataSetInsert(i + 1, i + 2);
TestData.writeData(dataset, conf);
}
InputFormat<RowData, ?> inputFormat1 = this.tableSource.getInputFormat();
assertThat(inputFormat1, instanceOf(MergeOnReadInputFormat.class));
List<RowData> actual = readData(inputFormat1);
final List<RowData> expected = TestData.dataSetInsert(1, 2, 3, 4, 5, 6);
TestData.assertRowDataEquals(actual, expected);
}
// -------------------------------------------------------------------------
// Utilities
// -------------------------------------------------------------------------