lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2112] Support reading pure logs file group for flink batch reader after compaction (#3202)

Browse Source
This commit is contained in:
Danny Chan
2021-07-02 16:29:22 +08:00
committed by GitHub
parent 6403547431
commit 7462fdefc3
3 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
View File

@@ -92,7 +92,7 @@ public class TestInputFormat {
TestData.writeData(TestData.DATA_SET_UPDATE_INSERT, conf);
// refresh the input format
this.tableSource.reloadActiveTimeline();
this.tableSource.reset();
inputFormat = this.tableSource.getInputFormat();
result = readData(inputFormat);
@@ -133,8 +133,12 @@ public class TestInputFormat {
conf.setBoolean(FlinkOptions.COMPACTION_ASYNC_ENABLED, false);
TestData.writeData(TestData.DATA_SET_UPDATE_INSERT, conf);
// write another commit using logs with separate partition
// so the file group has only logs
TestData.writeData(TestData.DATA_SET_INSERT_SEPARATE_PARTITION, conf);
// refresh the input format
this.tableSource.reloadActiveTimeline();
this.tableSource.reset();
inputFormat = this.tableSource.getInputFormat();
result = readData(inputFormat);
@@ -143,6 +147,10 @@ public class TestInputFormat {
expected = "[id1,Danny,24,1970-01-01T00:00:00.001,par1, "
+ "id10,Ella,38,1970-01-01T00:00:00.007,par4, "
+ "id11,Phoebe,52,1970-01-01T00:00:00.008,par4, "
+ "id12,Monica,27,1970-01-01T00:00:00.009,par5, "
+ "id13,Phoebe,31,1970-01-01T00:00:00.010,par5, "
+ "id14,Rachel,52,1970-01-01T00:00:00.011,par6, "
+ "id15,Ross,29,1970-01-01T00:00:00.012,par6, "
+ "id2,Stephen,34,1970-01-01T00:00:00.002,par1, "
+ "id3,Julian,54,1970-01-01T00:00:00.003,par2, "
+ "id4,Fabian,32,1970-01-01T00:00:00.004,par2, "

11
hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
View File

@@ -114,6 +114,17 @@ public class TestData {
TimestampData.fromEpochMillis(8), StringData.fromString("par4"))
);
public static List<RowData> DATA_SET_INSERT_SEPARATE_PARTITION = Arrays.asList(
insertRow(StringData.fromString("id12"), StringData.fromString("Monica"), 27,
TimestampData.fromEpochMillis(9), StringData.fromString("par5")),
insertRow(StringData.fromString("id13"), StringData.fromString("Phoebe"), 31,
TimestampData.fromEpochMillis(10), StringData.fromString("par5")),
insertRow(StringData.fromString("id14"), StringData.fromString("Rachel"), 52,
TimestampData.fromEpochMillis(11), StringData.fromString("par6")),
insertRow(StringData.fromString("id15"), StringData.fromString("Ross"), 29,
TimestampData.fromEpochMillis(12), StringData.fromString("par6"))
);
public static List<RowData> DATA_SET_INSERT_DUPLICATES = new ArrayList<>();
static {
IntStream.range(0, 5).forEach(i -> DATA_SET_INSERT_DUPLICATES.add(