lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1969] Support reading logs for MOR Hive rt table (#3033)

Browse Source
This commit is contained in:
Danny Chan
2021-07-14 14:43:30 +08:00
committed by GitHub
parent f0a2f378ea
commit ac75bda929
2 changed files with 73 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
View File

@@ -155,7 +155,7 @@ public class TestHoodieRealtimeRecordReader {
} else {
writer =
InputFormatTestUtil.writeDataBlockToLogFile(partitionDir, fs, schema, "fileid0", baseInstant,
instantTime, 100, 0, logVersion);
instantTime, 120, 0, logVersion);
}
long size = writer.getCurrentSize();
writer.close();
@@ -182,17 +182,21 @@ public class TestHoodieRealtimeRecordReader {
// use reader to read base Parquet File and log file, merge in flight and return latest commit
// here all 100 records should be updated, see above
// another 20 new insert records should also output with new commit time.
NullWritable key = recordReader.createKey();
ArrayWritable value = recordReader.createValue();
int recordCnt = 0;
while (recordReader.next(key, value)) {
Writable[] values = value.get();
// check if the record written is with latest commit, here "101"
assertEquals(latestInstant, values[0].toString());
key = recordReader.createKey();
value = recordReader.createValue();
recordCnt++;
}
recordReader.getPos();
assertEquals(1.0, recordReader.getProgress(), 0.05);
assertEquals(120, recordCnt);
recordReader.close();
} catch (Exception ioe) {
throw new HoodieException(ioe.getMessage(), ioe);