lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2074] Use while loop instead of recursive call in MergeOnReadInputFormat#MergeIterator to avoid StackOverflow (#3159)

Browse Source
This commit is contained in:
Danny Chan
2021-06-28 16:03:10 +08:00
committed by GitHub
parent e99a6b031b
commit d24341d10c
1 changed files with 21 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
View File

@@ -572,13 +572,13 @@ public class MergeOnReadInputFormat
@Override @Override
public boolean reachedEnd() throws IOException { public boolean reachedEnd() throws IOException {
if (!readLogs && !this.reader.reachedEnd()) { while (!readLogs && !this.reader.reachedEnd()) {
currentRecord = this.reader.nextRecord(); currentRecord = this.reader.nextRecord();
if (instantRange != null) { if (instantRange != null) {
boolean isInRange = instantRange.isInRange(currentRecord.getString(HOODIE_COMMIT_TIME_COL_POS).toString()); boolean isInRange = instantRange.isInRange(currentRecord.getString(HOODIE_COMMIT_TIME_COL_POS).toString());
if (!isInRange) { if (!isInRange) {
// filter base file by instant range // filter base file by instant range
return reachedEnd(); continue;
} }
} }
final String curKey = currentRecord.getString(HOODIE_RECORD_KEY_COL_POS).toString(); final String curKey = currentRecord.getString(HOODIE_RECORD_KEY_COL_POS).toString();
@@ -587,7 +587,7 @@ public class MergeOnReadInputFormat
Option<IndexedRecord> mergedAvroRecord = mergeRowWithLog(currentRecord, curKey); Option<IndexedRecord> mergedAvroRecord = mergeRowWithLog(currentRecord, curKey);
if (!mergedAvroRecord.isPresent()) { if (!mergedAvroRecord.isPresent()) {
// deleted // deleted
return reachedEnd(); continue;
} else { } else {
GenericRecord record = buildAvroRecordBySchema( GenericRecord record = buildAvroRecordBySchema(
mergedAvroRecord.get(), mergedAvroRecord.get(),
@@ -601,7 +601,8 @@ public class MergeOnReadInputFormat
// project the full record in base with required positions // project the full record in base with required positions
currentRecord = projection.project(currentRecord); currentRecord = projection.project(currentRecord);
return false; return false;
} else { }
// read the logs
readLogs = true; readLogs = true;
while (logKeysIterator.hasNext()) { while (logKeysIterator.hasNext()) {
final String curKey = logKeysIterator.next(); final String curKey = logKeysIterator.next();
@@ -622,7 +623,6 @@ public class MergeOnReadInputFormat
} }
return true; return true;
} }
}
@Override @Override
public RowData nextRecord() { public RowData nextRecord() {