lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2340] Merge the data set for flink bounded source when changelog mode turns off (#3513)

Browse Source
This commit is contained in:
Danny Chan
2021-08-21 07:21:35 +08:00
committed by GitHub
parent e39d0a2f28
commit c7c517f14c
4 changed files with 53 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
View File

@@ -197,7 +197,7 @@ public class HoodieTableSource implements
@Override
public ChangelogMode getChangelogMode() {
return conf.getBoolean(FlinkOptions.READ_AS_STREAMING)
return conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)
? ChangelogModes.FULL
// when all the changes are persisted or read as batch,
// use INSERT mode.

11
hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java
View File

@@ -91,6 +91,17 @@ public class FormatUtils {
}
}
/**
* Returns the RowKind of the given record, never null.
* Returns RowKind.INSERT when the given field value not found.
*/
public static RowKind getRowKindSafely(IndexedRecord record, int index) {
if (index == -1) {
return RowKind.INSERT;
}
return getRowKind(record, index);
}
public static GenericRecord buildAvroRecordBySchema(
IndexedRecord record,
Schema requiredSchema,

10
hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
View File

@@ -177,7 +177,8 @@ public class MergeOnReadInputFormat
}
} else if (!split.getBasePath().isPresent()) {
// log files only
if (conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)) {
if (conf.getBoolean(FlinkOptions.READ_AS_STREAMING)
&& conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)) {
this.iterator = new LogFileOnlyIterator(getUnMergedLogFileIterator(split));
} else {
this.iterator = new LogFileOnlyIterator(getLogFileIterator(split));
@@ -350,13 +351,18 @@ public class MergeOnReadInputFormat
// continue;
} else {
final IndexedRecord avroRecord = curAvroRecord.get();
final RowKind rowKind = FormatUtils.getRowKindSafely(avroRecord, tableState.getOperationPos());
if (rowKind == RowKind.DELETE && !emitDelete) {
// skip the delete record
continue;
}
GenericRecord requiredAvroRecord = buildAvroRecordBySchema(
avroRecord,
requiredSchema,
requiredPos,
recordBuilder);
currentRecord = (RowData) avroToRowDataConverter.convert(requiredAvroRecord);
FormatUtils.setRowKind(currentRecord, avroRecord, tableState.getOperationPos());
currentRecord.setRowKind(rowKind);
return true;
}
}