lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-727]: Copy default values of fields if not present when rewriting incoming record with new schema (#1427)

Browse Source
This commit is contained in:
Pratyaksh Sharma
2020-04-13 06:25:26 +05:30
committed by GitHub
parent 5d717a28f4
commit 6d7ca2cf7e
3 changed files with 84 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
View File

@@ -212,7 +212,7 @@ public abstract class AbstractRealtimeRecordReader {
throw new HoodieException("Field " + fn + " not found in log schema. Query cannot proceed! "
+ "Derived Schema Fields: " + new ArrayList<>(schemaFieldsMap.keySet()));
} else {
projectedFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
projectedFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
}
}
@@ -367,7 +367,7 @@ public abstract class AbstractRealtimeRecordReader {
Field field = schemaFieldsMap.get(columnName.toLowerCase());
if (field != null) {
hiveSchemaFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
hiveSchemaFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
} else {
// Hive has some extra virtual columns like BLOCK__OFFSET__INSIDE__FILE which do not exist in table schema.
// They will get skipped as they won't be found in the original schema.