lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2322] Use correct meta columns while preparing dataset for bulk insert (#3504)

Browse Source
This commit is contained in:
Sagar Sumit
2021-08-19 21:37:12 +05:30
committed by GitHub
parent 9762e4c08c
commit 37c29e75dc
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java
View File

@@ -145,10 +145,14 @@ public class HoodieDatasetBulkInsertHelper {
functions.lit("").cast(DataTypes.StringType)); functions.lit("").cast(DataTypes.StringType));
List<Column> originalFields = List<Column> originalFields =
Arrays.stream(rowsWithMetaCols.schema().fields()).filter(field -> !field.name().contains("_hoodie_")).map(f -> new Column(f.name())).collect(Collectors.toList()); Arrays.stream(rowsWithMetaCols.schema().fields())
.filter(field -> !HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(field.name()))
.map(f -> new Column(f.name())).collect(Collectors.toList());
List<Column> metaFields = List<Column> metaFields =
Arrays.stream(rowsWithMetaCols.schema().fields()).filter(field -> field.name().contains("_hoodie_")).map(f -> new Column(f.name())).collect(Collectors.toList()); Arrays.stream(rowsWithMetaCols.schema().fields())
.filter(field -> HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(field.name()))
.map(f -> new Column(f.name())).collect(Collectors.toList());
// reorder such that all meta columns are at the beginning followed by original columns // reorder such that all meta columns are at the beginning followed by original columns
List<Column> allCols = new ArrayList<>(); List<Column> allCols = new ArrayList<>();