From 37c29e75dcbd9f045bb1737bb734e2b408081182 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Thu, 19 Aug 2021 21:37:12 +0530 Subject: [PATCH] [HUDI-2322] Use correct meta columns while preparing dataset for bulk insert (#3504) --- .../org/apache/hudi/HoodieDatasetBulkInsertHelper.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java index 0ccd33d7d..47e97aaeb 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java @@ -145,10 +145,14 @@ public class HoodieDatasetBulkInsertHelper { functions.lit("").cast(DataTypes.StringType)); List originalFields = - Arrays.stream(rowsWithMetaCols.schema().fields()).filter(field -> !field.name().contains("_hoodie_")).map(f -> new Column(f.name())).collect(Collectors.toList()); + Arrays.stream(rowsWithMetaCols.schema().fields()) + .filter(field -> !HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(field.name())) + .map(f -> new Column(f.name())).collect(Collectors.toList()); List metaFields = - Arrays.stream(rowsWithMetaCols.schema().fields()).filter(field -> field.name().contains("_hoodie_")).map(f -> new Column(f.name())).collect(Collectors.toList()); + Arrays.stream(rowsWithMetaCols.schema().fields()) + .filter(field -> HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(field.name())) + .map(f -> new Column(f.name())).collect(Collectors.toList()); // reorder such that all meta columns are at the beginning followed by original columns List allCols = new ArrayList<>();