[HUDI-2009] Fixing extra commit metadata in row writer path (#3075)

hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java

@@ -53,6 +53,6 @@ public class DefaultSource extends BaseDefaultSource implements TableProvider {
     // 1st arg to createHooodieConfig is not really reuqired to be set. but passing it anyways.
     HoodieWriteConfig config = DataSourceUtils.createHoodieConfig(properties.get(HoodieWriteConfig.AVRO_SCHEMA.key()), path, tblName, properties);
     return new HoodieDataSourceInternalTable(instantTime, config, schema, getSparkSession(),
-        getConfiguration(), arePartitionRecordsSorted);
+        getConfiguration(), properties, arePartitionRecordsSorted);
   }
 }

hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWrite.java

@@ -18,6 +18,7 @@
 
 package org.apache.hudi.spark3.internal;
 
+import org.apache.hudi.DataSourceUtils;
 import org.apache.hudi.client.HoodieInternalWriteStatus;
 import org.apache.hudi.common.model.HoodieWriteStat;
 import org.apache.hudi.common.model.WriteOperationType;
@@ -33,7 +34,9 @@ import org.apache.spark.sql.connector.write.PhysicalWriteInfo;
 import org.apache.spark.sql.types.StructType;
 
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -47,15 +50,17 @@ public class HoodieDataSourceInternalBatchWrite implements BatchWrite {
   private final StructType structType;
   private final boolean arePartitionRecordsSorted;
   private final DataSourceInternalWriterHelper dataSourceInternalWriterHelper;
+  private Map<String, String> extraMetadata = new HashMap<>();
 
   public HoodieDataSourceInternalBatchWrite(String instantTime, HoodieWriteConfig writeConfig, StructType structType,
-      SparkSession jss, Configuration hadoopConfiguration, boolean arePartitionRecordsSorted) {
+      SparkSession jss, Configuration hadoopConfiguration, Map<String, String> properties, boolean arePartitionRecordsSorted) {
     this.instantTime = instantTime;
     this.writeConfig = writeConfig;
     this.structType = structType;
     this.arePartitionRecordsSorted = arePartitionRecordsSorted;
+    this.extraMetadata = DataSourceUtils.getExtraMetadata(properties);
     this.dataSourceInternalWriterHelper = new DataSourceInternalWriterHelper(instantTime, writeConfig, structType,
-        jss, hadoopConfiguration);
+        jss, hadoopConfiguration, extraMetadata);
   }
 
   @Override

hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWriteBuilder.java

@@ -26,6 +26,8 @@ import org.apache.spark.sql.connector.write.BatchWrite;
 import org.apache.spark.sql.connector.write.WriteBuilder;
 import org.apache.spark.sql.types.StructType;
 
+import java.util.Map;
+
 /**
  * Implementation of {@link WriteBuilder} for datasource "hudi.spark3.internal" to be used in datasource implementation
  * of bulk insert.
@@ -38,20 +40,22 @@ public class HoodieDataSourceInternalBatchWriteBuilder implements WriteBuilder {
   private final SparkSession jss;
   private final Configuration hadoopConfiguration;
   private final boolean arePartitionRecordsSorted;
+  private final Map<String, String> properties;
 
   public HoodieDataSourceInternalBatchWriteBuilder(String instantTime, HoodieWriteConfig writeConfig, StructType structType,
-      SparkSession jss, Configuration hadoopConfiguration, boolean arePartitionRecordsSorted) {
+      SparkSession jss, Configuration hadoopConfiguration, Map<String, String> properties, boolean arePartitionRecordsSorted) {
     this.instantTime = instantTime;
     this.writeConfig = writeConfig;
     this.structType = structType;
     this.jss = jss;
     this.hadoopConfiguration = hadoopConfiguration;
     this.arePartitionRecordsSorted = arePartitionRecordsSorted;
+    this.properties = properties;
   }
 
   @Override
   public BatchWrite buildForBatch() {
     return new HoodieDataSourceInternalBatchWrite(instantTime, writeConfig, structType, jss,
-        hadoopConfiguration, arePartitionRecordsSorted);
+        hadoopConfiguration, properties, arePartitionRecordsSorted);
   }
 }

hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalTable.java

@@ -29,6 +29,7 @@ import org.apache.spark.sql.connector.write.WriteBuilder;
 import org.apache.spark.sql.types.StructType;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -42,15 +43,18 @@ class HoodieDataSourceInternalTable implements SupportsWrite {
   private final SparkSession jss;
   private final Configuration hadoopConfiguration;
   private final boolean arePartitionRecordsSorted;
+  private final Map<String, String> properties;
 
   public HoodieDataSourceInternalTable(String instantTime, HoodieWriteConfig config,
-      StructType schema, SparkSession jss, Configuration hadoopConfiguration, boolean arePartitionRecordsSorted) {
+      StructType schema, SparkSession jss, Configuration hadoopConfiguration, Map<String, String> properties,
+                                       boolean arePartitionRecordsSorted) {
     this.instantTime = instantTime;
     this.writeConfig = config;
     this.structType = schema;
     this.jss = jss;
     this.hadoopConfiguration = hadoopConfiguration;
     this.arePartitionRecordsSorted = arePartitionRecordsSorted;
+    this.properties = properties;
   }
 
   @Override
@@ -75,6 +79,6 @@ class HoodieDataSourceInternalTable implements SupportsWrite {
   @Override
   public WriteBuilder newWriteBuilder(LogicalWriteInfo logicalWriteInfo) {
     return new HoodieDataSourceInternalBatchWriteBuilder(instantTime, writeConfig, structType, jss,
-        hadoopConfiguration, arePartitionRecordsSorted);
+        hadoopConfiguration, properties, arePartitionRecordsSorted);
   }
 }