[HUDI-332]Add operation type (insert/upsert/bulkinsert/delete) to HoodieCommitMetadata (#1157)

[HUDI-332]Add operation type (insert/upsert/bulkinsert/delete) to HoodieCommitMetadata (#1157)

hudi-common/src/main/avro/HoodieCommitMetadata.avsc

@@ -133,6 +133,11 @@
          "name":"version",
          "type":["int", "null"],
          "default": 1
+      },
+      {
+         "name":"operationType",
+         "type": ["null","string"],
+         "default":null
       }
    ]
 }

hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java

@@ -49,6 +49,8 @@ public class HoodieCommitMetadata implements Serializable {
 
   private Map<String, String> extraMetadata;
 
+  private WriteOperationType operationType = WriteOperationType.UNKNOWN;
+
   // for ser/deser
   public HoodieCommitMetadata() {
     this(false);
@@ -106,6 +108,14 @@ public class HoodieCommitMetadata implements Serializable {
     return filePaths;
   }
 
+  public void setOperationType(WriteOperationType type) {
+    this.operationType = type;
+  }
+
+  public WriteOperationType getOperationType() {
+    return this.operationType;
+  }
+
   public HashMap<String, String> getFileIdAndFullPaths(String basePath) {
     HashMap<String, String> fullPaths = new HashMap<>();
     for (Map.Entry<String, String> entry : getFileIdAndRelativePaths().entrySet()) {

hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.model;
+
+import org.apache.hudi.exception.HoodieException;
+
+import java.util.Locale;
+
+/**
+ * The supported write operation types, used by commitMetadata.
+ */
+public enum WriteOperationType {
+  // directly insert
+  INSERT("insert"),
+  INSERT_PREPPED("insert_prepped"),
+  // update and insert
+  UPSERT("upsert"),
+  UPSERT_PREPPED("upsert_prepped"),
+  // bulk insert
+  BULK_INSERT("bulk_insert"),
+  BULK_INSERT_PREPPED("bulk_insert_prepped"),
+  // delete
+  DELETE("delete"),
+  // used for old version
+  UNKNOWN("unknown");
+
+  private final String value;
+
+  WriteOperationType(String value) {
+    this.value = value;
+  }
+
+  /**
+   * Convert string value to WriteOperationType.
+   */
+  public static WriteOperationType fromValue(String value) {
+    switch (value.toLowerCase(Locale.ROOT)) {
+      case "insert":
+        return INSERT;
+      case "insert_prepped":
+        return INSERT_PREPPED;
+      case "upsert":
+        return UPSERT;
+      case "upsert_prepped":
+        return UPSERT_PREPPED;
+      case "bulk_insert":
+        return BULK_INSERT;
+      case "bulk_insert_prepped":
+        return BULK_INSERT_PREPPED;
+      case "delete":
+        return DELETE;
+      default:
+        throw new HoodieException("Invalid value of Type.");
+    }
+  }
+}

hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java

@@ -18,6 +18,7 @@
 
 package org.apache.hudi.common.model;
 
+import org.apache.hudi.common.util.FileIOUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -46,4 +47,25 @@ public class TestHoodieCommitMetadata {
     Assert.assertEquals(0, (long) metadata.getTotalScanTime());
     Assert.assertTrue(metadata.getTotalLogFilesCompacted() > 0);
   }
+
+  @Test
+  public void testCompatibilityWithoutOperationType() throws Exception {
+    // test compatibility of old version file
+    String serializedCommitMetadata =
+        FileIOUtils.readAsUTFString(TestHoodieCommitMetadata.class.getResourceAsStream("/old-version.commit"));
+    HoodieCommitMetadata metadata =
+        HoodieCommitMetadata.fromJsonString(serializedCommitMetadata, HoodieCommitMetadata.class);
+    Assert.assertTrue(metadata.getOperationType() == WriteOperationType.UNKNOWN);
+
+    // test operate type
+    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
+    commitMetadata.setOperationType(WriteOperationType.INSERT);
+    Assert.assertTrue(commitMetadata.getOperationType() == WriteOperationType.INSERT);
+
+    // test serialized
+    serializedCommitMetadata = commitMetadata.toJsonString();
+    metadata =
+      HoodieCommitMetadata.fromJsonString(serializedCommitMetadata, HoodieCommitMetadata.class);
+    Assert.assertTrue(metadata.getOperationType() == WriteOperationType.INSERT);
+  }
 }

hudi-common/src/test/resources/old-version.commit

@@ -0,0 +1,20 @@
+{
+  "partitionToWriteStats" : {
+    "americas/brazil/sao_paulo" : []
+  },
+  "compacted" : false,
+  "extraMetadataMap" : {
+  },
+  "extraMetadata" : {
+  },
+  "fileIdAndRelativePaths" : {
+  },
+  "totalRecordsDeleted" : 0,
+  "totalLogRecordsCompacted" : 0,
+  "totalScanTime" : 0,
+  "totalCreateTime" : 4895,
+  "totalUpsertTime" : 0,
+  "totalCompactedRecordsUpdated" : 0,
+  "totalLogFilesCompacted" : 0,
+  "totalLogFilesSize" : 0
+}