[HUDI-1338] Adding Delete support to test suite framework (#2172)

- Adding Delete support to test suite. 
         Added DeleteNode 
         Added support to generate delete records

docker/demo/config/test-suite/complex-dag-cow.yaml

@@ -59,15 +59,13 @@ first_hive_sync:
   deps: first_upsert
 first_hive_query:
   config:
-    hive_props:
-      prop2: "set spark.yarn.queue="
-      prop3: "set hive.strict.checks.large.query=false"
-      prop4: "set hive.stats.autogather=false"
+    queue_name: "adhoc"
+    engine: "mr"
     hive_queries:
       query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
       result1: 0
       query2: "select count(*) from testdb.table1"
-      result2: 11600
+      result2: 11300
   type: HiveQueryNode
   deps: first_hive_sync
 second_upsert:
@@ -82,14 +80,55 @@ second_upsert:
   deps: first_hive_query
 second_hive_query:
   config:
-    hive_props:
-      prop2: "set mapred.job.queue.name="
-      prop3: "set hive.strict.checks.large.query=false"
-      prop4: "set hive.stats.autogather=false"
+    queue_name: "adhoc"
+    engine: "mr"
     hive_queries:
       query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
       result1: 0
       query2: "select count(*) from testdb.table1"
-      result2: 11900
+      result2: 11600
   type: HiveQueryNode
   deps: second_upsert
+fourth_insert:
+  config:
+    record_size: 70000
+    num_insert_partitions: 1
+    repeat_count: 1
+    num_records_insert: 1000
+  deps: second_hive_query
+  type: InsertNode
+third_hive_query:
+  config:
+    queue_name: "adhoc"
+    engine: "mr"
+    hive_queries:
+      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
+      result1: 0
+      query2: "select count(*) from testdb.table1"
+      result2: 12600
+  type: HiveQueryNode
+  deps: fourth_insert
+first_delete:
+  config:
+    record_size: 70000
+    num_partitions_delete: 1
+    num_records_delete: 200
+  deps: third_hive_query
+  type: DeleteNode
+fourth_hive_sync:
+  config:
+    queue_name: "adhoc"
+    engine: "mr"
+  type: HiveSyncNode
+  deps: first_delete
+fourth_hive_query:
+  config:
+    queue_name: "adhoc"
+    engine: "mr"
+    hive_queries:
+      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
+      result1: 0
+      query2: "select count(*) from testdb.table1"
+      result2: 12400
+  type: HiveQueryNode
+  deps: fourth_hive_sync

docker/demo/config/test-suite/source.avsc

@@ -46,6 +46,10 @@
   }, {
     "name" : "fare",
     "type" : "double"
-  } ]
+  }, {
+   "name" : "_hoodie_is_deleted",
+   "type" : "boolean",
+    "default" : false
+  }]
 }
 

docker/demo/config/test-suite/target.avsc

@@ -46,6 +46,10 @@
   }, {
     "name" : "fare",
     "type" : "double"
+  }, {
+   "name" : "_hoodie_is_deleted",
+   "type" : "boolean",
+    "default" : false
   }, {
     "name" : "haversine_distance",
     "type" : "double"

docker/demo/config/test-suite/test.properties

@@ -0,0 +1,27 @@
+hoodie.insert.shuffle.parallelism=100
+hoodie.upsert.shuffle.parallelism=100
+hoodie.bulkinsert.shuffle.parallelism=100
+
+hoodie.deltastreamer.source.test.num_partitions=100
+hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
+hoodie.deltastreamer.source.test.max_unique_records=100000000
+hoodie.embed.timeline.server=false
+hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
+
+hoodie.datasource.write.recordkey.field=_row_key
+hoodie.datasource.write.keygenerator.class=org.apache.hudi.keygen.TimestampBasedKeyGenerator
+hoodie.datasource.write.partitionpath.field=timestamp
+
+hoodie.deltastreamer.source.dfs.root=/user/hive/warehouse/hudi-integ-test-suite/input
+hoodie.deltastreamer.schemaprovider.target.schema.file=file:/var/hoodie/ws/docker/demo/config/test-suite/source.avsc
+hoodie.deltastreamer.schemaprovider.source.schema.file=file:/var/hoodie/ws/docker/demo/config/test-suite/source.avsc
+hoodie.deltastreamer.keygen.timebased.timestamp.type=UNIX_TIMESTAMP
+hoodie.deltastreamer.keygen.timebased.output.dateformat=yyyy/MM/dd
+
+hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000/
+hoodie.datasource.hive_sync.database=testdb
+hoodie.datasource.hive_sync.table=table1
+hoodie.datasource.hive_sync.assume_date_partitioning=false
+hoodie.datasource.hive_sync.partition_fields=_hoodie_partition_path
+hoodie.datasource.hive_sync.partition_extractor_class=org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor
+