[HUDI-1331] Adding support for validating entire dataset and long running tests in test suite framework (#2168)

* trigger rebuild * [HUDI-1156] Remove unused dependencies from HoodieDeltaStreamerWrapper Class (#1927) * Adding support for validating records and long running tests in test sutie framework * Adding partial validate node * Fixing spark session initiation in Validate nodes * Fixing validation * Adding hive table validation to ValidateDatasetNode * Rebasing with latest commits from master * Addressing feedback * Addressing comments Co-authored-by: lamber-ken <lamberken@163.com> Co-authored-by: linshan-ma <mabin194046@163.com>
2020-12-26 12:29:24 -05:00
parent 3ec9270e8e
commit 8cf6a7223f
33 changed files with 1101 additions and 518 deletions
--- a/docker/demo/config/test-suite/complex-dag-cow.yaml
+++ b/docker/demo/config/test-suite/complex-dag-cow.yaml
@@ -13,122 +13,56 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-first_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 1000
-  type: InsertNode
-  deps: none
-second_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 10000
-  deps: first_insert
-  type: InsertNode
-third_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 300
-  deps: second_insert
-  type: InsertNode
-first_rollback:
-  config:
-  deps: third_insert
-  type: RollbackNode
-first_upsert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    num_records_insert: 300
-    repeat_count: 1
-    num_records_upsert: 100
-    num_upsert_partitions: 10
-  type: UpsertNode
-  deps: first_rollback
-first_hive_sync:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-  type: HiveSyncNode
-  deps: first_upsert
-first_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 11300
-  type: HiveQueryNode
-  deps: first_hive_sync
-second_upsert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    num_records_insert: 300
-    repeat_count: 1
-    num_records_upsert: 100
-    num_upsert_partitions: 10
-  type: UpsertNode
-  deps: first_hive_query
-second_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 11600
-  type: HiveQueryNode
-  deps: second_upsert
-fourth_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 1000
-  deps: second_hive_query
-  type: InsertNode
-third_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 12600
-  type: HiveQueryNode
-  deps: fourth_insert
-first_delete:
-  config:
-    record_size: 70000
-    num_partitions_delete: 1
-    num_records_delete: 200
-  deps: third_hive_query
-  type: DeleteNode
-fourth_hive_sync:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-  type: HiveSyncNode
-  deps: first_delete
-fourth_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 12400
-  type: HiveQueryNode
-  deps: fourth_hive_sync
+dag_name: cow-long-running-example.yaml
+dag_rounds: 2
+dag_intermittent_delay_mins: 1
+dag_content:
+  first_insert:
+    config:
+      record_size: 100
+      num_partitions_insert: 1
+      repeat_count: 1
+      num_records_insert: 1000
+    type: InsertNode
+    deps: none
+  second_insert:
+    config:
+      record_size: 100
+      num_partitions_insert: 1
+      repeat_count: 1
+      num_records_insert: 10000
+    deps: first_insert
+    type: InsertNode
+  third_insert:
+    config:
+      record_size: 100
+      num_partitions_insert: 1
+      repeat_count: 1
+      num_records_insert: 300
+    deps: second_insert
+    type: InsertNode
+  first_validate:
+    config:
+    type: ValidateDatasetNode
+    deps: third_insert
+  first_upsert:
+    config:
+      record_size: 100
+      num_partitions_insert: 1
+      num_records_insert: 300
+      repeat_count: 1
+      num_records_upsert: 100
+      num_partitions_upsert: 1
+    type: UpsertNode
+    deps: first_validate
+  first_delete:
+    config:
+      num_partitions_delete: 1
+      num_records_delete: 2000
+    type: DeleteNode
+    deps: first_upsert
+  second_validate:
+    config:
+      delete_input_data: true
+    type: ValidateDatasetNode
+    deps: first_delete