[HUDI-3659] Reducing the validation frequency with integ tests (#5067)

docker/demo/config/test-suite/cow-spark-long-running.yaml

@@ -25,11 +25,6 @@ dag_content:
       num_records_insert: 10000
     type: SparkInsertNode
     deps: none
-  first_validate:
-    config:
-      validate_hive: false
-    type: ValidateDatasetNode
-    deps: first_insert
   first_upsert:
     config:
       record_size: 200
@@ -39,7 +34,7 @@ dag_content:
       num_records_upsert: 3000
       num_partitions_upsert: 50
     type: SparkUpsertNode
-    deps: first_validate
+    deps: first_insert
   first_delete:
     config:
       num_partitions_delete: 50
@@ -48,6 +43,7 @@ dag_content:
     deps: first_upsert
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: false
       delete_input_data: true
     type: ValidateDatasetNode

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml

@@ -47,11 +47,6 @@ dag_content:
       engine: "mr"
     type: HiveSyncNode
     deps: third_insert
-  first_validate:
-    config:
-      validate_hive: false
-    type: ValidateDatasetNode
-    deps: first_hive_sync
   first_upsert:
     config:
       record_size: 1000
@@ -61,7 +56,7 @@ dag_content:
       num_records_upsert: 100
       num_partitions_upsert: 1
     type: UpsertNode
-    deps: first_validate
+    deps: first_hive_sync
   first_delete:
     config:
       num_partitions_delete: 50
@@ -76,6 +71,7 @@ dag_content:
     deps: first_delete
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: true
       delete_input_data: true
     type: ValidateDatasetNode

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml

@@ -59,6 +59,7 @@ dag_content:
     deps: first_upsert
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: false
       delete_input_data: true
     type: ValidateDatasetNode

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml

@@ -59,6 +59,7 @@ dag_content:
     deps: first_upsert
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: false
       delete_input_data: true
     type: ValidateDatasetNode

docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml

@@ -62,6 +62,7 @@ dag_content:
     deps: first_upsert
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: false
       delete_input_data: false
     type: ValidateDatasetNode

docker/demo/config/test-suite/detlastreamer-long-running-example.yaml

@@ -41,11 +41,6 @@ dag_content:
       num_records_insert: 300
     deps: second_insert
     type: InsertNode
-  first_validate:
-    config:
-      validate_hive: false
-    type: ValidateDatasetNode
-    deps: third_insert
   first_upsert:
     config:
       record_size: 1000
@@ -55,7 +50,7 @@ dag_content:
       num_records_upsert: 100
       num_partitions_upsert: 1
     type: UpsertNode
-    deps: first_validate
+    deps: third_insert
   first_delete:
     config:
       num_partitions_delete: 1
@@ -64,6 +59,7 @@ dag_content:
     deps: first_upsert
   second_validate:
     config:
+      validate_once_every_itr : 5
       validate_hive: false
       delete_input_data: true
     type: ValidateDatasetNode