[HUDI-1331] Adding support for validating entire dataset and long running tests in test suite framework (#2168)

* trigger rebuild * [HUDI-1156] Remove unused dependencies from HoodieDeltaStreamerWrapper Class (#1927) * Adding support for validating records and long running tests in test sutie framework * Adding partial validate node * Fixing spark session initiation in Validate nodes * Fixing validation * Adding hive table validation to ValidateDatasetNode * Rebasing with latest commits from master * Addressing feedback * Addressing comments Co-authored-by: lamber-ken <lamberken@163.com> Co-authored-by: linshan-ma <mabin194046@163.com>
2020-12-26 12:29:24 -05:00
parent 3ec9270e8e
commit 8cf6a7223f
33 changed files with 1101 additions and 518 deletions
--- a/docker/demo/config/test-suite/complex-dag-mor.yaml
+++ b/docker/demo/config/test-suite/complex-dag-mor.yaml
@@ -13,103 +13,107 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-first_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 100
-  type: InsertNode
-  deps: none
-second_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 100
-  deps: first_insert
-  type: InsertNode
-third_insert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    repeat_count: 1
-    num_records_insert: 300
-  deps: second_insert
-  type: InsertNode
-first_rollback:
-  config:
-  deps: third_insert
-  type: RollbackNode
-first_upsert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    num_records_insert: 300
-    repeat_count: 1
-    num_records_upsert: 100
-    num_upsert_partitions: 10
-  type: UpsertNode
-  deps: first_rollback
-first_hive_sync:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-  type: HiveSyncNode
-  deps: first_upsert
-first_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-  type: HiveQueryNode
-  deps: first_hive_sync
-second_upsert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    num_records_insert: 300
-    repeat_count: 1
-    num_records_upsert: 100
-    num_upsert_partitions: 10
-  type: UpsertNode
-  deps: first_hive_query
-second_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 1100
-  type: HiveQueryNode
-  deps: second_upsert
-first_schedule_compact:
-  config:
-  type: ScheduleCompactNode
-  deps: second_hive_query
-third_upsert:
-  config:
-    record_size: 70000
-    num_insert_partitions: 1
-    num_records_insert: 300
-    repeat_count: 1
-    num_records_upsert: 100
-    num_upsert_partitions: 10
-  type: UpsertNode
-  deps: first_schedule_compact
-first_compact:
-  config:
-  type: CompactNode
-  deps: first_schedule_compact
-third_hive_query:
-  config:
-    queue_name: "adhoc"
-    engine: "mr"
-    hive_queries:
-      query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
-      result1: 0
-      query2: "select count(*) from testdb.table1"
-      result2: 1400
-  type: HiveQueryNode
-  deps: first_compact
+dag_name: complex-dag-mor.yaml
+dag_rounds: 1
+dag_intermittent_delay_mins: 10
+dag_content:
+  first_insert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      repeat_count: 5
+      num_records_insert: 100
+    type: InsertNode
+    deps: none
+  second_insert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      repeat_count: 5
+      num_records_insert: 100
+    deps: first_insert
+    type: InsertNode
+  third_insert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      repeat_count: 2
+      num_records_insert: 300
+    deps: second_insert
+    type: InsertNode
+  first_rollback:
+    config:
+    deps: third_insert
+    type: RollbackNode
+  first_upsert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      num_records_insert: 300
+      repeat_count: 1
+      num_records_upsert: 100
+      num_partitions_upsert: 10
+    type: UpsertNode
+    deps: first_rollback
+  first_hive_sync:
+    config:
+      queue_name: "adhoc"
+      engine: "mr"
+    type: HiveSyncNode
+    deps: first_upsert
+  first_hive_query:
+    config:
+      queue_name: "adhoc"
+      engine: "mr"
+    type: HiveQueryNode
+    deps: first_hive_sync
+  second_upsert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      num_records_insert: 300
+      repeat_count: 1
+      num_records_upsert: 100
+      num_partitions_upsert: 10
+    type: UpsertNode
+    deps: first_hive_query
+  second_hive_query:
+    config:
+      queue_name: "adhoc"
+      engine: "mr"
+      hive_queries:
+        query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
+        result1: 0
+        query2: "select count(*) from testdb.table1"
+        result2: 1100
+    type: HiveQueryNode
+    deps: second_upsert
+  first_schedule_compact:
+    config:
+    type: ScheduleCompactNode
+    deps: second_hive_query
+  third_upsert:
+    config:
+      record_size: 70000
+      num_partitions_insert: 1
+      num_records_insert: 300
+      repeat_count: 1
+      num_records_upsert: 100
+      num_partitions_upsert: 10
+    type: UpsertNode
+    deps: first_schedule_compact
+  first_compact:
+    config:
+    type: CompactNode
+    deps: first_schedule_compact
+  third_hive_query:
+    config:
+      queue_name: "adhoc"
+      engine: "mr"
+      hive_queries:
+        query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
+        result1: 0
+        query2: "select count(*) from testdb.table1"
+        result2: 1400
+    type: HiveQueryNode
+    deps: first_compact