lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3623] Removing hive sync node from non hive yamls (#5040)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-03-14 15:39:26 -07:00
committed by GitHub
parent 22c3ce73db
commit 30cf39301e
11 changed files with 18 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docker/demo/config/test-suite/cow-spark-long-running.yaml
View File

@@ -25,17 +25,11 @@ dag_content:
num_records_insert: 10000
type: SparkInsertNode
deps: none
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_insert
first_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: first_hive_sync
deps: first_insert
first_upsert:
config:
record_size: 200
@@ -52,18 +46,12 @@ dag_content:
num_records_delete: 8000
type: SparkDeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: true
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete
last_validate:
config:
execute_itr_count: 30

16
docker/demo/config/test-suite/cow-spark-simple.yaml
View File

@@ -25,17 +25,11 @@ dag_content:
num_records_insert: 100
type: SparkInsertNode
deps: none
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_insert
first_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: first_hive_sync
deps: first_insert
first_upsert:
config:
record_size: 1000
@@ -52,15 +46,9 @@ dag_content:
num_records_delete: 30
type: SparkDeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete

8
docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml
View File

@@ -57,18 +57,12 @@ dag_content:
num_records_delete: 8000
type: DeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: true
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete
last_validate:
config:
execute_itr_count: 30

8
docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml
View File

@@ -57,18 +57,12 @@ dag_content:
num_records_delete: 8000
type: DeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: true
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete
last_validate:
config:
execute_itr_count: 50

8
docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml
View File

@@ -60,18 +60,12 @@ dag_content:
num_records_delete: 8000
type: DeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete
last_validate:
config:
execute_itr_count: 20

18
docker/demo/config/test-suite/detlastreamer-long-running-example.yaml
View File

@@ -41,17 +41,11 @@ dag_content:
num_records_insert: 300
deps: second_insert
type: InsertNode
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: third_insert
first_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: first_hive_sync
deps: third_insert
first_upsert:
config:
record_size: 1000
@@ -68,22 +62,14 @@ dag_content:
num_records_delete: 8000
type: DeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: true
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete
last_validate:
config:
execute_itr_count: 50
validate_clean: true
validate_archival: true
type: ValidateAsyncOperations
deps: second_validate

16
docker/demo/config/test-suite/insert-overwrite-table.yaml
View File

@@ -53,19 +53,13 @@ dag_content:
num_partitions_upsert: 10
type: SparkUpsertNode
deps: second_insert
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: second_upsert
first_insert_overwrite_table:
config:
record_size: 1000
repeat_count: 10
num_records_insert: 10
type: SparkInsertOverwriteTableNode
deps: first_hive_sync
deps: second_upsert
delete_all_input_except_last:
config:
delete_input_data_except_latest: true
@@ -89,16 +83,10 @@ dag_content:
num_partitions_upsert: 10
type: SparkUpsertNode
deps: third_insert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: third_upsert
second_validate:
config:
validate_full_data : true
validate_hive: false
delete_input_data: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: third_upsert

16
docker/demo/config/test-suite/insert-overwrite.yaml
View File

@@ -54,12 +54,6 @@ dag_content:
num_partitions_upsert: 10
type: SparkUpsertNode
deps: second_insert
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: second_upsert
first_insert_overwrite:
config:
record_size: 1000
@@ -67,7 +61,7 @@ dag_content:
repeat_count: 1
num_records_insert: 10
type: SparkInsertOverwriteNode
deps: first_hive_sync
deps: second_upsert
delete_all_input_except_last:
config:
delete_input_data_except_latest: true
@@ -91,16 +85,10 @@ dag_content:
num_partitions_upsert: 10
type: SparkUpsertNode
deps: third_insert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: third_upsert
second_validate:
config:
validate_full_data : true
validate_hive: false
delete_input_data: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: third_upsert

16
docker/demo/config/test-suite/simple-clustering.yaml
View File

@@ -47,30 +47,18 @@ dag_content:
num_records_delete: 9000
type: DeleteNode
deps: third_insert
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
first_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: first_hive_sync
deps: first_delete
first_cluster:
config:
execute_itr_count: 25
type: ClusteringNode
deps: first_validate
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_cluster
second_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_cluster

16
docker/demo/config/test-suite/simple-deltastreamer.yaml
View File

@@ -41,17 +41,11 @@ dag_content:
num_records_insert: 300
deps: second_insert
type: InsertNode
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: third_insert
first_validate:
config:
validate_hive: false
type: ValidateDatasetNode
deps: first_hive_sync
deps: third_insert
first_upsert:
config:
record_size: 1000
@@ -68,15 +62,9 @@ dag_content:
num_records_delete: 2000
type: DeleteNode
deps: first_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: true
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete

8
docker/demo/config/test-suite/spark-clustering.yaml
View File

@@ -59,15 +59,9 @@ dag_content:
num_records_delete: 16000
type: SparkDeleteNode
deps: second_upsert
second_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_delete
second_validate:
config:
validate_hive: false
delete_input_data: false
type: ValidateDatasetNode
deps: second_hive_sync
deps: first_delete