[HUDI-3623] Removing hive sync node from non hive yamls (#5040)
This commit is contained in:
committed by
GitHub
parent
22c3ce73db
commit
30cf39301e
@@ -54,12 +54,6 @@ dag_content:
|
||||
num_partitions_upsert: 10
|
||||
type: SparkUpsertNode
|
||||
deps: second_insert
|
||||
first_hive_sync:
|
||||
config:
|
||||
queue_name: "adhoc"
|
||||
engine: "mr"
|
||||
type: HiveSyncNode
|
||||
deps: second_upsert
|
||||
first_insert_overwrite:
|
||||
config:
|
||||
record_size: 1000
|
||||
@@ -67,7 +61,7 @@ dag_content:
|
||||
repeat_count: 1
|
||||
num_records_insert: 10
|
||||
type: SparkInsertOverwriteNode
|
||||
deps: first_hive_sync
|
||||
deps: second_upsert
|
||||
delete_all_input_except_last:
|
||||
config:
|
||||
delete_input_data_except_latest: true
|
||||
@@ -91,16 +85,10 @@ dag_content:
|
||||
num_partitions_upsert: 10
|
||||
type: SparkUpsertNode
|
||||
deps: third_insert
|
||||
second_hive_sync:
|
||||
config:
|
||||
queue_name: "adhoc"
|
||||
engine: "mr"
|
||||
type: HiveSyncNode
|
||||
deps: third_upsert
|
||||
second_validate:
|
||||
config:
|
||||
validate_full_data : true
|
||||
validate_hive: false
|
||||
delete_input_data: false
|
||||
type: ValidateDatasetNode
|
||||
deps: second_hive_sync
|
||||
deps: third_upsert
|
||||
|
||||
Reference in New Issue
Block a user