lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4084] Add support to test async table services with integ test suite framework (#5557)

Browse Source 
* Add support to test async table services with integ test suite framework

* Make await time for validation configurable
This commit is contained in:
Sivabalan Narayanan
2022-05-23 23:05:56 -04:00
committed by GitHub
parent 47b764ec33
commit af1128acf9
17 changed files with 612 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml
View File

@@ -74,10 +74,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: true
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: second_hive_sync
last_validate:
config:
execute_itr_count: 50
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml
View File

@@ -62,10 +62,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 30
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml
View File

@@ -62,10 +62,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 50
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/deltastreamer-medium-clustering.yaml
View File

@@ -64,10 +64,12 @@ dag_content:
config:
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 20
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/deltastreamer-medium-full-dataset-validation.yaml
View File

@@ -65,10 +65,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: false
delete_input_data: false
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 20
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/detlastreamer-long-running-example.yaml
View File

@@ -62,10 +62,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 50
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/spark-long-running-non-partitioned.yaml
View File

@@ -45,10 +45,12 @@ dag_content:
config:
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 6
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate

2
docker/demo/config/test-suite/spark-long-running.yaml
View File

@@ -46,10 +46,12 @@ dag_content:
validate_once_every_itr : 5
validate_hive: false
delete_input_data: true
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateDatasetNode
deps: first_delete
last_validate:
config:
execute_itr_count: 30
max_wait_time_for_deltastreamer_catch_up_ms: 600000
type: ValidateAsyncOperations
deps: second_validate