lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2006] Adding more yaml templates to test suite (#3073)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-06-29 23:05:46 -04:00
committed by GitHub
parent 202887b8ca
commit 5564c7ec01
6 changed files with 322 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
docker/generate_test_suite.sh
View File

@@ -16,13 +16,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
NUM_ITR=1
MEDIUM_NUM_ITR=20
LONG_NUM_ITR=50
DELAY_MINS=1
TABLE_TYPE=COPY_ON_WRITE
INCLUDE_LONG_TEST_SUITE=false
INCLUDE_MEDIUM_TEST_SUITE=false
INCLUDE_CLUSTER_YAML=false
CLUSTER_NUM_ITR=2
CLUSTER_NUM_ITR=30
CLUSTER_DELAY_MINS=1
CLUSTER_ITR_COUNT=1
CLUSTER_ITR_COUNT=15
EXECUTE_TEST_SUITE=true
JAR_NAME=hudi-integ-test-bundle-0.9.0-SNAPSHOT.jar
INPUT_PATH="/user/hive/warehouse/hudi-integ-test-suite/input/"
OUTPUT_PATH="/user/hive/warehouse/hudi-integ-test-suite/output/"
@@ -35,8 +39,18 @@ do
key="$1"
case $key in
--num_iterations)
NUM_ITR="$2"
--execute_test_suite)
EXECUTE_TEST_SUITE="$2"
shift # past argument
shift # past value
;;
--medium_num_iterations)
MEDIUM_NUM_ITR="$2"
shift # past argument
shift # past value
;;
--long_num_iterations)
LONG_NUM_ITR="$2"
shift # past argument
shift # past value
;;
@@ -50,6 +64,16 @@ case $key in
shift # past argument
shift # past value
;;
--include_long_test_suite_yaml)
INCLUDE_LONG_TEST_SUITE="$2"
shift # past argument
shift # past value
;;
--include_medium_test_suite_yaml)
INCLUDE_MEDIUM_TEST_SUITE="$2"
shift # past argument
shift # past value
;;
--include_cluster_yaml)
INCLUDE_CLUSTER_YAML="$2"
shift # past argument
@@ -97,13 +121,23 @@ esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
echo "Num Iterations = ${NUM_ITR}"
echo "Include Medium test suite $INCLUDE_MEDIUM_TEST_SUITE"
if $INCLUDE_MEDIUM_TEST_SUITE ; then
echo "Medium test suite iterations = ${MEDIUM_NUM_ITR}"
fi
echo "Include Long test suite $INCLUDE_LONG_TEST_SUITE"
if $INCLUDE_LONG_TEST_SUITE ; then
echo "Long test suite iterations = ${LONG_NUM_ITR}"
fi
echo "Intermittent delay in mins = ${DELAY_MINS}"
echo "Table type = ${TABLE_TYPE}"
echo "Include cluster yaml $INCLUDE_CLUSTER_YAML"
echo "Cluster total itr count $CLUSTER_NUM_ITR"
echo "Cluster delay mins $CLUSTER_DELAY_MINS"
echo "Cluster exec itr count $CLUSTER_ITR_COUNT"
if $INCLUDE_CLUSTER_YAML ; then
echo "Cluster total itr count $CLUSTER_NUM_ITR"
echo "Cluster delay mins $CLUSTER_DELAY_MINS"
echo "Cluster exec itr count $CLUSTER_ITR_COUNT"
fi
echo "Jar name $JAR_NAME"
INPUT_PATH=$(echo "$INPUT_PATH" | sed "s|\/|\\\/|g")
echo "Input path $INPUT_PATH"
@@ -125,22 +159,56 @@ if [ ! -d "demo/config/test-suite/staging" ]; then
mkdir demo/config/test-suite/staging
fi
cp demo/config/test-suite/templates/long-running.yaml.template demo/config/test-suite/staging/long-running.yaml
cp demo/config/test-suite/templates/sanity.yaml.template demo/config/test-suite/staging/sanity.yaml
sed -i '' "s/NAME/$TABLE_TYPE/" demo/config/test-suite/staging/long-running.yaml
sed -i '' "s/num_iterations/$NUM_ITR/" demo/config/test-suite/staging/long-running.yaml
sed -i '' "s/delay_in_mins/$DELAY_MINS/" demo/config/test-suite/staging/long-running.yaml
sed -i '' "s/NAME/$TABLE_TYPE/" demo/config/test-suite/staging/sanity.yaml
cp demo/config/test-suite/templates/test.properties.template demo/config/test-suite/staging/test.properties
sed -i '' "s/INPUT_PATH/$INPUT_PATH/" demo/config/test-suite/staging/test.properties
cp demo/config/test-suite/templates/spark_command.txt.template demo/config/test-suite/staging/long_running_spark_command.sh
cp demo/config/test-suite/templates/spark_command.txt.template demo/config/test-suite/staging/sanity_spark_command.sh
sed -i '' "s/JAR_NAME/$JAR_NAME/" demo/config/test-suite/staging/long_running_spark_command.sh
sed -i '' "s/INPUT_PATH/$INPUT_PATH/" demo/config/test-suite/staging/long_running_spark_command.sh
sed -i '' "s/OUTPUT_PATH/$OUTPUT_PATH/" demo/config/test-suite/staging/long_running_spark_command.sh
sed -i '' "s/input_yaml/long-running.yaml/" demo/config/test-suite/staging/long_running_spark_command.sh
sed -i '' "s/TABLE_TYPE/$TABLE_TYPE/" demo/config/test-suite/staging/long_running_spark_command.sh
sed -i '' "s/JAR_NAME/$JAR_NAME/" demo/config/test-suite/staging/sanity_spark_command.sh
sed -i '' "s/INPUT_PATH/$INPUT_PATH/" demo/config/test-suite/staging/sanity_spark_command.sh
sed -i '' "s/OUTPUT_PATH/$OUTPUT_PATH/" demo/config/test-suite/staging/sanity_spark_command.sh
sed -i '' "s/input_yaml/sanity.yaml/" demo/config/test-suite/staging/sanity_spark_command.sh
sed -i '' "s/TABLE_TYPE/$TABLE_TYPE/" demo/config/test-suite/staging/sanity_spark_command.sh
if $INCLUDE_MEDIUM_TEST_SUITE ; then
cp demo/config/test-suite/templates/medium_test_suite.yaml.template demo/config/test-suite/staging/medium_test_suite.yaml
sed -i '' "s/NAME/$TABLE_TYPE/" demo/config/test-suite/staging/medium_test_suite.yaml
sed -i '' "s/medium_num_iterations/$MEDIUM_NUM_ITR/" demo/config/test-suite/staging/medium_test_suite.yaml
sed -i '' "s/delay_in_mins/$DELAY_MINS/" demo/config/test-suite/staging/medium_test_suite.yaml
cp demo/config/test-suite/templates/spark_command.txt.template demo/config/test-suite/staging/medium_test_suite_spark_command.sh
sed -i '' "s/JAR_NAME/$JAR_NAME/" demo/config/test-suite/staging/medium_test_suite_spark_command.sh
sed -i '' "s/INPUT_PATH/$INPUT_PATH/" demo/config/test-suite/staging/medium_test_suite_spark_command.sh
sed -i '' "s/OUTPUT_PATH/$OUTPUT_PATH/" demo/config/test-suite/staging/medium_test_suite_spark_command.sh
sed -i '' "s/input_yaml/medium_test_suite.yaml/" demo/config/test-suite/staging/medium_test_suite_spark_command.sh
sed -i '' "s/TABLE_TYPE/$TABLE_TYPE/" demo/config/test-suite/staging/medium_test_suite_spark_command.sh
fi
if $INCLUDE_LONG_TEST_SUITE ; then
cp demo/config/test-suite/templates/long_test_suite.yaml.template demo/config/test-suite/staging/long_test_suite.yaml
sed -i '' "s/NAME/$TABLE_TYPE/" demo/config/test-suite/staging/long_test_suite.yaml
sed -i '' "s/long_num_iterations/$LONG_NUM_ITR/" demo/config/test-suite/staging/long_test_suite.yaml
sed -i '' "s/delay_in_mins/$DELAY_MINS/" demo/config/test-suite/staging/long_test_suite.yaml
cp demo/config/test-suite/templates/spark_command.txt.template demo/config/test-suite/staging/long_test_suite_spark_command.sh
sed -i '' "s/JAR_NAME/$JAR_NAME/" demo/config/test-suite/staging/long_test_suite_spark_command.sh
sed -i '' "s/INPUT_PATH/$INPUT_PATH/" demo/config/test-suite/staging/long_test_suite_spark_command.sh
sed -i '' "s/OUTPUT_PATH/$OUTPUT_PATH/" demo/config/test-suite/staging/long_test_suite_spark_command.sh
sed -i '' "s/input_yaml/long_test_suite.yaml/" demo/config/test-suite/staging/long_test_suite_spark_command.sh
sed -i '' "s/TABLE_TYPE/$TABLE_TYPE/" demo/config/test-suite/staging/long_test_suite_spark_command.sh
fi
if $INCLUDE_CLUSTER_YAML ; then
@@ -148,7 +216,7 @@ if $INCLUDE_CLUSTER_YAML ; then
sed -i '' "s/NAME/$TABLE_TYPE/" demo/config/test-suite/staging/clustering.yaml
sed -i '' "s/clustering_num_iterations/$CLUSTER_NUM_ITR/" demo/config/test-suite/staging/clustering.yaml
sed -i '' "s/clustering_delay_in_mins/$CLUSTER_DELAY_MINS/" demo/config/test-suite/staging/clustering.yaml
sed -i '' "s/delay_in_mins/$CLUSTER_DELAY_MINS/" demo/config/test-suite/staging/clustering.yaml
sed -i '' "s/clustering_itr_count/$CLUSTER_ITR_COUNT/" demo/config/test-suite/staging/clustering.yaml
cp demo/config/test-suite/templates/spark_command.txt.template demo/config/test-suite/staging/clustering_spark_command.sh
@@ -162,11 +230,27 @@ if $INCLUDE_CLUSTER_YAML ; then
fi
docker cp $CUR_DIR/../packaging/hudi-integ-test-bundle/target/$JAR_NAME adhoc-2:/opt/
docker exec -it adhoc-2 /bin/bash rm -rf /opt/staging*
docker cp demo/config/test-suite/staging/ adhoc-2:/opt/
docker exec -it adhoc-2 /bin/bash /opt/staging/long_running_spark_command.sh
if $EXECUTE_TEST_SUITE ; then
docker cp $CUR_DIR/../packaging/hudi-integ-test-bundle/target/$JAR_NAME adhoc-2:/opt/
docker exec -it adhoc-2 /bin/bash rm -rf /opt/staging*
docker cp demo/config/test-suite/staging/ adhoc-2:/opt/
docker exec -it adhoc-2 /bin/bash echo "\n============================== Executing sanity test suite ============================== "
docker exec -it adhoc-2 /bin/bash /opt/staging/sanity_spark_command.sh
if [ -f demo/config/test-suite/staging/medium_test_suite_spark_command.sh ]; then
docker exec -it adhoc-2 /bin/bash echo "\n\n\n============================== Executing medium test suite ============================== "
docker exec -it adhoc-2 /bin/bash /opt/staging/medium_test_suite_spark_command.sh
fi
if [ -f demo/config/test-suite/staging/long_test_suite_spark_command.sh ]; then
docker exec -it adhoc-2 /bin/bash echo "\n\n\n============================== Executing long test suite ============================== "
docker exec -it adhoc-2 /bin/bash /opt/staging/long_test_suite_spark_command.sh
fi
if [ -f demo/config/test-suite/staging/clustering_spark_command.sh ]; then
docker exec -it adhoc-2 /bin/bash echo "\n\n\n============================== Executing clustering test suite ============================== "
docker exec -it adhoc-2 /bin/bash /opt/staging/clustering_spark_command.sh
fi
if [ -f demo/config/test-suite/staging/clustering_spark_command.sh ]; then
docker exec -it adhoc-2 /bin/bash /opt/staging/clustering_spark_command.sh
fi