diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark231.yml b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml similarity index 97% rename from docker/compose/docker-compose_hadoop284_hive233_spark231.yml rename to docker/compose/docker-compose_hadoop284_hive233_spark244.yml index 8da8edb9b..fa9b09a9a 100644 --- a/docker/compose/docker-compose_hadoop284_hive233_spark231.yml +++ b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml @@ -22,7 +22,7 @@ services: hostname: namenode container_name: namenode environment: - - CLUSTER_NAME=hudi_hadoop284_hive232_spark231 + - CLUSTER_NAME=hudi_hadoop284_hive232_spark244 ports: - "50070:50070" - "8020:8020" @@ -39,7 +39,7 @@ services: container_name: datanode1 hostname: datanode1 environment: - - CLUSTER_NAME=hudi_hadoop284_hive232_spark231 + - CLUSTER_NAME=hudi_hadoop284_hive232_spark244 env_file: - ./hadoop.env ports: @@ -61,7 +61,7 @@ services: hostname: historyserver container_name: historyserver environment: - - CLUSTER_NAME=hudi_hadoop284_hive232_spark231 + - CLUSTER_NAME=hudi_hadoop284_hive232_spark244 depends_on: - "namenode" links: @@ -128,7 +128,7 @@ services: - ${HUDI_WS}:/var/hoodie/ws sparkmaster: - image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:latest hostname: sparkmaster container_name: sparkmaster env_file: @@ -145,7 +145,7 @@ services: - "namenode" spark-worker-1: - image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:latest hostname: spark-worker-1 container_name: spark-worker-1 env_file: @@ -218,7 +218,7 @@ services: command: worker adhoc-1: - image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest hostname: adhoc-1 container_name: adhoc-1 env_file: @@ -239,7 +239,7 @@ services: - ${HUDI_WS}:/var/hoodie/ws adhoc-2: - image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest hostname: adhoc-2 container_name: adhoc-2 env_file: diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml index 21ae28263..55e5c6540 100644 --- a/docker/hoodie/hadoop/pom.xml +++ b/docker/hoodie/hadoop/pom.xml @@ -50,7 +50,7 @@ false true - 2.3.1 + 2.4.4 2.3.3 2.8.4 0.217 diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile index 600f90d3e..547459352 100644 --- a/docker/hoodie/hadoop/spark_base/Dockerfile +++ b/docker/hoodie/hadoop/spark_base/Dockerfile @@ -24,7 +24,7 @@ ENV ENABLE_INIT_DAEMON true ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon ENV INIT_DAEMON_STEP spark_master_init -ARG SPARK_VERSION=2.3.1 +ARG SPARK_VERSION=2.4.4 ARG SPARK_HADOOP_VERSION=2.7 ENV SPARK_VERSION ${SPARK_VERSION} diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile b/docker/hoodie/hadoop/sparkadhoc/Dockerfile index e0f2113b9..fde286012 100644 --- a/docker/hoodie/hadoop/sparkadhoc/Dockerfile +++ b/docker/hoodie/hadoop/sparkadhoc/Dockerfile @@ -18,7 +18,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 -ARG SPARK_VERSION=2.3.1 +ARG SPARK_VERSION=2.4.4 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} ARG PRESTO_VERSION=0.217 diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile b/docker/hoodie/hadoop/sparkmaster/Dockerfile index 01f5f9d33..94281cce5 100644 --- a/docker/hoodie/hadoop/sparkmaster/Dockerfile +++ b/docker/hoodie/hadoop/sparkmaster/Dockerfile @@ -18,7 +18,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 -ARG SPARK_VERSION=2.3.1 +ARG SPARK_VERSION=2.4.4 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} COPY master.sh /opt/spark diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile b/docker/hoodie/hadoop/sparkworker/Dockerfile index 70e75ad21..fef3ec943 100644 --- a/docker/hoodie/hadoop/sparkworker/Dockerfile +++ b/docker/hoodie/hadoop/sparkworker/Dockerfile @@ -18,7 +18,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 -ARG SPARK_VERSION=2.3.1 +ARG SPARK_VERSION=2.4.4 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} COPY worker.sh /opt/spark diff --git a/docker/setup_demo.sh b/docker/setup_demo.sh index 4f4072455..2456e3da7 100755 --- a/docker/setup_demo.sh +++ b/docker/setup_demo.sh @@ -20,10 +20,10 @@ WS_ROOT=`dirname $PWD` # restart cluster -HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml down -HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml pull +HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark244.yml down +HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark244.yml pull sleep 5 -HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml up -d +HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark244.yml up -d sleep 15 docker exec -it adhoc-1 /bin/bash /var/hoodie/ws/docker/demo/setup_demo_container.sh diff --git a/docker/stop_demo.sh b/docker/stop_demo.sh index e795f25af..4927b8650 100755 --- a/docker/stop_demo.sh +++ b/docker/stop_demo.sh @@ -21,7 +21,7 @@ # set up root directory WS_ROOT=`dirname $PWD` # shut down cluster -HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark231.yml down +HUDI_WS=${WS_ROOT} docker-compose -f compose/docker-compose_hadoop284_hive233_spark244.yml down # remove houst mount directory rm -rf /tmp/hadoop_data diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml index bb7cec56f..af82818af 100644 --- a/hudi-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -131,7 +131,7 @@ ${project.basedir}/compose_env - ${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark231.yml + ${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark244.yml false ${skipITs} true @@ -201,7 +201,7 @@ ${docker.compose.skip} unix:///var/run/docker.sock - ${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark231.yml + ${dockerCompose.file} true ${dockerCompose.envFile} @@ -214,7 +214,7 @@ ${docker.compose.skip} - ${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark231.yml + ${dockerCompose.file} true