diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml index 3c1acbdfe..086004f12 100644 --- a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml +++ b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml @@ -184,7 +184,7 @@ services: presto-coordinator-1: container_name: presto-coordinator-1 hostname: presto-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.268:latest + image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest ports: - '8090:8090' environment: @@ -201,25 +201,25 @@ services: command: coordinator presto-worker-1: - container_name: presto-worker-1 - hostname: presto-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.268:latest - depends_on: ["presto-coordinator-1"] - environment: - - PRESTO_JVM_MAX_HEAP=512M - - PRESTO_QUERY_MAX_MEMORY=1GB - - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB - - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB - - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB - - TERM=xterm - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: worker + container_name: presto-worker-1 + hostname: presto-worker-1 + image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest + depends_on: [ "presto-coordinator-1" ] + environment: + - PRESTO_JVM_MAX_HEAP=512M + - PRESTO_QUERY_MAX_MEMORY=1GB + - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB + - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB + - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB + - TERM=xterm + links: + - "hivemetastore" + - "hiveserver" + - "hive-metastore-postgresql" + - "namenode" + volumes: + - ${HUDI_WS}:/var/hoodie/ws + command: worker trino-coordinator-1: container_name: trino-coordinator-1 diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml index de3bd3d57..b029abafa 100644 --- a/docker/hoodie/hadoop/pom.xml +++ b/docker/hoodie/hadoop/pom.xml @@ -57,7 +57,7 @@ 2.4.4 2.3.3 2.8.4 - 0.268 + 0.271 368 1.4.13 true diff --git a/docker/hoodie/hadoop/prestobase/Dockerfile b/docker/hoodie/hadoop/prestobase/Dockerfile index 12b644aa0..accedb94d 100644 --- a/docker/hoodie/hadoop/prestobase/Dockerfile +++ b/docker/hoodie/hadoop/prestobase/Dockerfile @@ -22,7 +22,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest as hadoop-base -ARG PRESTO_VERSION=0.268 +ARG PRESTO_VERSION=0.271 ENV PRESTO_VERSION ${PRESTO_VERSION} ENV PRESTO_HOME /opt/presto-server-${PRESTO_VERSION} @@ -79,6 +79,15 @@ RUN chmod +x /usr/local/bin/entrypoint.sh ADD target/ /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/ ENV HUDI_PRESTO_BUNDLE /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/hudi-presto-bundle.jar RUN cp ${HUDI_PRESTO_BUNDLE} ${PRESTO_HOME}/plugin/hive-hadoop2/ +# TODO: the latest master of Presto relies on hudi-presto-bundle, while current Presto releases +# rely on hudi-common and hudi-hadoop-mr 0.9.0, which are pulled in plugin/hive-hadoop2/ in the +# docker setup, making it hard to test the latest changes in Hudi due to class conflict. +# To get around the conflicts due to older Hudi jars below, they are removed for integration tests, +# so the hudi-presto-bundle build can be used solely for testing. This temporary logic must be +# removed once Presto has a new release depending on hudi-presto-bundle and we upgrade docker setup +# to that release version. +RUN rm ${PRESTO_HOME}/plugin/hive-hadoop2/hudi-common-* +RUN rm ${PRESTO_HOME}/plugin/hive-hadoop2/hudi-hadoop-mr-* VOLUME ["${PRESTO_LOG_DIR}"]