lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3588] Remove hudi-common and hudi-hadoop-mr jars in Presto Docker image (#4997)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-03-15 18:49:30 -07:00
committed by GitHub
parent 55dca969f9
commit 296a0e6bcf
3 changed files with 31 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docker/compose/docker-compose_hadoop284_hive233_spark244.yml
View File

@@ -184,7 +184,7 @@ services:
presto-coordinator-1: presto-coordinator-1:
container_name: presto-coordinator-1 container_name: presto-coordinator-1
hostname: presto-coordinator-1 hostname: presto-coordinator-1
image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.268:latest image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest
ports: ports:
- '8090:8090' - '8090:8090'
environment: environment:
@@ -203,7 +203,7 @@ services:
presto-worker-1: presto-worker-1:
container_name: presto-worker-1 container_name: presto-worker-1
hostname: presto-worker-1 hostname: presto-worker-1
image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.268:latest image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.271:latest
depends_on: [ "presto-coordinator-1" ] depends_on: [ "presto-coordinator-1" ]
environment: environment:
- PRESTO_JVM_MAX_HEAP=512M - PRESTO_JVM_MAX_HEAP=512M

2
docker/hoodie/hadoop/pom.xml
View File

@@ -57,7 +57,7 @@
<docker.spark.version>2.4.4</docker.spark.version> <docker.spark.version>2.4.4</docker.spark.version>
<docker.hive.version>2.3.3</docker.hive.version> <docker.hive.version>2.3.3</docker.hive.version>
<docker.hadoop.version>2.8.4</docker.hadoop.version> <docker.hadoop.version>2.8.4</docker.hadoop.version>
<docker.presto.version>0.268</docker.presto.version> <docker.presto.version>0.271</docker.presto.version>
<docker.trino.version>368</docker.trino.version> <docker.trino.version>368</docker.trino.version>
<dockerfile.maven.version>1.4.13</dockerfile.maven.version> <dockerfile.maven.version>1.4.13</dockerfile.maven.version>
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>

11
docker/hoodie/hadoop/prestobase/Dockerfile
View File

@@ -22,7 +22,7 @@ ARG HADOOP_VERSION=2.8.4
ARG HIVE_VERSION=2.3.3 ARG HIVE_VERSION=2.3.3
FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest as hadoop-base FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest as hadoop-base
ARG PRESTO_VERSION=0.268 ARG PRESTO_VERSION=0.271
ENV PRESTO_VERSION ${PRESTO_VERSION} ENV PRESTO_VERSION ${PRESTO_VERSION}
ENV PRESTO_HOME /opt/presto-server-${PRESTO_VERSION} ENV PRESTO_HOME /opt/presto-server-${PRESTO_VERSION}
@@ -79,6 +79,15 @@ RUN chmod +x /usr/local/bin/entrypoint.sh
ADD target/ /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/ ADD target/ /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/
ENV HUDI_PRESTO_BUNDLE /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/hudi-presto-bundle.jar ENV HUDI_PRESTO_BUNDLE /var/hoodie/ws/docker/hoodie/hadoop/prestobase/target/hudi-presto-bundle.jar
RUN cp ${HUDI_PRESTO_BUNDLE} ${PRESTO_HOME}/plugin/hive-hadoop2/ RUN cp ${HUDI_PRESTO_BUNDLE} ${PRESTO_HOME}/plugin/hive-hadoop2/
# TODO: the latest master of Presto relies on hudi-presto-bundle, while current Presto releases
# rely on hudi-common and hudi-hadoop-mr 0.9.0, which are pulled in plugin/hive-hadoop2/ in the
# docker setup, making it hard to test the latest changes in Hudi due to class conflict.
# To get around the conflicts due to older Hudi jars below, they are removed for integration tests,
# so the hudi-presto-bundle build can be used solely for testing. This temporary logic must be
# removed once Presto has a new release depending on hudi-presto-bundle and we upgrade docker setup
# to that release version.
RUN rm ${PRESTO_HOME}/plugin/hive-hadoop2/hudi-common-*
RUN rm ${PRESTO_HOME}/plugin/hive-hadoop2/hudi-hadoop-mr-*
VOLUME ["${PRESTO_LOG_DIR}"] VOLUME ["${PRESTO_LOG_DIR}"]