From 243c58f77cfc875159a0ac568efb0e3dfff199a6 Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Wed, 17 Apr 2019 09:53:22 -0700 Subject: [PATCH] Move to apachehudi dockerhub repository & use openjdk docker containers --- ...cker-compose_hadoop284_hive233_spark231.yml | 18 +++++++++--------- docker/hoodie/hadoop/base/Dockerfile | 6 ++---- docker/hoodie/hadoop/base/pom.xml | 4 ++-- docker/hoodie/hadoop/datanode/Dockerfile | 2 +- docker/hoodie/hadoop/datanode/pom.xml | 4 ++-- docker/hoodie/hadoop/historyserver/Dockerfile | 2 +- docker/hoodie/hadoop/historyserver/pom.xml | 4 ++-- docker/hoodie/hadoop/hive_base/Dockerfile | 2 +- docker/hoodie/hadoop/hive_base/pom.xml | 4 ++-- docker/hoodie/hadoop/namenode/Dockerfile | 2 +- docker/hoodie/hadoop/namenode/pom.xml | 4 ++-- docker/hoodie/hadoop/spark_base/Dockerfile | 2 +- docker/hoodie/hadoop/spark_base/pom.xml | 4 ++-- docker/hoodie/hadoop/sparkadhoc/Dockerfile | 2 +- docker/hoodie/hadoop/sparkadhoc/pom.xml | 4 ++-- docker/hoodie/hadoop/sparkmaster/Dockerfile | 2 +- docker/hoodie/hadoop/sparkmaster/pom.xml | 4 ++-- docker/hoodie/hadoop/sparkworker/Dockerfile | 2 +- docker/hoodie/hadoop/sparkworker/pom.xml | 4 ++-- 19 files changed, 37 insertions(+), 39 deletions(-) diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark231.yml b/docker/compose/docker-compose_hadoop284_hive233_spark231.yml index 6eb874f00..bbb9f10e9 100644 --- a/docker/compose/docker-compose_hadoop284_hive233_spark231.yml +++ b/docker/compose/docker-compose_hadoop284_hive233_spark231.yml @@ -3,7 +3,7 @@ version: "3.3" services: namenode: - image: varadarb/hudi-hadoop_2.8.4-namenode:latest + image: apachehudi/hudi-hadoop_2.8.4-namenode:latest hostname: namenode container_name: namenode volumes: @@ -22,7 +22,7 @@ services: retries: 3 datanode1: - image: varadarb/hudi-hadoop_2.8.4-datanode:latest + image: apachehudi/hudi-hadoop_2.8.4-datanode:latest container_name: datanode1 hostname: datanode1 environment: @@ -46,7 +46,7 @@ services: - /tmp/hadoop_data:/hadoop/dfs/data historyserver: - image: varadarb/hudi-hadoop_2.8.4-history:latest + image: apachehudi/hudi-hadoop_2.8.4-history:latest hostname: historyserver container_name: historyserver environment: @@ -75,7 +75,7 @@ services: container_name: hive-metastore-postgresql hivemetastore: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest hostname: hivemetastore container_name: hivemetastore links: @@ -98,7 +98,7 @@ services: - "namenode" hiveserver: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest hostname: hiveserver container_name: hiveserver env_file: @@ -117,7 +117,7 @@ services: - ${HUDI_WS}:/var/hoodie/ws sparkmaster: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.3.1:latest hostname: sparkmaster container_name: sparkmaster env_file: @@ -134,7 +134,7 @@ services: - "namenode" spark-worker-1: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.3.1:latest hostname: spark-worker-1 container_name: spark-worker-1 env_file: @@ -171,7 +171,7 @@ services: - ALLOW_PLAINTEXT_LISTENER=yes adhoc-1: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest hostname: adhoc-1 container_name: adhoc-1 env_file: @@ -191,7 +191,7 @@ services: - ${HUDI_WS}:/var/hoodie/ws adhoc-2: - image: varadarb/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest + image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.3.1:latest hostname: adhoc-2 container_name: adhoc-2 env_file: diff --git a/docker/hoodie/hadoop/base/Dockerfile b/docker/hoodie/hadoop/base/Dockerfile index 0d0a70884..26029d91d 100644 --- a/docker/hoodie/hadoop/base/Dockerfile +++ b/docker/hoodie/hadoop/base/Dockerfile @@ -1,19 +1,17 @@ -FROM frolvlad/alpine-java +FROM openjdk:8u212-jdk-slim-stretch MAINTAINER Hoodie USER root # Default to UTF-8 file.encoding ENV LANG C.UTF-8 -# Updating & Installing packages -RUN apk add net-tools curl bash perl procps - ARG HADOOP_VERSION=2.8.4 ARG HADOOP_URL=https://www.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz ENV HADOOP_VERSION ${HADOOP_VERSION} ENV HADOOP_URL ${HADOOP_URL} RUN set -x \ + && DEBIAN_FRONTEND=noninteractive apt-get -yq update && apt-get -yq install curl wget netcat procps \ && echo "Fetch URL2 is : ${HADOOP_URL}" \ && curl -fSL "${HADOOP_URL}" -o /tmp/hadoop.tar.gz \ && curl -fSL "${HADOOP_URL}.asc" -o /tmp/hadoop.tar.gz.asc \ diff --git a/docker/hoodie/hadoop/base/pom.xml b/docker/hoodie/hadoop/base/pom.xml index 4bd40c0a9..78c71e1df 100644 --- a/docker/hoodie/hadoop/base/pom.xml +++ b/docker/hoodie/hadoop/base/pom.xml @@ -62,7 +62,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-base + apachehudi/hudi-hadoop_${docker.hadoop.version}-base true latest @@ -78,7 +78,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-base + apachehudi/hudi-hadoop_${docker.hadoop.version}-base true ${project.version} diff --git a/docker/hoodie/hadoop/datanode/Dockerfile b/docker/hoodie/hadoop/datanode/Dockerfile index 703a622cf..caabd3cab 100644 --- a/docker/hoodie/hadoop/datanode/Dockerfile +++ b/docker/hoodie/hadoop/datanode/Dockerfile @@ -1,6 +1,6 @@ ARG HADOOP_VERSION=2.8.4 ARG HADOOP_DN_PORT=50075 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-base:latest +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest ENV HADOOP_DN_PORT ${HADOOP_DN_PORT} diff --git a/docker/hoodie/hadoop/datanode/pom.xml b/docker/hoodie/hadoop/datanode/pom.xml index 70691ce79..b44fe3ef9 100644 --- a/docker/hoodie/hadoop/datanode/pom.xml +++ b/docker/hoodie/hadoop/datanode/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-datanode + apachehudi/hudi-hadoop_${docker.hadoop.version}-datanode true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-datanode + apachehudi/hudi-hadoop_${docker.hadoop.version}-datanode true ${project.version} diff --git a/docker/hoodie/hadoop/historyserver/Dockerfile b/docker/hoodie/hadoop/historyserver/Dockerfile index d2e05e6da..81100635b 100644 --- a/docker/hoodie/hadoop/historyserver/Dockerfile +++ b/docker/hoodie/hadoop/historyserver/Dockerfile @@ -1,6 +1,6 @@ ARG HADOOP_VERSION=2.8.4 ARG HADOOP_HISTORY_PORT=8188 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-base:latest +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest ENV HADOOP_HISTORY_PORT ${HADOOP_HISTORY_PORT} diff --git a/docker/hoodie/hadoop/historyserver/pom.xml b/docker/hoodie/hadoop/historyserver/pom.xml index 4f3b7b9a6..8770d3891 100644 --- a/docker/hoodie/hadoop/historyserver/pom.xml +++ b/docker/hoodie/hadoop/historyserver/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-history + apachehudi/hudi-hadoop_${docker.hadoop.version}-history true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-history + apachehudi/hudi-hadoop_${docker.hadoop.version}-history true ${project.version} diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile index 40ab1d10c..516497722 100644 --- a/docker/hoodie/hadoop/hive_base/Dockerfile +++ b/docker/hoodie/hadoop/hive_base/Dockerfile @@ -1,5 +1,5 @@ ARG HADOOP_VERSION=2.8.4 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-base:latest +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest ENV HIVE_HOME /opt/hive ENV PATH $HIVE_HOME/bin:$PATH diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml index c8eebbb06..064f19dc0 100644 --- a/docker/hoodie/hadoop/hive_base/pom.xml +++ b/docker/hoodie/hadoop/hive_base/pom.xml @@ -81,7 +81,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version} true latest @@ -97,7 +97,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version} true ${project.version} diff --git a/docker/hoodie/hadoop/namenode/Dockerfile b/docker/hoodie/hadoop/namenode/Dockerfile index 648032977..ff10aa6aa 100644 --- a/docker/hoodie/hadoop/namenode/Dockerfile +++ b/docker/hoodie/hadoop/namenode/Dockerfile @@ -1,6 +1,6 @@ ARG HADOOP_VERSION=2.8.4 ARG HADOOP_WEBHDFS_PORT=50070 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-base:latest +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest ENV HADOOP_WEBHDFS_PORT ${HADOOP_WEBHDFS_PORT} diff --git a/docker/hoodie/hadoop/namenode/pom.xml b/docker/hoodie/hadoop/namenode/pom.xml index 841ac62cc..130ebf667 100644 --- a/docker/hoodie/hadoop/namenode/pom.xml +++ b/docker/hoodie/hadoop/namenode/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-namenode + apachehudi/hudi-hadoop_${docker.hadoop.version}-namenode true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-namenode + apachehudi/hudi-hadoop_${docker.hadoop.version}-namenode true ${project.version} diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile index e35b3996d..7b0917d9b 100644 --- a/docker/hoodie/hadoop/spark_base/Dockerfile +++ b/docker/hoodie/hadoop/spark_base/Dockerfile @@ -1,6 +1,6 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION} +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION} ENV ENABLE_INIT_DAEMON true ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon diff --git a/docker/hoodie/hadoop/spark_base/pom.xml b/docker/hoodie/hadoop/spark_base/pom.xml index 841d58a97..19ab633b5 100644 --- a/docker/hoodie/hadoop/spark_base/pom.xml +++ b/docker/hoodie/hadoop/spark_base/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkbase_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkbase_${docker.spark.version} true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkbase_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkbase_${docker.spark.version} true ${project.version} diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile b/docker/hoodie/hadoop/sparkadhoc/Dockerfile index 8a731305d..22e0b4915 100644 --- a/docker/hoodie/hadoop/sparkadhoc/Dockerfile +++ b/docker/hoodie/hadoop/sparkadhoc/Dockerfile @@ -1,7 +1,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 ARG SPARK_VERSION=2.3.1 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} COPY adhoc.sh /opt/spark diff --git a/docker/hoodie/hadoop/sparkadhoc/pom.xml b/docker/hoodie/hadoop/sparkadhoc/pom.xml index 4e5b76624..5b714a873 100644 --- a/docker/hoodie/hadoop/sparkadhoc/pom.xml +++ b/docker/hoodie/hadoop/sparkadhoc/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkadhoc_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkadhoc_${docker.spark.version} true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkadhoc_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkadhoc_${docker.spark.version} true ${project.version} diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile b/docker/hoodie/hadoop/sparkmaster/Dockerfile index acfba8c69..8d1178c72 100644 --- a/docker/hoodie/hadoop/sparkmaster/Dockerfile +++ b/docker/hoodie/hadoop/sparkmaster/Dockerfile @@ -1,7 +1,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 ARG SPARK_VERSION=2.3.1 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} COPY master.sh /opt/spark diff --git a/docker/hoodie/hadoop/sparkmaster/pom.xml b/docker/hoodie/hadoop/sparkmaster/pom.xml index bce4511eb..56d2867f1 100644 --- a/docker/hoodie/hadoop/sparkmaster/pom.xml +++ b/docker/hoodie/hadoop/sparkmaster/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkmaster_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkmaster_${docker.spark.version} true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkmaster_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkmaster_${docker.spark.version} true ${project.version} diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile b/docker/hoodie/hadoop/sparkworker/Dockerfile index acbddc077..e595fdaa6 100644 --- a/docker/hoodie/hadoop/sparkworker/Dockerfile +++ b/docker/hoodie/hadoop/sparkworker/Dockerfile @@ -1,7 +1,7 @@ ARG HADOOP_VERSION=2.8.4 ARG HIVE_VERSION=2.3.3 ARG SPARK_VERSION=2.3.1 -FROM varadarb/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} +FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} COPY worker.sh /opt/spark diff --git a/docker/hoodie/hadoop/sparkworker/pom.xml b/docker/hoodie/hadoop/sparkworker/pom.xml index 8c0c77af9..dc0282a5e 100644 --- a/docker/hoodie/hadoop/sparkworker/pom.xml +++ b/docker/hoodie/hadoop/sparkworker/pom.xml @@ -61,7 +61,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkworker_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkworker_${docker.spark.version} true latest @@ -77,7 +77,7 @@ ${docker.build.skip} false - varadarb/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkworker_${docker.spark.version} + apachehudi/hudi-hadoop_${docker.hadoop.version}-hive_${docker.hive.version}-sparkworker_${docker.spark.version} true ${project.version}