[HUDI-2785] Add Trino setup in Docker Demo (#4300)
* [HUDI-2785] Add Trino setup in Docker Demo * Update docker account and remove unnecessary configs * Adjust sparkadhoc Dockerfile
This commit is contained in:
66
docker/hoodie/hadoop/trinobase/Dockerfile
Normal file
66
docker/hoodie/hadoop/trinobase/Dockerfile
Normal file
@@ -0,0 +1,66 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster
|
||||
|
||||
ARG HADOOP_VERSION=2.8.4
|
||||
ARG HIVE_VERSION=2.3.3
|
||||
FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-java11:latest as hadoop-base
|
||||
|
||||
ENV TRINO_VERSION=368
|
||||
ENV TRINO_HOME=/usr/local/trino
|
||||
ENV BASE_URL=https://repo1.maven.org/maven2
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y \
|
||||
curl \
|
||||
tar \
|
||||
sudo \
|
||||
rsync \
|
||||
python \
|
||||
wget \
|
||||
python3-pip \
|
||||
python-dev \
|
||||
build-essential \
|
||||
uuid-runtime \
|
||||
less
|
||||
|
||||
ENV JAVA_HOME /usr/java/default
|
||||
ENV PATH $PATH:$JAVA_HOME/bin
|
||||
|
||||
WORKDIR /usr/local/bin
|
||||
RUN wget -q ${BASE_URL}/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar
|
||||
RUN chmod +x trino-cli-${TRINO_VERSION}-executable.jar
|
||||
RUN mv trino-cli-${TRINO_VERSION}-executable.jar trino-cli
|
||||
|
||||
WORKDIR /usr/local
|
||||
RUN wget -q ${BASE_URL}/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz
|
||||
RUN tar xvzf trino-server-${TRINO_VERSION}.tar.gz -C /usr/local/
|
||||
RUN ln -s /usr/local/trino-server-${TRINO_VERSION} $TRINO_HOME
|
||||
|
||||
ENV TRINO_BASE_WS /var/hoodie/ws/docker/hoodie/hadoop/trinobase
|
||||
RUN mkdir -p ${TRINO_BASE_WS}/target/
|
||||
ADD target/ ${TRINO_BASE_WS}/target/
|
||||
ENV HUDI_TRINO_BUNDLE ${TRINO_BASE_WS}/target/hudi-trino-bundle.jar
|
||||
RUN cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/
|
||||
|
||||
ADD scripts ${TRINO_HOME}/scripts
|
||||
RUN chmod +x ${TRINO_HOME}/scripts/trino.sh
|
||||
|
||||
RUN mkdir -p $TRINO_HOME/data
|
||||
VOLUME ["$TRINO_HOME/data"]
|
||||
116
docker/hoodie/hadoop/trinobase/pom.xml
Normal file
116
docker/hoodie/hadoop/trinobase/pom.xml
Normal file
@@ -0,0 +1,116 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||
~ or more contributor license agreements. See the NOTICE file
|
||||
~ distributed with this work for additional information
|
||||
~ regarding copyright ownership. The ASF licenses this file
|
||||
~ to you under the Apache License, Version 2.0 (the
|
||||
~ "License"); you may not use this file except in compliance
|
||||
~ with the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing,
|
||||
~ software distributed under the License is distributed on an
|
||||
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
~ KIND, either express or implied. See the License for the
|
||||
~ specific language governing permissions and limitations
|
||||
~ under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hudi-hadoop-docker</artifactId>
|
||||
<groupId>org.apache.hudi</groupId>
|
||||
<version>0.11.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<packaging>pom</packaging>
|
||||
<artifactId>hudi-hadoop-trinobase-docker</artifactId>
|
||||
<description>Trino Base Docker Image with Hudi</description>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<checkstyle.skip>true</checkstyle.skip>
|
||||
<main.basedir>${project.parent.parent.basedir}</main.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hudi</groupId>
|
||||
<artifactId>hudi-hadoop-base-java11-docker</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<copy file="${project.basedir}/../../../../packaging/hudi-trino-bundle/target/hudi-trino-bundle-${project.version}.jar"
|
||||
tofile="target/hudi-trino-bundle.jar"/>
|
||||
</target>
|
||||
</configuration>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!-- Build Docker image -->
|
||||
<plugin>
|
||||
<groupId>com.spotify</groupId>
|
||||
<artifactId>dockerfile-maven-plugin</artifactId>
|
||||
<version>${dockerfile.maven.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>tag-latest</id>
|
||||
<phase>pre-integration-test</phase>
|
||||
<goals>
|
||||
<goal>build</goal>
|
||||
<goal>tag</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<skip>${docker.build.skip}</skip>
|
||||
<pullNewerImage>false</pullNewerImage>
|
||||
<repository>
|
||||
apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version}
|
||||
</repository>
|
||||
<force>true</force>
|
||||
<tag>latest</tag>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>tag-version</id>
|
||||
<phase>pre-integration-test</phase>
|
||||
<goals>
|
||||
<goal>build</goal>
|
||||
<goal>tag</goal>
|
||||
<!-- <goal>push</goal> -->
|
||||
</goals>
|
||||
<configuration>
|
||||
<skip>${docker.build.skip}</skip>
|
||||
<pullNewerImage>false</pullNewerImage>
|
||||
<repository>
|
||||
apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version}
|
||||
</repository>
|
||||
<force>true</force>
|
||||
<tag>${project.version}</tag>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
21
docker/hoodie/hadoop/trinobase/scripts/trino.sh
Normal file
21
docker/hoodie/hadoop/trinobase/scripts/trino.sh
Normal file
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
/usr/local/trino/bin/launcher run
|
||||
Reference in New Issue
Block a user