8d04268264
The purpose of this pull request is to implement changes required on Hudi side to get Bootstrapped tables integrated with Presto. The testing was done against presto 0.232 and following changes were identified to make it work: Annotation UseRecordReaderFromInputFormat is required on HoodieParquetInputFormat as well, because the reading for bootstrapped tables needs to happen through record reader to be able to perform the merge. On presto side, this annotation is already handled. We need to internally maintain VIRTUAL_COLUMN_NAMES because presto's internal hive version hive-apache-1.2.2 has VirutalColumn as a class, versus the one we depend on in hudi which is an enum. Dependency changes in hudi-presto-bundle to avoid runtime exceptions.
222 lines
9.0 KiB
XML
222 lines
9.0 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
|
contributor license agreements. See the NOTICE file distributed with
|
|
this work for additional information regarding copyright ownership.
|
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
|
(the "License"); you may not use this file except in compliance with
|
|
the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<parent>
|
|
<artifactId>hudi</artifactId>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<version>0.6.0-SNAPSHOT</version>
|
|
<relativePath>../../pom.xml</relativePath>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<artifactId>hudi-presto-bundle</artifactId>
|
|
<packaging>jar</packaging>
|
|
|
|
<properties>
|
|
<checkstyle.skip>true</checkstyle.skip>
|
|
<main.basedir>${project.parent.basedir}</main.basedir>
|
|
</properties>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.rat</groupId>
|
|
<artifactId>apache-rat-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<version>${maven-shade-plugin.version}</version>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
<configuration>
|
|
<createSourcesJar>true</createSourcesJar>
|
|
<dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml
|
|
</dependencyReducedPomLocation>
|
|
<transformers>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer">
|
|
</transformer>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
|
|
<addHeader>true</addHeader>
|
|
</transformer>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
|
|
<resource>META-INF/LICENSE</resource>
|
|
<file>target/classes/META-INF/LICENSE</file>
|
|
</transformer>
|
|
</transformers>
|
|
<artifactSet>
|
|
<includes>
|
|
<include>org.apache.hudi:hudi-common</include>
|
|
<include>org.apache.hudi:hudi-hadoop-mr</include>
|
|
|
|
<include>org.apache.parquet:parquet-avro</include>
|
|
<include>org.apache.avro:avro</include>
|
|
<include>com.esotericsoftware:kryo-shaded</include>
|
|
<include>org.objenesis:objenesis</include>
|
|
<include>com.esotericsoftware:minlog</include>
|
|
<include>org.apache.hbase:hbase-client</include>
|
|
<include>org.apache.hbase:hbase-common</include>
|
|
<include>org.apache.hbase:hbase-protocol</include>
|
|
<include>org.apache.hbase:hbase-server</include>
|
|
<include>org.apache.hbase:hbase-annotations</include>
|
|
<include>org.apache.htrace:htrace-core</include>
|
|
<include>com.yammer.metrics:metrics-core</include>
|
|
<include>com.google.guava:guava</include>
|
|
<include>commons-lang:commons-lang</include>
|
|
<include>com.google.protobuf:protobuf-java</include>
|
|
</includes>
|
|
</artifactSet>
|
|
<relocations>
|
|
|
|
<relocation>
|
|
<pattern>org.apache.avro.</pattern>
|
|
<shadedPattern>org.apache.hudi.org.apache.avro.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.esotericsoftware.kryo.</pattern>
|
|
<shadedPattern>org.apache.hudi.com.esotericsoftware.kryo.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.objenesis.</pattern>
|
|
<shadedPattern>org.apache.hudi.org.objenesis.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.esotericsoftware.minlog.</pattern>
|
|
<shadedPattern>org.apache.hudi.com.esotericsoftware.minlog.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern> org.apache.hadoop.hbase.</pattern>
|
|
<shadedPattern>org.apache.hudi.org.apache.hadoop.hbase.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern> org.apache.htrace.</pattern>
|
|
<shadedPattern>org.apache.hudi.org.apache.htrace.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.yammer.metrics.</pattern>
|
|
<shadedPattern>org.apache.hudi.com.yammer.metrics.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.common.</pattern>
|
|
<shadedPattern>${presto.bundle.bootstrap.shade.prefix}com.google.common.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.apache.commons.lang.</pattern>
|
|
<shadedPattern>${presto.bundle.bootstrap.shade.prefix}org.apache.commons.lang.</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.protobuf.</pattern>
|
|
<shadedPattern>${presto.bundle.bootstrap.shade.prefix}com.google.protobuf.</shadedPattern>
|
|
</relocation>
|
|
</relocations>
|
|
<createDependencyReducedPom>false</createDependencyReducedPom>
|
|
<filters>
|
|
<filter>
|
|
<artifact>*:*</artifact>
|
|
<excludes>
|
|
<exclude>META-INF/*.SF</exclude>
|
|
<exclude>META-INF/*.DSA</exclude>
|
|
<exclude>META-INF/*.RSA</exclude>
|
|
<exclude>META-INF/services/javax.*</exclude>
|
|
</excludes>
|
|
</filter>
|
|
</filters>
|
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
</resource>
|
|
<resource>
|
|
<directory>src/test/resources</directory>
|
|
</resource>
|
|
</resources>
|
|
</build>
|
|
|
|
<dependencies>
|
|
<!-- Hoodie -->
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-common</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-hadoop-mr-bundle</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
|
|
<!-- Parquet -->
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-avro</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
|
|
<!--Guava needs to be shaded because HBase 1.2.3 depends on an earlier guava version i.e 12.0.1 and hits runtime
|
|
issues with the guava version present in Presto runtime-->
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<version>12.0.1</version>
|
|
<scope>${presto.bundle.bootstrap.scope}</scope>
|
|
</dependency>
|
|
|
|
<!--commons-lang needs to be shaded because HBase 1.2.3 needs it at runtime, but Presto runtime does not have this
|
|
dependency-->
|
|
<dependency>
|
|
<groupId>commons-lang</groupId>
|
|
<artifactId>commons-lang</artifactId>
|
|
<version>2.6</version>
|
|
<scope>${presto.bundle.bootstrap.scope}</scope>
|
|
</dependency>
|
|
|
|
<!--protobuf needs to be shaded because HBase 1.2.3 needs it at runtime, but Presto runtime does not have this
|
|
dependency-->
|
|
<dependency>
|
|
<groupId>com.google.protobuf</groupId>
|
|
<artifactId>protobuf-java</artifactId>
|
|
<version>2.5.0</version>
|
|
<scope>${presto.bundle.bootstrap.scope}</scope>
|
|
</dependency>
|
|
|
|
</dependencies>
|
|
<profiles>
|
|
<profile>
|
|
<id>presto-shade-unbundle-bootstrap</id>
|
|
<properties>
|
|
<presto.bundle.bootstrap.scope>provided</presto.bundle.bootstrap.scope>
|
|
<presto.bundle.bootstrap.shade.prefix></presto.bundle.bootstrap.shade.prefix>
|
|
</properties>
|
|
</profile>
|
|
</profiles>
|
|
</project>
|