[HUDI-1684] Tweak hudi-flink-bundle module pom and reorganize the pacakges for hudi-flink module (#2669)
* Add required dependencies for hudi-flink-bundle module * Some packages reorganization of hudi-flink module
This commit is contained in:
Danny Chan
@@ -31,6 +31,8 @@
|
||||
<properties>
|
||||
<checkstyle.skip>true</checkstyle.skip>
|
||||
<main.basedir>${project.parent.basedir}</main.basedir>
|
||||
<flink.bundle.hive.shade.prefix>org.apache.hudi.</flink.bundle.hive.shade.prefix>
|
||||
<javax.servlet.version>3.1.0</javax.servlet.version>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
@@ -70,6 +72,7 @@
|
||||
<include>org.apache.hudi:hudi-client-common</include>
|
||||
<include>org.apache.hudi:hudi-flink-client</include>
|
||||
<include>org.apache.hudi:hudi-flink_${scala.binary.version}</include>
|
||||
<include>org.apache.hudi:hudi-hadoop-mr</include>
|
||||
<include>org.apache.hudi:hudi-timeline-service</include>
|
||||
|
||||
<include>com.yammer.metrics:metrics-core</include>
|
||||
@@ -81,7 +84,14 @@
|
||||
<include>org.apache.httpcomponents:httpcore</include>
|
||||
<include>org.apache.httpcomponents:fluent-hc</include>
|
||||
<include>org.antlr:stringtemplate</include>
|
||||
<!-- Parquet -->
|
||||
<include>org.apache.parquet:parquet-avro</include>
|
||||
<include>org.apache.parquet:parquet-hadoop</include>
|
||||
<include>org.apache.parquet:parquet-column</include>
|
||||
<include>org.apache.parquet:parquet-common</include>
|
||||
<include>org.apache.parquet:parquet-format</include>
|
||||
<include>org.apache.parquet:parquet-encoding</include>
|
||||
|
||||
<include>org.apache.avro:avro</include>
|
||||
<include>joda-time:joda-time</include>
|
||||
<include>com.fasterxml.jackson.core:jackson-annotations</include>
|
||||
@@ -103,11 +113,25 @@
|
||||
<include>io.prometheus:simpleclient_pushgateway</include>
|
||||
<include>io.prometheus:simpleclient_common</include>
|
||||
<include>com.yammer.metrics:metrics-core</include>
|
||||
<include>org.apache.flink:flink-connector-kafka_${scala.binary.version}</include>
|
||||
<include>org.apache.flink:flink-connector-kafka-base_${scala.binary.version}</include>
|
||||
<include>org.apache.kafka:kafka_${scala.binary.version}</include>
|
||||
<include>com.101tec:zkclient</include>
|
||||
<include>org.apache.kafka:kafka-clients</include>
|
||||
|
||||
<!-- Used for HUDI TimelineService -->
|
||||
<include>org.eclipse.jetty:*</include>
|
||||
<include>org.eclipse.jetty.websocket:*</include>
|
||||
<include>javax.servlet:javax.servlet-api</include>
|
||||
|
||||
<include>org.apache.flink:flink-connector-kafka_${scala.binary.version}</include>
|
||||
<include>org.apache.flink:flink-connector-kafka-base_${scala.binary.version}</include>
|
||||
<include>org.apache.flink:flink-hadoop-compatibility_${scala.binary.version}</include>
|
||||
<include>org.apache.flink:flink-avro</include>
|
||||
<include>org.apache.flink:flink-json</include>
|
||||
<include>org.apache.flink:flink-parquet_${scala.binary.version}</include>
|
||||
|
||||
<include>org.apache.hive:hive-common</include>
|
||||
<include>org.apache.hive:hive-exec</include>
|
||||
<include>org.apache.hive:hive-metastore</include>
|
||||
<include>org.apache.hive:hive-jdbc</include>
|
||||
|
||||
<include>org.apache.hbase:hbase-client</include>
|
||||
<include>org.apache.hbase:hbase-common</include>
|
||||
@@ -125,6 +149,34 @@
|
||||
<pattern>com.beust.jcommander.</pattern>
|
||||
<shadedPattern>org.apache.hudi.com.beust.jcommander.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hive.jdbc.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.jdbc.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hadoop.hive.metastore.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hive.common.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.common.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hadoop.hive.common.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.common.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hadoop.hive.conf.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hive.service.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.service.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.hadoop.hive.service.</pattern>
|
||||
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.service.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.codahale.metrics.</pattern>
|
||||
<shadedPattern>org.apache.hudi.com.codahale.metrics.</shadedPattern>
|
||||
@@ -133,6 +185,10 @@
|
||||
<pattern>org.apache.commons.codec.</pattern>
|
||||
<shadedPattern>org.apache.hudi.org.apache.commons.codec.</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.eclipse.jetty.</pattern>
|
||||
<shadedPattern>org.apache.hudi.org.apache.jetty.</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<filters>
|
||||
<filter>
|
||||
@@ -142,6 +198,8 @@
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
<exclude>META-INF/services/javax.*</exclude>
|
||||
<!-- exclude the test dynamic factory -->
|
||||
<exclude>META-INF/services/org.apache.flink.table.factories.Factory</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
@@ -183,6 +241,21 @@
|
||||
<artifactId>hudi-flink_${scala.binary.version}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hudi</groupId>
|
||||
<artifactId>hudi-hadoop-mr</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hudi</groupId>
|
||||
<artifactId>hudi-timeline-service</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<version>${javax.servlet.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Flink -->
|
||||
<dependency>
|
||||
@@ -196,6 +269,30 @@
|
||||
<version>${flink.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-avro</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-parquet_${scala.binary.version}</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-json</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Parquet -->
|
||||
<dependency>
|
||||
@@ -208,9 +305,69 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
<!-- Override the version to be same with Flink avro -->
|
||||
<version>1.10.0</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Hive -->
|
||||
<dependency>
|
||||
<groupId>${hive.groupid}</groupId>
|
||||
<artifactId>hive-exec</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.mail</groupId>
|
||||
<artifactId>mail</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty.aggregate</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>${hive.groupid}</groupId>
|
||||
<artifactId>hive-metastore</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet.jsp</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>${hive.groupid}</groupId>
|
||||
<artifactId>hive-jdbc</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.servlet.jsp</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>${hive.groupid}</groupId>
|
||||
<artifactId>hive-common</artifactId>
|
||||
<version>${hive.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty.orbit</groupId>
|
||||
<artifactId>javax.servlet</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
|
||||
Reference in New Issue
Block a user