1
0

[HUDI-159]: Pom cleanup and removal of com.twitter.parquet

- Redo all classes based on org.parquet only
 - remove unuused dependencies like parquet-hadoop, common-configuration2
 - timeline-service does not build a fat jar anymore
 - Fix utilities and hadoop-mr bundles based on above
This commit is contained in:
vinoth chandar
2019-08-25 05:34:51 -07:00
committed by Balaji Varadarajan
parent 6edf0b9def
commit cd090871a1
29 changed files with 600 additions and 326 deletions

View File

@@ -25,6 +25,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-hadoop-mr-bundle</artifactId>
<packaging>jar</packaging>
<dependencies>
<!-- Hoodie -->
@@ -58,16 +59,6 @@
<artifactId>parquet-avro</artifactId>
</dependency>
<!-- Parquet (Twitter) -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop-bundle</artifactId>
</dependency>
<!-- Apache Commons -->
<dependency>
<groupId>commons-logging</groupId>
@@ -82,12 +73,6 @@
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>com.twitter.common</groupId>
<artifactId>objectsize</artifactId>
<version>0.0.12</version>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -239,15 +224,16 @@
<includes>
<include>org.apache.hudi:hudi-common</include>
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>com.twitter:parquet-avro</include>
<include>com.twitter:parquet-hadoop-bundle</include>
<include>com.twitter.common:objectsize</include>
<include>commons-logging:commons-logging</include>
<include>commons-io:commons-io</include>
<include>commons-lang:commons-lang</include>
<include>commons-pool:commons-pool</include>
<include>commons-codec:commons-codec</include>
<include>com.esotericsoftware:kryo-shaded</include>
<include>org.objenesis:objenesis</include>
<include>com.esotericsoftware:minlog</include>
<include>commons-codec:commons-codec</include>
<include>org.apache.parquet:parquet-avro</include>
</includes>
</artifactSet>
<filters>

View File

@@ -62,12 +62,6 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!-- Parquet (Twitter) -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<!-- Thrift -->
<dependency>
<groupId>org.apache.thrift</groupId>

View File

@@ -50,12 +50,6 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!-- Parquet (Twitter) -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<!-- Thrift -->
<dependency>
<groupId>org.apache.thrift</groupId>

View File

@@ -296,10 +296,6 @@
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
</dependency>
<!-- Hadoop -->
<dependency>

View File

@@ -75,11 +75,22 @@
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-timeline-service</include>
<include>com.beust:jcommander</include>
<include>io.javalin:javalin</include>
<include>org.jetbrains.kotlin:*</include>
<include>org.eclipse.jetty:*</include>
<include>org.eclipse.jetty.websocket:*</include>
<include>org.rocksdb:rocksdbjni</include>
<include>org.apache.httpcomponents:httpclient</include>
<include>org.apache.httpcomponents:fluent-hc</include>
<include>org.antlr:stringtemplate</include>
<include>commons-io:commons-io</include>
<include>commons-logging:commons-logging</include>
<include>org.apache.parquet:parquet-avro</include>
<include>com.twitter:bijection-avro_2.11</include>
<include>com.twitter:bijection-core_2.11</include>
<include>org.apache.parquet:parquet-avro</include>
<include>com.twitter:parquet-avro</include>
<include>com.twitter.common:objectsize</include>
<include>io.confluent:kafka-avro-serializer</include>
<include>io.confluent:common-config</include>
<include>io.confluent:common-utils</include>
@@ -263,10 +274,6 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
</dependency>
<!-- Spark -->
<dependency>