1
0

[HUDI-884] Shade avro and parquet-avro in hudi-hive-sync-bundle (#1618)

Co-authored-by: Mehrotra <uditme@amazon.com>
This commit is contained in:
Udit Mehrotra
2020-05-12 11:40:31 -07:00
committed by GitHub
parent e8ffc6f0aa
commit 404c7e82d9
2 changed files with 28 additions and 2 deletions

View File

@@ -125,8 +125,15 @@ public class ITTestHoodieDemo extends ITTestBase {
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+ " --target-base-path " + COW_BASE_PATH + " --target-table " + COW_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties"
+ " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+ String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME),
+ " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider ",
"spark-submit --class org.apache.hudi.hive.HiveSyncTool " + HUDI_HIVE_SYNC_BUNDLE
+ " --database default"
+ " --table " + COW_TABLE_NAME
+ " --base-path " + COW_BASE_PATH
+ " --user hive"
+ " --pass hive"
+ " --jdbc-url jdbc:hive2://hiveserver:10000"
+ " --partitioned-by dt",
("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+ " --table-type MERGE_ON_READ "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "

View File

@@ -69,12 +69,19 @@
<include>org.apache.hudi:hudi-hive-sync</include>
<include>com.beust:jcommander</include>
<include>org.apache.avro:avro</include>
<include>org.apache.parquet:parquet-avro</include>
<include>com.esotericsoftware:kryo-shaded</include>
<include>org.objenesis:objenesis</include>
<include>com.esotericsoftware:minlog</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>org.apache.avro.</pattern>
<shadedPattern>org.apache.hudi.org.apache.avro.</shadedPattern>
</relocation>
</relocations>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
@@ -120,5 +127,17 @@
<artifactId>hudi-hive-sync</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</project>