[HUDI-884] Shade avro and parquet-avro in hudi-hive-sync-bundle (#1618)
Co-authored-by: Mehrotra <uditme@amazon.com>
This commit is contained in:
@@ -125,8 +125,15 @@ public class ITTestHoodieDemo extends ITTestBase {
|
||||
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
|
||||
+ " --target-base-path " + COW_BASE_PATH + " --target-table " + COW_TABLE_NAME
|
||||
+ " --props /var/demo/config/dfs-source.properties"
|
||||
+ " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
|
||||
+ String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME),
|
||||
+ " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider ",
|
||||
"spark-submit --class org.apache.hudi.hive.HiveSyncTool " + HUDI_HIVE_SYNC_BUNDLE
|
||||
+ " --database default"
|
||||
+ " --table " + COW_TABLE_NAME
|
||||
+ " --base-path " + COW_BASE_PATH
|
||||
+ " --user hive"
|
||||
+ " --pass hive"
|
||||
+ " --jdbc-url jdbc:hive2://hiveserver:10000"
|
||||
+ " --partitioned-by dt",
|
||||
("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
|
||||
+ " --table-type MERGE_ON_READ "
|
||||
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
|
||||
|
||||
@@ -69,12 +69,19 @@
|
||||
<include>org.apache.hudi:hudi-hive-sync</include>
|
||||
|
||||
<include>com.beust:jcommander</include>
|
||||
<include>org.apache.avro:avro</include>
|
||||
<include>org.apache.parquet:parquet-avro</include>
|
||||
<include>com.esotericsoftware:kryo-shaded</include>
|
||||
<include>org.objenesis:objenesis</include>
|
||||
<include>com.esotericsoftware:minlog</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>org.apache.avro.</pattern>
|
||||
<shadedPattern>org.apache.hudi.org.apache.avro.</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
<filters>
|
||||
<filter>
|
||||
@@ -120,5 +127,17 @@
|
||||
<artifactId>hudi-hive-sync</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.parquet</groupId>
|
||||
<artifactId>parquet-avro</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
Reference in New Issue
Block a user