lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-159]: Pom cleanup and removal of com.twitter.parquet

Browse Source 
- Redo all classes based on org.parquet only
 - remove unuused dependencies like parquet-hadoop, common-configuration2
 - timeline-service does not build a fat jar anymore
 - Fix utilities and hadoop-mr bundles based on above
This commit is contained in:
vinoth chandar
2019-08-25 05:34:51 -07:00
committed by Balaji Varadarajan
parent 6edf0b9def
commit cd090871a1
29 changed files with 600 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
hudi-hadoop-mr/pom.xml
View File

@@ -49,28 +49,12 @@
<artifactId>parquet-avro</artifactId>
</dependency>
<!-- Parquet (Twitter) -->
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop-bundle</artifactId>
</dependency>
<!-- Apache Commons -->
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>com.twitter.common</groupId>
<artifactId>objectsize</artifactId>
<version>0.0.12</version>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -113,12 +97,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

6
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
View File

@@ -53,9 +53,9 @@ import org.apache.hudi.common.util.LogReaderUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import parquet.avro.AvroSchemaConverter;
import parquet.hadoop.ParquetFileReader;
import parquet.schema.MessageType;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.schema.MessageType;
/**
* Record Reader implementation to merge fresh avro data with base parquet data, to support real