lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1040] Make Hudi support Spark 3 (#2208)

Browse Source 
* Fix flaky MOR unit test

* Update Spark APIs to make it be compatible with both spark2 & spark3

* Refactor bulk insert v2 part to make Hudi be able to compile with Spark3

* Add spark3 profile to handle fasterxml & spark version

* Create hudi-spark-common module & refactor hudi-spark related modules

Co-authored-by: Wenning Ding <wenningd@amazon.com>
This commit is contained in:
wenningd
2020-12-09 15:52:23 -08:00
committed by GitHub
parent 3a91d26d62
commit fce1453fa6
79 changed files with 1040 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
pom.xml
View File

@@ -39,7 +39,7 @@
<module>hudi-cli</module>
<module>hudi-client</module>
<module>hudi-hadoop-mr</module>
<module>hudi-spark</module>
<module>hudi-spark-datasource</module>
<module>hudi-timeline-service</module>
<module>hudi-utilities</module>
<module>hudi-sync</module>
@@ -84,6 +84,9 @@
<java.version>1.8</java.version>
<fasterxml.version>2.6.7</fasterxml.version>
<fasterxml.jackson.databind.version>2.6.7.3</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>2.6.7.1</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>2.7.4</fasterxml.jackson.dataformat.yaml.version>
<kafka.version>2.0.0</kafka.version>
<glassfish.version>2.17</glassfish.version>
<parquet.version>1.10.1</parquet.version>
@@ -103,9 +106,12 @@
<http.version>4.4.1</http.version>
<spark.version>2.4.4</spark.version>
<flink.version>1.11.2</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.0.0</spark3.version>
<avro.version>1.8.2</avro.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<scala12.version>2.12.10</scala12.version>
<apache-rat-plugin.version>0.12</apache-rat-plugin.version>
<scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
<scalatest.version>3.0.1</scalatest.version>
@@ -432,7 +438,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${fasterxml.version}.3</version>
<version>${fasterxml.jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
@@ -442,7 +448,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
<version>${fasterxml.version}.1</version>
<version>${fasterxml.jackson.module.scala.version}</version>
</dependency>
<!-- Glassfish -->
@@ -1306,7 +1312,7 @@
<profile>
<id>scala-2.12</id>
<properties>
<scala.version>2.12.10</scala.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
</properties>
<activation>
@@ -1341,6 +1347,25 @@
</plugins>
</build>
</profile>
<profile>
<id>spark3</id>
<properties>
<spark.version>${spark3.version}</spark.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<kafka.version>2.4.1</kafka.version>
<fasterxml.version>2.10.0</fasterxml.version>
<fasterxml.jackson.databind.version>2.10.0</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>2.10.0</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>2.10.0</fasterxml.jackson.dataformat.yaml.version>
</properties>
<activation>
<property>
<name>spark3</name>
</property>
</activation>
</profile>
</profiles>
</project>