lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2811] Support Spark 3.2 (#4270)

Browse Source
This commit is contained in:
Yann Byron
2021-12-28 16:12:44 +08:00
committed by GitHub
parent 32505d5adb
commit 05942e018c
36 changed files with 596 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
pom.xml
View File

@@ -118,7 +118,7 @@
<sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
<flink.version>1.13.1</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.1.2</spark3.version>
<spark3.version>3.2.0</spark3.version>
<spark2bundle.version></spark2bundle.version>
<spark3bundle.version>3</spark3bundle.version>
<hudi.spark.module>hudi-spark2</hudi.spark.module>
@@ -1515,11 +1515,36 @@
</activation>
</profile>
<profile>
<id>spark3.1.x</id>
<properties>
<spark3.version>3.1.2</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<scalatest.version>3.1.0</scalatest.version>
<kafka.version>2.4.1</kafka.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<activation>
<property>
<name>spark3</name>
</property>
</activation>
</profile>
<profile>
<id>spark3.0.x</id>
<!-- for spark 3.0.x we need override the follow propeprties to package and run test-->
<properties>
<spark3.version>3.0.0</spark3.version>
<spark3.version>3.0.3</spark3.version>
<spark.version>${spark3.version}</spark.version>
<scalatest.version>3.0.1</scalatest.version>
</properties>