lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4186] Support Hudi with Spark 3.3.0 (#5943)

Browse Source 
Co-authored-by: Shawn Chang <yxchang@amazon.com>
This commit is contained in:
Shawn Chang
2022-07-27 14:47:49 -07:00
committed by GitHub
parent 924c30c7ea
commit cdaec5a8da
99 changed files with 10865 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
pom.xml
View File

@@ -121,7 +121,7 @@
<http.version>4.4.1</http.version>
<spark.version>${spark2.version}</spark.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.2.1</spark3.version>
<spark3.version>3.3.0</spark3.version>
<sparkbundle.version></sparkbundle.version>
<flink1.15.version>1.15.1</flink1.15.version>
<flink1.14.version>1.14.5</flink1.14.version>
@@ -142,6 +142,7 @@
<flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_${scala.binary.version}</flink.hadoop.compatibility.artifactId>
<spark31.version>3.1.3</spark31.version>
<spark32.version>3.2.1</spark32.version>
<spark33.version>3.3.0</spark33.version>
<hudi.spark.module>hudi-spark2</hudi.spark.module>
<hudi.spark.common.module>hudi-spark2-common</hudi.spark.common.module>
<avro.version>1.8.2</avro.version>
@@ -1664,23 +1665,24 @@
</activation>
</profile>
<!-- spark3 is an alias of spark3.2 -->
<!-- spark3 is an alias of spark3.3 -->
<profile>
<id>spark3</id>
<properties>
<spark3.version>3.2.1</spark3.version>
<spark3.version>3.3.0</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>3</sparkbundle.version>
<sparkbundle.version>3.3</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<hudi.spark.module>hudi-spark3.3.x</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
<avro.version>1.10.2</avro.version>
<orc.version>1.6.12</orc.version>
<avro.version>1.11.0</avro.version>
<orc.version>1.7.4</orc.version>
<antlr.version>4.8</antlr.version>
<fasterxml.spark3.version>2.13.3</fasterxml.spark3.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
@@ -1690,7 +1692,7 @@
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3</module>
<module>hudi-spark-datasource/hudi-spark3.3.x</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
@@ -1740,7 +1742,7 @@
<sparkbundle.version>3.2</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<hudi.spark.module>hudi-spark3.2.x</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
@@ -1757,7 +1759,7 @@
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3</module>
<module>hudi-spark-datasource/hudi-spark3.2.x</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
@@ -1767,6 +1769,42 @@
</activation>
</profile>
<profile>
<id>spark3.3</id>
<properties>
<spark3.version>3.3.0</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>3.3</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3.3.x</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
<avro.version>1.11.0</avro.version>
<orc.version>1.7.4</orc.version>
<antlr.version>4.8</antlr.version>
<fasterxml.spark3.version>2.13.3</fasterxml.spark3.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3.3.x</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
<property>
<name>spark3.3</name>
</property>
</activation>
</profile>
<profile>
<id>flink1.15</id>
<properties>