lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3681] Provision additional hudi-spark-bundle with different versions (#5171)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-03-30 17:35:56 -07:00
committed by GitHub
parent 2d73c8ae86
commit 9830005e9b
3 changed files with 102 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
pom.xml
View File

@@ -92,6 +92,7 @@
<fasterxml.jackson.dataformat.yaml.version>2.7.4</fasterxml.jackson.dataformat.yaml.version>
<fasterxml.spark3.version>2.10.0</fasterxml.spark3.version>
<kafka.version>2.0.0</kafka.version>
<kafka.spark3.version>2.4.1</kafka.spark3.version>
<pulsar.version>2.8.1</pulsar.version>
<confluent.version>5.3.4</confluent.version>
<glassfish.version>2.17</glassfish.version>
@@ -135,6 +136,7 @@
<apache-rat-plugin.version>0.12</apache-rat-plugin.version>
<scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
<scalatest.version>3.0.1</scalatest.version>
<scalatest.spark3.version>3.1.0</scalatest.spark3.version>
<surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire.properties</surefire-log4j.file>
<thrift.version>0.12.0</thrift.version>
<jetty.version>9.4.15.v20190215</jetty.version>
@@ -1588,6 +1590,7 @@
</build>
</profile>
<!-- Default is spark2 profile, and spark2 is an alias of spark2.4 -->
<profile>
<id>spark2</id>
<modules>
@@ -1603,25 +1606,44 @@
</property>
</activation>
</profile>
<profile>
<id>spark2.4</id>
<modules>
<module>hudi-spark-datasource/hudi-spark2</module>
<module>hudi-spark-datasource/hudi-spark2-common</module>
</modules>
<properties>
<sparkbundle.version>2.4</sparkbundle.version>
</properties>
<activation>
<property>
<name>spark2.4</name>
</property>
</activation>
</profile>
<!-- spark3 is an alias of spark3.2 -->
<profile>
<id>spark3</id>
<properties>
<spark3.version>3.2.1</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3.version}</sparkbundle.version>
<sparkbundle.version>3</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>3.1.0</scalatest.version>
<kafka.version>2.4.1</kafka.version>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
<avro.version>1.10.2</avro.version>
<orc.version>1.6.12</orc.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
@@ -1637,51 +1659,22 @@
</profile>
<profile>
<id>spark3.2.0</id>
<properties>
<spark3.version>3.2.0</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>3.1.0</scalatest.version>
<kafka.version>2.4.1</kafka.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
<property>
<name>spark3.2.0</name>
</property>
</activation>
</profile>
<profile>
<id>spark3.1.x</id>
<id>spark3.1</id>
<properties>
<spark3.version>3.1.3</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3.version}</sparkbundle.version>
<sparkbundle.version>3.1</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3.1.x</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>3.1.0</scalatest.version>
<kafka.version>2.4.1</kafka.version>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
@@ -1691,7 +1684,41 @@
</modules>
<activation>
<property>
<name>spark3.1.x</name>
<name>spark3.1</name>
</property>
</activation>
</profile>
<profile>
<id>spark3.2</id>
<properties>
<spark3.version>3.2.1</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>3.2</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
<avro.version>1.10.2</avro.version>
<orc.version>1.6.12</orc.version>
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
<property>
<name>spark3.2</name>
</property>
</activation>
</profile>