[HUDI-3195] Fix spark 3 pom (#4554)
- drop 3.0.x profile - update readme - update build CI bot.yml - fix spark 3 bundle name
This commit is contained in:
4
.github/workflows/bot.yml
vendored
4
.github/workflows/bot.yml
vendored
@@ -20,10 +20,6 @@ jobs:
|
||||
spark: "spark2"
|
||||
- scala: "scala-2.11"
|
||||
spark: "spark2,spark-shade-unbundle-avro"
|
||||
- scala: "scala-2.12"
|
||||
spark: "spark3.0.x"
|
||||
- scala: "scala-2.12"
|
||||
spark: "spark3.0.x,spark-shade-unbundle-avro"
|
||||
- scala: "scala-2.12"
|
||||
spark: "spark3.1.x"
|
||||
- scala: "scala-2.12"
|
||||
|
||||
@@ -83,14 +83,11 @@ mvn clean package -DskipTests -Dscala-2.12
|
||||
The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
|
||||
|
||||
```
|
||||
# Build against Spark 3.2.0 (default build shipped with the public jars)
|
||||
# Build against Spark 3.2.0 (the default build shipped with the public Spark 3 bundle)
|
||||
mvn clean package -DskipTests -Dspark3
|
||||
|
||||
# Build against Spark 3.1.2
|
||||
mvn clean package -DskipTests -Dspark3.1.x
|
||||
|
||||
# Build against Spark 3.0.3
|
||||
mvn clean package -DskipTests -Dspark3.0.x
|
||||
```
|
||||
|
||||
### Build without spark-avro module
|
||||
|
||||
31
pom.xml
31
pom.xml
@@ -115,12 +115,10 @@
|
||||
<prometheus.version>0.8.0</prometheus.version>
|
||||
<http.version>4.4.1</http.version>
|
||||
<spark.version>${spark2.version}</spark.version>
|
||||
<sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
|
||||
<sparkbundle.version></sparkbundle.version>
|
||||
<flink.version>1.13.1</flink.version>
|
||||
<spark2.version>2.4.4</spark2.version>
|
||||
<spark3.version>3.2.0</spark3.version>
|
||||
<spark2bundle.version></spark2bundle.version>
|
||||
<spark3bundle.version>3</spark3bundle.version>
|
||||
<hudi.spark.module>hudi-spark2</hudi.spark.module>
|
||||
<avro.version>1.8.2</avro.version>
|
||||
<scala11.version>2.11.12</scala11.version>
|
||||
@@ -1572,7 +1570,7 @@
|
||||
<id>spark3</id>
|
||||
<properties>
|
||||
<spark.version>${spark3.version}</spark.version>
|
||||
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
|
||||
<sparkbundle.version>${spark3.version}</sparkbundle.version>
|
||||
<scala.version>${scala12.version}</scala.version>
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
||||
@@ -1598,7 +1596,7 @@
|
||||
<properties>
|
||||
<spark3.version>3.1.2</spark3.version>
|
||||
<spark.version>${spark3.version}</spark.version>
|
||||
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
|
||||
<sparkbundle.version>${spark3.version}</sparkbundle.version>
|
||||
<scala.version>${scala12.version}</scala.version>
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
||||
@@ -1618,29 +1616,6 @@
|
||||
</activation>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark3.0.x</id>
|
||||
<properties>
|
||||
<spark3.version>3.0.3</spark3.version>
|
||||
<spark.version>${spark3.version}</spark.version>
|
||||
<scala.version>${scala12.version}</scala.version>
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
||||
<scalatest.version>3.0.1</scalatest.version>
|
||||
<kafka.version>2.4.1</kafka.version>
|
||||
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
|
||||
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
|
||||
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
|
||||
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
|
||||
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
|
||||
</properties>
|
||||
<activation>
|
||||
<property>
|
||||
<name>spark3.0.x</name>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>skipShadeSources</id>
|
||||
<properties>
|
||||
|
||||
Reference in New Issue
Block a user