[HUDI-3195] Fix spark 3 pom (#4554)
- drop 3.0.x profile - update readme - update build CI bot.yml - fix spark 3 bundle name
This commit is contained in:
4
.github/workflows/bot.yml
vendored
4
.github/workflows/bot.yml
vendored
@@ -20,10 +20,6 @@ jobs:
|
|||||||
spark: "spark2"
|
spark: "spark2"
|
||||||
- scala: "scala-2.11"
|
- scala: "scala-2.11"
|
||||||
spark: "spark2,spark-shade-unbundle-avro"
|
spark: "spark2,spark-shade-unbundle-avro"
|
||||||
- scala: "scala-2.12"
|
|
||||||
spark: "spark3.0.x"
|
|
||||||
- scala: "scala-2.12"
|
|
||||||
spark: "spark3.0.x,spark-shade-unbundle-avro"
|
|
||||||
- scala: "scala-2.12"
|
- scala: "scala-2.12"
|
||||||
spark: "spark3.1.x"
|
spark: "spark3.1.x"
|
||||||
- scala: "scala-2.12"
|
- scala: "scala-2.12"
|
||||||
|
|||||||
@@ -83,14 +83,11 @@ mvn clean package -DskipTests -Dscala-2.12
|
|||||||
The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
|
The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
|
||||||
|
|
||||||
```
|
```
|
||||||
# Build against Spark 3.2.0 (default build shipped with the public jars)
|
# Build against Spark 3.2.0 (the default build shipped with the public Spark 3 bundle)
|
||||||
mvn clean package -DskipTests -Dspark3
|
mvn clean package -DskipTests -Dspark3
|
||||||
|
|
||||||
# Build against Spark 3.1.2
|
# Build against Spark 3.1.2
|
||||||
mvn clean package -DskipTests -Dspark3.1.x
|
mvn clean package -DskipTests -Dspark3.1.x
|
||||||
|
|
||||||
# Build against Spark 3.0.3
|
|
||||||
mvn clean package -DskipTests -Dspark3.0.x
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Build without spark-avro module
|
### Build without spark-avro module
|
||||||
|
|||||||
31
pom.xml
31
pom.xml
@@ -115,12 +115,10 @@
|
|||||||
<prometheus.version>0.8.0</prometheus.version>
|
<prometheus.version>0.8.0</prometheus.version>
|
||||||
<http.version>4.4.1</http.version>
|
<http.version>4.4.1</http.version>
|
||||||
<spark.version>${spark2.version}</spark.version>
|
<spark.version>${spark2.version}</spark.version>
|
||||||
<sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
|
<sparkbundle.version></sparkbundle.version>
|
||||||
<flink.version>1.13.1</flink.version>
|
<flink.version>1.13.1</flink.version>
|
||||||
<spark2.version>2.4.4</spark2.version>
|
<spark2.version>2.4.4</spark2.version>
|
||||||
<spark3.version>3.2.0</spark3.version>
|
<spark3.version>3.2.0</spark3.version>
|
||||||
<spark2bundle.version></spark2bundle.version>
|
|
||||||
<spark3bundle.version>3</spark3bundle.version>
|
|
||||||
<hudi.spark.module>hudi-spark2</hudi.spark.module>
|
<hudi.spark.module>hudi-spark2</hudi.spark.module>
|
||||||
<avro.version>1.8.2</avro.version>
|
<avro.version>1.8.2</avro.version>
|
||||||
<scala11.version>2.11.12</scala11.version>
|
<scala11.version>2.11.12</scala11.version>
|
||||||
@@ -1572,7 +1570,7 @@
|
|||||||
<id>spark3</id>
|
<id>spark3</id>
|
||||||
<properties>
|
<properties>
|
||||||
<spark.version>${spark3.version}</spark.version>
|
<spark.version>${spark3.version}</spark.version>
|
||||||
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
|
<sparkbundle.version>${spark3.version}</sparkbundle.version>
|
||||||
<scala.version>${scala12.version}</scala.version>
|
<scala.version>${scala12.version}</scala.version>
|
||||||
<scala.binary.version>2.12</scala.binary.version>
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
||||||
@@ -1598,7 +1596,7 @@
|
|||||||
<properties>
|
<properties>
|
||||||
<spark3.version>3.1.2</spark3.version>
|
<spark3.version>3.1.2</spark3.version>
|
||||||
<spark.version>${spark3.version}</spark.version>
|
<spark.version>${spark3.version}</spark.version>
|
||||||
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
|
<sparkbundle.version>${spark3.version}</sparkbundle.version>
|
||||||
<scala.version>${scala12.version}</scala.version>
|
<scala.version>${scala12.version}</scala.version>
|
||||||
<scala.binary.version>2.12</scala.binary.version>
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
||||||
@@ -1618,29 +1616,6 @@
|
|||||||
</activation>
|
</activation>
|
||||||
</profile>
|
</profile>
|
||||||
|
|
||||||
<profile>
|
|
||||||
<id>spark3.0.x</id>
|
|
||||||
<properties>
|
|
||||||
<spark3.version>3.0.3</spark3.version>
|
|
||||||
<spark.version>${spark3.version}</spark.version>
|
|
||||||
<scala.version>${scala12.version}</scala.version>
|
|
||||||
<scala.binary.version>2.12</scala.binary.version>
|
|
||||||
<hudi.spark.module>hudi-spark3</hudi.spark.module>
|
|
||||||
<scalatest.version>3.0.1</scalatest.version>
|
|
||||||
<kafka.version>2.4.1</kafka.version>
|
|
||||||
<fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
|
|
||||||
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
|
|
||||||
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
|
|
||||||
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
|
|
||||||
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
|
|
||||||
</properties>
|
|
||||||
<activation>
|
|
||||||
<property>
|
|
||||||
<name>spark3.0.x</name>
|
|
||||||
</property>
|
|
||||||
</activation>
|
|
||||||
</profile>
|
|
||||||
|
|
||||||
<profile>
|
<profile>
|
||||||
<id>skipShadeSources</id>
|
<id>skipShadeSources</id>
|
||||||
<properties>
|
<properties>
|
||||||
|
|||||||
Reference in New Issue
Block a user