diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index b1ec4456a..c8e226f47 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -14,39 +14,45 @@ jobs:
build:
runs-on: ubuntu-latest
strategy:
- max-parallel: 7
+ max-parallel: 8
matrix:
include:
- # Spark 2.4.4
+ # Spark 2.4.4, scala 2.11
- scalaProfile: "scala-2.11"
- sparkProfile: "spark2"
+ sparkProfile: "spark2.4"
+ sparkVersion: "2.4.4"
+
+ # Spark 2.4.4, scala 2.12
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark2.4"
sparkVersion: "2.4.4"
# Spark 3.1.x
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3.1.x"
+ sparkProfile: "spark3.1"
sparkVersion: "3.1.0"
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3.1.x"
+ sparkProfile: "spark3.1"
sparkVersion: "3.1.1"
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3.1.x"
+ sparkProfile: "spark3.1"
sparkVersion: "3.1.2"
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3.1.x"
+ sparkProfile: "spark3.1"
sparkVersion: "3.1.3"
# Spark 3.2.x
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3.2.0"
+ sparkProfile: "spark3.2"
sparkVersion: "3.2.0"
- scalaProfile: "scala-2.12"
- sparkProfile: "spark3"
+ sparkProfile: "spark3.2"
sparkVersion: "3.2.1"
+
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
diff --git a/README.md b/README.md
index be38999ae..b8f38763b 100644
--- a/README.md
+++ b/README.md
@@ -70,24 +70,31 @@ To build the Javadoc for all Java and Scala classes:
mvn clean javadoc:aggregate -Pjavadocs
```
-### Build with Scala 2.12
+### Build with Different Spark versions
-The default Scala version supported is 2.11. To build for Scala 2.12 version, build using `scala-2.12` profile
+The default Spark version supported is 2.4.4. To build for different Spark versions and Scala 2.12, use the
+corresponding profile
+| Label | Artifact Name for Spark Bundle | Maven Profile Option | Notes |
+|--|--|--|--|
+| Spark 2.4, Scala 2.11 | hudi-spark2.4-bundle_2.11 | `-Pspark2.4` | For Spark 2.4.4, which is the same as the default |
+| Spark 2.4, Scala 2.12 | hudi-spark2.4-bundle_2.12 | `-Pspark2.4,scala-2.12` | For Spark 2.4.4, which is the same as the default and Scala 2.12 |
+| Spark 3.1, Scala 2.12 | hudi-spark3.1-bundle_2.12 | `-Pspark3.1` | For Spark 3.1.x |
+| Spark 3.2, Scala 2.12 | hudi-spark3.2-bundle_2.12 | `-Pspark3.2` | For Spark 3.2.x |
+| Spark 3, Scala 2.12 | hudi-spark3-bundle_2.12 | `-Pspark3` | This is the same as `Spark 3.2, Scala 2.12` |
+| Spark, Scala 2.11 | hudi-spark-bundle_2.11 | Default | The default profile, supporting Spark 2.4.4 |
+| Spark, Scala 2.12 | hudi-spark-bundle_2.12 | `-Pscala-2.12` | The default profile (for Spark 2.4.4) with Scala 2.12 |
+
+For example,
```
-mvn clean package -DskipTests -Dscala-2.12
-```
+# Build against Spark 3.2.x (the default build shipped with the public Spark 3 bundle)
+mvn clean package -DskipTests -Pspark3.2
-### Build with Spark 3
+# Build against Spark 3.1.x
+mvn clean package -DskipTests -Pspark3.1
-The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
-
-```
-# Build against Spark 3.2.1 (the default build shipped with the public Spark 3 bundle)
-mvn clean package -DskipTests -Dspark3
-
-# Build against Spark 3.1.2
-mvn clean package -DskipTests -Dspark3.1.x
+# Build against Spark 2.4.4 and Scala 2.12
+mvn clean package -DskipTests -Pspark2.4,scala-2.12
```
### What about "spark-avro" module?
diff --git a/pom.xml b/pom.xml
index 9c68ca13d..d81525f97 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@
2.7.4
2.10.0
2.0.0
+ 2.4.1
2.8.1
5.3.4
2.17
@@ -135,6 +136,7 @@
0.12
3.3.1
3.0.1
+ 3.1.0
file://${project.basedir}/src/test/resources/log4j-surefire.properties
0.12.0
9.4.15.v20190215
@@ -1588,6 +1590,7 @@
+
spark2
@@ -1603,25 +1606,44 @@
+
+
+ spark2.4
+
+ hudi-spark-datasource/hudi-spark2
+ hudi-spark-datasource/hudi-spark2-common
+
+
+ 2.4
+
+
+
+ spark2.4
+
+
+
+
+
spark3
3.2.1
${spark3.version}
- ${spark3.version}
+ 3
${scala12.version}
2.12
hudi-spark3
hudi-spark3-common
- 3.1.0
- 2.4.1
+ ${scalatest.spark3.version}
+ ${kafka.spark3.version}
1.12.2
1.10.2
1.6.12
${fasterxml.spark3.version}
${fasterxml.spark3.version}
${fasterxml.spark3.version}
- ${fasterxml.spark3.version}
+ ${fasterxml.spark3.version}
+
true
true
@@ -1637,51 +1659,22 @@
- spark3.2.0
-
- 3.2.0
- ${spark3.version}
- ${spark3.version}
- ${scala12.version}
- 2.12
- hudi-spark3
- hudi-spark3-common
- 3.1.0
- 2.4.1
- ${fasterxml.spark3.version}
- ${fasterxml.spark3.version}
- ${fasterxml.spark3.version}
- ${fasterxml.spark3.version}
- true
- true
-
-
- hudi-spark-datasource/hudi-spark3
- hudi-spark-datasource/hudi-spark3-common
-
-
-
- spark3.2.0
-
-
-
-
-
- spark3.1.x
+ spark3.1
3.1.3
${spark3.version}
- ${spark3.version}
+ 3.1
${scala12.version}
2.12
hudi-spark3.1.x
hudi-spark3-common
- 3.1.0
- 2.4.1
+ ${scalatest.spark3.version}
+ ${kafka.spark3.version}
${fasterxml.spark3.version}
${fasterxml.spark3.version}
${fasterxml.spark3.version}
- ${fasterxml.spark3.version}
+ ${fasterxml.spark3.version}
+
true
true
@@ -1691,7 +1684,41 @@
- spark3.1.x
+ spark3.1
+
+
+
+
+
+ spark3.2
+
+ 3.2.1
+ ${spark3.version}
+ 3.2
+ ${scala12.version}
+ 2.12
+ hudi-spark3
+ hudi-spark3-common
+ ${scalatest.spark3.version}
+ ${kafka.spark3.version}
+ 1.12.2
+ 1.10.2
+ 1.6.12
+ ${fasterxml.spark3.version}
+ ${fasterxml.spark3.version}
+ ${fasterxml.spark3.version}
+ ${fasterxml.spark3.version}
+
+ true
+ true
+
+
+ hudi-spark-datasource/hudi-spark3
+ hudi-spark-datasource/hudi-spark3-common
+
+
+
+ spark3.2