[HUDI-3681] Provision additional hudi-spark-bundle with different versions (#5171)

2022-03-30 17:35:56 -07:00
parent 2d73c8ae86
commit 9830005e9b
3 changed files with 102 additions and 62 deletions
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -14,39 +14,45 @@ jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
-      max-parallel: 7
+      max-parallel: 8
      matrix:
        include:
-          # Spark 2.4.4
+          # Spark 2.4.4, scala 2.11
          - scalaProfile: "scala-2.11"
-            sparkProfile: "spark2"
+            sparkProfile: "spark2.4"
+            sparkVersion: "2.4.4"
+
+          # Spark 2.4.4, scala 2.12
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark2.4"
            sparkVersion: "2.4.4"

          # Spark 3.1.x
          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3.1.x"
+            sparkProfile: "spark3.1"
            sparkVersion: "3.1.0"

          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3.1.x"
+            sparkProfile: "spark3.1"
            sparkVersion: "3.1.1"

          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3.1.x"
+            sparkProfile: "spark3.1"
            sparkVersion: "3.1.2"

          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3.1.x"
+            sparkProfile: "spark3.1"
            sparkVersion: "3.1.3"

          # Spark 3.2.x
          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3.2.0"
+            sparkProfile: "spark3.2"
            sparkVersion: "3.2.0"

          - scalaProfile: "scala-2.12"
-            sparkProfile: "spark3"
+            sparkProfile: "spark3.2"
            sparkVersion: "3.2.1"
+
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK 8
--- a/README.md
+++ b/README.md
@@ -70,24 +70,31 @@ To build the Javadoc for all Java and Scala classes:
 mvn clean javadoc:aggregate -Pjavadocs
 ```

-### Build with Scala 2.12
+### Build with Different Spark versions

-The default Scala version supported is 2.11. To build for Scala 2.12 version, build using `scala-2.12` profile
+The default Spark version supported is 2.4.4. To build for different Spark versions and Scala 2.12, use the
+corresponding profile

+| Label | Artifact Name for Spark Bundle | Maven Profile Option | Notes |
+|--|--|--|--|
+| Spark 2.4, Scala 2.11  | hudi-spark2.4-bundle_2.11 | `-Pspark2.4` | For Spark 2.4.4, which is the same as the default  |
+| Spark 2.4, Scala 2.12 | hudi-spark2.4-bundle_2.12 | `-Pspark2.4,scala-2.12` | For Spark 2.4.4, which is the same as the default and Scala 2.12 |
+| Spark 3.1, Scala 2.12 | hudi-spark3.1-bundle_2.12 | `-Pspark3.1` | For Spark 3.1.x |
+| Spark 3.2, Scala 2.12 | hudi-spark3.2-bundle_2.12 | `-Pspark3.2` | For Spark 3.2.x |
+| Spark 3, Scala 2.12 | hudi-spark3-bundle_2.12 | `-Pspark3` | This is the same as `Spark 3.2, Scala 2.12` |
+| Spark, Scala 2.11 | hudi-spark-bundle_2.11 | Default | The default profile, supporting Spark 2.4.4 |
+| Spark, Scala 2.12 | hudi-spark-bundle_2.12 | `-Pscala-2.12` | The default profile (for Spark 2.4.4) with Scala 2.12 |
+
+For example,
 ```
-mvn clean package -DskipTests -Dscala-2.12
-```
+# Build against Spark 3.2.x (the default build shipped with the public Spark 3 bundle)
+mvn clean package -DskipTests -Pspark3.2

-### Build with Spark 3
+# Build against Spark 3.1.x
+mvn clean package -DskipTests -Pspark3.1

-The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
-
-```
-# Build against Spark 3.2.1 (the default build shipped with the public Spark 3 bundle)
-mvn clean package -DskipTests -Dspark3
-
-# Build against Spark 3.1.2
-mvn clean package -DskipTests -Dspark3.1.x
+# Build against Spark 2.4.4 and Scala 2.12
+mvn clean package -DskipTests -Pspark2.4,scala-2.12
 ```

 ### What about "spark-avro" module? 
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@
    <fasterxml.jackson.dataformat.yaml.version>2.7.4</fasterxml.jackson.dataformat.yaml.version>
    <fasterxml.spark3.version>2.10.0</fasterxml.spark3.version>
    <kafka.version>2.0.0</kafka.version>
+    <kafka.spark3.version>2.4.1</kafka.spark3.version>
    <pulsar.version>2.8.1</pulsar.version>
    <confluent.version>5.3.4</confluent.version>
    <glassfish.version>2.17</glassfish.version>
@@ -135,6 +136,7 @@
    <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
    <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
    <scalatest.version>3.0.1</scalatest.version>
+    <scalatest.spark3.version>3.1.0</scalatest.spark3.version>
    <surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire.properties</surefire-log4j.file>
    <thrift.version>0.12.0</thrift.version>
    <jetty.version>9.4.15.v20190215</jetty.version>
@@ -1588,6 +1590,7 @@
      </build>
    </profile>

+    <!-- Default is spark2 profile, and spark2 is an alias of spark2.4 -->
    <profile>
      <id>spark2</id>
      <modules>
@@ -1603,25 +1606,44 @@
        </property>
      </activation>
    </profile>
+
+    <profile>
+      <id>spark2.4</id>
+      <modules>
+        <module>hudi-spark-datasource/hudi-spark2</module>
+        <module>hudi-spark-datasource/hudi-spark2-common</module>
+      </modules>
+      <properties>
+        <sparkbundle.version>2.4</sparkbundle.version>
+      </properties>
+      <activation>
+        <property>
+          <name>spark2.4</name>
+        </property>
+      </activation>
+    </profile>
+
+    <!-- spark3 is an alias of spark3.2 -->
    <profile>
      <id>spark3</id>
      <properties>
        <spark3.version>3.2.1</spark3.version>
        <spark.version>${spark3.version}</spark.version>
-        <sparkbundle.version>${spark3.version}</sparkbundle.version>
+        <sparkbundle.version>3</sparkbundle.version>
        <scala.version>${scala12.version}</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <hudi.spark.module>hudi-spark3</hudi.spark.module>
        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
-        <scalatest.version>3.1.0</scalatest.version>
-        <kafka.version>2.4.1</kafka.version>
+        <scalatest.version>${scalatest.spark3.version}</scalatest.version>
+        <kafka.version>${kafka.spark3.version}</kafka.version>
        <parquet.version>1.12.2</parquet.version>
        <avro.version>1.10.2</avro.version>
        <orc.version>1.6.12</orc.version>
        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
-        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
+        </fasterxml.jackson.dataformat.yaml.version>
        <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
        <skipITs>true</skipITs>
      </properties>
@@ -1637,51 +1659,22 @@
    </profile>

    <profile>
-      <id>spark3.2.0</id>
-      <properties>
-        <spark3.version>3.2.0</spark3.version>
-        <spark.version>${spark3.version}</spark.version>
-        <sparkbundle.version>${spark3.version}</sparkbundle.version>
-        <scala.version>${scala12.version}</scala.version>
-        <scala.binary.version>2.12</scala.binary.version>
-        <hudi.spark.module>hudi-spark3</hudi.spark.module>
-        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
-        <scalatest.version>3.1.0</scalatest.version>
-        <kafka.version>2.4.1</kafka.version>
-        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
-        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
-        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
-        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
-        <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
-        <skipITs>true</skipITs>
-      </properties>
-      <modules>
-        <module>hudi-spark-datasource/hudi-spark3</module>
-        <module>hudi-spark-datasource/hudi-spark3-common</module>
-      </modules>
-      <activation>
-        <property>
-          <name>spark3.2.0</name>
-        </property>
-      </activation>
-    </profile>
-
-    <profile>
-      <id>spark3.1.x</id>
+      <id>spark3.1</id>
      <properties>
        <spark3.version>3.1.3</spark3.version>
        <spark.version>${spark3.version}</spark.version>
-        <sparkbundle.version>${spark3.version}</sparkbundle.version>
+        <sparkbundle.version>3.1</sparkbundle.version>
        <scala.version>${scala12.version}</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <hudi.spark.module>hudi-spark3.1.x</hudi.spark.module>
        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
-        <scalatest.version>3.1.0</scalatest.version>
-        <kafka.version>2.4.1</kafka.version>
+        <scalatest.version>${scalatest.spark3.version}</scalatest.version>
+        <kafka.version>${kafka.spark3.version}</kafka.version>
        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
-        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
+        </fasterxml.jackson.dataformat.yaml.version>
        <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
        <skipITs>true</skipITs>
      </properties>
@@ -1691,7 +1684,41 @@
      </modules>
      <activation>
        <property>
-          <name>spark3.1.x</name>
+          <name>spark3.1</name>
+        </property>
+      </activation>
+    </profile>
+
+    <profile>
+      <id>spark3.2</id>
+      <properties>
+        <spark3.version>3.2.1</spark3.version>
+        <spark.version>${spark3.version}</spark.version>
+        <sparkbundle.version>3.2</sparkbundle.version>
+        <scala.version>${scala12.version}</scala.version>
+        <scala.binary.version>2.12</scala.binary.version>
+        <hudi.spark.module>hudi-spark3</hudi.spark.module>
+        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
+        <scalatest.version>${scalatest.spark3.version}</scalatest.version>
+        <kafka.version>${kafka.spark3.version}</kafka.version>
+        <parquet.version>1.12.2</parquet.version>
+        <avro.version>1.10.2</avro.version>
+        <orc.version>1.6.12</orc.version>
+        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
+        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
+        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
+        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}
+        </fasterxml.jackson.dataformat.yaml.version>
+        <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
+        <skipITs>true</skipITs>
+      </properties>
+      <modules>
+        <module>hudi-spark-datasource/hudi-spark3</module>
+        <module>hudi-spark-datasource/hudi-spark3-common</module>
+      </modules>
+      <activation>
+        <property>
+          <name>spark3.2</name>
        </property>
      </activation>
    </profile>