[HUDI-3574] Improve maven module configs for different spark profiles (#4970)

.github/workflows/bot.yml

@@ -18,16 +18,12 @@ jobs:
         include:
           - scala: "scala-2.11"
             spark: "spark2"
-            skipModules: ""
           - scala: "scala-2.11"
             spark: "spark2,spark-shade-unbundle-avro"
-            skipModules: ""
           - scala: "scala-2.12"
             spark: "spark3.1.x"
-            skipModules: "!hudi-spark-datasource/hudi-spark3"
           - scala: "scala-2.12"
             spark: "spark3.1.x,spark-shade-unbundle-avro"
-            skipModules: "!hudi-spark-datasource/hudi-spark3"
           - scala: "scala-2.12"
             spark: "spark3"
           - scala: "scala-2.12"
@@ -44,5 +40,4 @@ jobs:
         env:
           SCALA_PROFILE: ${{ matrix.scala }}
           SPARK_PROFILE: ${{ matrix.spark }}
-          SKIP_MODULES: ${{ matrix.skipModules }}
+        run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
-        run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -pl "$SKIP_MODULES" -DskipTests=true -Dmaven.javadoc.skip=true -B -V

hudi-spark-datasource/pom.xml

@@ -33,11 +33,6 @@
   <modules>
     <module>hudi-spark-common</module>
     <module>hudi-spark</module>
-    <module>hudi-spark2</module>
-    <module>hudi-spark3</module>
-    <module>hudi-spark3.1.x</module>
-    <module>hudi-spark3-common</module>
-    <module>hudi-spark2-common</module>
   </modules>
 
 </project>

pom.xml

@@ -1581,10 +1581,23 @@
     <!-- Exists for backwards compatibility; profile doesn't do anything -->
     <profile>
       <id>spark2</id>
+      <modules>
+        <module>hudi-spark-datasource/hudi-spark2</module>
+        <module>hudi-spark-datasource/hudi-spark2-common</module>
+      </modules>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>spark2</name>
+          <!-- add spark2 moudule to all profile -->
+          <value>!disabled</value>
+        </property>
+      </activation>
     </profile>
     <profile>
       <id>spark3</id>
       <properties>
+        <spark3.version>3.2.1</spark3.version>
         <spark.version>${spark3.version}</spark.version>
         <sparkbundle.version>${spark3.version}</sparkbundle.version>
         <scala.version>${scala12.version}</scala.version>
@@ -1603,6 +1616,10 @@
         <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
         <skipITs>true</skipITs>
       </properties>
+      <modules>
+        <module>hudi-spark-datasource/hudi-spark3</module>
+        <module>hudi-spark-datasource/hudi-spark3-common</module>
+      </modules>
       <activation>
         <property>
           <name>spark3</name>
@@ -1629,6 +1646,10 @@
         <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
         <skipITs>true</skipITs>
       </properties>
+      <modules>
+        <module>hudi-spark-datasource/hudi-spark3.1.x</module>
+        <module>hudi-spark-datasource/hudi-spark3-common</module>
+      </modules>
       <activation>
         <property>
           <name>spark3.1.x</name>