lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3574] Improve maven module configs for different spark profiles (#4970)

Browse Source
This commit is contained in:
ForwardXu
2022-03-08 17:01:05 +08:00
committed by GitHub
parent fe53bd2dea
commit 25385805aa
3 changed files with 22 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/bot.yml vendored
View File

@@ -18,16 +18,12 @@ jobs:
include:
- scala: "scala-2.11"
spark: "spark2"
skipModules: ""
- scala: "scala-2.11"
spark: "spark2,spark-shade-unbundle-avro"
skipModules: ""
- scala: "scala-2.12"
spark: "spark3.1.x"
skipModules: "!hudi-spark-datasource/hudi-spark3"
- scala: "scala-2.12"
spark: "spark3.1.x,spark-shade-unbundle-avro"
skipModules: "!hudi-spark-datasource/hudi-spark3"
- scala: "scala-2.12"
spark: "spark3"
- scala: "scala-2.12"
@@ -44,5 +40,4 @@ jobs:
env:
SCALA_PROFILE: ${{ matrix.scala }}
SPARK_PROFILE: ${{ matrix.spark }}
SKIP_MODULES: ${{ matrix.skipModules }}
run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -pl "$SKIP_MODULES" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V

5
hudi-spark-datasource/pom.xml
View File

@@ -33,11 +33,6 @@
<modules>
<module>hudi-spark-common</module>
<module>hudi-spark</module>
<module>hudi-spark2</module>
<module>hudi-spark3</module>
<module>hudi-spark3.1.x</module>
<module>hudi-spark3-common</module>
<module>hudi-spark2-common</module>
</modules>
</project>

21
pom.xml
View File

@@ -1581,10 +1581,23 @@
<!-- Exists for backwards compatibility; profile doesn't do anything -->
<profile>
<id>spark2</id>
<modules>
<module>hudi-spark-datasource/hudi-spark2</module>
<module>hudi-spark-datasource/hudi-spark2-common</module>
</modules>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>spark2</name>
<!-- add spark2 moudule to all profile -->
<value>!disabled</value>
</property>
</activation>
</profile>
<profile>
<id>spark3</id>
<properties>
<spark3.version>3.2.1</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
@@ -1603,6 +1616,10 @@
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
<property>
<name>spark3</name>
@@ -1629,6 +1646,10 @@
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
<modules>
<module>hudi-spark-datasource/hudi-spark3.1.x</module>
<module>hudi-spark-datasource/hudi-spark3-common</module>
</modules>
<activation>
<property>
<name>spark3.1.x</name>