lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[1568] Fixing spark3 bundles (#2625)

Browse Source 
- [1568] Fixing spark3 bundles
This commit is contained in:
Sivabalan Narayanan
2021-03-19 14:21:36 -04:00
committed by GitHub
parent f74828fca1
commit 55a489c769
5 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/hoodie/hadoop/hive_base/pom.xml
View File

@@ -58,7 +58,7 @@
<tasks>
<copy file="${project.basedir}/../../../../packaging/hudi-hadoop-mr-bundle/target/hudi-hadoop-mr-bundle-${project.version}.jar" tofile="target/hoodie-hadoop-mr-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-hive-sync-bundle/target/hudi-hive-sync-bundle-${project.version}.jar" tofile="target/hoodie-hive-sync-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-spark-bundle/target/hudi-spark-bundle_${scala.binary.version}-${project.version}.jar" tofile="target/hoodie-spark-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-spark-bundle/target/hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}-${project.version}.jar" tofile="target/hoodie-spark-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_${scala.binary.version}-${project.version}.jar" tofile="target/hoodie-utilities.jar" />
</tasks>
</configuration>

2
docker/hoodie/hadoop/pom.xml
View File

@@ -42,7 +42,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

2
packaging/hudi-spark-bundle/pom.xml
View File

@@ -23,7 +23,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<properties>

4
pom.xml
View File

@@ -106,9 +106,12 @@
<prometheus.version>0.8.0</prometheus.version>
<http.version>4.4.1</http.version>
<spark.version>${spark2.version}</spark.version>
<sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
<flink.version>1.11.2</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.0.0</spark3.version>
<spark2bundle.version></spark2bundle.version>
<spark3bundle.version>3</spark3bundle.version>
<avro.version>1.8.2</avro.version>
<scala11.version>2.11.12</scala11.version>
<scala12.version>2.12.10</scala12.version>
@@ -1459,6 +1462,7 @@
<id>spark3</id>
<properties>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<kafka.version>2.4.1</kafka.version>

6
scripts/release/deploy_staging_jars.sh
View File

@@ -21,7 +21,7 @@
## Variables with defaults (if not overwritten by environment)
##
MVN=${MVN:-mvn}
SPARK_VERSION=2
# fail immediately
set -o errexit
set -o nounset
@@ -44,6 +44,8 @@ else
do
if [[ $param =~ --scala_version\=(2\.1[1-2]) ]]; then
SCALA_VERSION=${BASH_REMATCH[1]}
elif [[ $param =~ --spark_version\=([2-3]) ]]; then
SPARK_VERSION=${BASH_REMATCH[0]}
fi
done
fi
@@ -54,5 +56,5 @@ cd ..
echo "Deploying to repository.apache.org with scala version ${SCALA_VERSION}"
COMMON_OPTIONS="-Dscala-${SCALA_VERSION} -Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
COMMON_OPTIONS="-Dscala-${SCALA_VERSION} -Dspark${SPARK_VERSION} -Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
$MVN clean deploy $COMMON_OPTIONS