lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1512] Fix spark 2 unit tests failure with Spark 3 (#2412)

Browse Source 
* [HUDI-1512] Fix spark 2 unit tests failure with Spark 3

* resolve comments

Co-authored-by: Wenning Ding <wenningd@amazon.com>
This commit is contained in:
wenningd
2021-01-21 07:04:28 -08:00
committed by GitHub
parent 81ccb0c71a
commit 976420c49a
2 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-spark-datasource/hudi-spark2/pom.xml
View File

@@ -124,6 +124,13 @@
<skip>false</skip> <skip>false</skip>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>${skip.hudi-spark2.unit.tests}</skipTests>
</configuration>
</plugin>
<plugin> <plugin>
<groupId>org.apache.rat</groupId> <groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId> <artifactId>apache-rat-plugin</artifactId>
@@ -144,7 +151,7 @@
<dependency> <dependency>
<groupId>org.scala-lang</groupId> <groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId> <artifactId>scala-library</artifactId>
<version>${scala.version}</version> <version>${scala11.version}</version>
</dependency> </dependency>
<!-- Hoodie --> <!-- Hoodie -->

9
pom.xml
View File

@@ -105,14 +105,15 @@
<metrics.version>4.1.1</metrics.version> <metrics.version>4.1.1</metrics.version>
<prometheus.version>0.8.0</prometheus.version> <prometheus.version>0.8.0</prometheus.version>
<http.version>4.4.1</http.version> <http.version>4.4.1</http.version>
<spark.version>2.4.4</spark.version> <spark.version>${spark2.version}</spark.version>
<flink.version>1.12.0</flink.version> <flink.version>1.12.0</flink.version>
<spark2.version>2.4.4</spark2.version> <spark2.version>2.4.4</spark2.version>
<spark3.version>3.0.0</spark3.version> <spark3.version>3.0.0</spark3.version>
<avro.version>1.8.2</avro.version> <avro.version>1.8.2</avro.version>
<scala.version>2.11.12</scala.version> <scala11.version>2.11.12</scala11.version>
<scala.binary.version>2.11</scala.binary.version>
<scala12.version>2.12.10</scala12.version> <scala12.version>2.12.10</scala12.version>
<scala.version>${scala11.version}</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<apache-rat-plugin.version>0.12</apache-rat-plugin.version> <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
<scala-maven-plugin.version>3.3.1</scala-maven-plugin.version> <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
<scalatest.version>3.0.1</scalatest.version> <scalatest.version>3.0.1</scalatest.version>
@@ -140,6 +141,7 @@
<presto.bundle.bootstrap.scope>compile</presto.bundle.bootstrap.scope> <presto.bundle.bootstrap.scope>compile</presto.bundle.bootstrap.scope>
<presto.bundle.bootstrap.shade.prefix>org.apache.hudi.</presto.bundle.bootstrap.shade.prefix> <presto.bundle.bootstrap.shade.prefix>org.apache.hudi.</presto.bundle.bootstrap.shade.prefix>
<shadeSources>true</shadeSources> <shadeSources>true</shadeSources>
<skip.hudi-spark2.unit.tests>false</skip.hudi-spark2.unit.tests>
</properties> </properties>
<scm> <scm>
@@ -1361,6 +1363,7 @@
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version> <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version> <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version> <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
</properties> </properties>
<activation> <activation>
<property> <property>