lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1869] Upgrading Spark3 To 3.1 (#3844)

Browse Source 
Co-authored-by: pengzhiwei <pengzhiwei2015@icloud.com>
This commit is contained in:
Yann Byron
2021-11-03 09:25:12 +08:00
committed by GitHub
parent dee3a14aae
commit 1f17467f73
22 changed files with 315 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-sync/hudi-hive-sync/pom.xml
View File

@@ -153,6 +153,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<!-- Needed for running HiveServer for Tests -->
<dependency>
<groupId>org.eclipse.jetty.aggregate</groupId>

14
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestParquet2SparkSchemaUtils.java
View File

@@ -36,7 +36,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestParquet2SparkSchemaUtils {
private final SparkToParquetSchemaConverter spark2ParquetConverter =
new SparkToParquetSchemaConverter(new SQLConf());
private final SparkSqlParser parser = new SparkSqlParser(new SQLConf());
private final SparkSqlParser parser = createSqlParser();
private static SparkSqlParser createSqlParser() {
try {
return SparkSqlParser.class.getDeclaredConstructor(SQLConf.class).newInstance(new SQLConf());
} catch (Exception ne) {
try { // For spark 3.1, there is no constructor with SQLConf, use the default constructor
return SparkSqlParser.class.getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
@Test
public void testConvertPrimitiveType() {