lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Fix CI by ignoring SparkContext error (#5468)

Browse Source 
Sets spark.driver.allowMultipleContexts = true when constructing Spark conf in UtilHelpers
This commit is contained in:
Y Ethan Guo
2022-04-29 11:19:07 -07:00
committed by GitHub
parent e421d536ea
commit a1d82b4dc5
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
View File

@@ -279,6 +279,7 @@ public class UtilHelpers {
sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true"); sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true");
sparkConf.set("spark.hadoop.mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); sparkConf.set("spark.hadoop.mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
sparkConf.set("spark.hadoop.mapred.output.compression.type", "BLOCK"); sparkConf.set("spark.hadoop.mapred.output.compression.type", "BLOCK");
sparkConf.set("spark.driver.allowMultipleContexts", "true");
additionalConfigs.forEach(sparkConf::set); additionalConfigs.forEach(sparkConf::set);
return SparkRDDWriteClient.registerClasses(sparkConf); return SparkRDDWriteClient.registerClasses(sparkConf);