From a1d82b4dc5050b67668e5de5aacf62cdcbd4cb2b Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Fri, 29 Apr 2022 11:19:07 -0700 Subject: [PATCH] [MINOR] Fix CI by ignoring SparkContext error (#5468) Sets spark.driver.allowMultipleContexts = true when constructing Spark conf in UtilHelpers --- .../src/main/java/org/apache/hudi/utilities/UtilHelpers.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java index 5d1fd1926..f389695f7 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java @@ -279,6 +279,7 @@ public class UtilHelpers { sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true"); sparkConf.set("spark.hadoop.mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); sparkConf.set("spark.hadoop.mapred.output.compression.type", "BLOCK"); + sparkConf.set("spark.driver.allowMultipleContexts", "true"); additionalConfigs.forEach(sparkConf::set); return SparkRDDWriteClient.registerClasses(sparkConf);