diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java b/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java index 74fce58f5..2dd446b8d 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java +++ b/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java @@ -48,7 +48,10 @@ public class SparkUtil { public static JavaSparkContext initJavaSparkConf(String name) { SparkConf sparkConf = new SparkConf().setAppName(name); - sparkConf.setMaster(DEFUALT_SPARK_MASTER); + String defMasterFromEnv = sparkConf.get("spark.master"); + if ((null == defMasterFromEnv) || (defMasterFromEnv.isEmpty())) { + sparkConf.setMaster(DEFUALT_SPARK_MASTER); + } sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); sparkConf.set("spark.driver.maxResultSize", "2g"); sparkConf.set("spark.eventLog.overwrite", "true");