From cce88b36d2861da21ddc719f1228be7e5e5d34e4 Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Mon, 10 Sep 2018 16:06:31 -0700 Subject: [PATCH] Use spark Master from environment if set --- .../src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java b/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java index 74fce58f5..2dd446b8d 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java +++ b/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java @@ -48,7 +48,10 @@ public class SparkUtil { public static JavaSparkContext initJavaSparkConf(String name) { SparkConf sparkConf = new SparkConf().setAppName(name); - sparkConf.setMaster(DEFUALT_SPARK_MASTER); + String defMasterFromEnv = sparkConf.get("spark.master"); + if ((null == defMasterFromEnv) || (defMasterFromEnv.isEmpty())) { + sparkConf.setMaster(DEFUALT_SPARK_MASTER); + } sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); sparkConf.set("spark.driver.maxResultSize", "2g"); sparkConf.set("spark.eventLog.overwrite", "true");