lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Use spark Master from environment if set

Browse Source
This commit is contained in:
Balaji Varadarajan
2018-09-10 16:06:31 -07:00
committed by vinoth chandar
parent 605af8a82f
commit cce88b36d2
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java
View File

@@ -48,7 +48,10 @@ public class SparkUtil {
public static JavaSparkContext initJavaSparkConf(String name) { public static JavaSparkContext initJavaSparkConf(String name) {
SparkConf sparkConf = new SparkConf().setAppName(name); SparkConf sparkConf = new SparkConf().setAppName(name);
sparkConf.setMaster(DEFUALT_SPARK_MASTER); String defMasterFromEnv = sparkConf.get("spark.master");
if ((null == defMasterFromEnv) || (defMasterFromEnv.isEmpty())) {
sparkConf.setMaster(DEFUALT_SPARK_MASTER);
}
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
sparkConf.set("spark.driver.maxResultSize", "2g"); sparkConf.set("spark.driver.maxResultSize", "2g");
sparkConf.set("spark.eventLog.overwrite", "true"); sparkConf.set("spark.eventLog.overwrite", "true");