lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Don't raise when spark-defaults.conf doesn't exist

Browse Source
This commit is contained in:
David Muto (pseudomuto)
2019-05-07 21:16:58 -04:00
committed by n3nash
parent e2dcef8606
commit 4b27cc72bb
1 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java
View File

@@ -21,6 +21,8 @@ import com.uber.hoodie.cli.commands.SparkMain;
import com.uber.hoodie.common.util.FSUtils; import com.uber.hoodie.common.util.FSUtils;
import java.io.File; import java.io.File;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
@@ -37,8 +39,13 @@ public class SparkUtil {
public static SparkLauncher initLauncher(String propertiesFile) throws URISyntaxException { public static SparkLauncher initLauncher(String propertiesFile) throws URISyntaxException {
String currentJar = new File(SparkUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath()) String currentJar = new File(SparkUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())
.getAbsolutePath(); .getAbsolutePath();
SparkLauncher sparkLauncher = new SparkLauncher().setAppResource(currentJar).setMainClass(SparkMain.class.getName()) SparkLauncher sparkLauncher = new SparkLauncher().setAppResource(currentJar)
.setPropertiesFile(propertiesFile); .setMainClass(SparkMain.class.getName());
if (StringUtils.isNotEmpty(propertiesFile)) {
sparkLauncher.setPropertiesFile(propertiesFile);
}
File libDirectory = new File(new File(currentJar).getParent(), "lib"); File libDirectory = new File(new File(currentJar).getParent(), "lib");
for (String library : libDirectory.list()) { for (String library : libDirectory.list()) {
sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath()); sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath());