lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-118]: Options provided for passing properties to Cleaner, compactor and importer commands

Browse Source
This commit is contained in:
Pratyaksh Sharma
2019-12-28 23:46:09 +05:30
committed by n3nash
parent ff1113f3b7
commit 290278fc6c
6 changed files with 138 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
View File

@@ -69,7 +69,7 @@ public class HoodieCompactor {
public int retry = 0;
@Parameter(names = {"--schedule", "-sc"}, description = "Schedule compaction", required = false)
public Boolean runSchedule = false;
@Parameter(names = {"--strategy", "-st"}, description = "Stratgey Class", required = false)
@Parameter(names = {"--strategy", "-st"}, description = "Strategy Class", required = false)
public String strategyClassName = null;
@Parameter(names = {"--help", "-h"}, help = true)
public Boolean help = false;

8
hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
View File

@@ -43,6 +43,7 @@ import org.apache.spark.Accumulator;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.launcher.SparkLauncher;
import org.apache.spark.sql.SparkSession;
import java.io.BufferedReader;
@@ -50,6 +51,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -114,7 +116,7 @@ public class UtilHelpers {
public static TypedProperties buildProperties(List<String> props) {
TypedProperties properties = new TypedProperties();
props.stream().forEach(x -> {
props.forEach(x -> {
String[] kv = x.split("=");
Preconditions.checkArgument(kv.length == 2);
properties.setProperty(kv[0], kv[1]);
@@ -122,6 +124,10 @@ public class UtilHelpers {
return properties;
}
public static void validateAndAddProperties(String[] configs, SparkLauncher sparkLauncher) {
Arrays.stream(configs).filter(config -> config.contains("=") && config.split("=").length == 2).forEach(sparkLauncher::addAppArgs);
}
/**
* Parse Schema from file.
*