lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-821] Fixing JCommander param parsing in deltastreamer (#1525)

Browse Source 
Co-authored-by: Alex Filipchik <alex.filipchik@csscompany.com>
This commit is contained in:
Alexander Filipchik
2020-04-21 20:12:34 -07:00
committed by GitHub
parent 62bd3e7ded
commit 2a56f82908
2 changed files with 2 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
View File

@@ -54,7 +54,6 @@ import org.apache.spark.sql.SparkSession;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
@@ -65,7 +64,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
/** /**
@@ -164,17 +162,6 @@ public class HoodieDeltaStreamer implements Serializable {
} }
} }
protected static class TransformersConverter implements IStringConverter<List<String>> {
@Override
public List<String> convert(String value) throws ParameterException {
return value == null ? null : Arrays.stream(value.split(","))
.map(String::trim)
.filter(s -> !s.isEmpty())
.collect(Collectors.toList());
}
}
public static class Config implements Serializable { public static class Config implements Serializable {
@Parameter(names = {"--target-base-path"}, @Parameter(names = {"--target-base-path"},
@@ -228,8 +215,7 @@ public class HoodieDeltaStreamer implements Serializable {
+ ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before " + ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before "
+ "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which " + "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which "
+ "allows a SQL query templated to be passed as a transformation function). " + "allows a SQL query templated to be passed as a transformation function). "
+ "Pass a comma-separated list of subclass names to chain the transformations.", + "Pass a comma-separated list of subclass names to chain the transformations.")
converter = TransformersConverter.class)
public List<String> transformerClassNames = null; public List<String> transformerClassNames = null;
@Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. " @Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. "

3
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java
View File

@@ -256,8 +256,7 @@ public class HoodieMultiTableDeltaStreamer {
+ ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before " + ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before "
+ "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which " + "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which "
+ "allows a SQL query templated to be passed as a transformation function). " + "allows a SQL query templated to be passed as a transformation function). "
+ "Pass a comma-separated list of subclass names to chain the transformations.", + "Pass a comma-separated list of subclass names to chain the transformations.")
converter = HoodieDeltaStreamer.TransformersConverter.class)
public List<String> transformerClassNames = null; public List<String> transformerClassNames = null;
@Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. " @Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. "