From 2a56f82908a8b8f788a7547d3c707c144696c1df Mon Sep 17 00:00:00 2001 From: Alexander Filipchik Date: Tue, 21 Apr 2020 20:12:34 -0700 Subject: [PATCH] [HUDI-821] Fixing JCommander param parsing in deltastreamer (#1525) Co-authored-by: Alex Filipchik --- .../deltastreamer/HoodieDeltaStreamer.java | 16 +--------------- .../HoodieMultiTableDeltaStreamer.java | 3 +-- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java index f849b9e91..3b714b66d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java @@ -54,7 +54,6 @@ import org.apache.spark.sql.SparkSession; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -65,7 +64,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import java.util.stream.IntStream; /** @@ -164,17 +162,6 @@ public class HoodieDeltaStreamer implements Serializable { } } - protected static class TransformersConverter implements IStringConverter> { - - @Override - public List convert(String value) throws ParameterException { - return value == null ? null : Arrays.stream(value.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList()); - } - } - public static class Config implements Serializable { @Parameter(names = {"--target-base-path"}, @@ -228,8 +215,7 @@ public class HoodieDeltaStreamer implements Serializable { + ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before " + "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which " + "allows a SQL query templated to be passed as a transformation function). " - + "Pass a comma-separated list of subclass names to chain the transformations.", - converter = TransformersConverter.class) + + "Pass a comma-separated list of subclass names to chain the transformations.") public List transformerClassNames = null; @Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. " diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java index 836d992be..58501f627 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java @@ -256,8 +256,7 @@ public class HoodieMultiTableDeltaStreamer { + ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before " + "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which " + "allows a SQL query templated to be passed as a transformation function). " - + "Pass a comma-separated list of subclass names to chain the transformations.", - converter = HoodieDeltaStreamer.TransformersConverter.class) + + "Pass a comma-separated list of subclass names to chain the transformations.") public List transformerClassNames = null; @Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. "