lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1137] Add option to configure different path selector

Browse Source
This commit is contained in:
Satish Kotha
2020-08-24 11:11:10 -07:00
committed by n3nash
parent 111a9753a0
commit ea983ff912
4 changed files with 21 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
View File

@@ -352,12 +352,17 @@ public class UtilHelpers {
}
}
public static DFSPathSelector createSourceSelector(String sourceSelectorClass, TypedProperties props,
public static DFSPathSelector createSourceSelector(TypedProperties props,
Configuration conf) throws IOException {
String sourceSelectorClass =
props.getString(DFSPathSelector.Config.SOURCE_INPUT_SELECTOR, DFSPathSelector.class.getName());
try {
return (DFSPathSelector) ReflectionUtils.loadClass(sourceSelectorClass,
DFSPathSelector selector = (DFSPathSelector) ReflectionUtils.loadClass(sourceSelectorClass,
new Class<?>[]{TypedProperties.class, Configuration.class},
props, conf);
LOG.info("Using path selector " + selector.getClass().getName());
return selector;
} catch (Throwable e) {
throw new IOException("Could not load source selector class " + sourceSelectorClass, e);
}

3
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java
View File

@@ -47,8 +47,7 @@ public class AvroDFSSource extends AvroSource {
SchemaProvider schemaProvider) throws IOException {
super(props, sparkContext, sparkSession, schemaProvider);
this.pathSelector = UtilHelpers
.createSourceSelector(DFSPathSelector.class.getName(), props, sparkContext
.hadoopConfiguration());
.createSourceSelector(props, sparkContext.hadoopConfiguration());
}
@Override

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java
View File

@@ -52,6 +52,7 @@ public class DFSPathSelector {
public static class Config {
public static final String ROOT_INPUT_PATH_PROP = "hoodie.deltastreamer.source.dfs.root";
public static final String SOURCE_INPUT_SELECTOR = "hoodie.deltastreamer.source.input.selector";
}
protected static final List<String> IGNORE_FILEPREFIX_LIST = Arrays.asList(".", "_");