lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1611] Added a configuration to allow specific directories to be filtered out during Metadata Table bootstrap. (#2565)

Browse Source
This commit is contained in:
Prashant Wason
2021-02-25 16:52:28 -08:00
committed by GitHub
parent 9f5e8cc7c3
commit 022df0d1b1
3 changed files with 38 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
View File

@@ -75,6 +75,10 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX + ".fallback.enable";
public static final String DEFAULT_ENABLE_FALLBACK = "true";
// Regex to filter out matching directories during bootstrap
public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX + ".dir.filter.regex";
public static final String DEFAULT_DIRECTORY_FILTER_REGEX = "";
public static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = "hoodie.assume.date.partitioning";
public static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
@@ -117,6 +121,10 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
return Boolean.parseBoolean(props.getProperty(METADATA_METRICS_ENABLE_PROP));
}
public String getDirectoryFilterRegex() {
return props.getProperty(DIRECTORY_FILTER_REGEX);
}
public static class Builder {
private final Properties props = new Properties();
@@ -194,6 +202,11 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
return this;
}
public Builder withDirectoryFilterRegex(String regex) {
props.setProperty(DIRECTORY_FILTER_REGEX, regex);
return this;
}
public HoodieMetadataConfig build() {
HoodieMetadataConfig config = new HoodieMetadataConfig(props);
setDefaultOnCondition(props, !props.containsKey(METADATA_ENABLE_PROP), METADATA_ENABLE_PROP,
@@ -222,6 +235,8 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
DEFAULT_ENABLE_FALLBACK);
setDefaultOnCondition(props, !props.containsKey(ENABLE_REUSE_PROP), ENABLE_REUSE_PROP,
DEFAULT_ENABLE_REUSE);
setDefaultOnCondition(props, !props.containsKey(DIRECTORY_FILTER_REGEX), DIRECTORY_FILTER_REGEX,
DEFAULT_DIRECTORY_FILTER_REGEX);
return config;
}
}