lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3515] Making rdd unpersist optional at the end of writes (#4898)

Browse Source 
Co-authored-by: 苏承祥 <sucx@tuya.com>
This commit is contained in:
苏承祥
2022-02-26 00:30:10 +08:00
committed by GitHub
parent b50f4b491c
commit 92cdc5987a
3 changed files with 68 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -447,6 +447,12 @@ public class HoodieWriteConfig extends HoodieConfig {
.sinceVersion("0.11.0")
.withDocumentation("Master control to disable all table services including archive, clean, compact, cluster, etc.");
public static final ConfigProperty<Boolean> RELEASE_RESOURCE_ENABLE = ConfigProperty
.key("hoodie.release.resource.on.completion.enable")
.defaultValue(true)
.sinceVersion("0.11.0")
.withDocumentation("Control to enable release all persist rdds when the spark job finish.");
private ConsistencyGuardConfig consistencyGuardConfig;
private FileSystemRetryConfig fileSystemRetryConfig;
@@ -1716,7 +1722,7 @@ public class HoodieWriteConfig extends HoodieConfig {
public String getMetricReporterMetricsNamePrefix() {
return getStringOrDefault(HoodieMetricsConfig.METRICS_REPORTER_PREFIX);
}
/**
* memory configs.
*/
@@ -1945,6 +1951,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return getBooleanOrDefault(TABLE_SERVICES_ENABLED);
}
public boolean areReleaseResourceEnabled() {
return getBooleanOrDefault(RELEASE_RESOURCE_ENABLE);
}
/**
* Layout configs.
*/
@@ -2315,6 +2325,11 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
public Builder withReleaseResourceEnabled(boolean enabled) {
writeConfig.setValue(RELEASE_RESOURCE_ENABLE, Boolean.toString(enabled));
return this;
}
public Builder withProperties(Properties properties) {
this.writeConfig.getProps().putAll(properties);
return this;