lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3515] Making rdd unpersist optional at the end of writes (#4898)

Browse Source 
Co-authored-by: 苏承祥 <sucx@tuya.com>
This commit is contained in:
苏承祥
2022-02-26 00:30:10 +08:00
committed by GitHub
parent b50f4b491c
commit 92cdc5987a
3 changed files with 68 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
View File

@@ -531,7 +531,11 @@ public class SparkRDDWriteClient<T extends HoodieRecordPayload> extends
@Override
protected void releaseResources() {
((HoodieSparkEngineContext) context).getJavaSparkContext().getPersistentRDDs().values()
.forEach(rdd -> rdd.unpersist());
// If we do not explicitly release the resource, spark will automatically manage the resource and clean it up automatically
// see: https://spark.apache.org/docs/latest/rdd-programming-guide.html#removing-data
if (config.areReleaseResourceEnabled()) {
((HoodieSparkEngineContext) context).getJavaSparkContext().getPersistentRDDs().values()
.forEach(JavaRDD::unpersist);
}
}
}