lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1688]hudi write should uncache rdd， when the write operation is finnished (#2673)

Browse Source
This commit is contained in:
xiarixiaoyao
2021-03-19 01:19:18 +08:00
committed by GitHub
parent f1e0018f12
commit d429169ff7
1 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
View File

@@ -221,6 +221,23 @@ private[hudi] object HoodieSparkSqlWriter {
val (writeSuccessful, compactionInstant) = val (writeSuccessful, compactionInstant) =
commitAndPerformPostOperations(writeResult, parameters, writeClient, tableConfig, jsc, commitAndPerformPostOperations(writeResult, parameters, writeClient, tableConfig, jsc,
TableInstantInfo(basePath, instantTime, commitActionType, operation)) TableInstantInfo(basePath, instantTime, commitActionType, operation))
def unpersistRdd(rdd: RDD[_]): Unit = {
if (sparkContext.getPersistentRDDs.contains(rdd.id)) {
try {
rdd.unpersist()
} catch {
case t: Exception => log.warn("Got excepting trying to unpersist rdd", t)
}
}
val parentRdds = rdd.dependencies.map(_.rdd)
parentRdds.foreach { parentRdd =>
unpersistRdd(parentRdd)
}
}
// it's safe to unpersist cached rdds here
unpersistRdd(writeResult.getWriteStatuses.rdd)
(writeSuccessful, common.util.Option.ofNullable(instantTime), compactionInstant, writeClient, tableConfig) (writeSuccessful, common.util.Option.ofNullable(instantTime), compactionInstant, writeClient, tableConfig)
} }
} }