lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2383] Clean the marker files after compaction (#3576)

Browse Source
This commit is contained in:
Shawy Geng
2021-09-24 03:40:58 +08:00
committed by GitHub
parent eeafd24f4c
commit b4b23e401b
3 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
View File

@@ -301,6 +301,8 @@ public class SparkRDDWriteClient<T extends HoodieRecordPayload> extends
finalizeWrite(table, compactionCommitTime, writeStats);
LOG.info("Committing Compaction " + compactionCommitTime + ". Finished with result " + metadata);
SparkCompactHelpers.newInstance().completeInflightCompaction(table, compactionCommitTime, metadata);
WriteMarkersFactory.get(config.getMarkersType(), table, compactionCommitTime)
.quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
if (compactionTimer != null) {
long durationInMs = metrics.getDurationInMs(compactionTimer.stop());