lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-294] Delete Paths written in Cleaner plan needs to be relative to partition-path (#1062)

Browse Source 
[HUDI-294] Delete Paths written in Cleaner plan needs to be relative to partition-path
This commit is contained in:
leesf
2019-12-04 02:11:03 +08:00
committed by Balaji Varadarajan
parent 845e261658
commit 98ab33bb6e
11 changed files with 425 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java
View File

@@ -23,11 +23,13 @@ import org.apache.hudi.avro.model.HoodieCleanerPlan;
import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.common.HoodieCleanStat;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
@@ -166,9 +168,10 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
logger.info("cleanerElaspsedTime (Minutes): " + durationInMs.get() / (1000 * 60));
}
HoodieTableMetaClient metaClient = createMetaClient(true);
// Create the metadata and save it
HoodieCleanMetadata metadata =
AvroUtils.convertCleanMetadata(cleanInstant.getTimestamp(), durationInMs, cleanStats);
CleanerUtils.convertCleanMetadata(metaClient, cleanInstant.getTimestamp(), durationInMs, cleanStats);
logger.info("Cleaned " + metadata.getTotalFilesDeleted() + " files. Earliest Retained :" + metadata.getEarliestCommitToRetain());
metrics.updateCleanMetrics(durationInMs.orElseGet(() -> -1L), metadata.getTotalFilesDeleted());