lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2925] Fix duplicate cleaning of same files when unfinished clean operations are present using a config. (#4212)

Browse Source 
Co-authored-by: sivabalan <n.siva.b@gmail.com>
This commit is contained in:
Prashant Wason
2022-02-21 18:53:03 -08:00
committed by GitHub
parent 0c950181aa
commit 0dee8edc97
7 changed files with 118 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
View File

@@ -35,6 +35,9 @@ import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataV1Mig
import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataV2MigrationHandler;
import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanMigrator;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
@@ -43,6 +46,9 @@ import java.util.Map;
import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
public class CleanerUtils {
private static final Logger LOG = LogManager.getLogger(CleanerUtils.class);
public static final Integer CLEAN_METADATA_VERSION_1 = CleanMetadataV1MigrationHandler.VERSION;
public static final Integer CLEAN_METADATA_VERSION_2 = CleanMetadataV2MigrationHandler.VERSION;
public static final Integer LATEST_CLEAN_METADATA_VERSION = CLEAN_METADATA_VERSION_2;
@@ -131,6 +137,7 @@ public class CleanerUtils {
// No need to do any special cleanup for failed operations during clean
return;
} else if (cleaningPolicy.isLazy()) {
LOG.info("Cleaned failed attempts if any");
// Perform rollback of failed operations for all types of actions during clean
rollbackFailedWritesFunc.apply();
return;
@@ -140,6 +147,7 @@ public class CleanerUtils {
case COMMIT_ACTION:
// For any other actions, perform rollback of failed writes
if (cleaningPolicy.isEager()) {
LOG.info("Cleaned failed attempts if any");
rollbackFailedWritesFunc.apply();
return;
}