From 7a6b071647fae632af2126b517ac869e81f539cb Mon Sep 17 00:00:00 2001 From: satishkotha Date: Mon, 1 Mar 2021 09:40:00 -0800 Subject: [PATCH] [HUDI-1644] Do not delete older rollback instants as part of rollback. Archival can take care of removing old instants cleanly (#2610) --- .../restore/BaseRestoreActionExecutor.java | 12 ----- .../rollback/BaseRollbackActionExecutor.java | 8 --- .../org/apache/hudi/common/fs/FSUtils.java | 24 --------- .../apache/hudi/common/fs/TestFSUtils.java | 54 +------------------ 4 files changed, 2 insertions(+), 96 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java index 6d38d03ea..3b722a7a4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java @@ -21,7 +21,6 @@ package org.apache.hudi.table.action.restore; import org.apache.hudi.avro.model.HoodieRestoreMetadata; import org.apache.hudi.avro.model.HoodieRollbackMetadata; import org.apache.hudi.common.engine.HoodieEngineContext; -import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -96,17 +95,6 @@ public abstract class BaseRestoreActionExecutor instants) { - // TODO - this should be archived when archival is made general for all meta-data - // skip MIN_CLEAN_TO_KEEP and delete rest - instants.skip(MIN_CLEAN_TO_KEEP).forEach(s -> { - try { - fs.delete(new Path(metaPath, s.getFileName()), false); - } catch (IOException e) { - throw new HoodieIOException("Could not delete clean meta files" + s.getFileName(), e); - } - }); - } - - public static void deleteOlderRollbackMetaFiles(FileSystem fs, String metaPath, Stream instants) { - // TODO - this should be archived when archival is made general for all meta-data - // skip MIN_ROLLBACK_TO_KEEP and delete rest - instants.skip(MIN_ROLLBACK_TO_KEEP).forEach(s -> { - try { - fs.delete(new Path(metaPath, s.getFileName()), false); - } catch (IOException e) { - throw new HoodieIOException("Could not delete rollback meta files " + s.getFileName(), e); - } - }); - } - public static void deleteInstantFile(FileSystem fs, String metaPath, HoodieInstant instant) { try { LOG.warn("try to delete instant file: " + instant); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index f1d80785c..1dc1f5085 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -18,22 +18,18 @@ package org.apache.hudi.common.fs; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableMetaClient; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.exception.HoodieException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -48,7 +44,6 @@ import java.util.stream.Stream; import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_FORMATTER; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -273,51 +268,6 @@ public class TestFSUtils extends HoodieCommonTestHarness { return "." + String.format("%s_%s%s.%d", fileId, baseCommitTime, logFileExtension, version); } - @Test - public void testDeleteOlderRollbackFiles() throws Exception { - String[] instantTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101", - "20160502020601", "20160502030601", "20160502040601", "20160502050601", "20160506030611", - "20160506040611", "20160506050611", "20160506060611"}; - List hoodieInstants = new ArrayList<>(); - // create rollback files - for (String instantTime : instantTimes) { - Files.createFile(Paths.get(basePath, - HoodieTableMetaClient.METAFOLDER_NAME, - instantTime + HoodieTimeline.ROLLBACK_EXTENSION)); - hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.ROLLBACK_ACTION, instantTime)); - } - - String metaPath = Paths.get(basePath, ".hoodie").toString(); - FSUtils.deleteOlderRollbackMetaFiles(FSUtils.getFs(basePath, new Configuration()), - metaPath, hoodieInstants.stream()); - File[] rollbackFiles = new File(metaPath).listFiles((dir, name) - -> name.contains(HoodieTimeline.ROLLBACK_EXTENSION)); - assertNotNull(rollbackFiles); - assertEquals(rollbackFiles.length, minRollbackToKeep); - } - - @Test - public void testDeleteOlderCleanMetaFiles() throws Exception { - String[] instantTimes = new String[]{"20160501010101", "20160501020101", "20160501030101", "20160501040101", - "20160502020601", "20160502030601", "20160502040601", "20160502050601", "20160506030611", - "20160506040611", "20160506050611", "20160506060611"}; - List hoodieInstants = new ArrayList<>(); - // create rollback files - for (String instantTime : instantTimes) { - Files.createFile(Paths.get(basePath, - HoodieTableMetaClient.METAFOLDER_NAME, - instantTime + HoodieTimeline.CLEAN_EXTENSION)); - hoodieInstants.add(new HoodieInstant(false, HoodieTimeline.CLEAN_ACTION, instantTime)); - } - String metaPath = Paths.get(basePath, ".hoodie").toString(); - FSUtils.deleteOlderCleanMetaFiles(FSUtils.getFs(basePath, new Configuration()), - metaPath, hoodieInstants.stream()); - File[] cleanFiles = new File(metaPath).listFiles((dir, name) - -> name.contains(HoodieTimeline.CLEAN_EXTENSION)); - assertNotNull(cleanFiles); - assertEquals(cleanFiles.length, minCleanToKeep); - } - @Test public void testFileNameRelatedFunctions() throws Exception { String instantTime = "20160501010101";