lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3884] Support archival beyond savepoint commits (#5837)

Browse Source 
Co-authored-by: sivabalan <n.siva.b@gmail.com>
This commit is contained in:
Sagar Sumit
2022-07-26 00:12:29 +05:30
committed by GitHub
parent eee6a02f77
commit 6e7ac45735
14 changed files with 364 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
View File

@@ -18,10 +18,15 @@
package org.apache.hudi.common.model;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.testutils.MockHoodieTimeline;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -47,4 +52,25 @@ public class TestHoodieFileGroup {
assertTrue(fileGroup.getLatestFileSlice().get().getBaseInstantTime().equals("001"));
assertTrue((new HoodieFileGroup(fileGroup)).getLatestFileSlice().get().getBaseInstantTime().equals("001"));
}
@Test
public void testCommittedFileSlicesWithSavepointAndHoles() {
MockHoodieTimeline activeTimeline = new MockHoodieTimeline(Stream.of(
new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "01"),
new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "01"),
new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "03"),
new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "03"),
new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "05") // this can be DELTA_COMMIT/REPLACE_COMMIT as well
).collect(Collectors.toList()));
HoodieFileGroup fileGroup = new HoodieFileGroup("", "data", activeTimeline.filterCompletedAndCompactionInstants());
for (int i = 0; i < 7; i++) {
HoodieBaseFile baseFile = new HoodieBaseFile("data_1_0" + i);
fileGroup.addBaseFile(baseFile);
}
List<FileSlice> allFileSlices = fileGroup.getAllFileSlices().collect(Collectors.toList());
assertEquals(6, allFileSlices.size());
assertTrue(!allFileSlices.stream().anyMatch(s -> s.getBaseInstantTime().equals("06")));
assertEquals(7, fileGroup.getAllFileSlicesIncludingInflight().count());
assertTrue(fileGroup.getLatestFileSlice().get().getBaseInstantTime().equals("05"));
}
}

51
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
View File

@@ -262,6 +262,57 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
assertEquals(instant7.getTimestamp(), timeline.getContiguousCompletedWriteTimeline().lastInstant().get().getTimestamp());
}
@Test
public void testTimelineWithSavepointAndHoles() {
timeline = new MockHoodieTimeline(Stream.of(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "01"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "01"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "05") // this can be DELTA_COMMIT/REPLACE_COMMIT as well
).collect(Collectors.toList()));
assertTrue(timeline.isBeforeTimelineStarts("00"));
assertTrue(timeline.isBeforeTimelineStarts("01"));
assertTrue(timeline.isBeforeTimelineStarts("02"));
assertTrue(timeline.isBeforeTimelineStarts("03"));
assertTrue(timeline.isBeforeTimelineStarts("04"));
assertFalse(timeline.isBeforeTimelineStarts("05"));
assertFalse(timeline.isBeforeTimelineStarts("06"));
// with an inflight savepoint in between
timeline = new MockHoodieTimeline(Stream.of(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "01"),
new HoodieInstant(State.INFLIGHT, HoodieTimeline.SAVEPOINT_ACTION, "01"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "05")
).collect(Collectors.toList()));
assertTrue(timeline.isBeforeTimelineStarts("00"));
assertTrue(timeline.isBeforeTimelineStarts("01"));
assertTrue(timeline.isBeforeTimelineStarts("02"));
assertTrue(timeline.isBeforeTimelineStarts("03"));
assertTrue(timeline.isBeforeTimelineStarts("04"));
assertFalse(timeline.isBeforeTimelineStarts("05"));
assertFalse(timeline.isBeforeTimelineStarts("06"));
// with a pending replacecommit after savepoints
timeline = new MockHoodieTimeline(Stream.of(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "01"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "01"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.SAVEPOINT_ACTION, "03"),
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "05"),
new HoodieInstant(State.INFLIGHT, HoodieTimeline.REPLACE_COMMIT_ACTION, "06")
).collect(Collectors.toList()));
assertTrue(timeline.isBeforeTimelineStarts("00"));
assertTrue(timeline.isBeforeTimelineStarts("01"));
assertTrue(timeline.isBeforeTimelineStarts("02"));
assertTrue(timeline.isBeforeTimelineStarts("03"));
assertTrue(timeline.isBeforeTimelineStarts("04"));
assertFalse(timeline.isBeforeTimelineStarts("05"));
assertFalse(timeline.isBeforeTimelineStarts("06"));
}
@Test
public void testTimelineGetOperations() {
List<HoodieInstant> allInstants = getAllInstants();

15
hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
View File

@@ -26,6 +26,7 @@ import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.avro.model.HoodieRestoreMetadata;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFileFormat;
@@ -35,6 +36,7 @@ import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.util.Option;
@@ -156,6 +158,10 @@ public class FileCreateUtils {
}
}
public static void createSavepointCommit(String basePath, String instantTime, HoodieSavepointMetadata savepointMetadata) throws IOException {
createMetaFile(basePath, instantTime, HoodieTimeline.SAVEPOINT_EXTENSION, TimelineMetadataUtils.serializeSavepointMetadata(savepointMetadata).get());
}
public static void createCommit(String basePath, String instantTime, FileSystem fs) throws IOException {
createMetaFile(basePath, instantTime, HoodieTimeline.COMMIT_EXTENSION, fs);
}
@@ -285,6 +291,10 @@ public class FileCreateUtils {
createMetaFile(basePath, instantTime, HoodieTimeline.INFLIGHT_COMPACTION_EXTENSION);
}
public static void createInflightSavepoint(String basePath, String instantTime) throws IOException {
createAuxiliaryMetaFile(basePath, instantTime, HoodieTimeline.INFLIGHT_SAVEPOINT_EXTENSION);
}
public static void createPartitionMetaFile(String basePath, String partitionPath) throws IOException {
Path parentPath = Paths.get(basePath, partitionPath);
Files.createDirectories(parentPath);
@@ -439,4 +449,9 @@ public class FileCreateUtils {
public static void deleteDeltaCommit(String basePath, String instantTime, FileSystem fs) throws IOException {
deleteMetaFile(basePath, instantTime, HoodieTimeline.DELTA_COMMIT_EXTENSION, fs);
}
public static void deleteSavepointCommit(String basePath, String instantTime, FileSystem fs) throws IOException {
deleteMetaFile(basePath, instantTime, HoodieTimeline.INFLIGHT_SAVEPOINT_EXTENSION, fs);
deleteMetaFile(basePath, instantTime, HoodieTimeline.SAVEPOINT_EXTENSION, fs);
}
}

23
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
View File

@@ -99,6 +99,7 @@ import static org.apache.hudi.common.testutils.FileCreateUtils.createInflightCom
import static org.apache.hudi.common.testutils.FileCreateUtils.createInflightDeltaCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.createInflightReplaceCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.createInflightRollbackFile;
import static org.apache.hudi.common.testutils.FileCreateUtils.createInflightSavepoint;
import static org.apache.hudi.common.testutils.FileCreateUtils.createMarkerFile;
import static org.apache.hudi.common.testutils.FileCreateUtils.createReplaceCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.createRequestedCleanFile;
@@ -109,6 +110,8 @@ import static org.apache.hudi.common.testutils.FileCreateUtils.createRequestedRe
import static org.apache.hudi.common.testutils.FileCreateUtils.createRequestedRollbackFile;
import static org.apache.hudi.common.testutils.FileCreateUtils.createRestoreFile;
import static org.apache.hudi.common.testutils.FileCreateUtils.createRollbackFile;
import static org.apache.hudi.common.testutils.FileCreateUtils.createSavepointCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.deleteSavepointCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.logFileName;
import static org.apache.hudi.common.util.CleanerUtils.convertCleanMetadata;
import static org.apache.hudi.common.util.CollectionUtils.createImmutableMap;
@@ -199,6 +202,12 @@ public class HoodieTestTable {
return this;
}
public HoodieTestTable addSavepointCommit(String instantTime, HoodieSavepointMetadata savepointMetadata) throws IOException {
createInflightSavepoint(basePath, instantTime);
createSavepointCommit(basePath, instantTime, savepointMetadata);
return this;
}
public HoodieCommitMetadata createCommitMetadata(WriteOperationType operationType, String commitTime,
HoodieTestTableState testTableState) {
String actionType = getCommitActionType(operationType, metaClient.getTableType());
@@ -394,7 +403,7 @@ public class HoodieTestTable {
public HoodieSavepointMetadata getSavepointMetadata(String instant, Map<String, List<String>> partitionToFilesMeta) {
HoodieSavepointMetadata savepointMetadata = new HoodieSavepointMetadata();
savepointMetadata.setSavepointedAt(Long.valueOf(instant));
savepointMetadata.setSavepointedAt(12345L);
Map<String, HoodieSavepointPartitionMetadata> partitionMetadataMap = new HashMap<>();
for (Map.Entry<String, List<String>> entry : partitionToFilesMeta.entrySet()) {
HoodieSavepointPartitionMetadata savepointPartitionMetadata = new HoodieSavepointPartitionMetadata();
@@ -404,6 +413,7 @@ public class HoodieTestTable {
}
savepointMetadata.setPartitionMetadata(partitionMetadataMap);
savepointMetadata.setSavepointedBy("test");
savepointMetadata.setComments("test_comment");
return savepointMetadata;
}
@@ -454,6 +464,17 @@ public class HoodieTestTable {
return this;
}
public HoodieTestTable addSavepoint(String instantTime, HoodieSavepointMetadata savepointMetadata) throws IOException {
createInflightSavepoint(basePath, instantTime);
createSavepointCommit(basePath, instantTime, savepointMetadata);
return this;
}
public HoodieTestTable deleteSavepoint(String instantTime) throws IOException {
deleteSavepointCommit(basePath, instantTime, fs);
return this;
}
public HoodieTestTable forCommit(String instantTime) {
currentInstantTime = instantTime;
return this;

6
hudi-common/src/test/java/org/apache/hudi/common/testutils/MockHoodieTimeline.java
View File

@@ -23,6 +23,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -38,4 +39,9 @@ public class MockHoodieTimeline extends HoodieActiveTimeline {
inflights.map(s -> new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, s)))
.sorted(Comparator.comparing(HoodieInstant::getFileName)).collect(Collectors.toList()));
}
public MockHoodieTimeline(List<HoodieInstant> instants) {
super();
this.setInstants(instants);
}
}