lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3435] Do not throw exception when instant to rollback does not exist in metadata table active timeline (#4821)

Browse Source
This commit is contained in:
Danny Chan
2022-03-26 11:42:54 +08:00
committed by GitHub
parent 51034fecf1
commit 0c09a973fb
5 changed files with 108 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
View File

@@ -294,24 +294,29 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
AtomicInteger commitTime = new AtomicInteger(1);
// trigger 2 regular writes(1 bootstrap commit). just 1 before archival can get triggered.
int i = 1;
for (; i <= 2; i++) {
for (int i = 1; i <= 2; i++) {
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
}
// expected num commits = 1 (bootstrap) + 2 (writes)
HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
HoodieActiveTimeline metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals(metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants(), 3);
assertEquals(3, metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants());
// trigger a async table service, archival should not kick in, even though conditions are met.
// trigger an async table service, archival should not kick in, even though conditions are met.
doCluster(testTable, "000000" + commitTime.getAndIncrement());
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals(metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants(), 4);
assertEquals(4, metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants());
// trigger a regular write operation. archival should kick in.
// start the timeline server for MARKERS cleaning up
getHoodieWriteClient(writeConfig);
// trigger a regular write operation. data set timeline archival should kick in.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
archiveDataTable(writeConfig, HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build());
// trigger a regular write operation. metadata timeline archival should kick in.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals(metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants(), 3);
assertEquals(4, metadataTimeline.getCommitsTimeline().filterCompletedInstants().countInstants());
}
@ParameterizedTest
@@ -942,7 +947,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
}
assertTrue(exceptionRaised, "Rollback of archived instants should fail");
assertFalse(exceptionRaised, "Metadata table should not archive instants that are in dataset active timeline");
// Since each rollback also creates a deltacommit, we can only support rolling back of half of the original
// instants present before rollback started.
assertTrue(numRollbacks >= Math.max(minArchiveCommitsDataset, minArchiveCommitsMetadata) / 2,

47
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
View File

@@ -1054,7 +1054,7 @@ public class TestHoodieTimelineArchiver extends HoodieClientTestHarness {
.setBasePath(HoodieTableMetadata.getMetadataTableBasePath(basePath))
.setLoadActiveTimelineOnLoad(true).build();
for (int i = 1; i <= 16; i++) {
for (int i = 1; i <= 17; i++) {
testTable.doWriteOperation("000000" + String.format("%02d", i), WriteOperationType.UPSERT,
i == 1 ? Arrays.asList("p1", "p2") : Collections.emptyList(), Arrays.asList("p1", "p2"), 2);
// archival
@@ -1075,6 +1075,30 @@ public class TestHoodieTimelineArchiver extends HoodieClientTestHarness {
IntStream.range(1, i + 1).forEach(j ->
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "0000000" + j))));
} else if (i == 8) {
// i == 8
// The instant "00000000000000" was archived since it's less than
// the earliest instant on the dataset active timeline,
// the dataset active timeline has instants of range [00000001 ~ 00000008]
// because when it does the archiving, no compaction instant on the
// metadata active timeline exists yet.
assertEquals(9, metadataTableInstants.size());
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "00000007001")));
IntStream.range(1, i + 1).forEach(j ->
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "0000000" + j))));
} else if (i <= 11) {
// In the metadata table timeline, the first delta commit is "00000007"
// because it equals with the earliest commit on the dataset timeline, after archival,
// delta commits "00000008" till "00000011" are added later on without archival or compaction
assertEquals(i - 5, metadataTableInstants.size());
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "00000007001")));
IntStream.range(7, i + 1).forEach(j ->
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION,
"000000" + String.format("%02d", j)))));
} else if (i <= 14) {
// In the metadata table timeline, the first delta commit is "00000007001"
// from metadata table compaction, after archival, delta commits "00000008"
@@ -1095,14 +1119,27 @@ public class TestHoodieTimelineArchiver extends HoodieClientTestHarness {
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION,
"000000" + String.format("%02d", j)))));
} else {
} else if (i == 16) {
// i == 16
// Only commit "00000015001" and delta commit "00000016" are in the active timeline
assertEquals(2, metadataTableInstants.size());
// dataset timeline has commits "00000015" and "00000016",
// the metadata timeline has commits [00000008, 00000016] and "00000015001"
assertEquals(10, metadataTableInstants.size());
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "00000015001")));
IntStream.range(8, 17).forEach(j ->
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION,
"000000" + String.format("%02d", j)))));
} else {
// i == 17
// Only commits [00000015, 00000017] and "00000015001" are on the metadata timeline
assertEquals(4, metadataTableInstants.size());
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "00000016")));
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "00000015001")));
IntStream.range(15, 18).forEach(j ->
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION,
"000000" + String.format("%02d", j)))));
}
}
}