lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3721] Delete MDT if necessary when trigger rollback to savepoint (#5173)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
YueZhang
2022-03-31 11:26:37 +08:00
committed by GitHub
parent 2c4554fada
commit 2dbb273d26
9 changed files with 106 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java
View File

@@ -22,6 +22,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.commands.TableCommand;
import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -29,6 +31,9 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
@@ -118,6 +123,54 @@ public class ITTestSavepointsCommand extends AbstractShellIntegrationTest {
new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "103")));
}
/**
* Test case of command 'savepoint rollback' with metadata table bootstrap.
*/
@Test
public void testRollbackToSavepointWithMetadataTableEnable() throws IOException {
// generate for savepoints
for (int i = 101; i < 105; i++) {
String instantTime = String.valueOf(i);
HoodieTestDataGenerator.createCommitFile(tablePath, instantTime, jsc.hadoopConfiguration());
}
// generate one savepoint at 102
String savepoint = "102";
HoodieTestDataGenerator.createSavepointFile(tablePath, savepoint, jsc.hadoopConfiguration());
// re-bootstrap metadata table
// delete first
String basePath = metaClient.getBasePath();
Path metadataTableBasePath = new Path(HoodieTableMetadata.getMetadataTableBasePath(basePath));
metaClient.getFs().delete(metadataTableBasePath, true);
// then bootstrap metadata table at instant 104
HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(HoodieCLI.basePath)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
assertTrue(HoodieCLI.fs.exists(metadataTableBasePath));
// roll back to savepoint
CommandResult cr = getShell().executeCommand(
String.format("savepoint rollback --savepoint %s --sparkMaster %s", savepoint, "local"));
assertAll("Command run failed",
() -> assertTrue(cr.isSuccess()),
() -> assertEquals(
String.format("Savepoint \"%s\" rolled back", savepoint), cr.getResult().toString()));
// there is 1 restore instant
HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
assertEquals(1, timeline.getRestoreTimeline().countInstants());
// 103 and 104 instant had rollback
assertFalse(timeline.getCommitTimeline().containsInstant(
new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "103")));
assertFalse(timeline.getCommitTimeline().containsInstant(
new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "104")));
}
/**
* Test case of command 'savepoint delete'.
*/