lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3721] Delete MDT if necessary when trigger rollback to savepoint (#5173)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
YueZhang
2022-03-31 11:26:37 +08:00
committed by GitHub
parent 2c4554fada
commit 2dbb273d26
9 changed files with 106 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
View File

@@ -425,11 +425,13 @@ public class SparkRDDWriteClient<T extends HoodieRecordPayload> extends
}
@Override
protected HoodieTable doInitTable(HoodieTableMetaClient metaClient, Option<String> instantTime) {
// Initialize Metadata Table to make sure it's bootstrapped _before_ the operation,
// if it didn't exist before
// See https://issues.apache.org/jira/browse/HUDI-3343 for more details
initializeMetadataTable(instantTime);
protected HoodieTable doInitTable(HoodieTableMetaClient metaClient, Option<String> instantTime, boolean initialMetadataTableIfNecessary) {
if (initialMetadataTableIfNecessary) {
// Initialize Metadata Table to make sure it's bootstrapped _before_ the operation,
// if it didn't exist before
// See https://issues.apache.org/jira/browse/HUDI-3343 for more details
initializeMetadataTable(instantTime);
}
// Create a Hoodie table which encapsulated the commits and files visible
return HoodieSparkTable.create(config, (HoodieSparkEngineContext) context, metaClient, config.isMetadataTableEnabled());

4
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
View File

@@ -291,7 +291,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase {
}
// Rollback to the original schema
client.restoreToInstant("004");
client.restoreToInstant("004", hoodieWriteConfig.isMetadataTableEnabled());
checkLatestDeltaCommit("004");
// Updates with original schema are now allowed
@@ -432,7 +432,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase {
// Revert to the older commit and ensure that the original schema can now
// be used for inserts and inserts.
client.restoreToInstant("003");
client.restoreToInstant("003", hoodieWriteConfig.isMetadataTableEnabled());
curTimeline = metaClient.reloadActiveTimeline().getCommitTimeline().filterCompletedInstants();
assertTrue(curTimeline.lastInstant().get().getTimestamp().equals("003"));
checkReadRecords("000", numRecords);

2
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
View File

@@ -1208,7 +1208,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(client);
// Restore
client.restoreToInstant("0000006");
client.restoreToInstant("0000006", writeConfig.isMetadataTableEnabled());
validateMetadata(client);
}
}

2
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -585,7 +585,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
client.savepoint("004", "user1","comment1");
client.restoreToInstant("004");
client.restoreToInstant("004", config.isMetadataTableEnabled());
assertFalse(metaClient.reloadActiveTimeline().getRollbackTimeline().lastInstant().isPresent());

8
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java
View File

@@ -150,7 +150,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction
// NOTE: First writer will have Metadata table DISABLED
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder(false, rollbackUsingMarkers, HoodieIndex.IndexType.SIMPLE);
addConfigsForPopulateMetaFields(cfgBuilder, true);
HoodieWriteConfig cfg = cfgBuilder.build();
@@ -480,7 +480,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction
copyOfRecords.clear();
// Rollback latest commit first
client.restoreToInstant("000");
client.restoreToInstant("000", cfg.isMetadataTableEnabled());
metaClient = HoodieTableMetaClient.reload(metaClient);
allFiles = listAllBaseFilesInPath(hoodieTable);
@@ -530,7 +530,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction
if (!restoreAfterCompaction) {
// restore to 002 and validate records.
client.restoreToInstant("002");
client.restoreToInstant("002", cfg.isMetadataTableEnabled());
validateRecords(cfg, metaClient, updates1);
} else {
// trigger compaction and then trigger couple of upserts followed by restore.
@@ -546,7 +546,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction
validateRecords(cfg, metaClient, updates5);
// restore to 003 and validate records.
client.restoreToInstant("003");
client.restoreToInstant("003", cfg.isMetadataTableEnabled());
validateRecords(cfg, metaClient, updates2);
}
}