lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2468] Metadata table support for rolling back the first commit (#3843)

Browse Source 
- Fix is to make Metadata table writer creation aware of the currently inflight action so that it can
  make some informed decision about whether bootstrapping is needed for the table and whether
  any pending action on the data timeline can be ignored.
This commit is contained in:
Manoj Govindassamy
2021-10-23 07:07:09 -07:00
committed by GitHub
parent 5ed35bff83
commit c9d641cc30
9 changed files with 201 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHBaseIndex.java
View File

@@ -337,7 +337,7 @@ public class TestHBaseIndex extends SparkClientFunctionalTestHarness {
public void testSimpleTagLocationAndUpdateWithRollback() throws Exception {
// Load to memory
HoodieWriteConfig config = getConfigBuilder(100, false, false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()).build();
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
SparkHoodieHBaseIndex index = new SparkHoodieHBaseIndex(config);
SparkRDDWriteClient writeClient = getHoodieWriteClient(config);
@@ -425,7 +425,7 @@ public class TestHBaseIndex extends SparkClientFunctionalTestHarness {
public void testEnsureTagLocationUsesCommitTimeline() throws Exception {
// Load to memory
HoodieWriteConfig config = getConfigBuilder(100, false, false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()).build();
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
SparkHoodieHBaseIndex index = new SparkHoodieHBaseIndex(config);
SparkRDDWriteClient writeClient = getHoodieWriteClient(config);

33
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
View File

@@ -459,6 +459,39 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Some operations are not feasible with test table infra. hence using write client to test those cases.
/**
* Rollback of the first commit should not trigger bootstrap errors at the metadata table.
*/
@ParameterizedTest
@EnumSource(HoodieTableType.class)
public void testFirstCommitRollback(HoodieTableType tableType) throws Exception {
init(tableType);
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, getWriteConfig(true, true))) {
// Write 1
String commitTime = "0000001";
List<HoodieRecord> records = dataGen.generateInserts(commitTime, 20);
client.startCommitWithTime(commitTime);
List<WriteStatus> writeStatuses = client.insert(jsc.parallelize(records, 1), commitTime).collect();
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
// Rollback the first commit
client.rollback(commitTime);
// Write 2
commitTime = "0000002";
records = dataGen.generateInserts(commitTime, 10);
client.startCommitWithTime(commitTime);
writeStatuses = client.upsert(jsc.parallelize(records, 1), commitTime).collect();
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
}
}
/**
* Test several table operations with restore. This test uses SparkRDDWriteClient.
* Once the restore support is ready in HoodieTestTable, then rewrite this test.