lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3732] Fixing rollback validation (#5157)

Browse Source 
* Fixing rollback validation

* Adding tests
This commit is contained in:
Sivabalan Narayanan
2022-03-31 04:55:24 -07:00
committed by GitHub
parent 80011df995
commit 73a21092f8
4 changed files with 49 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -1399,6 +1399,41 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
testInsertAndClustering(clusteringConfig, populateMetaFields, true, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
}
@Test
public void testRolblackOfRegularCommitWithPendingReplaceCommitInTimeline() throws Exception {
HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
.withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).withInlineClustering(true)
.withPreserveHoodieCommitMetadata(true).build();
// trigger clustering, but do not complete
testInsertAndClustering(clusteringConfig, true, false, false, SqlQueryEqualityPreCommitValidator.class.getName(), COUNT_SQL_QUERY_FOR_VALIDATION, "");
// trigger another partial commit, followed by valid commit. rollback of partial commit should succeed.
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
String commitTime1 = HoodieActiveTimeline.createNewInstantTime();
List<HoodieRecord> records1 = dataGen.generateInserts(commitTime1, 200);
client.startCommitWithTime(commitTime1);
JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(records1, 2);
JavaRDD<WriteStatus> statuses = client.upsert(insertRecordsRDD1, commitTime1);
List<WriteStatus> statusList = statuses.collect();
assertNoWriteErrors(statusList);
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterInflightsAndRequested().countInstants());
// trigger another commit. this should rollback latest partial commit.
records1 = dataGen.generateInserts(commitTime1, 200);
client.startCommitWithTime(commitTime1);
insertRecordsRDD1 = jsc.parallelize(records1, 2);
statuses = client.upsert(insertRecordsRDD1, commitTime1);
statusList = statuses.collect();
assertNoWriteErrors(statusList);
client.commit(commitTime1, statuses);
metaClient.reloadActiveTimeline();
// rollback should have succeeded. Essentially, the pending clustering should not hinder the rollback of regular commits.
assertEquals(1, metaClient.getActiveTimeline().getCommitsTimeline().filterInflightsAndRequested().countInstants());
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testInlineScheduleClustering(boolean scheduleInlineClustering) throws IOException {