lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1436]: Provide an option to trigger clean every nth commit (#4385)

Browse Source 
- Provided option to trigger clean every nth commit with default number of commits as 1 so that existing users are not affected.
Co-authored-by: sivabalan <n.siva.b@gmail.com>
This commit is contained in:
Pratyaksh Sharma
2022-03-22 05:36:30 +05:30
committed by GitHub
parent 26e5d2e6fc
commit ca0931d332
9 changed files with 521 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestHoodieConcatHandle.java
View File

@@ -117,7 +117,7 @@ public class TestHoodieConcatHandle extends HoodieJavaClientTestBase {
records1.add(new HoodieAvroRecord(new HoodieKey(insertRow2.getRowKey(), insertRow2.getPartitionPath()), insertRow2));
int startInstant = 1;
String firstCommitTime = makeNewCommitTime(startInstant++);
String firstCommitTime = makeNewCommitTime(startInstant++, "%09d");
// First insert
writeClient.startCommitWithTime(firstCommitTime);
writeClient.insert(records1, firstCommitTime);
@@ -145,7 +145,7 @@ public class TestHoodieConcatHandle extends HoodieJavaClientTestBase {
records2.add(new HoodieAvroRecord(new HoodieKey(insertRow1.getRowKey(), insertRow1.getPartitionPath()), insertRow1));
records2.add(new HoodieAvroRecord(new HoodieKey(insertRow2.getRowKey(), insertRow2.getPartitionPath()), insertRow2));
String newCommitTime = makeNewCommitTime(startInstant++);
String newCommitTime = makeNewCommitTime(startInstant++, "%09d");
writeClient.startCommitWithTime(newCommitTime);
// Second insert is the same as the _row_key of the first one,test allowDuplicateInserts
writeClient.insert(records2, newCommitTime);
@@ -183,7 +183,7 @@ public class TestHoodieConcatHandle extends HoodieJavaClientTestBase {
HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator(new String[]{partitionPath});
int startInstant = 1;
String firstCommitTime = makeNewCommitTime(startInstant++);
String firstCommitTime = makeNewCommitTime(startInstant++, "%09d");
List<HoodieRecord> records1 = dataGenerator.generateInserts(firstCommitTime, 100);
// First insert
@@ -200,7 +200,7 @@ public class TestHoodieConcatHandle extends HoodieJavaClientTestBase {
assertTrue(filter.mightContain(record.getRecordKey()));
}
String newCommitTime = makeNewCommitTime(startInstant++);
String newCommitTime = makeNewCommitTime(startInstant++, "%09d");
List<HoodieRecord> records2 = dataGenerator.generateUpdates(newCommitTime, 100);
writeClient.startCommitWithTime(newCommitTime);
// Second insert is the same as the _row_key of the first one,test allowDuplicateInserts

4
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
View File

@@ -123,7 +123,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestBase
// Prepare the AvroParquetIO
HoodieWriteConfig config = makeHoodieClientConfig();
int startInstant = 1;
String firstCommitTime = makeNewCommitTime(startInstant++);
String firstCommitTime = makeNewCommitTime(startInstant++, "%09d");
HoodieJavaWriteClient writeClient = getHoodieWriteClient(config);
writeClient.startCommitWithTime(firstCommitTime);
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -185,7 +185,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestBase
List<HoodieRecord> updatedRecords = Arrays.asList(updatedRecord1, insertedRecord1);
String newCommitTime = makeNewCommitTime(startInstant++);
String newCommitTime = makeNewCommitTime(startInstant++, "%09d");
metaClient = HoodieTableMetaClient.reload(metaClient);
writeClient.startCommitWithTime(newCommitTime);
List<WriteStatus> statuses = writeClient.upsert(updatedRecords, newCommitTime);