lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fix the filter condition is missing in the judgment condition of compaction instance (#3025)

Browse Source
This commit is contained in:
swuferhong
2021-06-17 05:28:53 +08:00
committed by GitHub
parent d519c74626
commit 5ce64a81bd
2 changed files with 25 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java
View File

@@ -63,7 +63,7 @@ public abstract class BaseScheduleCompactionActionExecutor<T extends HoodieRecor
+ ", Compaction scheduled at " + instantTime)); + ", Compaction scheduled at " + instantTime));
// Committed and pending compaction instants should have strictly lower timestamps // Committed and pending compaction instants should have strictly lower timestamps
List<HoodieInstant> conflictingInstants = table.getActiveTimeline() List<HoodieInstant> conflictingInstants = table.getActiveTimeline()
.getWriteTimeline().getInstants() .getWriteTimeline().filterCompletedAndCompactionInstants().getInstants()
.filter(instant -> HoodieTimeline.compareTimestamps( .filter(instant -> HoodieTimeline.compareTimestamps(
instant.getTimestamp(), HoodieTimeline.GREATER_THAN_OR_EQUALS, instantTime)) instant.getTimestamp(), HoodieTimeline.GREATER_THAN_OR_EQUALS, instantTime))
.collect(Collectors.toList()); .collect(Collectors.toList());

24
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java
View File

@@ -132,6 +132,30 @@ public class TestHoodieCompactor extends HoodieClientTestHarness {
} }
} }
@Test
public void testScheduleCompactionWithInflightInstant() {
HoodieWriteConfig config = getConfig();
try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
// insert 100 records.
String newCommitTime = "100";
writeClient.startCommitWithTime(newCommitTime);
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1);
writeClient.insert(recordsRDD, newCommitTime).collect();
// create one inflight instance.
newCommitTime = "102";
writeClient.startCommitWithTime(newCommitTime);
metaClient.getActiveTimeline().transitionRequestedToInflight(new HoodieInstant(State.REQUESTED,
HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime), Option.empty());
// create one compaction instance before exist inflight instance.
String compactionTime = "101";
writeClient.scheduleCompactionAtInstant(compactionTime, Option.empty());
}
}
@Test @Test
public void testWriteStatusContentsAfterCompaction() throws Exception { public void testWriteStatusContentsAfterCompaction() throws Exception {
// insert 100 records // insert 100 records