lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4408] Reuse old rollover file as base file for flink merge handle (#6120)

Browse Source
This commit is contained in:
Danny Chan
2022-07-16 20:46:23 +08:00
committed by GitHub
parent 0faa562b6f
commit 6aec9d754f
2 changed files with 21 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java
View File

@@ -408,16 +408,6 @@ public class StreamWriteFunction<I> extends AbstractStreamWriteFunction<I> {
&& this.buckets.values().stream().anyMatch(bucket -> bucket.records.size() > 0);
}
private void cleanWriteHandles() {
if (freshInstant(currentInstant)) {
// In rare cases, when a checkpoint was aborted and the instant time
// is reused, the merge handle generates a new file name
// with the reused instant time of last checkpoint, the write handles
// should be kept and reused in case data loss.
this.writeClient.cleanHandles();
}
}
@SuppressWarnings("unchecked, rawtypes")
private boolean flushBucket(DataBucket bucket) {
String instant = instantToWrite(true);
@@ -489,7 +479,7 @@ public class StreamWriteFunction<I> extends AbstractStreamWriteFunction<I> {
this.eventGateway.sendEventToCoordinator(event);
this.buckets.clear();
this.tracer.reset();
cleanWriteHandles();
this.writeClient.cleanHandles();
this.writeStatuses.addAll(writeStatus);
// blocks flushing until the coordinator starts a new instant
this.confirming = true;