lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1886] Avoid to generates corrupted files for flink sink (#2929)

Browse Source
This commit is contained in:
Danny Chan
2021-05-10 10:43:03 +08:00
committed by GitHub
parent bfbf993cbe
commit c1b331bcff
4 changed files with 49 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkCreateHandle.java
View File

@@ -55,6 +55,7 @@ public class FlinkCreateHandle<T extends HoodieRecordPayload, I, K, O>
private static final Logger LOG = LogManager.getLogger(FlinkCreateHandle.class);
private long lastFileSize = 0L;
private long totalRecordsWritten = 0L;
public FlinkCreateHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T, I, K, O> hoodieTable,
String partitionPath, String fileId, TaskContextSupplier taskContextSupplier) {
@@ -143,6 +144,7 @@ public class FlinkCreateHandle<T extends HoodieRecordPayload, I, K, O>
try {
setupWriteStatus();
// reset the write status
totalRecordsWritten += recordsWritten;
recordsWritten = 0;
recordsDeleted = 0;
insertRecordsWritten = 0;
@@ -169,7 +171,7 @@ public class FlinkCreateHandle<T extends HoodieRecordPayload, I, K, O>
@Override
public void finishWrite() {
LOG.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten);
LOG.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + totalRecordsWritten);
try {
fileWriter.close();
} catch (IOException e) {