lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1637] Avoid to rename for bucket update when there is only one flush action during a checkpoint (#2599)

Browse Source 
Some of the object storages do not have strong read-after-write
consistency, we should promote to remove the rename operations in the
future.
This commit is contained in:
Danny Chan
2021-02-25 10:21:27 +08:00
committed by GitHub
parent 77ba561a6b
commit 97864a48c1
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
View File

@@ -94,7 +94,8 @@ public class FlinkMergeHandle<T extends HoodieRecordPayload, I, K, O>
* Use the fileId + "-" + rollNumber as the new fileId of a mini-batch write. * Use the fileId + "-" + rollNumber as the new fileId of a mini-batch write.
*/ */
protected String generatesDataFileName() { protected String generatesDataFileName() {
return FSUtils.makeDataFileName(instantTime, writeToken, fileId + "-" + rollNumber, hoodieTable.getBaseFileExtension()); final String fileID = this.needBootStrap ? fileId : fileId + "-" + rollNumber;
return FSUtils.makeDataFileName(instantTime, writeToken, fileID, hoodieTable.getBaseFileExtension());
} }
public boolean isNeedBootStrap() { public boolean isNeedBootStrap() {
@@ -178,6 +179,12 @@ public class FlinkMergeHandle<T extends HoodieRecordPayload, I, K, O>
} }
public void finishWrite() { public void finishWrite() {
// The file visibility should be kept by the configured ConsistencyGuard instance.
if (rolloverPaths.size() == 1) {
// only one flush action, no need to roll over
return;
}
for (int i = 0; i < rolloverPaths.size() - 1; i++) { for (int i = 0; i < rolloverPaths.size() - 1; i++) {
Path path = rolloverPaths.get(i); Path path = rolloverPaths.get(i);
try { try {