lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

If there are multiple files under the same partition path and file ID, sort them according to the modification time of the files to avoid reading the files that failed to write before.

Browse Source
This commit is contained in:
jcxiaozf
2022-05-17 16:05:45 +08:00
committed by v-zhangjc9
parent 5c4908f006
commit eb4b741c38
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
View File

@@ -185,7 +185,11 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
String fileId = pair.getValue(); String fileId = pair.getValue();
HoodieFileGroup group = new HoodieFileGroup(pair.getKey(), fileId, timeline); HoodieFileGroup group = new HoodieFileGroup(pair.getKey(), fileId, timeline);
if (baseFiles.containsKey(pair)) { if (baseFiles.containsKey(pair)) {
baseFiles.get(pair).forEach(group::addBaseFile); // if there are multiple files under the same partition path and file ID, sort them according to the
// modification time of the files to avoid reading the files that failed to write before.
List<HoodieBaseFile> partitionFileIdPairBaseFiles = baseFiles.get(pair);
partitionFileIdPairBaseFiles.sort(Comparator.comparingLong(o -> o.getFileStatus().getModificationTime()));
partitionFileIdPairBaseFiles.forEach(group::addBaseFile);
} }
if (logFiles.containsKey(pair)) { if (logFiles.containsKey(pair)) {
logFiles.get(pair).forEach(group::addLogFile); logFiles.get(pair).forEach(group::addLogFile);