From eb4b741c38f34cb5725cd6fa8cae1acbe9d8351a Mon Sep 17 00:00:00 2001 From: jcxiaozf Date: Tue, 17 May 2022 16:05:45 +0800 Subject: [PATCH] If there are multiple files under the same partition path and file ID, sort them according to the modification time of the files to avoid reading the files that failed to write before. --- .../hudi/common/table/view/AbstractTableFileSystemView.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index dc6fc47b5..18e3ee35a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -185,7 +185,11 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV String fileId = pair.getValue(); HoodieFileGroup group = new HoodieFileGroup(pair.getKey(), fileId, timeline); if (baseFiles.containsKey(pair)) { - baseFiles.get(pair).forEach(group::addBaseFile); + // if there are multiple files under the same partition path and file ID, sort them according to the + // modification time of the files to avoid reading the files that failed to write before. + List partitionFileIdPairBaseFiles = baseFiles.get(pair); + partitionFileIdPairBaseFiles.sort(Comparator.comparingLong(o -> o.getFileStatus().getModificationTime())); + partitionFileIdPairBaseFiles.forEach(group::addBaseFile); } if (logFiles.containsKey(pair)) { logFiles.get(pair).forEach(group::addLogFile);