From 62b8a341ddae0ab80195c41c7a44b84c1fe23d31 Mon Sep 17 00:00:00 2001 From: hj2016 Date: Fri, 16 Apr 2021 13:59:19 +0800 Subject: [PATCH] [HUDI-1792] flink-client query error when processing files larger than 128mb (#2814) Co-authored-by: huangjing --- .../hudi/table/format/cow/CopyOnWriteInputFormat.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java index 77f3e2a10..477f54bf0 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java +++ b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java @@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -209,7 +210,13 @@ public class CopyOnWriteInputFormat extends FileInputFormat { // get the block locations and make sure they are in order with respect to their offset final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len); - Arrays.sort(blocks); + Arrays.sort(blocks, new Comparator() { + @Override + public int compare(BlockLocation o1, BlockLocation o2) { + long diff = o1.getLength() - o2.getOffset(); + return diff < 0L ? -1 : (diff > 0L ? 1 : 0); + } + }); long bytesUnassigned = len; long position = 0;