lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1792] flink-client query error when processing files larger than 128mb (#2814)

Browse Source 
Co-authored-by: huangjing <huangjing@clinbrain.com>
This commit is contained in:
hj2016
2021-04-16 13:59:19 +08:00
committed by GitHub
parent b6d949b48a
commit 62b8a341dd
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
View File

@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
@@ -209,7 +210,13 @@ public class CopyOnWriteInputFormat extends FileInputFormat<RowData> {
// get the block locations and make sure they are in order with respect to their offset // get the block locations and make sure they are in order with respect to their offset
final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len); final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
Arrays.sort(blocks); Arrays.sort(blocks, new Comparator<BlockLocation>() {
@Override
public int compare(BlockLocation o1, BlockLocation o2) {
long diff = o1.getLength() - o2.getOffset();
return diff < 0L ? -1 : (diff > 0L ? 1 : 0);
}
});
long bytesUnassigned = len; long bytesUnassigned = len;
long position = 0; long position = 0;