lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3438] Avoid getSmallFiles if hoodie.parquet.small.file.limit is 0 (#4823)

Browse Source 
Co-authored-by: Hui An <hui.an@shopee.com>
This commit is contained in:
RexAn
2022-02-18 21:57:04 +08:00
committed by GitHub
parent fba5822ee3
commit 5009138d04
5 changed files with 26 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java
View File

@@ -180,7 +180,14 @@ public class WriteProfile {
if (smallFilesMap.containsKey(partitionPath)) {
return smallFilesMap.get(partitionPath);
}
List<SmallFile> smallFiles = smallFilesProfile(partitionPath);
List<SmallFile> smallFiles = new ArrayList<>();
if (config.getParquetSmallFileLimit() <= 0) {
this.smallFilesMap.put(partitionPath, smallFiles);
return smallFiles;
}
smallFiles = smallFilesProfile(partitionPath);
this.smallFilesMap.put(partitionPath, smallFiles);
return smallFiles;
}