lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1398] Align insert file size for reducing IO (#2256)

Browse Source 
* [HUDI-1398] Align insert file size for reducing IO

Co-authored-by: zhang wen <wen.zhang@dmall.com>
This commit is contained in:
steven zhang
2020-12-29 11:52:35 +08:00
committed by GitHub
parent 0ecdec348e
commit 4c17528de0
2 changed files with 22 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java
View File

@@ -218,7 +218,11 @@ public class UpsertPartitioner<T extends HoodieRecordPayload<T>> extends Partiti
+ ", totalInsertBuckets => " + insertBuckets + ", recordsPerBucket => " + insertRecordsPerBucket);
for (int b = 0; b < insertBuckets; b++) {
bucketNumbers.add(totalBuckets);
recordsPerBucket.add(totalUnassignedInserts / insertBuckets);
if (b < insertBuckets - 1) {
recordsPerBucket.add(insertRecordsPerBucket);
} else {
recordsPerBucket.add(totalUnassignedInserts - (insertBuckets - 1) * insertRecordsPerBucket);
}
BucketInfo bucketInfo = new BucketInfo();
bucketInfo.bucketType = BucketType.INSERT;
bucketInfo.partitionPath = partitionPath;