lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

adding new config to separate shuffle and write parallelism

Browse Source
This commit is contained in:
Nishith Agarwal
2017-07-26 11:49:22 -07:00
committed by vinoth chandar
parent b1cf097b0c
commit e484e91807
2 changed files with 13 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
View File

@@ -233,7 +233,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
// into 1 RDD partition, while big ones spread evenly across multiple RDD partitions
return String
.format("%s+%s", record.getPartitionPath(), record.getRecordKey());
}, true, config.getInsertShuffleParallelism());
}, true, config.getBulkInsertShuffleParallelism());
JavaRDD<WriteStatus> writeStatusRDD = sortedRecords
.mapPartitionsWithIndex(new BulkInsertMapFunction<T>(commitTime, config, table), true)
.flatMap(writeStatuses -> writeStatuses.iterator());