lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add config for index parallelism and make clean public (#109)

Browse Source 
* Add config for index parallelism and make clean public

* Review comments on clean api modification
This commit is contained in:
prazanna
2017-03-21 17:36:46 -07:00
committed by GitHub
parent 21898907c1
commit f1b7afad21
5 changed files with 51 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hoodie-client/src/main/java/com/uber/hoodie/index/HoodieBloomIndex.java
View File

@@ -255,8 +255,11 @@ public class HoodieBloomIndex<T extends HoodieRecordPayload> extends HoodieIndex
for (long subparts : subpartitionCountMap.values()) {
totalSubparts += (int) subparts;
}
int joinParallelism = Math.max(totalSubparts, inputParallelism);
// If bloom index parallelism is set, use it to to check against the input parallelism and take the max
int indexParallelism = Math.max(inputParallelism, config.getBloomIndexParallelism());
int joinParallelism = Math.max(totalSubparts, indexParallelism);
logger.info("InputParallelism: ${" + inputParallelism + "}, " +
"IndexParallelism: ${" + config.getBloomIndexParallelism() + "}, " +
"TotalSubParts: ${" + totalSubparts + "}, " +
"Join Parallelism set to : " + joinParallelism);
return joinParallelism;