lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-772] Make UserDefinedBulkInsertPartitioner configurable for DataSource (#1500)

Browse Source
This commit is contained in:
Dongwook
2020-04-20 08:38:18 -07:00
committed by GitHub
parent 09fd6f64c5
commit ddd105bb31
4 changed files with 134 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -57,6 +57,7 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private static final String DEFAULT_PARALLELISM = "1500";
private static final String INSERT_PARALLELISM = "hoodie.insert.shuffle.parallelism";
private static final String BULKINSERT_PARALLELISM = "hoodie.bulkinsert.shuffle.parallelism";
private static final String BULKINSERT_USER_DEFINED_PARTITIONER_CLASS = "hoodie.bulkinsert.user.defined.partitioner.class";
private static final String UPSERT_PARALLELISM = "hoodie.upsert.shuffle.parallelism";
private static final String DELETE_PARALLELISM = "hoodie.delete.shuffle.parallelism";
private static final String DEFAULT_ROLLBACK_PARALLELISM = "100";
@@ -157,6 +158,10 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return Integer.parseInt(props.getProperty(BULKINSERT_PARALLELISM));
}
public String getUserDefinedBulkInsertPartitionerClass() {
return props.getProperty(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS);
}
public int getInsertShuffleParallelism() {
return Integer.parseInt(props.getProperty(INSERT_PARALLELISM));
}
@@ -603,6 +608,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return this;
}
public Builder withUserDefinedBulkInsertPartitionerClass(String className) {
props.setProperty(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS, className);
return this;
}
public Builder withParallelism(int insertShuffleParallelism, int upsertShuffleParallelism) {
props.setProperty(INSERT_PARALLELISM, String.valueOf(insertShuffleParallelism));
props.setProperty(UPSERT_PARALLELISM, String.valueOf(upsertShuffleParallelism));