lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4210] Create custom hbase index to solve data skew issue on hbase regions (#5797)

Browse Source
This commit is contained in:
冯健
2022-07-26 18:09:17 +08:00
committed by GitHub
parent 1ea1e659c2
commit e5faf2cc84
4 changed files with 53 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java
View File

@@ -183,6 +183,10 @@ public class HoodieHBaseIndexConfig extends HoodieConfig {
.noDefaultValue()
.withDocumentation("The value of hbase.master.kerberos.principal in hbase cluster.");
public static final ConfigProperty<Integer> BUCKET_NUMBER = ConfigProperty
.key("hoodie.index.hbase.bucket.number")
.defaultValue(8)
.withDocumentation("Only applicable when using RebalancedSparkHoodieHBaseIndex, same as hbase regions count can get the best performance");
/**
* @deprecated Use {@link #ZKQUORUM} and its methods instead

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -1553,6 +1553,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return getBooleanOrDefault(HoodieHBaseIndexConfig.UPDATE_PARTITION_PATH_ENABLE);
}
public int getHBaseIndexRegionCount() {
return getInt(HoodieHBaseIndexConfig.BUCKET_NUMBER);
}
public int getBloomIndexParallelism() {
return getInt(HoodieIndexConfig.BLOOM_INDEX_PARALLELISM);
}