lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3045] New clustering regex match config to choose partitions when building clustering plan (#4346)

Browse Source 
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
YueZhang
2022-01-12 15:23:55 +08:00
committed by GitHub
parent 017ddbbfac
commit 9fe28e56b4
4 changed files with 125 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
View File

@@ -87,6 +87,12 @@ public class HoodieClusteringConfig extends HoodieConfig {
.sinceVersion("0.7.0")
.withDocumentation("Files smaller than the size specified here are candidates for clustering");
public static final ConfigProperty<String> PARTITION_REGEX_PATTERN = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "partition.regex.pattern")
.noDefaultValue()
.sinceVersion("0.11.0")
.withDocumentation("Filter clustering partitions that matched regex pattern");
public static final ConfigProperty<String> PLAN_STRATEGY_CLASS_NAME = ConfigProperty
.key("hoodie.clustering.plan.strategy.class")
.defaultValue(SPARK_SIZED_BASED_CLUSTERING_PLAN_STRATEGY)
@@ -424,6 +430,11 @@ public class HoodieClusteringConfig extends HoodieConfig {
return this;
}
public Builder withClusteringPartitionRegexPattern(String pattern) {
clusteringConfig.setValue(PARTITION_REGEX_PATTERN, pattern);
return this;
}
public Builder withClusteringSkipPartitionsFromLatest(int clusteringSkipPartitionsFromLatest) {
clusteringConfig.setValue(PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST, String.valueOf(clusteringSkipPartitionsFromLatest));
return this;

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -1252,6 +1252,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return getLong(HoodieClusteringConfig.PLAN_STRATEGY_SMALL_FILE_LIMIT);
}
public String getClusteringPartitionFilterRegexPattern() {
return getString(HoodieClusteringConfig.PARTITION_REGEX_PATTERN);
}
public int getClusteringMaxNumGroups() {
return getInt(HoodieClusteringConfig.PLAN_STRATEGY_MAX_GROUPS);
}

14
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
View File

@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilter;
@@ -35,6 +36,7 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -70,6 +72,8 @@ public abstract class PartitionAwareClusteringPlanStrategy<T extends HoodieRecor
HoodieWriteConfig config = getWriteConfig();
List<String> partitionPaths = FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath());
// get regex matched partitions if set
partitionPaths = getRegexPatternMatchedPartitions(config, partitionPaths);
// filter the partition paths if needed to reduce list status
partitionPaths = filterPartitionPaths(partitionPaths);
@@ -108,4 +112,14 @@ public abstract class PartitionAwareClusteringPlanStrategy<T extends HoodieRecor
.setPreserveHoodieMetadata(getWriteConfig().isPreserveHoodieCommitMetadataForClustering())
.build());
}
public List<String> getRegexPatternMatchedPartitions(HoodieWriteConfig config, List<String> partitionPaths) {
String pattern = config.getClusteringPartitionFilterRegexPattern();
if (!StringUtils.isNullOrEmpty(pattern)) {
partitionPaths = partitionPaths.stream()
.filter(partition -> Pattern.matches(pattern, partition))
.collect(Collectors.toList());
}
return partitionPaths;
}
}