lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Restore 0.8.0 config keys with deprecated annotation (#3506)

Browse Source 
Co-authored-by: Sagar Sumit <sagarsumit09@gmail.com>
Co-authored-by: Vinoth Chandar <vinoth@apache.org>
This commit is contained in:
Udit Mehrotra
2021-08-19 13:36:40 -07:00
committed by GitHub
parent 37c29e75dc
commit c350d05dd3
137 changed files with 3460 additions and 1527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java
View File

@@ -79,7 +79,7 @@ public class HoodieWriteCommitHttpCallbackClient implements Closeable {
}
private String getApiKey() {
return writeConfig.getString(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_API_KEY);
return writeConfig.getString(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_API_KEY_VALUE);
}
private String getUrl() {
@@ -97,7 +97,7 @@ public class HoodieWriteCommitHttpCallbackClient implements Closeable {
}
private Integer getHttpTimeoutSeconds() {
return writeConfig.getInt(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_TIMEOUT_SECONDS);
return writeConfig.getInt(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_TIMEOUT_IN_SECONDS);
}
@Override

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java
View File

@@ -39,7 +39,7 @@ public class HoodieCommitCallbackFactory {
return (HoodieWriteCommitCallback) instance;
} else {
throw new HoodieCommitCallbackException(String.format("The value of the config option %s can not be null or "
+ "empty", HoodieWriteCommitCallbackConfig.CALLBACK_CLASS.key()));
+ "empty", HoodieWriteCommitCallbackConfig.CALLBACK_CLASS_NAME.key()));
}
}

118
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java
View File

@@ -44,57 +44,55 @@ import java.util.Properties;
+ " writers and new hudi writers in parallel, to validate the migration.")
public class HoodieBootstrapConfig extends HoodieConfig {
public static final ConfigProperty<String> BOOTSTRAP_BASE_PATH = ConfigProperty
public static final ConfigProperty<String> BASE_PATH = ConfigProperty
.key("hoodie.bootstrap.base.path")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Base path of the dataset that needs to be bootstrapped as a Hudi table");
@Deprecated
public static final String BOOTSTRAP_BASE_PATH_PROP = BOOTSTRAP_BASE_PATH.key();
public static final ConfigProperty<String> BOOTSTRAP_MODE_SELECTOR = ConfigProperty
public static final ConfigProperty<String> MODE_SELECTOR_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.mode.selector")
.defaultValue(MetadataOnlyBootstrapModeSelector.class.getCanonicalName())
.sinceVersion("0.6.0")
.withDocumentation("Selects the mode in which each file/partition in the bootstrapped dataset gets bootstrapped");
public static final ConfigProperty<String> FULL_BOOTSTRAP_INPUT_PROVIDER = ConfigProperty
public static final ConfigProperty<String> FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.full.input.provider")
.defaultValue("org.apache.hudi.bootstrap.SparkParquetBootstrapDataProvider")
.sinceVersion("0.6.0")
.withDocumentation("Class to use for reading the bootstrap dataset partitions/files, for Bootstrap mode FULL_RECORD");
public static final ConfigProperty<String> BOOTSTRAP_KEYGEN_CLASS = ConfigProperty
public static final ConfigProperty<String> KEYGEN_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.keygen.class")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Key generator implementation to be used for generating keys from the bootstrapped dataset");
public static final ConfigProperty<String> BOOTSTRAP_KEYGEN_TYPE = ConfigProperty
public static final ConfigProperty<String> KEYGEN_TYPE = ConfigProperty
.key("hoodie.bootstrap.keygen.type")
.defaultValue(KeyGeneratorType.SIMPLE.name())
.sinceVersion("0.9.0")
.withDocumentation("Type of build-in key generator, currently support SIMPLE, COMPLEX, TIMESTAMP, CUSTOM, NON_PARTITION, GLOBAL_DELETE");
public static final ConfigProperty<String> BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = ConfigProperty
public static final ConfigProperty<String> PARTITION_PATH_TRANSLATOR_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.partitionpath.translator.class")
.defaultValue(IdentityBootstrapPartitionPathTranslator.class.getName())
.sinceVersion("0.6.0")
.withDocumentation("Translates the partition paths from the bootstrapped data into how is laid out as a Hudi table.");
public static final ConfigProperty<String> BOOTSTRAP_PARALLELISM = ConfigProperty
public static final ConfigProperty<String> PARALLELISM_VALUE = ConfigProperty
.key("hoodie.bootstrap.parallelism")
.defaultValue("1500")
.sinceVersion("0.6.0")
.withDocumentation("Parallelism value to be used to bootstrap data into hudi");
public static final ConfigProperty<String> BOOTSTRAP_MODE_SELECTOR_REGEX = ConfigProperty
public static final ConfigProperty<String> PARTITION_SELECTOR_REGEX_PATTERN = ConfigProperty
.key("hoodie.bootstrap.mode.selector.regex")
.defaultValue(".*")
.sinceVersion("0.6.0")
.withDocumentation("Matches each bootstrap dataset partition against this regex and applies the mode below to it.");
public static final ConfigProperty<String> BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = ConfigProperty
public static final ConfigProperty<String> PARTITION_SELECTOR_REGEX_MODE = ConfigProperty
.key("hoodie.bootstrap.mode.selector.regex.mode")
.defaultValue(BootstrapMode.METADATA_ONLY.name())
.sinceVersion("0.6.0")
@@ -102,13 +100,87 @@ public class HoodieBootstrapConfig extends HoodieConfig {
+ "METADATA_ONLY will generate just skeleton base files with keys/footers, avoiding full cost of rewriting the dataset. "
+ "FULL_RECORD will perform a full copy/rewrite of the data as a Hudi table.");
public static final ConfigProperty<String> BOOTSTRAP_INDEX_CLASS = ConfigProperty
public static final ConfigProperty<String> INDEX_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.index.class")
.defaultValue(HFileBootstrapIndex.class.getName())
.sinceVersion("0.6.0")
.withDocumentation("Implementation to use, for mapping a skeleton base file to a boostrap base file.");
/**
* @deprecated Use {@link #BASE_PATH} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_INDEX_CLASS_PROP = BOOTSTRAP_INDEX_CLASS.key();
public static final String BOOTSTRAP_BASE_PATH_PROP = BASE_PATH.key();
/**
* @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_INDEX_CLASS_PROP = INDEX_CLASS_NAME.key();
/**
* @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = INDEX_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #MODE_SELECTOR_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_MODE_SELECTOR = MODE_SELECTOR_CLASS_NAME.key();
/**
* @deprecated Use {@link #FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String FULL_BOOTSTRAP_INPUT_PROVIDER = FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.key();
/**
* @deprecated Use {@link #FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_FULL_BOOTSTRAP_INPUT_PROVIDER = FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #KEYGEN_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_KEYGEN_CLASS = KEYGEN_CLASS_NAME.key();
/**
* @deprecated Use {@link #PARTITION_PATH_TRANSLATOR_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = PARTITION_PATH_TRANSLATOR_CLASS_NAME.key();
/**
* @deprecated Use {@link #PARTITION_PATH_TRANSLATOR_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = PARTITION_PATH_TRANSLATOR_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #PARALLELISM_VALUE} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_PARALLELISM = PARALLELISM_VALUE.key();
/**
* @deprecated Use {@link #PARALLELISM_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BOOTSTRAP_PARALLELISM = PARALLELISM_VALUE.defaultValue();
/**
* @deprecated Use {@link #PARTITION_SELECTOR_REGEX_PATTERN} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_MODE_SELECTOR_REGEX = PARTITION_SELECTOR_REGEX_PATTERN.key();
/**
* @deprecated Use {@link #PARTITION_SELECTOR_REGEX_MODE} and its methods instead
*/
@Deprecated
public static final String BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = PARTITION_SELECTOR_REGEX_MODE.key();
/**
* @deprecated Use {@link #PARTITION_SELECTOR_REGEX_PATTERN} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BOOTSTRAP_MODE_SELECTOR_REGEX = PARTITION_SELECTOR_REGEX_PATTERN.defaultValue();
/**
* @deprecated Use {@link #PARTITION_SELECTOR_REGEX_MODE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = PARTITION_SELECTOR_REGEX_MODE.defaultValue();
private HoodieBootstrapConfig() {
super();
@@ -130,48 +202,48 @@ public class HoodieBootstrapConfig extends HoodieConfig {
}
public Builder withBootstrapBasePath(String basePath) {
bootstrapConfig.setValue(BOOTSTRAP_BASE_PATH, basePath);
bootstrapConfig.setValue(BASE_PATH, basePath);
return this;
}
public Builder withBootstrapModeSelector(String partitionSelectorClass) {
bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR, partitionSelectorClass);
bootstrapConfig.setValue(MODE_SELECTOR_CLASS_NAME, partitionSelectorClass);
return this;
}
public Builder withFullBootstrapInputProvider(String partitionSelectorClass) {
bootstrapConfig.setValue(FULL_BOOTSTRAP_INPUT_PROVIDER, partitionSelectorClass);
bootstrapConfig.setValue(FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME, partitionSelectorClass);
return this;
}
public Builder withBootstrapKeyGenClass(String keyGenClass) {
bootstrapConfig.setValue(BOOTSTRAP_KEYGEN_CLASS, keyGenClass);
bootstrapConfig.setValue(KEYGEN_CLASS_NAME, keyGenClass);
return this;
}
public Builder withBootstrapKeyGenType(String keyGenType) {
bootstrapConfig.setValue(BOOTSTRAP_KEYGEN_TYPE, keyGenType);
bootstrapConfig.setValue(KEYGEN_TYPE, keyGenType);
return this;
}
public Builder withBootstrapPartitionPathTranslatorClass(String partitionPathTranslatorClass) {
bootstrapConfig
.setValue(BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS, partitionPathTranslatorClass);
.setValue(PARTITION_PATH_TRANSLATOR_CLASS_NAME, partitionPathTranslatorClass);
return this;
}
public Builder withBootstrapParallelism(int parallelism) {
bootstrapConfig.setValue(BOOTSTRAP_PARALLELISM, String.valueOf(parallelism));
bootstrapConfig.setValue(PARALLELISM_VALUE, String.valueOf(parallelism));
return this;
}
public Builder withBootstrapModeSelectorRegex(String regex) {
bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR_REGEX, regex);
bootstrapConfig.setValue(PARTITION_SELECTOR_REGEX_PATTERN, regex);
return this;
}
public Builder withBootstrapModeForRegexMatch(BootstrapMode modeForRegexMatch) {
bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, modeForRegexMatch.name());
bootstrapConfig.setValue(PARTITION_SELECTOR_REGEX_MODE, modeForRegexMatch.name());
return this;
}
@@ -182,7 +254,7 @@ public class HoodieBootstrapConfig extends HoodieConfig {
public HoodieBootstrapConfig build() {
// TODO: use infer function instead
bootstrapConfig.setDefaultValue(BOOTSTRAP_INDEX_CLASS, HoodieTableConfig.getDefaultBootstrapIndexClass(
bootstrapConfig.setDefaultValue(INDEX_CLASS_NAME, HoodieTableConfig.getDefaultBootstrapIndexClass(
bootstrapConfig.getProps()));
bootstrapConfig.setDefaults(HoodieBootstrapConfig.class.getName());
return bootstrapConfig;

184
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
View File

@@ -40,21 +40,21 @@ public class HoodieClusteringConfig extends HoodieConfig {
// Any strategy specific params can be saved with this prefix
public static final String CLUSTERING_STRATEGY_PARAM_PREFIX = "hoodie.clustering.plan.strategy.";
public static final ConfigProperty<String> CLUSTERING_TARGET_PARTITIONS = ConfigProperty
public static final ConfigProperty<String> DAYBASED_LOOKBACK_PARTITIONS = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.lookback.partitions")
.defaultValue("2")
.sinceVersion("0.7.0")
.withDocumentation("Number of partitions to list to create ClusteringPlan");
public static final ConfigProperty<String> CLUSTERING_PLAN_STRATEGY_CLASS = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_CLASS_NAME = ConfigProperty
.key("hoodie.clustering.plan.strategy.class")
.defaultValue("org.apache.hudi.client.clustering.plan.strategy.SparkRecentDaysClusteringPlanStrategy")
.sinceVersion("0.7.0")
.withDocumentation("Config to provide a strategy class (subclass of ClusteringPlanStrategy) to create clustering plan "
+ "i.e select what file groups are being clustered. Default strategy, looks at the last N (determined by "
+ CLUSTERING_TARGET_PARTITIONS.key() + ") day based partitions picks the small file slices within those partitions.");
+ DAYBASED_LOOKBACK_PARTITIONS.key() + ") day based partitions picks the small file slices within those partitions.");
public static final ConfigProperty<String> CLUSTERING_EXECUTION_STRATEGY_CLASS = ConfigProperty
public static final ConfigProperty<String> EXECUTION_STRATEGY_CLASS_NAME = ConfigProperty
.key("hoodie.clustering.execution.strategy.class")
.defaultValue("org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy")
.sinceVersion("0.7.0")
@@ -67,36 +67,32 @@ public class HoodieClusteringConfig extends HoodieConfig {
.defaultValue("false")
.sinceVersion("0.7.0")
.withDocumentation("Turn on inline clustering - clustering will be run after each write operation is complete");
@Deprecated
public static final String INLINE_CLUSTERING_PROP = INLINE_CLUSTERING.key();
public static final ConfigProperty<String> INLINE_CLUSTERING_MAX_COMMIT = ConfigProperty
public static final ConfigProperty<String> INLINE_CLUSTERING_MAX_COMMITS = ConfigProperty
.key("hoodie.clustering.inline.max.commits")
.defaultValue("4")
.sinceVersion("0.7.0")
.withDocumentation("Config to control frequency of clustering planning");
@Deprecated
public static final String INLINE_CLUSTERING_MAX_COMMIT_PROP = INLINE_CLUSTERING_MAX_COMMIT.key();
public static final ConfigProperty<String> ASYNC_CLUSTERING_MAX_COMMIT_PROP = ConfigProperty
public static final ConfigProperty<String> ASYNC_CLUSTERING_MAX_COMMITS = ConfigProperty
.key("hoodie.clustering.async.max.commits")
.defaultValue("4")
.sinceVersion("0.9.0")
.withDocumentation("Config to control frequency of async clustering");
public static final ConfigProperty<String> CLUSTERING_SKIP_PARTITIONS_FROM_LATEST = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.skipfromlatest.partitions")
.defaultValue("0")
.sinceVersion("0.9.0")
.withDocumentation("Number of partitions to skip from latest when choosing partitions to create ClusteringPlan");
public static final ConfigProperty<String> PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.skipfromlatest.partitions")
.defaultValue("0")
.sinceVersion("0.9.0")
.withDocumentation("Number of partitions to skip from latest when choosing partitions to create ClusteringPlan");
public static final ConfigProperty<String> CLUSTERING_PLAN_SMALL_FILE_LIMIT = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_SMALL_FILE_LIMIT = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "small.file.limit")
.defaultValue(String.valueOf(600 * 1024 * 1024L))
.sinceVersion("0.7.0")
.withDocumentation("Files smaller than the size specified here are candidates for clustering");
public static final ConfigProperty<String> CLUSTERING_MAX_BYTES_PER_GROUP = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "max.bytes.per.group")
.defaultValue(String.valueOf(2 * 1024 * 1024 * 1024L))
.sinceVersion("0.7.0")
@@ -104,46 +100,156 @@ public class HoodieClusteringConfig extends HoodieConfig {
+ " is defined by below two properties (CLUSTERING_MAX_BYTES_PER_GROUP * CLUSTERING_MAX_NUM_GROUPS)."
+ " Max amount of data to be included in one group");
public static final ConfigProperty<String> CLUSTERING_MAX_NUM_GROUPS = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_MAX_GROUPS = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "max.num.groups")
.defaultValue("30")
.sinceVersion("0.7.0")
.withDocumentation("Maximum number of groups to create as part of ClusteringPlan. Increasing groups will increase parallelism");
public static final ConfigProperty<String> CLUSTERING_TARGET_FILE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_TARGET_FILE_MAX_BYTES = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "target.file.max.bytes")
.defaultValue(String.valueOf(1024 * 1024 * 1024L))
.sinceVersion("0.7.0")
.withDocumentation("Each group can produce 'N' (CLUSTERING_MAX_GROUP_SIZE/CLUSTERING_TARGET_FILE_SIZE) output file groups");
public static final ConfigProperty<String> CLUSTERING_SORT_COLUMNS_PROPERTY = ConfigProperty
public static final ConfigProperty<String> PLAN_STRATEGY_SORT_COLUMNS = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "sort.columns")
.noDefaultValue()
.sinceVersion("0.7.0")
.withDocumentation("Columns to sort the data by when clustering");
public static final ConfigProperty<String> CLUSTERING_UPDATES_STRATEGY = ConfigProperty
public static final ConfigProperty<String> UPDATES_STRATEGY = ConfigProperty
.key("hoodie.clustering.updates.strategy")
.defaultValue("org.apache.hudi.client.clustering.update.strategy.SparkRejectUpdateStrategy")
.sinceVersion("0.7.0")
.withDocumentation("Determines how to handle updates, deletes to file groups that are under clustering."
+ " Default strategy just rejects the update");
@Deprecated
public static final String CLUSTERING_UPDATES_STRATEGY_PROP = CLUSTERING_UPDATES_STRATEGY.key();
public static final ConfigProperty<String> ASYNC_CLUSTERING_ENABLE = ConfigProperty
.key("hoodie.clustering.async.enabled")
.defaultValue("false")
.sinceVersion("0.7.0")
.withDocumentation("Enable running of clustering service, asynchronously as inserts happen on the table.");
@Deprecated
public static final String ASYNC_CLUSTERING_ENABLE_OPT_KEY = "hoodie.clustering.async.enabled";
public static final ConfigProperty<Boolean> CLUSTERING_PRESERVE_HOODIE_COMMIT_METADATA = ConfigProperty
public static final ConfigProperty<Boolean> PRESERVE_COMMIT_METADATA = ConfigProperty
.key("hoodie.clustering.preserve.commit.metadata")
.defaultValue(false)
.sinceVersion("0.9.0")
.withDocumentation("When rewriting data, preserves existing hoodie_commit_time");
/**
* @deprecated Use {@link #PLAN_STRATEGY_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_PLAN_STRATEGY_CLASS = PLAN_STRATEGY_CLASS_NAME.key();
/**
* @deprecated Use {@link #PLAN_STRATEGY_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_PLAN_STRATEGY_CLASS = PLAN_STRATEGY_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #EXECUTION_STRATEGY_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_EXECUTION_STRATEGY_CLASS = EXECUTION_STRATEGY_CLASS_NAME.key();
/**
* @deprecated Use {@link #EXECUTION_STRATEGY_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_EXECUTION_STRATEGY_CLASS = EXECUTION_STRATEGY_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #INLINE_CLUSTERING} and its methods instead
*/
@Deprecated
public static final String INLINE_CLUSTERING_PROP = INLINE_CLUSTERING.key();
/**
* @deprecated Use {@link #INLINE_CLUSTERING} and its methods instead
*/
@Deprecated
private static final String DEFAULT_INLINE_CLUSTERING = INLINE_CLUSTERING.defaultValue();
/**
* @deprecated Use {@link #INLINE_CLUSTERING_MAX_COMMITS} and its methods instead
*/
@Deprecated
public static final String INLINE_CLUSTERING_MAX_COMMIT_PROP = INLINE_CLUSTERING_MAX_COMMITS.key();
/**
* @deprecated Use {@link #INLINE_CLUSTERING_MAX_COMMITS} and its methods instead
*/
@Deprecated
private static final String DEFAULT_INLINE_CLUSTERING_NUM_COMMITS = INLINE_CLUSTERING_MAX_COMMITS.defaultValue();
/**
* @deprecated Use {@link #DAYBASED_LOOKBACK_PARTITIONS} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_TARGET_PARTITIONS = DAYBASED_LOOKBACK_PARTITIONS.key();
/**
* @deprecated Use {@link #DAYBASED_LOOKBACK_PARTITIONS} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_TARGET_PARTITIONS = DAYBASED_LOOKBACK_PARTITIONS.defaultValue();
/**
* @deprecated Use {@link #PLAN_STRATEGY_SMALL_FILE_LIMIT} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_PLAN_SMALL_FILE_LIMIT = PLAN_STRATEGY_SMALL_FILE_LIMIT.key();
/**
* @deprecated Use {@link #PLAN_STRATEGY_SMALL_FILE_LIMIT} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_PLAN_SMALL_FILE_LIMIT = PLAN_STRATEGY_SMALL_FILE_LIMIT.defaultValue();
/**
* @deprecated Use {@link #PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_MAX_BYTES_PER_GROUP = PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP.key();
/**
* @deprecated Use {@link #PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_MAX_GROUP_SIZE = PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP.defaultValue();
/**
* @deprecated Use {@link #PLAN_STRATEGY_MAX_GROUPS} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_MAX_NUM_GROUPS = PLAN_STRATEGY_MAX_GROUPS.key();
/**
* @deprecated Use {@link #PLAN_STRATEGY_MAX_GROUPS} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_MAX_NUM_GROUPS = PLAN_STRATEGY_MAX_GROUPS.defaultValue();
/**
* @deprecated Use {@link #PLAN_STRATEGY_TARGET_FILE_MAX_BYTES} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_TARGET_FILE_MAX_BYTES = PLAN_STRATEGY_TARGET_FILE_MAX_BYTES.key();
/**
* @deprecated Use {@link #PLAN_STRATEGY_TARGET_FILE_MAX_BYTES} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_TARGET_FILE_MAX_BYTES = PLAN_STRATEGY_TARGET_FILE_MAX_BYTES.defaultValue();
/**
* @deprecated Use {@link #PLAN_STRATEGY_SORT_COLUMNS} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_SORT_COLUMNS_PROPERTY = PLAN_STRATEGY_SORT_COLUMNS.key();
/**
* @deprecated Use {@link #UPDATES_STRATEGY} and its methods instead
*/
@Deprecated
public static final String CLUSTERING_UPDATES_STRATEGY_PROP = UPDATES_STRATEGY.key();
/**
* @deprecated Use {@link #UPDATES_STRATEGY} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLUSTERING_UPDATES_STRATEGY = UPDATES_STRATEGY.defaultValue();
/**
* @deprecated Use {@link #ASYNC_CLUSTERING_ENABLE} and its methods instead
*/
@Deprecated
public static final String ASYNC_CLUSTERING_ENABLE_OPT_KEY = ASYNC_CLUSTERING_ENABLE.key();
/** @deprecated Use {@link #ASYNC_CLUSTERING_ENABLE} and its methods instead */
@Deprecated
public static final String DEFAULT_ASYNC_CLUSTERING_ENABLE_OPT_VAL = ASYNC_CLUSTERING_ENABLE.defaultValue();
public HoodieClusteringConfig() {
super();
@@ -165,47 +271,47 @@ public class HoodieClusteringConfig extends HoodieConfig {
}
public Builder withClusteringPlanStrategyClass(String clusteringStrategyClass) {
clusteringConfig.setValue(CLUSTERING_PLAN_STRATEGY_CLASS, clusteringStrategyClass);
clusteringConfig.setValue(PLAN_STRATEGY_CLASS_NAME, clusteringStrategyClass);
return this;
}
public Builder withClusteringExecutionStrategyClass(String runClusteringStrategyClass) {
clusteringConfig.setValue(CLUSTERING_EXECUTION_STRATEGY_CLASS, runClusteringStrategyClass);
clusteringConfig.setValue(EXECUTION_STRATEGY_CLASS_NAME, runClusteringStrategyClass);
return this;
}
public Builder withClusteringTargetPartitions(int clusteringTargetPartitions) {
clusteringConfig.setValue(CLUSTERING_TARGET_PARTITIONS, String.valueOf(clusteringTargetPartitions));
clusteringConfig.setValue(DAYBASED_LOOKBACK_PARTITIONS, String.valueOf(clusteringTargetPartitions));
return this;
}
public Builder withClusteringSkipPartitionsFromLatest(int clusteringSkipPartitionsFromLatest) {
clusteringConfig.setValue(CLUSTERING_SKIP_PARTITIONS_FROM_LATEST, String.valueOf(clusteringSkipPartitionsFromLatest));
clusteringConfig.setValue(PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST, String.valueOf(clusteringSkipPartitionsFromLatest));
return this;
}
public Builder withClusteringPlanSmallFileLimit(long clusteringSmallFileLimit) {
clusteringConfig.setValue(CLUSTERING_PLAN_SMALL_FILE_LIMIT, String.valueOf(clusteringSmallFileLimit));
clusteringConfig.setValue(PLAN_STRATEGY_SMALL_FILE_LIMIT, String.valueOf(clusteringSmallFileLimit));
return this;
}
public Builder withClusteringSortColumns(String sortColumns) {
clusteringConfig.setValue(CLUSTERING_SORT_COLUMNS_PROPERTY, sortColumns);
clusteringConfig.setValue(PLAN_STRATEGY_SORT_COLUMNS, sortColumns);
return this;
}
public Builder withClusteringMaxBytesInGroup(long clusteringMaxGroupSize) {
clusteringConfig.setValue(CLUSTERING_MAX_BYTES_PER_GROUP, String.valueOf(clusteringMaxGroupSize));
clusteringConfig.setValue(PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP, String.valueOf(clusteringMaxGroupSize));
return this;
}
public Builder withClusteringMaxNumGroups(int maxNumGroups) {
clusteringConfig.setValue(CLUSTERING_MAX_NUM_GROUPS, String.valueOf(maxNumGroups));
clusteringConfig.setValue(PLAN_STRATEGY_MAX_GROUPS, String.valueOf(maxNumGroups));
return this;
}
public Builder withClusteringTargetFileMaxBytes(long targetFileSize) {
clusteringConfig.setValue(CLUSTERING_TARGET_FILE_MAX_BYTES, String.valueOf(targetFileSize));
clusteringConfig.setValue(PLAN_STRATEGY_TARGET_FILE_MAX_BYTES, String.valueOf(targetFileSize));
return this;
}
@@ -215,12 +321,12 @@ public class HoodieClusteringConfig extends HoodieConfig {
}
public Builder withInlineClusteringNumCommits(int numCommits) {
clusteringConfig.setValue(INLINE_CLUSTERING_MAX_COMMIT, String.valueOf(numCommits));
clusteringConfig.setValue(INLINE_CLUSTERING_MAX_COMMITS, String.valueOf(numCommits));
return this;
}
public Builder withAsyncClusteringMaxCommits(int numCommits) {
clusteringConfig.setValue(ASYNC_CLUSTERING_MAX_COMMIT_PROP, String.valueOf(numCommits));
clusteringConfig.setValue(ASYNC_CLUSTERING_MAX_COMMITS, String.valueOf(numCommits));
return this;
}
@@ -230,7 +336,7 @@ public class HoodieClusteringConfig extends HoodieConfig {
}
public Builder withClusteringUpdatesStrategy(String updatesStrategyClass) {
clusteringConfig.setValue(CLUSTERING_UPDATES_STRATEGY, updatesStrategyClass);
clusteringConfig.setValue(UPDATES_STRATEGY, updatesStrategyClass);
return this;
}
@@ -240,7 +346,7 @@ public class HoodieClusteringConfig extends HoodieConfig {
}
public Builder withPreserveHoodieCommitMetadata(Boolean preserveHoodieCommitMetadata) {
clusteringConfig.setValue(CLUSTERING_PRESERVE_HOODIE_COMMIT_METADATA, String.valueOf(preserveHoodieCommitMetadata));
clusteringConfig.setValue(PRESERVE_COMMIT_METADATA, String.valueOf(preserveHoodieCommitMetadata));
return this;
}

297
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
View File

@@ -56,24 +56,18 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("When enabled, the cleaner table service is invoked immediately after each commit,"
+ " to delete older file slices. It's recommended to enable this, to ensure metadata and data storage"
+ " growth is bounded.");
@Deprecated
public static final String AUTO_CLEAN_PROP = AUTO_CLEAN.key();
public static final ConfigProperty<String> ASYNC_CLEAN = ConfigProperty
.key("hoodie.clean.async")
.defaultValue("false")
.withDocumentation("Only applies when " + AUTO_CLEAN.key() + " is turned on. "
+ "When turned on runs cleaner async with writing, which can speed up overall write performance.");
@Deprecated
public static final String ASYNC_CLEAN_PROP = ASYNC_CLEAN.key();;
public static final ConfigProperty<String> CLEANER_COMMITS_RETAINED = ConfigProperty
.key("hoodie.cleaner.commits.retained")
.defaultValue("10")
.withDocumentation("Number of commits to retain, without cleaning. This will be retained for num_of_commits * time_between_commits "
+ "(scheduled). This also directly translates into how much data retention the table supports for incremental queries.");
@Deprecated
public static final String CLEANER_COMMITS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key();;
public static final ConfigProperty<String> CLEANER_POLICY = ConfigProperty
.key("hoodie.cleaner.policy")
@@ -82,48 +76,36 @@ public class HoodieCompactionConfig extends HoodieConfig {
+ " By default, cleaner spares the file slices written by the last N commits, determined by " + CLEANER_COMMITS_RETAINED.key()
+ " Long running query plans may often refer to older file slices and will break if those are cleaned, before the query has had"
+ " a chance to run. So, it is good to make sure that the data is retained for more than the maximum query execution time");
@Deprecated
public static final String CLEANER_POLICY_PROP = CLEANER_POLICY.key();
public static final ConfigProperty<String> INLINE_COMPACT = ConfigProperty
.key("hoodie.compact.inline")
.defaultValue("false")
.withDocumentation("When set to true, compaction service is triggered after each write. While being "
+ " simpler operationally, this adds extra latency on the write path.");
@Deprecated
public static final String INLINE_COMPACT_PROP = INLINE_COMPACT.key();
public static final ConfigProperty<String> INLINE_COMPACT_NUM_DELTA_COMMITS = ConfigProperty
.key("hoodie.compact.inline.max.delta.commits")
.defaultValue("5")
.withDocumentation("Number of delta commits after the last compaction, before scheduling of a new compaction is attempted.");
@Deprecated
public static final String INLINE_COMPACT_NUM_DELTA_COMMITS_PROP = INLINE_COMPACT_NUM_DELTA_COMMITS.key();
public static final ConfigProperty<String> INLINE_COMPACT_TIME_DELTA_SECONDS = ConfigProperty
.key("hoodie.compact.inline.max.delta.seconds")
.defaultValue(String.valueOf(60 * 60))
.withDocumentation("Number of elapsed seconds after the last compaction, before scheduling a new one.");
@Deprecated
public static final String INLINE_COMPACT_TIME_DELTA_SECONDS_PROP = INLINE_COMPACT_TIME_DELTA_SECONDS.key();
public static final ConfigProperty<String> INLINE_COMPACT_TRIGGER_STRATEGY = ConfigProperty
.key("hoodie.compact.inline.trigger.strategy")
.defaultValue(CompactionTriggerStrategy.NUM_COMMITS.name())
.withDocumentation("Controls how compaction scheduling is triggered, by time or num delta commits or combination of both. "
+ "Valid options: " + Arrays.stream(CompactionTriggerStrategy.values()).map(Enum::name).collect(Collectors.joining(",")));
@Deprecated
public static final String INLINE_COMPACT_TRIGGER_STRATEGY_PROP = INLINE_COMPACT_TRIGGER_STRATEGY.key();
public static final ConfigProperty<String> CLEANER_FILE_VERSIONS_RETAINED = ConfigProperty
.key("hoodie.cleaner.fileversions.retained")
.defaultValue("3")
.withDocumentation("When " + HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS.name() + " cleaning policy is used, "
+ " the minimum number of file slices to retain in each file group, during cleaning.");
@Deprecated
public static final String CLEANER_FILE_VERSIONS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key();
public static final ConfigProperty<String> CLEANER_INCREMENTAL_MODE = ConfigProperty
public static final ConfigProperty<String> CLEANER_INCREMENTAL_MODE_ENABLE = ConfigProperty
.key("hoodie.cleaner.incremental.mode")
.defaultValue("true")
.withDocumentation("When enabled, the plans for each cleaner service run is computed incrementally off the events "
@@ -136,26 +118,20 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("Archiving service moves older entries from timeline into an archived log after each write, to "
+ " keep the metadata overhead constant, even as the table size grows."
+ "This config controls the maximum number of instants to retain in the active timeline. ");
@Deprecated
public static final String MAX_COMMITS_TO_KEEP_PROP = MAX_COMMITS_TO_KEEP.key();
public static final ConfigProperty<String> MIN_COMMITS_TO_KEEP = ConfigProperty
.key("hoodie.keep.min.commits")
.defaultValue("20")
.withDocumentation("Similar to " + MAX_COMMITS_TO_KEEP.key() + ", but controls the minimum number of"
+ "instants to retain in the active timeline.");
@Deprecated
public static final String MIN_COMMITS_TO_KEEP_PROP = MIN_COMMITS_TO_KEEP.key();
public static final ConfigProperty<String> COMMITS_ARCHIVAL_BATCH_SIZE = ConfigProperty
.key("hoodie.commits.archival.batch")
.defaultValue(String.valueOf(10))
.withDocumentation("Archiving of instants is batched in best-effort manner, to pack more instants into a single"
+ " archive log. This config controls such archival batch size.");
@Deprecated
public static final String COMMITS_ARCHIVAL_BATCH_SIZE_PROP = COMMITS_ARCHIVAL_BATCH_SIZE.key();
public static final ConfigProperty<String> CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = ConfigProperty
public static final ConfigProperty<String> CLEANER_BOOTSTRAP_BASE_FILE_ENABLE = ConfigProperty
.key("hoodie.cleaner.delete.bootstrap.base.file")
.defaultValue("false")
.withDocumentation("When set to true, cleaner also deletes the bootstrap base file when it's skeleton base file is "
@@ -163,7 +139,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
+ " table receives updates/deletes. Another reason to turn this on, would be to ensure data residing in bootstrap "
+ " base files are also physically deleted, to comply with data privacy enforcement processes.");
public static final ConfigProperty<String> PARQUET_SMALL_FILE_LIMIT_BYTES = ConfigProperty
public static final ConfigProperty<String> PARQUET_SMALL_FILE_LIMIT = ConfigProperty
.key("hoodie.parquet.small.file.limit")
.defaultValue(String.valueOf(104857600))
.withDocumentation("During upsert operation, we opportunistically expand existing small files on storage, instead of writing"
@@ -177,10 +153,8 @@ public class HoodieCompactionConfig extends HoodieConfig {
+ " to bin pack records into partitions. If the previous commit is too small to make an accurate estimation, "
+ " Hudi will search commits in the reverse order, until we find a commit that has totalBytesWritten "
+ " larger than (PARQUET_SMALL_FILE_LIMIT_BYTES * this_threshold)");
@Deprecated
public static final String RECORD_SIZE_ESTIMATION_THRESHOLD_PROP = RECORD_SIZE_ESTIMATION_THRESHOLD.key();
public static final ConfigProperty<String> CLEANER_PARALLELISM = ConfigProperty
public static final ConfigProperty<String> CLEANER_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.cleaner.parallelism")
.defaultValue("200")
.withDocumentation("Parallelism for the cleaning operation. Increase this if cleaning becomes slow.");
@@ -191,8 +165,6 @@ public class HoodieCompactionConfig extends HoodieConfig {
.defaultValue(String.valueOf(500 * 1024))
.withDocumentation("Amount of MBs to spend during compaction run for the LogFileSizeBasedCompactionStrategy. "
+ "This value helps bound ingestion latency while compaction is run inline mode.");
@Deprecated
public static final String TARGET_IO_PER_COMPACTION_IN_MB_PROP = TARGET_IO_PER_COMPACTION_IN_MB.key();
public static final ConfigProperty<String> COMPACTION_STRATEGY = ConfigProperty
.key("hoodie.compaction.strategy")
@@ -200,34 +172,26 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("Compaction strategy decides which file groups are picked up for "
+ "compaction during each compaction run. By default. Hudi picks the log file "
+ "with most accumulated unmerged data");
@Deprecated
public static final String COMPACTION_STRATEGY_PROP = COMPACTION_STRATEGY.key();
public static final ConfigProperty<String> PAYLOAD_CLASS = ConfigProperty
public static final ConfigProperty<String> PAYLOAD_CLASS_NAME = ConfigProperty
.key("hoodie.compaction.payload.class")
.defaultValue(OverwriteWithLatestAvroPayload.class.getName())
.withDocumentation("This needs to be same as class used during insert/upserts. Just like writing, compaction also uses "
+ "the record payload class to merge records in the log against each other, merge again with the base file and "
+ "produce the final record to be written after compaction.");
@Deprecated
public static final String PAYLOAD_CLASS_PROP = PAYLOAD_CLASS.key();
public static final ConfigProperty<String> COMPACTION_LAZY_BLOCK_READ_ENABLED = ConfigProperty
public static final ConfigProperty<String> COMPACTION_LAZY_BLOCK_READ_ENABLE = ConfigProperty
.key("hoodie.compaction.lazy.block.read")
.defaultValue("false")
.withDocumentation("When merging the delta log files, this config helps to choose whether the log blocks "
+ "should be read lazily or not. Choose true to use lazy block reading (low memory usage, but incurs seeks to each block"
+ " header) or false for immediate block read (higher memory usage)");
@Deprecated
public static final String COMPACTION_LAZY_BLOCK_READ_ENABLED_PROP = COMPACTION_LAZY_BLOCK_READ_ENABLED.key();
public static final ConfigProperty<String> COMPACTION_REVERSE_LOG_READ_ENABLED = ConfigProperty
public static final ConfigProperty<String> COMPACTION_REVERSE_LOG_READ_ENABLE = ConfigProperty
.key("hoodie.compaction.reverse.log.read")
.defaultValue("false")
.withDocumentation("HoodieLogFormatReader reads a logfile in the forward direction starting from pos=0 to pos=file_length. "
+ "If this config is set to true, the reader reads the logfile in reverse direction, from pos=file_length to pos=0");
@Deprecated
public static final String COMPACTION_REVERSE_LOG_READ_ENABLED_PROP = COMPACTION_REVERSE_LOG_READ_ENABLED.key();
public static final ConfigProperty<String> FAILED_WRITES_CLEANER_POLICY = ConfigProperty
.key("hoodie.cleaner.policy.failed.writes")
@@ -235,21 +199,17 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("Cleaning policy for failed writes to be used. Hudi will delete any files written by "
+ "failed writes to re-claim space. Choose to perform this rollback of failed writes eagerly before "
+ "every writer starts (only supported for single writer) or lazily by the cleaner (required for multi-writers)");
@Deprecated
public static final String FAILED_WRITES_CLEANER_POLICY_PROP = FAILED_WRITES_CLEANER_POLICY.key();
public static final ConfigProperty<String> TARGET_PARTITIONS_PER_DAYBASED_COMPACTION = ConfigProperty
.key("hoodie.compaction.daybased.target.partitions")
.defaultValue("10")
.withDocumentation("Used by org.apache.hudi.io.compact.strategy.DayBasedCompactionStrategy to denote the number of "
+ "latest partitions to compact during a compaction run.");
@Deprecated
public static final String TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.key();
/**
* Configs related to specific table types.
*/
public static final ConfigProperty<String> COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = ConfigProperty
public static final ConfigProperty<String> COPY_ON_WRITE_INSERT_SPLIT_SIZE = ConfigProperty
.key("hoodie.copyonwrite.insert.split.size")
.defaultValue(String.valueOf(500000))
.withDocumentation("Number of inserts assigned for each partition/bucket for writing. "
@@ -257,20 +217,237 @@ public class HoodieCompactionConfig extends HoodieConfig {
+ " over provision to 500K. As long as auto-tuning of splits is turned on, this only affects the first "
+ " write, where there is no history to learn record sizes from.");
public static final ConfigProperty<String> COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = ConfigProperty
public static final ConfigProperty<String> COPY_ON_WRITE_AUTO_SPLIT_INSERTS = ConfigProperty
.key("hoodie.copyonwrite.insert.auto.split")
.defaultValue("true")
.withDocumentation("Config to control whether we control insert split sizes automatically based on average"
+ " record sizes. It's recommended to keep this turned on, since hand tuning is otherwise extremely"
+ " cumbersome.");
public static final ConfigProperty<String> COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = ConfigProperty
public static final ConfigProperty<String> COPY_ON_WRITE_RECORD_SIZE_ESTIMATE = ConfigProperty
.key("hoodie.copyonwrite.record.size.estimate")
.defaultValue(String.valueOf(1024))
.withDocumentation("The average record size. If not explicitly specified, hudi will compute the "
+ "record size estimate compute dynamically based on commit metadata. "
+ " This is critical in computing the insert parallelism and bin-packing inserts into small files.");
/** @deprecated Use {@link #CLEANER_POLICY} and its methods instead */
@Deprecated
public static final String CLEANER_POLICY_PROP = CLEANER_POLICY.key();
/** @deprecated Use {@link #AUTO_CLEAN} and its methods instead */
@Deprecated
public static final String AUTO_CLEAN_PROP = AUTO_CLEAN.key();
/** @deprecated Use {@link #ASYNC_CLEAN} and its methods instead */
@Deprecated
public static final String ASYNC_CLEAN_PROP = ASYNC_CLEAN.key();
/** @deprecated Use {@link #INLINE_COMPACT} and its methods instead */
@Deprecated
public static final String INLINE_COMPACT_PROP = INLINE_COMPACT.key();
/** @deprecated Use {@link #INLINE_COMPACT_NUM_DELTA_COMMITS} and its methods instead */
@Deprecated
public static final String INLINE_COMPACT_NUM_DELTA_COMMITS_PROP = INLINE_COMPACT_NUM_DELTA_COMMITS.key();
/** @deprecated Use {@link #INLINE_COMPACT_TIME_DELTA_SECONDS} and its methods instead */
@Deprecated
public static final String INLINE_COMPACT_TIME_DELTA_SECONDS_PROP = INLINE_COMPACT_TIME_DELTA_SECONDS.key();
/** @deprecated Use {@link #INLINE_COMPACT_TRIGGER_STRATEGY} and its methods instead */
@Deprecated
public static final String INLINE_COMPACT_TRIGGER_STRATEGY_PROP = INLINE_COMPACT_TRIGGER_STRATEGY.key();
/** @deprecated Use {@link #CLEANER_FILE_VERSIONS_RETAINED} and its methods instead */
@Deprecated
public static final String CLEANER_FILE_VERSIONS_RETAINED_PROP = CLEANER_FILE_VERSIONS_RETAINED.key();
/**
* @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods instead
*/
@Deprecated
public static final String CLEANER_COMMITS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key();
/**
* @deprecated Use {@link #CLEANER_INCREMENTAL_MODE_ENABLE} and its methods instead
*/
@Deprecated
public static final String CLEANER_INCREMENTAL_MODE = CLEANER_INCREMENTAL_MODE_ENABLE.key();
/**
* @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods instead
*/
@Deprecated
public static final String MAX_COMMITS_TO_KEEP_PROP = MAX_COMMITS_TO_KEEP.key();
/**
* @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods instead
*/
@Deprecated
public static final String MIN_COMMITS_TO_KEEP_PROP = MIN_COMMITS_TO_KEEP.key();
/**
* @deprecated Use {@link #COMMITS_ARCHIVAL_BATCH_SIZE} and its methods instead
*/
@Deprecated
public static final String COMMITS_ARCHIVAL_BATCH_SIZE_PROP = COMMITS_ARCHIVAL_BATCH_SIZE.key();
/**
* @deprecated Use {@link #CLEANER_BOOTSTRAP_BASE_FILE_ENABLE} and its methods instead
*/
@Deprecated
public static final String CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = CLEANER_BOOTSTRAP_BASE_FILE_ENABLE.key();
/**
* @deprecated Use {@link #PARQUET_SMALL_FILE_LIMIT} and its methods instead
*/
@Deprecated
public static final String PARQUET_SMALL_FILE_LIMIT_BYTES = PARQUET_SMALL_FILE_LIMIT.key();
/**
* @deprecated Use {@link #PARQUET_SMALL_FILE_LIMIT} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES = PARQUET_SMALL_FILE_LIMIT.defaultValue();
/**
* @deprecated Use {@link #RECORD_SIZE_ESTIMATION_THRESHOLD} and its methods instead
*/
@Deprecated
public static final String RECORD_SIZE_ESTIMATION_THRESHOLD_PROP = RECORD_SIZE_ESTIMATION_THRESHOLD.key();
/**
* @deprecated Use {@link #RECORD_SIZE_ESTIMATION_THRESHOLD} and its methods instead
*/
@Deprecated
public static final String DEFAULT_RECORD_SIZE_ESTIMATION_THRESHOLD = RECORD_SIZE_ESTIMATION_THRESHOLD.defaultValue();
/**
* @deprecated Use {@link #COPY_ON_WRITE_INSERT_SPLIT_SIZE} and its methods instead
*/
@Deprecated
public static final String COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = COPY_ON_WRITE_INSERT_SPLIT_SIZE.key();
/**
* @deprecated Use {@link #COPY_ON_WRITE_INSERT_SPLIT_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = COPY_ON_WRITE_INSERT_SPLIT_SIZE.defaultValue();
/**
* @deprecated Use {@link #COPY_ON_WRITE_AUTO_SPLIT_INSERTS} and its methods instead
*/
@Deprecated
public static final String COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = COPY_ON_WRITE_AUTO_SPLIT_INSERTS.key();
/**
* @deprecated Use {@link #COPY_ON_WRITE_AUTO_SPLIT_INSERTS} and its methods instead
*/
@Deprecated
public static final String DEFAULT_COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = COPY_ON_WRITE_AUTO_SPLIT_INSERTS.defaultValue();
/**
* @deprecated Use {@link #COPY_ON_WRITE_RECORD_SIZE_ESTIMATE} and its methods instead
*/
@Deprecated
public static final String COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key();
/**
* @deprecated Use {@link #COPY_ON_WRITE_RECORD_SIZE_ESTIMATE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.defaultValue();
/**
* @deprecated Use {@link #CLEANER_PARALLELISM_VALUE} and its methods instead
*/
@Deprecated
public static final String CLEANER_PARALLELISM = CLEANER_PARALLELISM_VALUE.key();
/**
* @deprecated Use {@link #CLEANER_PARALLELISM_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CLEANER_PARALLELISM = CLEANER_PARALLELISM_VALUE.defaultValue();
/**
* @deprecated Use {@link #TARGET_IO_PER_COMPACTION_IN_MB} and its methods instead
*/
@Deprecated
public static final String TARGET_IO_PER_COMPACTION_IN_MB_PROP = TARGET_IO_PER_COMPACTION_IN_MB.key();
/**
* @deprecated Use {@link #TARGET_IO_PER_COMPACTION_IN_MB} and its methods instead
*/
@Deprecated
public static final String DEFAULT_TARGET_IO_PER_COMPACTION_IN_MB = TARGET_IO_PER_COMPACTION_IN_MB.defaultValue();
/**
* @deprecated Use {@link #COMPACTION_STRATEGY} and its methods instead
*/
@Deprecated
public static final String COMPACTION_STRATEGY_PROP = COMPACTION_STRATEGY.key();
/** @deprecated Use {@link #COMPACTION_STRATEGY} and its methods instead */
@Deprecated
public static final String DEFAULT_COMPACTION_STRATEGY = COMPACTION_STRATEGY.defaultValue();
/** @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods instead */
@Deprecated
public static final String DEFAULT_PAYLOAD_CLASS = PAYLOAD_CLASS_NAME.defaultValue();
/** @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods instead */
@Deprecated
public static final String PAYLOAD_CLASS_PROP = PAYLOAD_CLASS_NAME.key();
/** @deprecated Use {@link #COMPACTION_LAZY_BLOCK_READ_ENABLE} and its methods instead */
@Deprecated
public static final String COMPACTION_LAZY_BLOCK_READ_ENABLED_PROP = COMPACTION_LAZY_BLOCK_READ_ENABLE.key();
/** @deprecated Use {@link #COMPACTION_LAZY_BLOCK_READ_ENABLE} and its methods instead */
@Deprecated
public static final String DEFAULT_COMPACTION_LAZY_BLOCK_READ_ENABLED = COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue();
/** @deprecated Use {@link #COMPACTION_REVERSE_LOG_READ_ENABLE} and its methods instead */
@Deprecated
public static final String COMPACTION_REVERSE_LOG_READ_ENABLED_PROP = COMPACTION_REVERSE_LOG_READ_ENABLE.key();
/** @deprecated Use {@link #COMPACTION_REVERSE_LOG_READ_ENABLE} and its methods instead */
@Deprecated
public static final String DEFAULT_COMPACTION_REVERSE_LOG_READ_ENABLED = COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue();
/** @deprecated Use {@link #CLEANER_POLICY} and its methods instead */
@Deprecated
private static final String DEFAULT_CLEANER_POLICY = CLEANER_POLICY.defaultValue();
/** @deprecated Use {@link #FAILED_WRITES_CLEANER_POLICY} and its methods instead */
@Deprecated
public static final String FAILED_WRITES_CLEANER_POLICY_PROP = FAILED_WRITES_CLEANER_POLICY.key();
/** @deprecated Use {@link #FAILED_WRITES_CLEANER_POLICY} and its methods instead */
@Deprecated
private static final String DEFAULT_FAILED_WRITES_CLEANER_POLICY = FAILED_WRITES_CLEANER_POLICY.defaultValue();
/** @deprecated Use {@link #AUTO_CLEAN} and its methods instead */
@Deprecated
private static final String DEFAULT_AUTO_CLEAN = AUTO_CLEAN.defaultValue();
/**
* @deprecated Use {@link #ASYNC_CLEAN} and its methods instead
*/
@Deprecated
private static final String DEFAULT_ASYNC_CLEAN = ASYNC_CLEAN.defaultValue();
/**
* @deprecated Use {@link #INLINE_COMPACT} and its methods instead
*/
@Deprecated
private static final String DEFAULT_INLINE_COMPACT = INLINE_COMPACT.defaultValue();
/**
* @deprecated Use {@link #CLEANER_INCREMENTAL_MODE_ENABLE} and its methods instead
*/
@Deprecated
private static final String DEFAULT_INCREMENTAL_CLEANER = CLEANER_INCREMENTAL_MODE_ENABLE.defaultValue();
/** @deprecated Use {@link #INLINE_COMPACT_NUM_DELTA_COMMITS} and its methods instead */
@Deprecated
private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = INLINE_COMPACT_NUM_DELTA_COMMITS.defaultValue();
/** @deprecated Use {@link #INLINE_COMPACT_TIME_DELTA_SECONDS} and its methods instead */
@Deprecated
private static final String DEFAULT_INLINE_COMPACT_TIME_DELTA_SECONDS = INLINE_COMPACT_TIME_DELTA_SECONDS.defaultValue();
/** @deprecated Use {@link #INLINE_COMPACT_TRIGGER_STRATEGY} and its methods instead */
@Deprecated
private static final String DEFAULT_INLINE_COMPACT_TRIGGER_STRATEGY = INLINE_COMPACT_TRIGGER_STRATEGY.defaultValue();
/** @deprecated Use {@link #CLEANER_FILE_VERSIONS_RETAINED} and its methods instead */
@Deprecated
private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = CLEANER_FILE_VERSIONS_RETAINED.defaultValue();
/** @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods instead */
@Deprecated
private static final String DEFAULT_CLEANER_COMMITS_RETAINED = CLEANER_COMMITS_RETAINED.defaultValue();
/** @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods instead */
@Deprecated
private static final String DEFAULT_MAX_COMMITS_TO_KEEP = MAX_COMMITS_TO_KEEP.defaultValue();
/**
* @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods instead
*/
@Deprecated
private static final String DEFAULT_MIN_COMMITS_TO_KEEP = MIN_COMMITS_TO_KEEP.defaultValue();
/**
* @deprecated Use {@link #COMMITS_ARCHIVAL_BATCH_SIZE} and its methods instead
*/
@Deprecated
private static final String DEFAULT_COMMITS_ARCHIVAL_BATCH_SIZE = COMMITS_ARCHIVAL_BATCH_SIZE.defaultValue();
/**
* @deprecated Use {@link #CLEANER_BOOTSTRAP_BASE_FILE_ENABLE} and its methods instead
*/
@Deprecated
private static final String DEFAULT_CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = CLEANER_BOOTSTRAP_BASE_FILE_ENABLE.defaultValue();
/** @deprecated Use {@link #TARGET_PARTITIONS_PER_DAYBASED_COMPACTION} and its methods instead */
@Deprecated
public static final String TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.key();
/** @deprecated Use {@link #TARGET_PARTITIONS_PER_DAYBASED_COMPACTION} and its methods instead */
@Deprecated
public static final String DEFAULT_TARGET_PARTITIONS_PER_DAYBASED_COMPACTION = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.defaultValue();
private HoodieCompactionConfig() {
super();
}
@@ -306,7 +483,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder withIncrementalCleaningMode(Boolean incrementalCleaningMode) {
compactionConfig.setValue(CLEANER_INCREMENTAL_MODE, String.valueOf(incrementalCleaningMode));
compactionConfig.setValue(CLEANER_INCREMENTAL_MODE_ENABLE, String.valueOf(incrementalCleaningMode));
return this;
}
@@ -342,7 +519,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder compactionSmallFileSize(long smallFileLimitBytes) {
compactionConfig.setValue(PARQUET_SMALL_FILE_LIMIT_BYTES, String.valueOf(smallFileLimitBytes));
compactionConfig.setValue(PARQUET_SMALL_FILE_LIMIT, String.valueOf(smallFileLimitBytes));
return this;
}
@@ -352,22 +529,22 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder insertSplitSize(int insertSplitSize) {
compactionConfig.setValue(COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE, String.valueOf(insertSplitSize));
compactionConfig.setValue(COPY_ON_WRITE_INSERT_SPLIT_SIZE, String.valueOf(insertSplitSize));
return this;
}
public Builder autoTuneInsertSplits(boolean autoTuneInsertSplits) {
compactionConfig.setValue(COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS, String.valueOf(autoTuneInsertSplits));
compactionConfig.setValue(COPY_ON_WRITE_AUTO_SPLIT_INSERTS, String.valueOf(autoTuneInsertSplits));
return this;
}
public Builder approxRecordSize(int recordSizeEstimate) {
compactionConfig.setValue(COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE, String.valueOf(recordSizeEstimate));
compactionConfig.setValue(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE, String.valueOf(recordSizeEstimate));
return this;
}
public Builder withCleanerParallelism(int cleanerParallelism) {
compactionConfig.setValue(CLEANER_PARALLELISM, String.valueOf(cleanerParallelism));
compactionConfig.setValue(CLEANER_PARALLELISM_VALUE, String.valueOf(cleanerParallelism));
return this;
}
@@ -377,7 +554,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder withPayloadClass(String payloadClassName) {
compactionConfig.setValue(PAYLOAD_CLASS, payloadClassName);
compactionConfig.setValue(PAYLOAD_CLASS_NAME, payloadClassName);
return this;
}
@@ -397,12 +574,12 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder withCompactionLazyBlockReadEnabled(Boolean compactionLazyBlockReadEnabled) {
compactionConfig.setValue(COMPACTION_LAZY_BLOCK_READ_ENABLED, String.valueOf(compactionLazyBlockReadEnabled));
compactionConfig.setValue(COMPACTION_LAZY_BLOCK_READ_ENABLE, String.valueOf(compactionLazyBlockReadEnabled));
return this;
}
public Builder withCompactionReverseLogReadEnabled(Boolean compactionReverseLogReadEnabled) {
compactionConfig.setValue(COMPACTION_REVERSE_LOG_READ_ENABLED, String.valueOf(compactionReverseLogReadEnabled));
compactionConfig.setValue(COMPACTION_REVERSE_LOG_READ_ENABLE, String.valueOf(compactionReverseLogReadEnabled));
return this;
}
@@ -417,7 +594,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
}
public Builder withCleanBootstrapBaseFileEnabled(Boolean cleanBootstrapSourceFileEnabled) {
compactionConfig.setValue(CLEANER_BOOTSTRAP_BASE_FILE_ENABLED, String.valueOf(cleanBootstrapSourceFileEnabled));
compactionConfig.setValue(CLEANER_BOOTSTRAP_BASE_FILE_ENABLE, String.valueOf(cleanBootstrapSourceFileEnabled));
return this;
}

271
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java
View File

@@ -36,138 +36,114 @@ import java.util.Properties;
+ "records as either inserts or updates to older records.")
public class HoodieHBaseIndexConfig extends HoodieConfig {
public static final ConfigProperty<String> HBASE_ZKQUORUM = ConfigProperty
public static final ConfigProperty<String> ZKQUORUM = ConfigProperty
.key("hoodie.index.hbase.zkquorum")
.noDefaultValue()
.withDocumentation("Only applies if index type is HBASE. HBase ZK Quorum url to connect to");
@Deprecated
public static final String HBASE_ZKQUORUM_PROP = HBASE_ZKQUORUM.key();
public static final ConfigProperty<String> HBASE_ZKPORT = ConfigProperty
public static final ConfigProperty<String> ZKPORT = ConfigProperty
.key("hoodie.index.hbase.zkport")
.noDefaultValue()
.withDocumentation("Only applies if index type is HBASE. HBase ZK Quorum port to connect to");
@Deprecated
public static final String HBASE_ZKPORT_PROP = HBASE_ZKPORT.key();
public static final ConfigProperty<String> HBASE_TABLENAME = ConfigProperty
public static final ConfigProperty<String> TABLENAME = ConfigProperty
.key("hoodie.index.hbase.table")
.noDefaultValue()
.withDocumentation("Only applies if index type is HBASE. HBase Table name to use as the index. "
+ "Hudi stores the row_key and [partition_path, fileID, commitTime] mapping in the table");
@Deprecated
public static final String HBASE_TABLENAME_PROP = HBASE_TABLENAME.key();
public static final ConfigProperty<Integer> HBASE_GET_BATCH_SIZE = ConfigProperty
public static final ConfigProperty<Integer> GET_BATCH_SIZE = ConfigProperty
.key("hoodie.index.hbase.get.batch.size")
.defaultValue(100)
.withDocumentation("Controls the batch size for performing gets against HBase. "
+ "Batching improves throughput, by saving round trips.");
@Deprecated
public static final String HBASE_GET_BATCH_SIZE_PROP = HBASE_GET_BATCH_SIZE.key();
public static final ConfigProperty<String> HBASE_ZK_ZNODEPARENT = ConfigProperty
public static final ConfigProperty<String> ZK_NODE_PATH = ConfigProperty
.key("hoodie.index.hbase.zknode.path")
.noDefaultValue()
.withDocumentation("Only applies if index type is HBASE. This is the root znode that will contain "
+ "all the znodes created/used by HBase");
public static final ConfigProperty<Integer> HBASE_PUT_BATCH_SIZE = ConfigProperty
public static final ConfigProperty<Integer> PUT_BATCH_SIZE = ConfigProperty
.key("hoodie.index.hbase.put.batch.size")
.defaultValue(100)
.withDocumentation("Controls the batch size for performing puts against HBase. "
+ "Batching improves throughput, by saving round trips.");
@Deprecated
public static final String HBASE_PUT_BATCH_SIZE_PROP = HBASE_PUT_BATCH_SIZE.key();
public static final ConfigProperty<String> HBASE_INDEX_QPS_ALLOCATOR_CLASS = ConfigProperty
public static final ConfigProperty<String> QPS_ALLOCATOR_CLASS_NAME = ConfigProperty
.key("hoodie.index.hbase.qps.allocator.class")
.defaultValue(DefaultHBaseQPSResourceAllocator.class.getName())
.withDocumentation("Property to set which implementation of HBase QPS resource allocator to be used, which"
+ "controls the batching rate dynamically.");
public static final ConfigProperty<String> HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE = ConfigProperty
public static final ConfigProperty<String> PUT_BATCH_SIZE_AUTO_COMPUTE = ConfigProperty
.key("hoodie.index.hbase.put.batch.size.autocompute")
.defaultValue("false")
.withDocumentation("Property to set to enable auto computation of put batch size");
@Deprecated
public static final String HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP = HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE.key();
public static final ConfigProperty<Float> HBASE_QPS_FRACTION = ConfigProperty
public static final ConfigProperty<Float> QPS_FRACTION = ConfigProperty
.key("hoodie.index.hbase.qps.fraction")
.defaultValue(0.5f)
.withDocumentation("Property to set the fraction of the global share of QPS that should be allocated to this job. Let's say there are 3"
+ " jobs which have input size in terms of number of rows required for HbaseIndexing as x, 2x, 3x respectively. Then"
+ " this fraction for the jobs would be (0.17) 1/6, 0.33 (2/6) and 0.5 (3/6) respectively."
+ " Default is 50%, which means a total of 2 jobs can run using HbaseIndex without overwhelming Region Servers.");
@Deprecated
public static final String HBASE_QPS_FRACTION_PROP = HBASE_QPS_FRACTION.key();
public static final ConfigProperty<Integer> HBASE_MAX_QPS_PER_REGION_SERVER = ConfigProperty
public static final ConfigProperty<Integer> MAX_QPS_PER_REGION_SERVER = ConfigProperty
.key("hoodie.index.hbase.max.qps.per.region.server")
.defaultValue(1000)
.withDocumentation("Property to set maximum QPS allowed per Region Server. This should be same across various jobs. This is intended to\n"
+ " limit the aggregate QPS generated across various jobs to an Hbase Region Server. It is recommended to set this\n"
+ " value based on global indexing throughput needs and most importantly, how much the HBase installation in use is\n"
+ " able to tolerate without Region Servers going down.");
@Deprecated
public static final String HBASE_MAX_QPS_PER_REGION_SERVER_PROP = HBASE_MAX_QPS_PER_REGION_SERVER.key();
public static final ConfigProperty<Boolean> HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = ConfigProperty
public static final ConfigProperty<Boolean> COMPUTE_QPS_DYNAMICALLY = ConfigProperty
.key("hoodie.index.hbase.dynamic_qps")
.defaultValue(false)
.withDocumentation("Property to decide if HBASE_QPS_FRACTION_PROP is dynamically calculated based on write volume.");
public static final ConfigProperty<String> HBASE_MIN_QPS_FRACTION = ConfigProperty
public static final ConfigProperty<String> MIN_QPS_FRACTION = ConfigProperty
.key("hoodie.index.hbase.min.qps.fraction")
.noDefaultValue()
.withDocumentation("Minimum for HBASE_QPS_FRACTION_PROP to stabilize skewed write workloads");
@Deprecated
public static final String HBASE_MIN_QPS_FRACTION_PROP = HBASE_MIN_QPS_FRACTION.key();
public static final ConfigProperty<String> HBASE_MAX_QPS_FRACTION = ConfigProperty
public static final ConfigProperty<String> MAX_QPS_FRACTION = ConfigProperty
.key("hoodie.index.hbase.max.qps.fraction")
.noDefaultValue()
.withDocumentation("Maximum for HBASE_QPS_FRACTION_PROP to stabilize skewed write workloads");
@Deprecated
public static final String HBASE_MAX_QPS_FRACTION_PROP = HBASE_MAX_QPS_FRACTION.key();
public static final ConfigProperty<Integer> HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = ConfigProperty
public static final ConfigProperty<Integer> DESIRED_PUTS_TIME_IN_SECONDS = ConfigProperty
.key("hoodie.index.hbase.desired_puts_time_in_secs")
.defaultValue(600)
.withDocumentation("");
public static final ConfigProperty<String> HBASE_SLEEP_MS_PUT_BATCH = ConfigProperty
public static final ConfigProperty<String> SLEEP_MS_FOR_PUT_BATCH = ConfigProperty
.key("hoodie.index.hbase.sleep.ms.for.put.batch")
.noDefaultValue()
.withDocumentation("");
@Deprecated
public static final String HBASE_SLEEP_MS_PUT_BATCH_PROP = HBASE_SLEEP_MS_PUT_BATCH.key();
public static final ConfigProperty<String> HBASE_SLEEP_MS_GET_BATCH = ConfigProperty
public static final ConfigProperty<String> SLEEP_MS_FOR_GET_BATCH = ConfigProperty
.key("hoodie.index.hbase.sleep.ms.for.get.batch")
.noDefaultValue()
.withDocumentation("");
@Deprecated
public static final String HBASE_SLEEP_MS_GET_BATCH_PROP = HBASE_SLEEP_MS_GET_BATCH.key();;
public static final ConfigProperty<Integer> HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS = ConfigProperty
public static final ConfigProperty<Integer> ZK_SESSION_TIMEOUT_MS = ConfigProperty
.key("hoodie.index.hbase.zk.session_timeout_ms")
.defaultValue(60 * 1000)
.withDocumentation("Session timeout value to use for Zookeeper failure detection, for the HBase client."
+ "Lower this value, if you want to fail faster.");
public static final ConfigProperty<Integer> HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS = ConfigProperty
public static final ConfigProperty<Integer> ZK_CONNECTION_TIMEOUT_MS = ConfigProperty
.key("hoodie.index.hbase.zk.connection_timeout_ms")
.defaultValue(15 * 1000)
.withDocumentation("Timeout to use for establishing connection with zookeeper, from HBase client.");
public static final ConfigProperty<String> HBASE_ZK_PATH_QPS_ROOT = ConfigProperty
public static final ConfigProperty<String> ZKPATH_QPS_ROOT = ConfigProperty
.key("hoodie.index.hbase.zkpath.qps_root")
.defaultValue("/QPS_ROOT")
.withDocumentation("chroot in zookeeper, to use for all qps allocation co-ordination.");
public static final ConfigProperty<Boolean> HBASE_INDEX_UPDATE_PARTITION_PATH = ConfigProperty
public static final ConfigProperty<Boolean> UPDATE_PARTITION_PATH_ENABLE = ConfigProperty
.key("hoodie.hbase.index.update.partition.path")
.defaultValue(false)
.withDocumentation("Only applies if index type is HBASE. "
@@ -175,12 +151,175 @@ public class HoodieHBaseIndexConfig extends HoodieConfig {
+ "this config when set, will delete old record in old paritition "
+ "and will insert it as new record in new partition.");
public static final ConfigProperty<Boolean> HBASE_INDEX_ROLLBACK_SYNC = ConfigProperty
public static final ConfigProperty<Boolean> ROLLBACK_SYNC_ENABLE = ConfigProperty
.key("hoodie.index.hbase.rollback.sync")
.defaultValue(false)
.withDocumentation("When set to true, the rollback method will delete the last failed task index. "
+ "The default value is false. Because deleting the index will add extra load on the Hbase cluster for each rollback");
/**
* @deprecated Use {@link #ZKQUORUM} and its methods instead
*/
@Deprecated
public static final String HBASE_ZKQUORUM_PROP = ZKQUORUM.key();
/**
* @deprecated Use {@link #ZKPORT} and its methods instead
*/
@Deprecated
public static final String HBASE_ZKPORT_PROP = ZKPORT.key();
/**
* @deprecated Use {@link #TABLENAME} and its methods instead
*/
@Deprecated
public static final String HBASE_TABLENAME_PROP = TABLENAME.key();
/**
* @deprecated Use {@link #GET_BATCH_SIZE} and its methods instead
*/
@Deprecated
public static final String HBASE_GET_BATCH_SIZE_PROP = GET_BATCH_SIZE.key();
/**
* @deprecated Use {@link #ZK_NODE_PATH} and its methods instead
*/
@Deprecated
public static final String HBASE_ZK_ZNODEPARENT = ZK_NODE_PATH.key();
/**
* @deprecated Use {@link #PUT_BATCH_SIZE} and its methods instead
*/
@Deprecated
public static final String HBASE_PUT_BATCH_SIZE_PROP = PUT_BATCH_SIZE.key();
/**
* @deprecated Use {@link #QPS_ALLOCATOR_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String HBASE_INDEX_QPS_ALLOCATOR_CLASS = QPS_ALLOCATOR_CLASS_NAME.key();
/**
* @deprecated Use {@link #QPS_ALLOCATOR_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HBASE_INDEX_QPS_ALLOCATOR_CLASS = QPS_ALLOCATOR_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #PUT_BATCH_SIZE_AUTO_COMPUTE} and its methods instead
*/
@Deprecated
public static final String HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP = PUT_BATCH_SIZE_AUTO_COMPUTE.key();
/**
* @deprecated Use {@link #PUT_BATCH_SIZE_AUTO_COMPUTE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE = PUT_BATCH_SIZE_AUTO_COMPUTE.defaultValue();
/**
* @deprecated Use {@link #MAX_QPS_FRACTION} and its methods instead
*/
@Deprecated
public static final String HBASE_QPS_FRACTION_PROP = QPS_FRACTION.key();
/**
* @deprecated Use {@link #MAX_QPS_PER_REGION_SERVER} and its methods instead
*/
@Deprecated
public static final String HBASE_MAX_QPS_PER_REGION_SERVER_PROP = MAX_QPS_PER_REGION_SERVER.key();
@Deprecated
public static final int DEFAULT_HBASE_BATCH_SIZE = 100;
/**
* @deprecated Use {@link #MAX_QPS_PER_REGION_SERVER} and its methods instead
*/
@Deprecated
public static final int DEFAULT_HBASE_MAX_QPS_PER_REGION_SERVER = MAX_QPS_PER_REGION_SERVER.defaultValue();
/**
* @deprecated Use {@link #QPS_FRACTION} and its methods instead
*/
@Deprecated
public static final float DEFAULT_HBASE_QPS_FRACTION = QPS_FRACTION.defaultValue();
/**
* @deprecated Use {@link #COMPUTE_QPS_DYNAMICALLY} and its methods instead
*/
@Deprecated
public static final String HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = COMPUTE_QPS_DYNAMICALLY.key();
/**
* @deprecated Use {@link #COMPUTE_QPS_DYNAMICALLY} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = COMPUTE_QPS_DYNAMICALLY.defaultValue();
/**
* @deprecated Use {@link #MIN_QPS_FRACTION} and its methods instead
*/
@Deprecated
public static final String HBASE_MIN_QPS_FRACTION_PROP = MIN_QPS_FRACTION.key();
/**
* @deprecated Use {@link #MAX_QPS_FRACTION} and its methods instead
*/
@Deprecated
public static final String HBASE_MAX_QPS_FRACTION_PROP = MAX_QPS_FRACTION.key();
/**
* @deprecated Use {@link #DESIRED_PUTS_TIME_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final String HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = DESIRED_PUTS_TIME_IN_SECONDS.key();
/**
* @deprecated Use {@link #DESIRED_PUTS_TIME_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = DESIRED_PUTS_TIME_IN_SECONDS.defaultValue();
/**
* @deprecated Use {@link #SLEEP_MS_FOR_PUT_BATCH} and its methods instead
*/
@Deprecated
public static final String HBASE_SLEEP_MS_PUT_BATCH_PROP = SLEEP_MS_FOR_PUT_BATCH.key();
/**
* @deprecated Use {@link #SLEEP_MS_FOR_GET_BATCH} and its methods instead
*/
@Deprecated
public static final String HBASE_SLEEP_MS_GET_BATCH_PROP = SLEEP_MS_FOR_GET_BATCH.key();
/**
* @deprecated Use {@link #ZK_SESSION_TIMEOUT_MS} and its methods instead
*/
@Deprecated
public static final String HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS = ZK_SESSION_TIMEOUT_MS.key();
/**
* @deprecated Use {@link #ZK_SESSION_TIMEOUT_MS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_ZK_SESSION_TIMEOUT_MS = ZK_SESSION_TIMEOUT_MS.defaultValue();
/**
* @deprecated Use {@link #ZK_CONNECTION_TIMEOUT_MS} and its methods instead
*/
@Deprecated
public static final String HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS = ZK_CONNECTION_TIMEOUT_MS.key();
/**
* @deprecated Use {@link #ZK_CONNECTION_TIMEOUT_MS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_ZK_CONNECTION_TIMEOUT_MS = ZK_CONNECTION_TIMEOUT_MS.defaultValue();
/**
* @deprecated Use {@link #ZKPATH_QPS_ROOT} and its methods instead
*/
@Deprecated
public static final String HBASE_ZK_PATH_QPS_ROOT = ZKPATH_QPS_ROOT.key();
/**
* @deprecated Use {@link #ZKPATH_QPS_ROOT} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HBASE_ZK_PATH_QPS_ROOT = ZKPATH_QPS_ROOT.defaultValue();
/**
* @deprecated Use {@link #UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final String HBASE_INDEX_UPDATE_PARTITION_PATH = UPDATE_PARTITION_PATH_ENABLE.key();
/**
* @deprecated Use {@link #UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final Boolean DEFAULT_HBASE_INDEX_UPDATE_PARTITION_PATH = UPDATE_PARTITION_PATH_ENABLE.defaultValue();
/**
* @deprecated Use {@link #ROLLBACK_SYNC_ENABLE} and its methods instead
*/
@Deprecated
public static final String HBASE_INDEX_ROLLBACK_SYNC = ROLLBACK_SYNC_ENABLE.key();
/**
* @deprecated Use {@link #ROLLBACK_SYNC_ENABLE} and its methods instead
*/
@Deprecated
public static final Boolean DEFAULT_HBASE_INDEX_ROLLBACK_SYNC = ROLLBACK_SYNC_ENABLE.defaultValue();
private HoodieHBaseIndexConfig() {
super();
}
@@ -206,102 +345,102 @@ public class HoodieHBaseIndexConfig extends HoodieConfig {
}
public HoodieHBaseIndexConfig.Builder hbaseZkQuorum(String zkString) {
hBaseIndexConfig.setValue(HBASE_ZKQUORUM, zkString);
hBaseIndexConfig.setValue(ZKQUORUM, zkString);
return this;
}
public HoodieHBaseIndexConfig.Builder hbaseZkPort(int port) {
hBaseIndexConfig.setValue(HBASE_ZKPORT, String.valueOf(port));
hBaseIndexConfig.setValue(ZKPORT, String.valueOf(port));
return this;
}
public HoodieHBaseIndexConfig.Builder hbaseTableName(String tableName) {
hBaseIndexConfig.setValue(HBASE_TABLENAME, tableName);
hBaseIndexConfig.setValue(TABLENAME, tableName);
return this;
}
public Builder hbaseZkZnodeQPSPath(String zkZnodeQPSPath) {
hBaseIndexConfig.setValue(HBASE_ZK_PATH_QPS_ROOT, zkZnodeQPSPath);
hBaseIndexConfig.setValue(ZKPATH_QPS_ROOT, zkZnodeQPSPath);
return this;
}
public Builder hbaseIndexGetBatchSize(int getBatchSize) {
hBaseIndexConfig.setValue(HBASE_GET_BATCH_SIZE, String.valueOf(getBatchSize));
hBaseIndexConfig.setValue(GET_BATCH_SIZE, String.valueOf(getBatchSize));
return this;
}
public Builder hbaseIndexPutBatchSize(int putBatchSize) {
hBaseIndexConfig.setValue(HBASE_PUT_BATCH_SIZE, String.valueOf(putBatchSize));
hBaseIndexConfig.setValue(PUT_BATCH_SIZE, String.valueOf(putBatchSize));
return this;
}
public Builder hbaseIndexPutBatchSizeAutoCompute(boolean putBatchSizeAutoCompute) {
hBaseIndexConfig.setValue(HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE, String.valueOf(putBatchSizeAutoCompute));
hBaseIndexConfig.setValue(PUT_BATCH_SIZE_AUTO_COMPUTE, String.valueOf(putBatchSizeAutoCompute));
return this;
}
public Builder hbaseIndexDesiredPutsTime(int desiredPutsTime) {
hBaseIndexConfig.setValue(HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS, String.valueOf(desiredPutsTime));
hBaseIndexConfig.setValue(DESIRED_PUTS_TIME_IN_SECONDS, String.valueOf(desiredPutsTime));
return this;
}
public Builder hbaseIndexShouldComputeQPSDynamically(boolean shouldComputeQPsDynamically) {
hBaseIndexConfig.setValue(HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY, String.valueOf(shouldComputeQPsDynamically));
hBaseIndexConfig.setValue(COMPUTE_QPS_DYNAMICALLY, String.valueOf(shouldComputeQPsDynamically));
return this;
}
public Builder hbaseIndexQPSFraction(float qpsFraction) {
hBaseIndexConfig.setValue(HBASE_QPS_FRACTION, String.valueOf(qpsFraction));
hBaseIndexConfig.setValue(QPS_FRACTION, String.valueOf(qpsFraction));
return this;
}
public Builder hbaseIndexMinQPSFraction(float minQPSFraction) {
hBaseIndexConfig.setValue(HBASE_MIN_QPS_FRACTION, String.valueOf(minQPSFraction));
hBaseIndexConfig.setValue(MIN_QPS_FRACTION, String.valueOf(minQPSFraction));
return this;
}
public Builder hbaseIndexMaxQPSFraction(float maxQPSFraction) {
hBaseIndexConfig.setValue(HBASE_MAX_QPS_FRACTION, String.valueOf(maxQPSFraction));
hBaseIndexConfig.setValue(MAX_QPS_FRACTION, String.valueOf(maxQPSFraction));
return this;
}
public Builder hbaseIndexSleepMsBetweenPutBatch(int sleepMsBetweenPutBatch) {
hBaseIndexConfig.setValue(HBASE_SLEEP_MS_PUT_BATCH, String.valueOf(sleepMsBetweenPutBatch));
hBaseIndexConfig.setValue(SLEEP_MS_FOR_PUT_BATCH, String.valueOf(sleepMsBetweenPutBatch));
return this;
}
public Builder hbaseIndexSleepMsBetweenGetBatch(int sleepMsBetweenGetBatch) {
hBaseIndexConfig.setValue(HBASE_SLEEP_MS_GET_BATCH, String.valueOf(sleepMsBetweenGetBatch));
hBaseIndexConfig.setValue(SLEEP_MS_FOR_GET_BATCH, String.valueOf(sleepMsBetweenGetBatch));
return this;
}
public Builder hbaseIndexUpdatePartitionPath(boolean updatePartitionPath) {
hBaseIndexConfig.setValue(HBASE_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath));
hBaseIndexConfig.setValue(UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath));
return this;
}
public Builder hbaseIndexRollbackSync(boolean rollbackSync) {
hBaseIndexConfig.setValue(HBASE_INDEX_ROLLBACK_SYNC, String.valueOf(rollbackSync));
hBaseIndexConfig.setValue(ROLLBACK_SYNC_ENABLE, String.valueOf(rollbackSync));
return this;
}
public Builder withQPSResourceAllocatorType(String qpsResourceAllocatorClass) {
hBaseIndexConfig.setValue(HBASE_INDEX_QPS_ALLOCATOR_CLASS, qpsResourceAllocatorClass);
hBaseIndexConfig.setValue(QPS_ALLOCATOR_CLASS_NAME, qpsResourceAllocatorClass);
return this;
}
public Builder hbaseIndexZkSessionTimeout(int zkSessionTimeout) {
hBaseIndexConfig.setValue(HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS, String.valueOf(zkSessionTimeout));
hBaseIndexConfig.setValue(ZK_SESSION_TIMEOUT_MS, String.valueOf(zkSessionTimeout));
return this;
}
public Builder hbaseIndexZkConnectionTimeout(int zkConnectionTimeout) {
hBaseIndexConfig.setValue(HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS, String.valueOf(zkConnectionTimeout));
hBaseIndexConfig.setValue(ZK_CONNECTION_TIMEOUT_MS, String.valueOf(zkConnectionTimeout));
return this;
}
public Builder hbaseZkZnodeParent(String zkZnodeParent) {
hBaseIndexConfig.setValue(HBASE_ZK_ZNODEPARENT, zkZnodeParent);
hBaseIndexConfig.setValue(ZK_NODE_PATH, zkZnodeParent);
return this;
}
@@ -317,7 +456,7 @@ public class HoodieHBaseIndexConfig extends HoodieConfig {
*/
public HoodieHBaseIndexConfig.Builder hbaseIndexMaxQPSPerRegionServer(int maxQPSPerRegionServer) {
// This should be same across various jobs
hBaseIndexConfig.setValue(HoodieHBaseIndexConfig.HBASE_MAX_QPS_PER_REGION_SERVER,
hBaseIndexConfig.setValue(HoodieHBaseIndexConfig.MAX_QPS_PER_REGION_SERVER,
String.valueOf(maxQPSPerRegionServer));
return this;
}

247
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
View File

@@ -34,11 +34,11 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_ZKQUORUM;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_ZKPORT;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_TABLENAME;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_GET_BATCH_SIZE;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_PUT_BATCH_SIZE;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.GET_BATCH_SIZE;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.PUT_BATCH_SIZE;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.TABLENAME;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.ZKPORT;
import static org.apache.hudi.config.HoodieHBaseIndexConfig.ZKQUORUM;
/**
* Indexing related config.
@@ -57,19 +57,15 @@ public class HoodieIndexConfig extends HoodieConfig {
+ "Possible options are [BLOOM | GLOBAL_BLOOM |SIMPLE | GLOBAL_SIMPLE | INMEMORY | HBASE]. "
+ "Bloom filters removes the dependency on a external system "
+ "and is stored in the footer of the Parquet Data Files");
@Deprecated
public static final String INDEX_TYPE_PROP = INDEX_TYPE.key();
public static final ConfigProperty<String> INDEX_CLASS = ConfigProperty
public static final ConfigProperty<String> INDEX_CLASS_NAME = ConfigProperty
.key("hoodie.index.class")
.defaultValue("")
.withDocumentation("Full path of user-defined index class and must be a subclass of HoodieIndex class. "
+ "It will take precedence over the hoodie.index.type configuration if specified");
@Deprecated
public static final String INDEX_CLASS_PROP = INDEX_CLASS.key();
// ***** Bloom Index configs *****
public static final ConfigProperty<String> BLOOM_FILTER_NUM_ENTRIES = ConfigProperty
public static final ConfigProperty<String> BLOOM_FILTER_NUM_ENTRIES_VALUE = ConfigProperty
.key("hoodie.index.bloom.num_entries")
.defaultValue("60000")
.withDocumentation("Only applies if index type is BLOOM. "
@@ -81,7 +77,7 @@ public class HoodieIndexConfig extends HoodieConfig {
+ "increase the size every base file linearly (roughly 4KB for every 50000 entries). "
+ "This config is also used with DYNAMIC bloom filter which determines the initial size for the bloom.");
public static final ConfigProperty<String> BLOOM_FILTER_FPP = ConfigProperty
public static final ConfigProperty<String> BLOOM_FILTER_FPP_VALUE = ConfigProperty
.key("hoodie.index.bloom.fpp")
.defaultValue("0.000000001")
.withDocumentation("Only applies if index type is BLOOM. "
@@ -97,8 +93,6 @@ public class HoodieIndexConfig extends HoodieConfig {
.withDocumentation("Only applies if index type is BLOOM. "
+ "This is the amount of parallelism for index lookup, which involves a shuffle. "
+ "By default, this is auto computed based on input workload characteristics.");
@Deprecated
public static final String BLOOM_INDEX_PARALLELISM_PROP = BLOOM_INDEX_PARALLELISM.key();
public static final ConfigProperty<String> BLOOM_INDEX_PRUNE_BY_RANGES = ConfigProperty
.key("hoodie.bloom.index.prune.by.ranges")
@@ -108,8 +102,6 @@ public class HoodieIndexConfig extends HoodieConfig {
+ "if the key has a monotonously increasing prefix, such as timestamp. "
+ "If the record key is completely random, it is better to turn this off, since range pruning will only "
+ " add extra overhead to the index lookup.");
@Deprecated
public static final String BLOOM_INDEX_PRUNE_BY_RANGES_PROP = BLOOM_INDEX_PRUNE_BY_RANGES.key();
public static final ConfigProperty<String> BLOOM_INDEX_USE_CACHING = ConfigProperty
.key("hoodie.bloom.index.use.caching")
@@ -117,8 +109,6 @@ public class HoodieIndexConfig extends HoodieConfig {
.withDocumentation("Only applies if index type is BLOOM."
+ "When true, the input RDD will cached to speed up index lookup by reducing IO "
+ "for computing parallelism or affected partitions");
@Deprecated
public static final String BLOOM_INDEX_USE_CACHING_PROP = BLOOM_INDEX_USE_CACHING.key();
public static final ConfigProperty<String> BLOOM_INDEX_TREE_BASED_FILTER = ConfigProperty
.key("hoodie.bloom.index.use.treebased.filter")
@@ -126,8 +116,6 @@ public class HoodieIndexConfig extends HoodieConfig {
.withDocumentation("Only applies if index type is BLOOM. "
+ "When true, interval tree based file pruning optimization is enabled. "
+ "This mode speeds-up file-pruning based on key ranges when compared with the brute-force mode");
@Deprecated
public static final String BLOOM_INDEX_TREE_BASED_FILTER_PROP = BLOOM_INDEX_TREE_BASED_FILTER.key();
// TODO: On by default. Once stable, we will remove the other mode.
public static final ConfigProperty<String> BLOOM_INDEX_BUCKETIZED_CHECKING = ConfigProperty
@@ -136,17 +124,15 @@ public class HoodieIndexConfig extends HoodieConfig {
.withDocumentation("Only applies if index type is BLOOM. "
+ "When true, bucketized bloom filtering is enabled. "
+ "This reduces skew seen in sort based bloom index lookup");
@Deprecated
public static final String BLOOM_INDEX_BUCKETIZED_CHECKING_PROP = BLOOM_INDEX_BUCKETIZED_CHECKING.key();
public static final ConfigProperty<String> BLOOM_INDEX_FILTER_TYPE = ConfigProperty
public static final ConfigProperty<String> BLOOM_FILTER_TYPE = ConfigProperty
.key("hoodie.bloom.index.filter.type")
.defaultValue(BloomFilterTypeCode.DYNAMIC_V0.name())
.withDocumentation("Filter type used. Default is BloomFilterTypeCode.DYNAMIC_V0. "
+ "Available values are [BloomFilterTypeCode.SIMPLE , BloomFilterTypeCode.DYNAMIC_V0]. "
+ "Dynamic bloom filters auto size themselves based on number of keys.");
public static final ConfigProperty<String> HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = ConfigProperty
public static final ConfigProperty<String> BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = ConfigProperty
.key("hoodie.bloom.index.filter.dynamic.max.entries")
.defaultValue("100000")
.withDocumentation("The threshold for the maximum number of keys to record in a dynamic Bloom filter row. "
@@ -158,24 +144,18 @@ public class HoodieIndexConfig extends HoodieConfig {
.withDocumentation("Only applies if index type is SIMPLE. "
+ "When true, the incoming writes will cached to speed up index lookup by reducing IO "
+ "for computing parallelism or affected partitions");
@Deprecated
public static final String SIMPLE_INDEX_USE_CACHING_PROP = SIMPLE_INDEX_USE_CACHING.key();
public static final ConfigProperty<String> SIMPLE_INDEX_PARALLELISM = ConfigProperty
.key("hoodie.simple.index.parallelism")
.defaultValue("50")
.withDocumentation("Only applies if index type is SIMPLE. "
+ "This is the amount of parallelism for index lookup, which involves a Spark Shuffle");
@Deprecated
public static final String SIMPLE_INDEX_PARALLELISM_PROP = SIMPLE_INDEX_PARALLELISM.key();
public static final ConfigProperty<String> GLOBAL_SIMPLE_INDEX_PARALLELISM = ConfigProperty
.key("hoodie.global.simple.index.parallelism")
.defaultValue("100")
.withDocumentation("Only applies if index type is GLOBAL_SIMPLE. "
+ "This is the amount of parallelism for index lookup, which involves a Spark Shuffle");
@Deprecated
public static final String GLOBAL_SIMPLE_INDEX_PARALLELISM_PROP = GLOBAL_SIMPLE_INDEX_PARALLELISM.key();
// 1B bloom filter checks happen in 250 seconds. 500ms to read a bloom filter.
// 10M checks in 2500ms, thus amortizing the cost of reading bloom filter across partitions.
@@ -186,16 +166,14 @@ public class HoodieIndexConfig extends HoodieConfig {
+ "This configuration controls the “bucket” size which tracks the number of record-key checks made against "
+ "a single file and is the unit of work allocated to each partition performing bloom filter lookup. "
+ "A higher value would amortize the fixed cost of reading a bloom filter to memory.");
@Deprecated
public static final String BLOOM_INDEX_KEYS_PER_BUCKET_PROP = BLOOM_INDEX_KEYS_PER_BUCKET.key();
public static final ConfigProperty<String> BLOOM_INDEX_INPUT_STORAGE_LEVEL = ConfigProperty
public static final ConfigProperty<String> BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE = ConfigProperty
.key("hoodie.bloom.index.input.storage.level")
.defaultValue("MEMORY_AND_DISK_SER")
.withDocumentation("Only applies when #bloomIndexUseCaching is set. Determine what level of persistence is used to cache input RDDs. "
+ "Refer to org.apache.spark.storage.StorageLevel for different values");
public static final ConfigProperty<String> SIMPLE_INDEX_INPUT_STORAGE_LEVEL = ConfigProperty
public static final ConfigProperty<String> SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE = ConfigProperty
.key("hoodie.simple.index.input.storage.level")
.defaultValue("MEMORY_AND_DISK_SER")
.withDocumentation("Only applies when #simpleIndexUseCaching is set. Determine what level of persistence is used to cache input RDDs. "
@@ -209,7 +187,7 @@ public class HoodieIndexConfig extends HoodieConfig {
* <p>
* When set to false, a record will be updated to the old partition.
*/
public static final ConfigProperty<String> BLOOM_INDEX_UPDATE_PARTITION_PATH = ConfigProperty
public static final ConfigProperty<String> BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE = ConfigProperty
.key("hoodie.bloom.index.update.partition.path")
.defaultValue("true")
.withDocumentation("Only applies if index type is GLOBAL_BLOOM. "
@@ -217,26 +195,195 @@ public class HoodieIndexConfig extends HoodieConfig {
+ "inserting the incoming record into the new partition and deleting the original record in the old partition. "
+ "When set to false, the original record will only be updated in the old partition");
public static final ConfigProperty<String> SIMPLE_INDEX_UPDATE_PARTITION_PATH = ConfigProperty
public static final ConfigProperty<String> SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE = ConfigProperty
.key("hoodie.simple.index.update.partition.path")
.defaultValue("true")
.withDocumentation("Similar to " + BLOOM_INDEX_UPDATE_PARTITION_PATH + ", but for simple index.");
.withDocumentation("Similar to " + BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE + ", but for simple index.");
/**
* Deprecated configs. These are now part of {@link HoodieHBaseIndexConfig}.
*/
@Deprecated
public static final String HBASE_ZKQUORUM_PROP = HBASE_ZKQUORUM.key();
public static final String HBASE_ZKQUORUM_PROP = ZKQUORUM.key();
@Deprecated
public static final String HBASE_ZKPORT_PROP = HBASE_ZKPORT.key();
public static final String HBASE_ZKPORT_PROP = ZKPORT.key();
@Deprecated
public static final String HBASE_ZK_ZNODEPARENT = HoodieHBaseIndexConfig.HBASE_ZK_ZNODEPARENT.key();
public static final String HBASE_ZK_ZNODEPARENT = HoodieHBaseIndexConfig.ZK_NODE_PATH.key();
@Deprecated
public static final String HBASE_TABLENAME_PROP = HBASE_TABLENAME.key();
public static final String HBASE_TABLENAME_PROP = TABLENAME.key();
@Deprecated
public static final String HBASE_GET_BATCH_SIZE_PROP = HBASE_GET_BATCH_SIZE.key();
public static final String HBASE_GET_BATCH_SIZE_PROP = GET_BATCH_SIZE.key();
@Deprecated
public static final String HBASE_PUT_BATCH_SIZE_PROP = HBASE_PUT_BATCH_SIZE.key();
public static final String HBASE_PUT_BATCH_SIZE_PROP = PUT_BATCH_SIZE.key();
@Deprecated
public static final String DEFAULT_HBASE_BATCH_SIZE = "100";
/** @deprecated Use {@link #INDEX_TYPE} and its methods instead */
@Deprecated
public static final String INDEX_TYPE_PROP = INDEX_TYPE.key();
/**
* @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String INDEX_CLASS_PROP = INDEX_CLASS_NAME.key();
/**
* @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_INDEX_CLASS = INDEX_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #BLOOM_FILTER_NUM_ENTRIES_VALUE} and its methods instead
*/
@Deprecated
public static final String BLOOM_FILTER_NUM_ENTRIES = BLOOM_FILTER_NUM_ENTRIES_VALUE.key();
/**
* @deprecated Use {@link #BLOOM_FILTER_NUM_ENTRIES_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_FILTER_NUM_ENTRIES = BLOOM_FILTER_NUM_ENTRIES_VALUE.defaultValue();
/**
* @deprecated Use {@link #BLOOM_FILTER_FPP_VALUE} and its methods instead
*/
@Deprecated
public static final String BLOOM_FILTER_FPP = BLOOM_FILTER_FPP_VALUE.key();
/**
* @deprecated Use {@link #BLOOM_FILTER_FPP_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_FILTER_FPP = BLOOM_FILTER_FPP_VALUE.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_PARALLELISM} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_PARALLELISM_PROP = BLOOM_INDEX_PARALLELISM.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_PARALLELISM} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_PARALLELISM = BLOOM_INDEX_PARALLELISM.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_PRUNE_BY_RANGES} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_PRUNE_BY_RANGES_PROP = BLOOM_INDEX_PRUNE_BY_RANGES.key();
/** @deprecated Use {@link #BLOOM_INDEX_PRUNE_BY_RANGES} and its methods instead */
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_PRUNE_BY_RANGES = BLOOM_INDEX_PRUNE_BY_RANGES.defaultValue();
/** @deprecated Use {@link #BLOOM_INDEX_USE_CACHING} and its methods instead */
@Deprecated
public static final String BLOOM_INDEX_USE_CACHING_PROP = BLOOM_INDEX_USE_CACHING.key();
/** @deprecated Use {@link #BLOOM_INDEX_USE_CACHING} and its methods instead */
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_USE_CACHING = BLOOM_INDEX_USE_CACHING.defaultValue();
/** @deprecated Use {@link #BLOOM_INDEX_TREE_BASED_FILTER} and its methods instead */
@Deprecated
public static final String BLOOM_INDEX_TREE_BASED_FILTER_PROP = BLOOM_INDEX_TREE_BASED_FILTER.key();
/** @deprecated Use {@link #BLOOM_INDEX_TREE_BASED_FILTER} and its methods instead */
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_TREE_BASED_FILTER = BLOOM_INDEX_TREE_BASED_FILTER.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_BUCKETIZED_CHECKING} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_BUCKETIZED_CHECKING_PROP = BLOOM_INDEX_BUCKETIZED_CHECKING.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_BUCKETIZED_CHECKING} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_BUCKETIZED_CHECKING = BLOOM_INDEX_BUCKETIZED_CHECKING.defaultValue();
/**
* @deprecated Use {@link #BLOOM_FILTER_TYPE} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_FILTER_TYPE = BLOOM_FILTER_TYPE.key();
/**
* @deprecated Use {@link #BLOOM_FILTER_TYPE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_FILTER_TYPE = BLOOM_FILTER_TYPE.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES} and its methods instead
*/
@Deprecated
public static final String HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.defaultValue();
/**
* @deprecated Use {@link #SIMPLE_INDEX_USE_CACHING} and its methods instead
*/
@Deprecated
public static final String SIMPLE_INDEX_USE_CACHING_PROP = SIMPLE_INDEX_USE_CACHING.key();
/**
* @deprecated Use {@link #SIMPLE_INDEX_USE_CACHING} and its methods instead
*/
@Deprecated
public static final String DEFAULT_SIMPLE_INDEX_USE_CACHING = SIMPLE_INDEX_USE_CACHING.defaultValue();
/** @deprecated Use {@link #SIMPLE_INDEX_PARALLELISM} and its methods instead */
@Deprecated
public static final String SIMPLE_INDEX_PARALLELISM_PROP = SIMPLE_INDEX_PARALLELISM.key();
/** @deprecated Use {@link #SIMPLE_INDEX_PARALLELISM} and its methods instead */
@Deprecated
public static final String DEFAULT_SIMPLE_INDEX_PARALLELISM = SIMPLE_INDEX_PARALLELISM.defaultValue();
/** @deprecated Use {@link #GLOBAL_SIMPLE_INDEX_PARALLELISM} and its methods instead */
@Deprecated
public static final String GLOBAL_SIMPLE_INDEX_PARALLELISM_PROP = GLOBAL_SIMPLE_INDEX_PARALLELISM.key();
/**
* @deprecated Use {@link #GLOBAL_SIMPLE_INDEX_PARALLELISM} and its methods instead
*/
@Deprecated
public static final String DEFAULT_GLOBAL_SIMPLE_INDEX_PARALLELISM = GLOBAL_SIMPLE_INDEX_PARALLELISM.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_KEYS_PER_BUCKET} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_KEYS_PER_BUCKET_PROP = BLOOM_INDEX_KEYS_PER_BUCKET.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_KEYS_PER_BUCKET} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_KEYS_PER_BUCKET = BLOOM_INDEX_KEYS_PER_BUCKET.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_INPUT_STORAGE_LEVEL = BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_INPUT_STORAGE_LEVEL = BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE.defaultValue();
/**
* @deprecated Use {@link #SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead
*/
@Deprecated
public static final String SIMPLE_INDEX_INPUT_STORAGE_LEVEL = SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE.key();
/**
* @deprecated Use {@link #SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_SIMPLE_INDEX_INPUT_STORAGE_LEVEL = SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE.defaultValue();
/**
* @deprecated Use {@link #BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final String BLOOM_INDEX_UPDATE_PARTITION_PATH = BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE.key();
/**
* @deprecated Use {@link #BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_BLOOM_INDEX_UPDATE_PARTITION_PATH = BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE.defaultValue();
/**
* @deprecated Use {@link #SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final String SIMPLE_INDEX_UPDATE_PARTITION_PATH = SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE.key();
/**
* @deprecated Use {@link #SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_SIMPLE_INDEX_UPDATE_PARTITION_PATH = SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE.defaultValue();
private EngineType engineType;
@@ -280,7 +427,7 @@ public class HoodieIndexConfig extends HoodieConfig {
}
public Builder withIndexClass(String indexClass) {
hoodieIndexConfig.setValue(INDEX_CLASS, indexClass);
hoodieIndexConfig.setValue(INDEX_CLASS_NAME, indexClass);
return this;
}
@@ -290,12 +437,12 @@ public class HoodieIndexConfig extends HoodieConfig {
}
public Builder bloomFilterNumEntries(int numEntries) {
hoodieIndexConfig.setValue(BLOOM_FILTER_NUM_ENTRIES, String.valueOf(numEntries));
hoodieIndexConfig.setValue(BLOOM_FILTER_NUM_ENTRIES_VALUE, String.valueOf(numEntries));
return this;
}
public Builder bloomFilterFPP(double fpp) {
hoodieIndexConfig.setValue(BLOOM_FILTER_FPP, String.valueOf(fpp));
hoodieIndexConfig.setValue(BLOOM_FILTER_FPP_VALUE, String.valueOf(fpp));
return this;
}
@@ -330,12 +477,12 @@ public class HoodieIndexConfig extends HoodieConfig {
}
public Builder withBloomIndexInputStorageLevel(String level) {
hoodieIndexConfig.setValue(BLOOM_INDEX_INPUT_STORAGE_LEVEL, level);
hoodieIndexConfig.setValue(BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE, level);
return this;
}
public Builder withBloomIndexUpdatePartitionPath(boolean updatePartitionPath) {
hoodieIndexConfig.setValue(BLOOM_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath));
hoodieIndexConfig.setValue(BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath));
return this;
}
@@ -350,7 +497,7 @@ public class HoodieIndexConfig extends HoodieConfig {
}
public Builder withSimpleIndexInputStorageLevel(String level) {
hoodieIndexConfig.setValue(SIMPLE_INDEX_INPUT_STORAGE_LEVEL, level);
hoodieIndexConfig.setValue(SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE, level);
return this;
}
@@ -360,7 +507,7 @@ public class HoodieIndexConfig extends HoodieConfig {
}
public Builder withGlobalSimpleIndexUpdatePartitionPath(boolean updatePartitionPath) {
hoodieIndexConfig.setValue(SIMPLE_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath));
hoodieIndexConfig.setValue(SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath));
return this;
}

55
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
View File

@@ -70,8 +70,6 @@ public class HoodieLockConfig extends HoodieConfig {
.sinceVersion("0.8.0")
.withDocumentation("Initial amount of time to wait between retries to acquire locks, "
+ " subsequent retries will exponentially backoff.");
@Deprecated
public static final String LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS.key();
public static final ConfigProperty<String> LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS = ConfigProperty
.key(LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY)
@@ -79,72 +77,54 @@ public class HoodieLockConfig extends HoodieConfig {
.sinceVersion("0.8.0")
.withDocumentation("Maximum amount of time to wait between retries by lock provider client. This bounds"
+ " the maximum delay from the exponential backoff. Currently used by ZK based lock provider only.");
@Deprecated
public static final String LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS.key();
public static final ConfigProperty<String> LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS = ConfigProperty
.key(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY)
.defaultValue(String.valueOf(10000L))
.sinceVersion("0.8.0")
.withDocumentation("Amount of time to wait between retries on the lock provider by the lock manager");
@Deprecated
public static final String LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS.key();
public static final ConfigProperty<String> LOCK_ACQUIRE_NUM_RETRIES = ConfigProperty
.key(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY)
.defaultValue(DEFAULT_LOCK_ACQUIRE_NUM_RETRIES)
.sinceVersion("0.8.0")
.withDocumentation("Maximum number of times to retry lock acquire, at each lock provider");
@Deprecated
public static final String LOCK_ACQUIRE_NUM_RETRIES_PROP = LOCK_ACQUIRE_NUM_RETRIES.key();
public static final ConfigProperty<String> LOCK_ACQUIRE_CLIENT_NUM_RETRIES = ConfigProperty
.key(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY)
.defaultValue(String.valueOf(0))
.sinceVersion("0.8.0")
.withDocumentation("Maximum number of times to retry to acquire lock additionally from the lock manager.");
@Deprecated
public static final String LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP = LOCK_ACQUIRE_CLIENT_NUM_RETRIES.key();
public static final ConfigProperty<Integer> LOCK_ACQUIRE_WAIT_TIMEOUT_MS = ConfigProperty
.key(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY)
.defaultValue(60 * 1000)
.sinceVersion("0.8.0")
.withDocumentation("Timeout in ms, to wait on an individual lock acquire() call, at the lock provider.");
@Deprecated
public static final String LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP = LOCK_ACQUIRE_WAIT_TIMEOUT_MS.key();
public static final ConfigProperty<String> FILESYSTEM_LOCK_PATH = ConfigProperty
.key(FILESYSTEM_LOCK_PATH_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("For DFS based lock providers, path to store the locks under.");
@Deprecated
public static final String FILESYSTEM_LOCK_PATH_PROP = FILESYSTEM_LOCK_PATH.key();
public static final ConfigProperty<String> HIVE_DATABASE_NAME = ConfigProperty
.key(HIVE_DATABASE_NAME_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("For Hive based lock provider, the Hive database to acquire lock against");
@Deprecated
public static final String HIVE_DATABASE_NAME_PROP = HIVE_DATABASE_NAME.key();
public static final ConfigProperty<String> HIVE_TABLE_NAME = ConfigProperty
.key(HIVE_TABLE_NAME_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("For Hive based lock provider, the Hive table to acquire lock against");
@Deprecated
public static final String HIVE_TABLE_NAME_PROP = HIVE_TABLE_NAME.key();
public static final ConfigProperty<String> HIVE_METASTORE_URI = ConfigProperty
.key(HIVE_METASTORE_URI_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("For Hive based lock provider, the Hive metastore URI to acquire locks against.");
@Deprecated
public static final String HIVE_METASTORE_URI_PROP = HIVE_METASTORE_URI.key();
public static final ConfigProperty<String> ZK_BASE_PATH = ConfigProperty
.key(ZK_BASE_PATH_PROP_KEY)
@@ -152,40 +132,30 @@ public class HoodieLockConfig extends HoodieConfig {
.sinceVersion("0.8.0")
.withDocumentation("The base path on Zookeeper under which to create lock related ZNodes. "
+ "This should be same for all concurrent writers to the same table");
@Deprecated
public static final String ZK_BASE_PATH_PROP = ZK_BASE_PATH.key();
public static final ConfigProperty<Integer> ZK_SESSION_TIMEOUT_MS = ConfigProperty
.key(ZK_SESSION_TIMEOUT_MS_PROP_KEY)
.defaultValue(DEFAULT_ZK_SESSION_TIMEOUT_MS)
.sinceVersion("0.8.0")
.withDocumentation("Timeout in ms, to wait after losing connection to ZooKeeper, before the session is expired");
@Deprecated
public static final String ZK_SESSION_TIMEOUT_MS_PROP = ZK_SESSION_TIMEOUT_MS.key();
public static final ConfigProperty<Integer> ZK_CONNECTION_TIMEOUT_MS = ConfigProperty
.key(ZK_CONNECTION_TIMEOUT_MS_PROP_KEY)
.defaultValue(DEFAULT_ZK_CONNECTION_TIMEOUT_MS)
.sinceVersion("0.8.0")
.withDocumentation("Timeout in ms, to wait for establishing connection with Zookeeper.");
@Deprecated
public static final String ZK_CONNECTION_TIMEOUT_MS_PROP = ZK_CONNECTION_TIMEOUT_MS.key();
public static final ConfigProperty<String> ZK_CONNECT_URL = ConfigProperty
.key(ZK_CONNECT_URL_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("Zookeeper URL to connect to.");
@Deprecated
public static final String ZK_CONNECT_URL_PROP = ZK_CONNECT_URL.key();
public static final ConfigProperty<String> ZK_PORT = ConfigProperty
.key(ZK_PORT_PROP_KEY)
.noDefaultValue()
.sinceVersion("0.8.0")
.withDocumentation("Zookeeper port to connect to.");
@Deprecated
public static final String ZK_PORT_PROP = ZK_PORT.key();
public static final ConfigProperty<String> ZK_LOCK_KEY = ConfigProperty
.key(ZK_LOCK_KEY_PROP_KEY)
@@ -193,28 +163,35 @@ public class HoodieLockConfig extends HoodieConfig {
.sinceVersion("0.8.0")
.withDocumentation("Key name under base_path at which to create a ZNode and acquire lock. "
+ "Final path on zk will look like base_path/lock_key. We recommend setting this to the table name");
@Deprecated
public static final String ZK_LOCK_KEY_PROP = ZK_LOCK_KEY.key();
// Pluggable type of lock provider
public static final ConfigProperty<String> LOCK_PROVIDER_CLASS = ConfigProperty
public static final ConfigProperty<String> LOCK_PROVIDER_CLASS_NAME = ConfigProperty
.key(LOCK_PREFIX + "provider")
.defaultValue(ZookeeperBasedLockProvider.class.getName())
.sinceVersion("0.8.0")
.withDocumentation("Lock provider class name, user can provide their own implementation of LockProvider "
+ "which should be subclass of org.apache.hudi.common.lock.LockProvider");
@Deprecated
public static final String LOCK_PROVIDER_CLASS_PROP = LOCK_PROVIDER_CLASS.key();
// Pluggable strategies to use when resolving conflicts
public static final ConfigProperty<String> WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS = ConfigProperty
public static final ConfigProperty<String> WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME = ConfigProperty
.key(LOCK_PREFIX + "conflict.resolution.strategy")
.defaultValue(SimpleConcurrentFileWritesConflictResolutionStrategy.class.getName())
.sinceVersion("0.8.0")
.withDocumentation("Lock provider class name, this should be subclass of "
+ "org.apache.hudi.client.transaction.ConflictResolutionStrategy");
/** @deprecated Use {@link #WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME} and its methods instead */
@Deprecated
public static final String WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_PROP = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS.key();
public static final String WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_PROP = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.key();
/** @deprecated Use {@link #WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME} and its methods instead */
@Deprecated
public static final String DEFAULT_WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.defaultValue();
/** @deprecated Use {@link #LOCK_PROVIDER_CLASS_NAME} and its methods instead */
@Deprecated
public static final String LOCK_PROVIDER_CLASS_PROP = LOCK_PROVIDER_CLASS_NAME.key();
/** @deprecated Use {@link #LOCK_PROVIDER_CLASS_NAME} and its methods instead */
@Deprecated
public static final String DEFAULT_LOCK_PROVIDER_CLASS = LOCK_PROVIDER_CLASS_NAME.defaultValue();
private HoodieLockConfig() {
super();
@@ -241,7 +218,7 @@ public class HoodieLockConfig extends HoodieConfig {
}
public HoodieLockConfig.Builder withLockProvider(Class<? extends LockProvider> lockProvider) {
lockConfig.setValue(LOCK_PROVIDER_CLASS, lockProvider.getName());
lockConfig.setValue(LOCK_PROVIDER_CLASS_NAME, lockProvider.getName());
return this;
}
@@ -321,7 +298,7 @@ public class HoodieLockConfig extends HoodieConfig {
}
public HoodieLockConfig.Builder withConflictResolutionStrategy(ConflictResolutionStrategy conflictResolutionStrategy) {
lockConfig.setValue(WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS, conflictResolutionStrategy.getClass().getName());
lockConfig.setValue(WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME, conflictResolutionStrategy.getClass().getName());
return this;
}

47
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
View File

@@ -46,8 +46,6 @@ public class HoodieMemoryConfig extends HoodieConfig {
.defaultValue(String.valueOf(0.6))
.withDocumentation("This fraction is multiplied with the user memory fraction (1 - spark.memory.fraction) "
+ "to get a final fraction of heap space to use during merge");
@Deprecated
public static final String MAX_MEMORY_FRACTION_FOR_MERGE_PROP = MAX_MEMORY_FRACTION_FOR_MERGE.key();
// Default max memory fraction during compaction, excess spills to disk
public static final ConfigProperty<String> MAX_MEMORY_FRACTION_FOR_COMPACTION = ConfigProperty
@@ -58,8 +56,6 @@ public class HoodieMemoryConfig extends HoodieConfig {
+ "less than or equal to the number of entries in the corresponding parquet file. This can lead to "
+ "OOM in the Scanner. Hence, a spillable map helps alleviate the memory pressure. Use this config to "
+ "set the max allowable inMemory footprint of the spillable map");
@Deprecated
public static final String MAX_MEMORY_FRACTION_FOR_COMPACTION_PROP = MAX_MEMORY_FRACTION_FOR_COMPACTION.key();
// Default memory size (1GB) per compaction (used if SparkEnv is absent), excess spills to disk
public static final long DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES = 1024 * 1024 * 1024L;
@@ -70,29 +66,21 @@ public class HoodieMemoryConfig extends HoodieConfig {
.key("hoodie.memory.merge.max.size")
.defaultValue(DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES)
.withDocumentation("Maximum amount of memory used for merge operations, before spilling to local storage.");
@Deprecated
public static final String MAX_MEMORY_FOR_MERGE_PROP = MAX_MEMORY_FOR_MERGE.key();
public static final ConfigProperty<String> MAX_MEMORY_FOR_COMPACTION = ConfigProperty
.key("hoodie.memory.compaction.max.size")
.noDefaultValue()
.withDocumentation("Maximum amount of memory used for compaction operations, before spilling to local storage.");
@Deprecated
public static final String MAX_MEMORY_FOR_COMPACTION_PROP = MAX_MEMORY_FOR_COMPACTION.key();
public static final ConfigProperty<Integer> MAX_DFS_STREAM_BUFFER_SIZE = ConfigProperty
.key("hoodie.memory.dfs.buffer.max.size")
.defaultValue(16 * 1024 * 1024)
.withDocumentation("Property to control the max memory for dfs input stream buffer size");
@Deprecated
public static final String MAX_DFS_STREAM_BUFFER_SIZE_PROP = MAX_DFS_STREAM_BUFFER_SIZE.key();
public static final ConfigProperty<String> SPILLABLE_MAP_BASE_PATH = ConfigProperty
.key("hoodie.memory.spillable.map.path")
.defaultValue("/tmp/")
.withDocumentation("Default file path prefix for spillable map");
@Deprecated
public static final String SPILLABLE_MAP_BASE_PATH_PROP = SPILLABLE_MAP_BASE_PATH.key();
public static final ConfigProperty<Double> WRITESTATUS_FAILURE_FRACTION = ConfigProperty
.key("hoodie.memory.writestatus.failure.fraction")
@@ -100,8 +88,43 @@ public class HoodieMemoryConfig extends HoodieConfig {
.withDocumentation("Property to control how what fraction of the failed record, exceptions we report back to driver. "
+ "Default is 10%. If set to 100%, with lot of failures, this can cause memory pressure, cause OOMs and "
+ "mask actual data errors.");
/** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_MERGE} and its methods instead */
@Deprecated
public static final String MAX_MEMORY_FRACTION_FOR_MERGE_PROP = MAX_MEMORY_FRACTION_FOR_MERGE.key();
/** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_MERGE} and its methods instead */
@Deprecated
public static final String DEFAULT_MAX_MEMORY_FRACTION_FOR_MERGE = MAX_MEMORY_FRACTION_FOR_MERGE.defaultValue();
/** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_COMPACTION} and its methods instead */
@Deprecated
public static final String MAX_MEMORY_FRACTION_FOR_COMPACTION_PROP = MAX_MEMORY_FRACTION_FOR_COMPACTION.key();
/** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_COMPACTION} and its methods instead */
@Deprecated
public static final String DEFAULT_MAX_MEMORY_FRACTION_FOR_COMPACTION = MAX_MEMORY_FRACTION_FOR_COMPACTION.defaultValue();
/** @deprecated Use {@link #MAX_MEMORY_FOR_MERGE} and its methods instead */
@Deprecated
public static final String MAX_MEMORY_FOR_MERGE_PROP = MAX_MEMORY_FOR_MERGE.key();
/** @deprecated Use {@link #MAX_MEMORY_FOR_COMPACTION} and its methods instead */
@Deprecated
public static final String MAX_MEMORY_FOR_COMPACTION_PROP = MAX_MEMORY_FOR_COMPACTION.key();
/** @deprecated Use {@link #MAX_DFS_STREAM_BUFFER_SIZE} and its methods instead */
@Deprecated
public static final String MAX_DFS_STREAM_BUFFER_SIZE_PROP = MAX_DFS_STREAM_BUFFER_SIZE.key();
/** @deprecated Use {@link #MAX_DFS_STREAM_BUFFER_SIZE} and its methods instead */
@Deprecated
public static final int DEFAULT_MAX_DFS_STREAM_BUFFER_SIZE = MAX_DFS_STREAM_BUFFER_SIZE.defaultValue();
/** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead */
@Deprecated
public static final String SPILLABLE_MAP_BASE_PATH_PROP = SPILLABLE_MAP_BASE_PATH.key();
/** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead */
@Deprecated
public static final String DEFAULT_SPILLABLE_MAP_BASE_PATH = SPILLABLE_MAP_BASE_PATH.defaultValue();
/** @deprecated Use {@link #WRITESTATUS_FAILURE_FRACTION} and its methods instead */
@Deprecated
public static final String WRITESTATUS_FAILURE_FRACTION_PROP = WRITESTATUS_FAILURE_FRACTION.key();
/** @deprecated Use {@link #WRITESTATUS_FAILURE_FRACTION} and its methods instead */
@Deprecated
public static final double DEFAULT_WRITESTATUS_FAILURE_FRACTION = WRITESTATUS_FAILURE_FRACTION.defaultValue();
private HoodieMemoryConfig() {
super();

119
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
View File

@@ -43,13 +43,13 @@ public class HoodieMetricsConfig extends HoodieConfig {
public static final String METRIC_PREFIX = "hoodie.metrics";
public static final ConfigProperty<Boolean> METRICS_ON = ConfigProperty
public static final ConfigProperty<Boolean> TURN_METRICS_ON = ConfigProperty
.key(METRIC_PREFIX + ".on")
.defaultValue(false)
.sinceVersion("0.5.0")
.withDocumentation("Turn on/off metrics reporting. off by default.");
public static final ConfigProperty<MetricsReporterType> METRICS_REPORTER_TYPE = ConfigProperty
public static final ConfigProperty<MetricsReporterType> METRICS_REPORTER_TYPE_VALUE = ConfigProperty
.key(METRIC_PREFIX + ".reporter.type")
.defaultValue(MetricsReporterType.GRAPHITE)
.sinceVersion("0.5.0")
@@ -58,13 +58,13 @@ public class HoodieMetricsConfig extends HoodieConfig {
// Graphite
public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite";
public static final ConfigProperty<String> GRAPHITE_SERVER_HOST = ConfigProperty
public static final ConfigProperty<String> GRAPHITE_SERVER_HOST_NAME = ConfigProperty
.key(GRAPHITE_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.0")
.withDocumentation("Graphite host to connect to");
public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT = ConfigProperty
public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT_NUM = ConfigProperty
.key(GRAPHITE_PREFIX + ".port")
.defaultValue(4756)
.sinceVersion("0.5.0")
@@ -73,38 +73,119 @@ public class HoodieMetricsConfig extends HoodieConfig {
// Jmx
public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
public static final ConfigProperty<String> JMX_HOST = ConfigProperty
public static final ConfigProperty<String> JMX_HOST_NAME = ConfigProperty
.key(JMX_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.1")
.withDocumentation("Jmx host to connect to");
public static final ConfigProperty<Integer> JMX_PORT = ConfigProperty
public static final ConfigProperty<Integer> JMX_PORT_NUM = ConfigProperty
.key(JMX_PREFIX + ".port")
.defaultValue(9889)
.sinceVersion("0.5.1")
.withDocumentation("Jmx port to connect to");
public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX = ConfigProperty
public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty
.key(GRAPHITE_PREFIX + ".metric.prefix")
.noDefaultValue()
.sinceVersion("0.5.1")
.withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g");
// User defined
public static final ConfigProperty<String> METRICS_REPORTER_CLASS = ConfigProperty
public static final ConfigProperty<String> METRICS_REPORTER_CLASS_NAME = ConfigProperty
.key(METRIC_PREFIX + ".reporter.class")
.defaultValue("")
.sinceVersion("0.6.0")
.withDocumentation("");
// Enable metrics collection from executors
public static final ConfigProperty<String> ENABLE_EXECUTOR_METRICS = ConfigProperty
public static final ConfigProperty<String> EXECUTOR_METRICS_ENABLE = ConfigProperty
.key(METRIC_PREFIX + ".executor.enable")
.noDefaultValue()
.sinceVersion("0.7.0")
.withDocumentation("");
/**
* @deprecated Use {@link #TURN_METRICS_ON} and its methods instead
*/
@Deprecated
public static final String METRICS_ON = TURN_METRICS_ON.key();
/**
* @deprecated Use {@link #TURN_METRICS_ON} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_METRICS_ON = TURN_METRICS_ON.defaultValue();
/**
* @deprecated Use {@link #METRICS_REPORTER_TYPE_VALUE} and its methods instead
*/
@Deprecated
public static final String METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.key();
/**
* @deprecated Use {@link #METRICS_REPORTER_TYPE_VALUE} and its methods instead
*/
@Deprecated
public static final MetricsReporterType DEFAULT_METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String JMX_HOST = JMX_HOST_NAME.key();
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String JMX_PORT = JMX_PORT_NUM.key();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key();
/**
* @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String METRICS_REPORTER_CLASS = METRICS_REPORTER_CLASS_NAME.key();
/**
* @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_METRICS_REPORTER_CLASS = METRICS_REPORTER_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #EXECUTOR_METRICS_ENABLE} and its methods instead
*/
@Deprecated
public static final String ENABLE_EXECUTOR_METRICS = EXECUTOR_METRICS_ENABLE.key();
private HoodieMetricsConfig() {
super();
}
@@ -130,47 +211,47 @@ public class HoodieMetricsConfig extends HoodieConfig {
}
public Builder on(boolean metricsOn) {
hoodieMetricsConfig.setValue(METRICS_ON, String.valueOf(metricsOn));
hoodieMetricsConfig.setValue(TURN_METRICS_ON, String.valueOf(metricsOn));
return this;
}
public Builder withReporterType(String reporterType) {
hoodieMetricsConfig.setValue(METRICS_REPORTER_TYPE, reporterType);
hoodieMetricsConfig.setValue(METRICS_REPORTER_TYPE_VALUE, reporterType);
return this;
}
public Builder toGraphiteHost(String host) {
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST, host);
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host);
return this;
}
public Builder onGraphitePort(int port) {
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT, String.valueOf(port));
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port));
return this;
}
public Builder toJmxHost(String host) {
hoodieMetricsConfig.setValue(JMX_HOST, host);
hoodieMetricsConfig.setValue(JMX_HOST_NAME, host);
return this;
}
public Builder onJmxPort(String port) {
hoodieMetricsConfig.setValue(JMX_PORT, port);
hoodieMetricsConfig.setValue(JMX_PORT_NUM, port);
return this;
}
public Builder usePrefix(String prefix) {
hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX, prefix);
hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix);
return this;
}
public Builder withReporterClass(String className) {
hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS, className);
hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS_NAME, className);
return this;
}
public Builder withExecutorMetrics(boolean enable) {
hoodieMetricsConfig.setValue(ENABLE_EXECUTOR_METRICS, String.valueOf(enable));
hoodieMetricsConfig.setValue(EXECUTOR_METRICS_ENABLE, String.valueOf(enable));
return this;
}
@@ -178,7 +259,7 @@ public class HoodieMetricsConfig extends HoodieConfig {
hoodieMetricsConfig.setDefaults(HoodieMetricsConfig.class.getName());
MetricsReporterType reporterType = MetricsReporterType.valueOf(hoodieMetricsConfig.getString(METRICS_REPORTER_TYPE));
MetricsReporterType reporterType = MetricsReporterType.valueOf(hoodieMetricsConfig.getString(METRICS_REPORTER_TYPE_VALUE));
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.DATADOG,
HoodieMetricsDatadogConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());

97
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java
View File

@@ -43,63 +43,124 @@ public class HoodieMetricsDatadogConfig extends HoodieConfig {
public static final String DATADOG_PREFIX = METRIC_PREFIX + ".datadog";
public static final ConfigProperty<Integer> DATADOG_REPORT_PERIOD_SECONDS = ConfigProperty
public static final ConfigProperty<Integer> REPORT_PERIOD_IN_SECONDS = ConfigProperty
.key(DATADOG_PREFIX + ".report.period.seconds")
.defaultValue(30)
.sinceVersion("0.6.0")
.withDocumentation("Datadog reporting period in seconds. Default to 30.");
public static final ConfigProperty<String> DATADOG_API_SITE = ConfigProperty
public static final ConfigProperty<String> API_SITE_VALUE = ConfigProperty
.key(DATADOG_PREFIX + ".api.site")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog API site: EU or US");
public static final ConfigProperty<String> DATADOG_API_KEY = ConfigProperty
public static final ConfigProperty<String> API_KEY = ConfigProperty
.key(DATADOG_PREFIX + ".api.key")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog API key");
public static final ConfigProperty<Boolean> DATADOG_API_KEY_SKIP_VALIDATION = ConfigProperty
public static final ConfigProperty<Boolean> API_KEY_SKIP_VALIDATION = ConfigProperty
.key(DATADOG_PREFIX + ".api.key.skip.validation")
.defaultValue(false)
.sinceVersion("0.6.0")
.withDocumentation("Before sending metrics via Datadog API, whether to skip validating Datadog API key or not. "
+ "Default to false.");
public static final ConfigProperty<String> DATADOG_API_KEY_SUPPLIER = ConfigProperty
public static final ConfigProperty<String> API_KEY_SUPPLIER = ConfigProperty
.key(DATADOG_PREFIX + ".api.key.supplier")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog API key supplier to supply the API key at runtime. "
+ "This will take effect if hoodie.metrics.datadog.api.key is not set.");
public static final ConfigProperty<Integer> DATADOG_API_TIMEOUT_SECONDS = ConfigProperty
public static final ConfigProperty<Integer> API_TIMEOUT_IN_SECONDS = ConfigProperty
.key(DATADOG_PREFIX + ".api.timeout.seconds")
.defaultValue(3)
.sinceVersion("0.6.0")
.withDocumentation("Datadog API timeout in seconds. Default to 3.");
public static final ConfigProperty<String> DATADOG_METRIC_PREFIX = ConfigProperty
public static final ConfigProperty<String> METRIC_PREFIX_VALUE = ConfigProperty
.key(DATADOG_PREFIX + ".metric.prefix")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog metric prefix to be prepended to each metric name with a dot as delimiter. "
+ "For example, if it is set to foo, foo. will be prepended.");
public static final ConfigProperty<String> DATADOG_METRIC_HOST = ConfigProperty
public static final ConfigProperty<String> METRIC_HOST_NAME = ConfigProperty
.key(DATADOG_PREFIX + ".metric.host")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog metric host to be sent along with metrics data.");
public static final ConfigProperty<String> DATADOG_METRIC_TAGS = ConfigProperty
public static final ConfigProperty<String> METRIC_TAG_VALUES = ConfigProperty
.key(DATADOG_PREFIX + ".metric.tags")
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Datadog metric tags (comma-delimited) to be sent along with metrics data.");
/**
* @deprecated Use {@link #REPORT_PERIOD_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final String DATADOG_REPORT_PERIOD_SECONDS = REPORT_PERIOD_IN_SECONDS.key();
/**
* @deprecated Use {@link #REPORT_PERIOD_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_DATADOG_REPORT_PERIOD_SECONDS = REPORT_PERIOD_IN_SECONDS.defaultValue();
/**
* @deprecated Use {@link #API_SITE_VALUE} and its methods instead
*/
@Deprecated
public static final String DATADOG_API_SITE = API_SITE_VALUE.key();
/**
* @deprecated Use {@link #API_KEY} and its methods instead
*/
@Deprecated
public static final String DATADOG_API_KEY = API_KEY.key();
/**
* @deprecated Use {@link #API_KEY_SKIP_VALIDATION} and its methods instead
*/
@Deprecated
public static final String DATADOG_API_KEY_SKIP_VALIDATION = API_KEY_SKIP_VALIDATION.key();
/**
* @deprecated Use {@link #API_KEY_SKIP_VALIDATION} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_DATADOG_API_KEY_SKIP_VALIDATION = API_KEY_SKIP_VALIDATION.defaultValue();
/**
* @deprecated Use {@link #API_KEY_SUPPLIER} and its methods instead
*/
@Deprecated
public static final String DATADOG_API_KEY_SUPPLIER = API_KEY_SUPPLIER.key();
/**
* @deprecated Use {@link #API_TIMEOUT_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final String DATADOG_API_TIMEOUT_SECONDS = API_TIMEOUT_IN_SECONDS.key();
/**
* @deprecated Use {@link #API_TIMEOUT_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_DATADOG_API_TIMEOUT_SECONDS = API_TIMEOUT_IN_SECONDS.defaultValue();
/**
* @deprecated Use {@link #METRIC_PREFIX_VALUE} and its methods instead
*/
@Deprecated
public static final String DATADOG_METRIC_PREFIX = METRIC_PREFIX_VALUE.key();
/**
* @deprecated Use {@link #METRIC_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DATADOG_METRIC_HOST = METRIC_HOST_NAME.key();
/**
* @deprecated Use {@link #METRIC_TAG_VALUES} and its methods instead
*/
@Deprecated
public static final String DATADOG_METRIC_TAGS = METRIC_TAG_VALUES.key();
private HoodieMetricsDatadogConfig() {
super();
}
@@ -118,47 +179,47 @@ public class HoodieMetricsDatadogConfig extends HoodieConfig {
}
public Builder withDatadogReportPeriodSeconds(int period) {
metricsDatadogConfig.setValue(DATADOG_REPORT_PERIOD_SECONDS, String.valueOf(period));
metricsDatadogConfig.setValue(REPORT_PERIOD_IN_SECONDS, String.valueOf(period));
return this;
}
public Builder withDatadogApiSite(String apiSite) {
metricsDatadogConfig.setValue(DATADOG_API_SITE, apiSite);
metricsDatadogConfig.setValue(API_SITE_VALUE, apiSite);
return this;
}
public Builder withDatadogApiKey(String apiKey) {
metricsDatadogConfig.setValue(DATADOG_API_KEY, apiKey);
metricsDatadogConfig.setValue(API_KEY, apiKey);
return this;
}
public Builder withDatadogApiKeySkipValidation(boolean skip) {
metricsDatadogConfig.setValue(DATADOG_API_KEY_SKIP_VALIDATION, String.valueOf(skip));
metricsDatadogConfig.setValue(API_KEY_SKIP_VALIDATION, String.valueOf(skip));
return this;
}
public Builder withDatadogApiKeySupplier(String apiKeySupplier) {
metricsDatadogConfig.setValue(DATADOG_API_KEY_SUPPLIER, apiKeySupplier);
metricsDatadogConfig.setValue(API_KEY_SUPPLIER, apiKeySupplier);
return this;
}
public Builder withDatadogApiTimeoutSeconds(int timeout) {
metricsDatadogConfig.setValue(DATADOG_API_TIMEOUT_SECONDS, String.valueOf(timeout));
metricsDatadogConfig.setValue(API_TIMEOUT_IN_SECONDS, String.valueOf(timeout));
return this;
}
public Builder withDatadogPrefix(String prefix) {
metricsDatadogConfig.setValue(DATADOG_METRIC_PREFIX, prefix);
metricsDatadogConfig.setValue(METRIC_PREFIX_VALUE, prefix);
return this;
}
public Builder withDatadogHost(String host) {
metricsDatadogConfig.setValue(DATADOG_METRIC_HOST, host);
metricsDatadogConfig.setValue(METRIC_HOST_NAME, host);
return this;
}
public Builder withDatadogTags(String tags) {
metricsDatadogConfig.setValue(DATADOG_METRIC_TAGS, tags);
metricsDatadogConfig.setValue(METRIC_TAG_VALUES, tags);
return this;
}

85
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java
View File

@@ -36,37 +36,37 @@ public class HoodieMetricsPrometheusConfig extends HoodieConfig {
// Prometheus PushGateWay
public static final String PUSHGATEWAY_PREFIX = METRIC_PREFIX + ".pushgateway";
public static final ConfigProperty<String> PUSHGATEWAY_HOST = ConfigProperty
public static final ConfigProperty<String> PUSHGATEWAY_HOST_NAME = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.6.0")
.withDocumentation("Hostname of the prometheus push gateway");
public static final ConfigProperty<Integer> PUSHGATEWAY_PORT = ConfigProperty
public static final ConfigProperty<Integer> PUSHGATEWAY_PORT_NUM = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".port")
.defaultValue(9091)
.sinceVersion("0.6.0")
.withDocumentation("Port for the push gateway.");
public static final ConfigProperty<Integer> PUSHGATEWAY_REPORT_PERIOD_SECONDS = ConfigProperty
public static final ConfigProperty<Integer> PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".report.period.seconds")
.defaultValue(30)
.sinceVersion("0.6.0")
.withDocumentation("Reporting interval in seconds.");
public static final ConfigProperty<Boolean> PUSHGATEWAY_DELETE_ON_SHUTDOWN = ConfigProperty
public static final ConfigProperty<Boolean> PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".delete.on.shutdown")
.defaultValue(true)
.sinceVersion("0.6.0")
.withDocumentation("");
public static final ConfigProperty<String> PUSHGATEWAY_JOB_NAME = ConfigProperty
public static final ConfigProperty<String> PUSHGATEWAY_JOBNAME = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".job.name")
.defaultValue("")
.sinceVersion("0.6.0")
.withDocumentation("Name of the push gateway job.");
public static final ConfigProperty<Boolean> PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = ConfigProperty
public static final ConfigProperty<Boolean> PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX = ConfigProperty
.key(PUSHGATEWAY_PREFIX + ".random.job.name.suffix")
.defaultValue(true)
.sinceVersion("0.6.0")
@@ -75,12 +75,83 @@ public class HoodieMetricsPrometheusConfig extends HoodieConfig {
// Prometheus HttpServer
public static final String PROMETHEUS_PREFIX = METRIC_PREFIX + ".prometheus";
public static final ConfigProperty<Integer> PROMETHEUS_PORT = ConfigProperty
public static final ConfigProperty<Integer> PROMETHEUS_PORT_NUM = ConfigProperty
.key(PROMETHEUS_PREFIX + ".port")
.defaultValue(9090)
.sinceVersion("0.6.0")
.withDocumentation("Port for prometheus server.");
/**
* @deprecated Use {@link #PUSHGATEWAY_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_HOST = PUSHGATEWAY_HOST_NAME.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PUSHGATEWAY_HOST = PUSHGATEWAY_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #PUSHGATEWAY_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_PORT = PUSHGATEWAY_PORT_NUM.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_PUSHGATEWAY_PORT = PUSHGATEWAY_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_REPORT_PERIOD_SECONDS = PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_PUSHGATEWAY_REPORT_PERIOD_SECONDS = PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS.defaultValue();
/**
* @deprecated Use {@link #PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_DELETE_ON_SHUTDOWN = PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_PUSHGATEWAY_DELETE_ON_SHUTDOWN = PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE.defaultValue();
/**
* @deprecated Use {@link #PUSHGATEWAY_JOBNAME} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_JOB_NAME = PUSHGATEWAY_JOBNAME.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_JOBNAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PUSHGATEWAY_JOB_NAME = PUSHGATEWAY_JOBNAME.defaultValue();
/**
* @deprecated Use {@link #PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX} and its methods instead
*/
@Deprecated
public static final String PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX.key();
/**
* @deprecated Use {@link #PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX.defaultValue();
/**
* @deprecated Use {@link #PROMETHEUS_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String PROMETHEUS_PORT = PROMETHEUS_PORT_NUM.key();
/**
* @deprecated Use {@link #PROMETHEUS_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_PROMETHEUS_PORT = PROMETHEUS_PORT_NUM.defaultValue();
private HoodieMetricsPrometheusConfig() {
super();
}

12
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java
View File

@@ -40,21 +40,17 @@ import static org.apache.hudi.common.model.HoodiePayloadProps.PAYLOAD_ORDERING_F
+ "control merges based on specific business fields in the data.")
public class HoodiePayloadConfig extends HoodieConfig {
public static final ConfigProperty<String> PAYLOAD_ORDERING_FIELD = ConfigProperty
public static final ConfigProperty<String> ORDERING_FIELD = ConfigProperty
.key(PAYLOAD_ORDERING_FIELD_PROP_KEY)
.defaultValue("ts")
.withDocumentation("Table column/field name to order records that have the same key, before "
+ "merging and writing to storage.");
@Deprecated
public static final String PAYLOAD_ORDERING_FIELD_PROP = PAYLOAD_ORDERING_FIELD.key();
public static final ConfigProperty<String> PAYLOAD_EVENT_TIME_FIELD = ConfigProperty
public static final ConfigProperty<String> EVENT_TIME_FIELD = ConfigProperty
.key(PAYLOAD_EVENT_TIME_FIELD_PROP_KEY)
.defaultValue("ts")
.withDocumentation("Table column/field name to derive timestamp associated with the records. This can"
+ "be useful for e.g, determining the freshness of the table.");
@Deprecated
public static final String PAYLOAD_EVENT_TIME_FIELD_PROP = PAYLOAD_EVENT_TIME_FIELD.key();
private HoodiePayloadConfig() {
super();
@@ -81,12 +77,12 @@ public class HoodiePayloadConfig extends HoodieConfig {
}
public Builder withPayloadOrderingField(String payloadOrderingField) {
payloadConfig.setValue(PAYLOAD_ORDERING_FIELD, String.valueOf(payloadOrderingField));
payloadConfig.setValue(ORDERING_FIELD, String.valueOf(payloadOrderingField));
return this;
}
public Builder withPayloadEventTimeField(String payloadEventTimeField) {
payloadConfig.setValue(PAYLOAD_EVENT_TIME_FIELD, String.valueOf(payloadEventTimeField));
payloadConfig.setValue(EVENT_TIME_FIELD, String.valueOf(payloadEventTimeField));
return this;
}

33
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java
View File

@@ -22,6 +22,7 @@ import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import javax.annotation.concurrent.Immutable;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
@@ -33,26 +34,26 @@ import java.util.Properties;
@Immutable
public class HoodiePreCommitValidatorConfig extends HoodieConfig {
public static final ConfigProperty<String> PRE_COMMIT_VALIDATORS = ConfigProperty
public static final ConfigProperty<String> VALIDATOR_CLASS_NAMES = ConfigProperty
.key("hoodie.precommit.validators")
.defaultValue("")
.withDocumentation("Comma separated list of class names that can be invoked to validate commit");
public static final String VALIDATOR_TABLE_VARIABLE = "<TABLE_NAME>";
public static final ConfigProperty<String> PRE_COMMIT_VALIDATORS_EQUALITY_SQL_QUERIES = ConfigProperty
public static final ConfigProperty<String> EQUALITY_SQL_QUERIES = ConfigProperty
.key("hoodie.precommit.validators.equality.sql.queries")
.defaultValue("")
.withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit."
+ " Multiple queries separated by ';' delimiter are supported."
+ " Example: \"select count(*) from \\<TABLE_NAME\\>"
+ " Multiple queries separated by ';' delimiter are supported."
+ " Example: \"select count(*) from \\<TABLE_NAME\\>"
+ " Note \\<TABLE_NAME\\> is replaced by table state before and after commit.");
public static final ConfigProperty<String> PRE_COMMIT_VALIDATORS_SINGLE_VALUE_SQL_QUERIES = ConfigProperty
public static final ConfigProperty<String> SINGLE_VALUE_SQL_QUERIES = ConfigProperty
.key("hoodie.precommit.validators.single.value.sql.queries")
.defaultValue("")
.withDocumentation("Spark SQL queries to run on table before committing new data to validate state after commit."
+ "Multiple queries separated by ';' delimiter are supported."
+ "Expected result is included as part of query separated by '#'. Example query: 'query1#result1:query2#result2'"
.withDocumentation("Spark SQL queries to run on table before committing new data to validate state after commit."
+ "Multiple queries separated by ';' delimiter are supported."
+ "Expected result is included as part of query separated by '#'. Example query: 'query1#result1:query2#result2'"
+ "Note \\<TABLE_NAME\\> variable is expected to be present in query.");
/**
@@ -61,11 +62,11 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig {
* Example query: 'select count(*) from \<TABLE_NAME\> where col=null'
* Note \<TABLE_NAME\> variable is expected to be present in query.
*/
public static final ConfigProperty<String> PRE_COMMIT_VALIDATORS_INEQUALITY_SQL_QUERIES = ConfigProperty
public static final ConfigProperty<String> INEQUALITY_SQL_QUERIES = ConfigProperty
.key("hoodie.precommit.validators.inequality.sql.queries")
.defaultValue("")
.withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit."
+ "Multiple queries separated by ';' delimiter are supported."
.withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit."
+ "Multiple queries separated by ';' delimiter are supported."
+ "Example query: 'select count(*) from \\<TABLE_NAME\\> where col=null'"
+ "Note \\<TABLE_NAME\\> variable is expected to be present in query.");
@@ -94,22 +95,22 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig {
}
public Builder withPreCommitValidator(String preCommitValidators) {
preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS, preCommitValidators);
preCommitValidatorConfig.setValue(VALIDATOR_CLASS_NAMES, preCommitValidators);
return this;
}
public Builder withPrecommitValidatorEqualitySqlQueries(String preCommitValidators) {
preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_EQUALITY_SQL_QUERIES, preCommitValidators);
preCommitValidatorConfig.setValue(EQUALITY_SQL_QUERIES, preCommitValidators);
return this;
}
public Builder withPrecommitValidatorSingleResultSqlQueries(String preCommitValidators) {
preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_SINGLE_VALUE_SQL_QUERIES, preCommitValidators);
preCommitValidatorConfig.setValue(SINGLE_VALUE_SQL_QUERIES, preCommitValidators);
return this;
}
public Builder withPrecommitValidatorInequalitySqlQueries(String preCommitValidators) {
preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_INEQUALITY_SQL_QUERIES, preCommitValidators);
preCommitValidatorConfig.setValue(INEQUALITY_SQL_QUERIES, preCommitValidators);
return this;
}

165
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java
View File

@@ -39,25 +39,25 @@ import java.util.Properties;
description = "Configurations that control aspects around writing, sizing, reading base and log files.")
public class HoodieStorageConfig extends HoodieConfig {
public static final ConfigProperty<String> PARQUET_FILE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> PARQUET_MAX_FILE_SIZE = ConfigProperty
.key("hoodie.parquet.max.file.size")
.defaultValue(String.valueOf(120 * 1024 * 1024))
.withDocumentation("Target size for parquet files produced by Hudi write phases. "
+ "For DFS, this needs to be aligned with the underlying filesystem block size for optimal performance.");
public static final ConfigProperty<String> PARQUET_BLOCK_SIZE_BYTES = ConfigProperty
public static final ConfigProperty<String> PARQUET_BLOCK_SIZE = ConfigProperty
.key("hoodie.parquet.block.size")
.defaultValue(String.valueOf(120 * 1024 * 1024))
.withDocumentation("Parquet RowGroup size. It's recommended to make this large enough that scan costs can be"
+ " amortized by packing enough column values into a single row group.");
public static final ConfigProperty<String> PARQUET_PAGE_SIZE_BYTES = ConfigProperty
public static final ConfigProperty<String> PARQUET_PAGE_SIZE = ConfigProperty
.key("hoodie.parquet.page.size")
.defaultValue(String.valueOf(1 * 1024 * 1024))
.withDocumentation("Parquet page size. Page is the unit of read within a parquet file. "
+ "Within a block, pages are compressed separately.");
public static final ConfigProperty<String> ORC_FILE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> ORC_FILE_MAX_SIZE = ConfigProperty
.key("hoodie.orc.max.file.size")
.defaultValue(String.valueOf(120 * 1024 * 1024))
.withDocumentation("Target file size for ORC base files.");
@@ -69,64 +69,175 @@ public class HoodieStorageConfig extends HoodieConfig {
public static final ConfigProperty<String> ORC_BLOCK_SIZE = ConfigProperty
.key("hoodie.orc.block.size")
.defaultValue(ORC_FILE_MAX_BYTES.defaultValue())
.defaultValue(ORC_FILE_MAX_SIZE.defaultValue())
.withDocumentation("ORC block size, recommended to be aligned with the target file size.");
public static final ConfigProperty<String> HFILE_FILE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> HFILE_MAX_FILE_SIZE = ConfigProperty
.key("hoodie.hfile.max.file.size")
.defaultValue(String.valueOf(120 * 1024 * 1024))
.withDocumentation("Target file size for HFile base files.");
public static final ConfigProperty<String> HFILE_BLOCK_SIZE_BYTES = ConfigProperty
public static final ConfigProperty<String> HFILE_BLOCK_SIZE = ConfigProperty
.key("hoodie.hfile.block.size")
.defaultValue(String.valueOf(1024 * 1024))
.withDocumentation("Lower values increase the size of metadata tracked within HFile, but can offer potentially "
+ "faster lookup times.");
// used to size log files
public static final ConfigProperty<String> LOGFILE_SIZE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> LOGFILE_MAX_SIZE = ConfigProperty
.key("hoodie.logfile.max.size")
.defaultValue(String.valueOf(1024 * 1024 * 1024)) // 1 GB
.withDocumentation("LogFile max size. This is the maximum size allowed for a log file "
+ "before it is rolled over to the next version.");
// used to size data blocks in log file
public static final ConfigProperty<String> LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = ConfigProperty
public static final ConfigProperty<String> LOGFILE_DATA_BLOCK_MAX_SIZE = ConfigProperty
.key("hoodie.logfile.data.block.max.size")
.defaultValue(String.valueOf(256 * 1024 * 1024))
.withDocumentation("LogFile Data block max size. This is the maximum size allowed for a single data block "
+ "to be appended to a log file. This helps to make sure the data appended to the log file is broken up "
+ "into sizable blocks to prevent from OOM errors. This size should be greater than the JVM memory.");
public static final ConfigProperty<String> PARQUET_COMPRESSION_RATIO = ConfigProperty
public static final ConfigProperty<String> PARQUET_COMPRESSION_RATIO_FRACTION = ConfigProperty
.key("hoodie.parquet.compression.ratio")
.defaultValue(String.valueOf(0.1))
.withDocumentation("Expected compression of parquet data used by Hudi, when it tries to size new parquet files. "
+ "Increase this value, if bulk_insert is producing smaller than expected sized files");
// Default compression codec for parquet
public static final ConfigProperty<String> PARQUET_COMPRESSION_CODEC = ConfigProperty
public static final ConfigProperty<String> PARQUET_COMPRESSION_CODEC_NAME = ConfigProperty
.key("hoodie.parquet.compression.codec")
.defaultValue("gzip")
.withDocumentation("Compression Codec for parquet files");
public static final ConfigProperty<String> HFILE_COMPRESSION_ALGORITHM = ConfigProperty
public static final ConfigProperty<String> HFILE_COMPRESSION_ALGORITHM_NAME = ConfigProperty
.key("hoodie.hfile.compression.algorithm")
.defaultValue("GZ")
.withDocumentation("Compression codec to use for hfile base files.");
public static final ConfigProperty<String> ORC_COMPRESSION_CODEC = ConfigProperty
public static final ConfigProperty<String> ORC_COMPRESSION_CODEC_NAME = ConfigProperty
.key("hoodie.orc.compression.codec")
.defaultValue("ZLIB")
.withDocumentation("Compression codec to use for ORC base files.");
// Default compression ratio for log file to parquet, general 3x
public static final ConfigProperty<String> LOGFILE_TO_PARQUET_COMPRESSION_RATIO = ConfigProperty
public static final ConfigProperty<String> LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION = ConfigProperty
.key("hoodie.logfile.to.parquet.compression.ratio")
.defaultValue(String.valueOf(0.35))
.withDocumentation("Expected additional compression as records move from log files to parquet. Used for merge_on_read "
+ "table to send inserts into log files & control the size of compacted parquet file.");
/**
* @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead
*/
@Deprecated
public static final String PARQUET_FILE_MAX_BYTES = PARQUET_MAX_FILE_SIZE.key();
/**
* @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PARQUET_FILE_MAX_BYTES = PARQUET_MAX_FILE_SIZE.defaultValue();
/**
* @deprecated Use {@link #PARQUET_BLOCK_SIZE} and its methods instead
*/
@Deprecated
public static final String PARQUET_BLOCK_SIZE_BYTES = PARQUET_BLOCK_SIZE.key();
/**
* @deprecated Use {@link #PARQUET_BLOCK_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PARQUET_BLOCK_SIZE_BYTES = PARQUET_BLOCK_SIZE.defaultValue();
/**
* @deprecated Use {@link #PARQUET_PAGE_SIZE} and its methods instead
*/
@Deprecated
public static final String PARQUET_PAGE_SIZE_BYTES = PARQUET_PAGE_SIZE.key();
/**
* @deprecated Use {@link #PARQUET_PAGE_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PARQUET_PAGE_SIZE_BYTES = PARQUET_PAGE_SIZE.defaultValue();
/**
* @deprecated Use {@link #HFILE_MAX_FILE_SIZE} and its methods instead
*/
@Deprecated
public static final String HFILE_FILE_MAX_BYTES = HFILE_MAX_FILE_SIZE.key();
/**
* @deprecated Use {@link #HFILE_MAX_FILE_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HFILE_FILE_MAX_BYTES = HFILE_MAX_FILE_SIZE.defaultValue();
/**
* @deprecated Use {@link #HFILE_BLOCK_SIZE} and its methods instead
*/
@Deprecated
public static final String HFILE_BLOCK_SIZE_BYTES = HFILE_BLOCK_SIZE.defaultValue();
/**
* @deprecated Use {@link #HFILE_BLOCK_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HFILE_BLOCK_SIZE_BYTES = HFILE_BLOCK_SIZE.defaultValue();
/**
* @deprecated Use {@link #LOGFILE_MAX_SIZE} and its methods instead
*/
@Deprecated
public static final String LOGFILE_SIZE_MAX_BYTES = LOGFILE_MAX_SIZE.key();
/**
* @deprecated Use {@link #LOGFILE_MAX_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_LOGFILE_SIZE_MAX_BYTES = LOGFILE_MAX_SIZE.defaultValue();
/**
* @deprecated Use {@link #LOGFILE_DATA_BLOCK_MAX_SIZE} and its methods instead
*/
@Deprecated
public static final String LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = LOGFILE_DATA_BLOCK_MAX_SIZE.key();
/**
* @deprecated Use {@link #LOGFILE_DATA_BLOCK_MAX_SIZE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = LOGFILE_DATA_BLOCK_MAX_SIZE.defaultValue();
/**
* @deprecated Use {@link #PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead
*/
@Deprecated
public static final String PARQUET_COMPRESSION_RATIO = PARQUET_COMPRESSION_RATIO_FRACTION.key();
/**
* @deprecated Use {@link #PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead
*/
@Deprecated
public static final String DEFAULT_STREAM_COMPRESSION_RATIO = PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue();
/**
* @deprecated Use {@link #PARQUET_COMPRESSION_CODEC_NAME} and its methods instead
*/
@Deprecated
public static final String PARQUET_COMPRESSION_CODEC = PARQUET_COMPRESSION_CODEC_NAME.key();
/**
* @deprecated Use {@link #HFILE_COMPRESSION_ALGORITHM_NAME} and its methods instead
*/
@Deprecated
public static final String HFILE_COMPRESSION_ALGORITHM = HFILE_COMPRESSION_ALGORITHM_NAME.key();
/**
* @deprecated Use {@link #PARQUET_COMPRESSION_CODEC_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_PARQUET_COMPRESSION_CODEC = PARQUET_COMPRESSION_CODEC_NAME.defaultValue();
/**
* @deprecated Use {@link #HFILE_COMPRESSION_ALGORITHM_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_HFILE_COMPRESSION_ALGORITHM = HFILE_COMPRESSION_ALGORITHM_NAME.defaultValue();
/**
* @deprecated Use {@link #LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead
*/
@Deprecated
public static final String LOGFILE_TO_PARQUET_COMPRESSION_RATIO = LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key();
/**
* @deprecated Use {@link #LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead
*/
@Deprecated
public static final String DEFAULT_LOGFILE_TO_PARQUET_COMPRESSION_RATIO = LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue();
private HoodieStorageConfig() {
super();
}
@@ -152,62 +263,62 @@ public class HoodieStorageConfig extends HoodieConfig {
}
public Builder parquetMaxFileSize(long maxFileSize) {
storageConfig.setValue(PARQUET_FILE_MAX_BYTES, String.valueOf(maxFileSize));
storageConfig.setValue(PARQUET_MAX_FILE_SIZE, String.valueOf(maxFileSize));
return this;
}
public Builder parquetBlockSize(int blockSize) {
storageConfig.setValue(PARQUET_BLOCK_SIZE_BYTES, String.valueOf(blockSize));
storageConfig.setValue(PARQUET_BLOCK_SIZE, String.valueOf(blockSize));
return this;
}
public Builder parquetPageSize(int pageSize) {
storageConfig.setValue(PARQUET_PAGE_SIZE_BYTES, String.valueOf(pageSize));
storageConfig.setValue(PARQUET_PAGE_SIZE, String.valueOf(pageSize));
return this;
}
public Builder hfileMaxFileSize(long maxFileSize) {
storageConfig.setValue(HFILE_FILE_MAX_BYTES, String.valueOf(maxFileSize));
storageConfig.setValue(HFILE_MAX_FILE_SIZE, String.valueOf(maxFileSize));
return this;
}
public Builder hfileBlockSize(int blockSize) {
storageConfig.setValue(HFILE_BLOCK_SIZE_BYTES, String.valueOf(blockSize));
storageConfig.setValue(HFILE_BLOCK_SIZE, String.valueOf(blockSize));
return this;
}
public Builder logFileDataBlockMaxSize(int dataBlockSize) {
storageConfig.setValue(LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES, String.valueOf(dataBlockSize));
storageConfig.setValue(LOGFILE_DATA_BLOCK_MAX_SIZE, String.valueOf(dataBlockSize));
return this;
}
public Builder logFileMaxSize(int logFileSize) {
storageConfig.setValue(LOGFILE_SIZE_MAX_BYTES, String.valueOf(logFileSize));
storageConfig.setValue(LOGFILE_MAX_SIZE, String.valueOf(logFileSize));
return this;
}
public Builder parquetCompressionRatio(double parquetCompressionRatio) {
storageConfig.setValue(PARQUET_COMPRESSION_RATIO, String.valueOf(parquetCompressionRatio));
storageConfig.setValue(PARQUET_COMPRESSION_RATIO_FRACTION, String.valueOf(parquetCompressionRatio));
return this;
}
public Builder parquetCompressionCodec(String parquetCompressionCodec) {
storageConfig.setValue(PARQUET_COMPRESSION_CODEC, parquetCompressionCodec);
storageConfig.setValue(PARQUET_COMPRESSION_CODEC_NAME, parquetCompressionCodec);
return this;
}
public Builder hfileCompressionAlgorithm(String hfileCompressionAlgorithm) {
storageConfig.setValue(HFILE_COMPRESSION_ALGORITHM, hfileCompressionAlgorithm);
storageConfig.setValue(HFILE_COMPRESSION_ALGORITHM_NAME, hfileCompressionAlgorithm);
return this;
}
public Builder logFileToParquetCompressionRatio(double logFileToParquetCompressionRatio) {
storageConfig.setValue(LOGFILE_TO_PARQUET_COMPRESSION_RATIO, String.valueOf(logFileToParquetCompressionRatio));
storageConfig.setValue(LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION, String.valueOf(logFileToParquetCompressionRatio));
return this;
}
public Builder orcMaxFileSize(long maxFileSize) {
storageConfig.setValue(ORC_FILE_MAX_BYTES, String.valueOf(maxFileSize));
storageConfig.setValue(ORC_FILE_MAX_SIZE, String.valueOf(maxFileSize));
return this;
}
@@ -222,7 +333,7 @@ public class HoodieStorageConfig extends HoodieConfig {
}
public Builder orcCompressionCodec(String orcCompressionCodec) {
storageConfig.setValue(ORC_COMPRESSION_CODEC, orcCompressionCodec);
storageConfig.setValue(ORC_COMPRESSION_CODEC_NAME, orcCompressionCodec);
return this;
}

66
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java
View File

@@ -38,20 +38,18 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig {
public static final String CALLBACK_PREFIX = "hoodie.write.commit.callback.";
public static final ConfigProperty<Boolean> CALLBACK_ON = ConfigProperty
public static final ConfigProperty<Boolean> TURN_CALLBACK_ON = ConfigProperty
.key(CALLBACK_PREFIX + "on")
.defaultValue(false)
.sinceVersion("0.6.0")
.withDocumentation("Turn commit callback on/off. off by default.");
public static final ConfigProperty<String> CALLBACK_CLASS = ConfigProperty
public static final ConfigProperty<String> CALLBACK_CLASS_NAME = ConfigProperty
.key(CALLBACK_PREFIX + "class")
.defaultValue("org.apache.hudi.callback.impl.HoodieWriteCommitHttpCallback")
.sinceVersion("0.6.0")
.withDocumentation("Full path of callback class and must be a subclass of HoodieWriteCommitCallback class, "
+ "org.apache.hudi.callback.impl.HoodieWriteCommitHttpCallback by default");
@Deprecated
public static final String CALLBACK_CLASS_PROP = CALLBACK_CLASS.key();
// ***** HTTP callback configs *****
public static final ConfigProperty<String> CALLBACK_HTTP_URL = ConfigProperty
@@ -59,21 +57,65 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig {
.noDefaultValue()
.sinceVersion("0.6.0")
.withDocumentation("Callback host to be sent along with callback messages");
@Deprecated
public static final String CALLBACK_HTTP_URL_PROP = CALLBACK_HTTP_URL.key();
public static final ConfigProperty<String> CALLBACK_HTTP_API_KEY = ConfigProperty
public static final ConfigProperty<String> CALLBACK_HTTP_API_KEY_VALUE = ConfigProperty
.key(CALLBACK_PREFIX + "http.api.key")
.defaultValue("hudi_write_commit_http_callback")
.sinceVersion("0.6.0")
.withDocumentation("Http callback API key. hudi_write_commit_http_callback by default");
public static final ConfigProperty<Integer> CALLBACK_HTTP_TIMEOUT_SECONDS = ConfigProperty
public static final ConfigProperty<Integer> CALLBACK_HTTP_TIMEOUT_IN_SECONDS = ConfigProperty
.key(CALLBACK_PREFIX + "http.timeout.seconds")
.defaultValue(3)
.sinceVersion("0.6.0")
.withDocumentation("Callback timeout in seconds. 3 by default");
/**
* @deprecated Use {@link #TURN_CALLBACK_ON} and its methods instead
*/
@Deprecated
public static final String CALLBACK_ON = TURN_CALLBACK_ON.key();
/**
* @deprecated Use {@link #TURN_CALLBACK_ON} and its methods instead
*/
@Deprecated
public static final boolean DEFAULT_CALLBACK_ON = TURN_CALLBACK_ON.defaultValue();
/**
* @deprecated Use {@link #CALLBACK_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String CALLBACK_CLASS_PROP = CALLBACK_CLASS_NAME.key();
/**
* @deprecated Use {@link #CALLBACK_CLASS_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CALLBACK_CLASS_PROP = CALLBACK_CLASS_NAME.defaultValue();
/**
* @deprecated Use {@link #CALLBACK_HTTP_URL} and its methods instead
*/
@Deprecated
public static final String CALLBACK_HTTP_URL_PROP = CALLBACK_HTTP_URL.key();
/**
* @deprecated Use {@link #CALLBACK_HTTP_API_KEY_VALUE} and its methods instead
*/
@Deprecated
public static final String CALLBACK_HTTP_API_KEY = CALLBACK_HTTP_API_KEY_VALUE.key();
/**
* @deprecated Use {@link #CALLBACK_HTTP_API_KEY_VALUE} and its methods instead
*/
@Deprecated
public static final String DEFAULT_CALLBACK_HTTP_API_KEY = CALLBACK_HTTP_API_KEY_VALUE.defaultValue();
/**
* @deprecated Use {@link #CALLBACK_HTTP_TIMEOUT_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final String CALLBACK_HTTP_TIMEOUT_SECONDS = CALLBACK_HTTP_TIMEOUT_IN_SECONDS.key();
/**
* @deprecated Use {@link #CALLBACK_HTTP_TIMEOUT_IN_SECONDS} and its methods instead
*/
@Deprecated
public static final int DEFAULT_CALLBACK_HTTP_TIMEOUT_SECONDS = CALLBACK_HTTP_TIMEOUT_IN_SECONDS.defaultValue();
private HoodieWriteCommitCallbackConfig() {
super();
}
@@ -99,12 +141,12 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig {
}
public HoodieWriteCommitCallbackConfig.Builder writeCommitCallbackOn(String callbackOn) {
writeCommitCallbackConfig.setValue(CALLBACK_ON, callbackOn);
writeCommitCallbackConfig.setValue(TURN_CALLBACK_ON, callbackOn);
return this;
}
public HoodieWriteCommitCallbackConfig.Builder withCallbackClass(String callbackClass) {
writeCommitCallbackConfig.setValue(CALLBACK_CLASS, callbackClass);
writeCommitCallbackConfig.setValue(CALLBACK_CLASS_NAME, callbackClass);
return this;
}
@@ -114,12 +156,12 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig {
}
public Builder withCallbackHttpTimeoutSeconds(String timeoutSeconds) {
writeCommitCallbackConfig.setValue(CALLBACK_HTTP_TIMEOUT_SECONDS, timeoutSeconds);
writeCommitCallbackConfig.setValue(CALLBACK_HTTP_TIMEOUT_IN_SECONDS, timeoutSeconds);
return this;
}
public Builder withCallbackHttpApiKey(String apiKey) {
writeCommitCallbackConfig.setValue(CALLBACK_HTTP_API_KEY, apiKey);
writeCommitCallbackConfig.setValue(CALLBACK_HTTP_API_KEY_VALUE, apiKey);
return this;
}

837
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

File diff suppressed because it is too large Load Diff

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java
View File

@@ -186,7 +186,7 @@ public class KeyGenUtils {
*/
public static KeyGenerator createKeyGeneratorByClassName(TypedProperties props) throws IOException {
KeyGenerator keyGenerator = null;
String keyGeneratorClass = props.getString(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), null);
String keyGeneratorClass = props.getString(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), null);
if (!StringUtils.isNullOrEmpty(keyGeneratorClass)) {
try {
keyGenerator = (KeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass, props);

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java
View File

@@ -41,7 +41,7 @@ import java.util.Objects;
/**
* Factory help to create {@link org.apache.hudi.keygen.KeyGenerator}.
* <p>
* This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS} firstly, this ensures the class prop
* This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS_NAME} firstly, this ensures the class prop
* will not be overwritten by {@link KeyGeneratorType}
*/
public class HoodieAvroKeyGeneratorFactory {

8
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
View File

@@ -67,7 +67,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP;
import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP;
@@ -299,9 +299,9 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
LOG.info("Creating a new metadata table in " + metadataWriteConfig.getBasePath() + " at instant " + createInstantTime);
HoodieTableMetaClient.withPropertyBuilder()
.setTableType(HoodieTableType.MERGE_ON_READ)
.setTableName(tableName)
.setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue())
.setTableType(HoodieTableType.MERGE_ON_READ)
.setTableName(tableName)
.setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue())
.setPayloadClassName(HoodieMetadataPayload.class.getName())
.setBaseFileFormat(HoodieFileFormat.HFILE.toString())
.initTable(hadoopConf.get(), metadataWriteConfig.getBasePath());

10
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
View File

@@ -18,10 +18,6 @@
package org.apache.hudi.table;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
@@ -71,6 +67,10 @@ import org.apache.hudi.table.action.bootstrap.HoodieBootstrapWriteMetadata;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -114,7 +114,7 @@ public abstract class HoodieTable<T extends HoodieRecordPayload, I, K, O> implem
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(config.getMetadataConfig().getProps())
.build();
this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath(),
FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue());
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue());
this.viewManager = FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig(), () -> metadata);
this.metaClient = metaClient;

8
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
View File

@@ -18,9 +18,6 @@
package org.apache.hudi.table;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieArchivedMetaEntry;
import org.apache.hudi.client.utils.MetadataConversionUtils;
import org.apache.hudi.common.engine.HoodieEngineContext;
@@ -49,6 +46,9 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -204,7 +204,7 @@ public class HoodieTimelineArchiveLog<T extends HoodieAvroPayload, I, K, O> {
// instant on the metadata table. This is required for metadata table sync.
if (config.isMetadataTableEnabled()) {
try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(),
config.getBasePath(), FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue())) {
config.getBasePath(), FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue())) {
Option<String> lastSyncedInstantTime = tableMetadata.getUpdateTime();
if (lastSyncedInstantTime.isPresent()) {

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java
View File

@@ -32,9 +32,9 @@ public abstract class BaseOneToTwoUpgradeHandler implements UpgradeHandler {
@Override
public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) {
Map<ConfigProperty, String> tablePropsToAdd = new HashMap<>();
tablePropsToAdd.put(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP, getPartitionColumns(config));
tablePropsToAdd.put(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()));
tablePropsToAdd.put(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP, config.getString(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP));
tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config));
tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()));
tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT));
return tablePropsToAdd;
}