diff --git a/docker/demo/sparksql-incremental.commands b/docker/demo/sparksql-incremental.commands index e53203b14..da61347ec 100644 --- a/docker/demo/sparksql-incremental.commands +++ b/docker/demo/sparksql-incremental.commands @@ -42,7 +42,7 @@ spark.sql("select key, `_hoodie_partition_path` as datestr, symbol, ts, open, cl option(DataSourceWriteOptions.RECORDKEY_FIELD.key(), "key"). option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key(), "datestr"). option(DataSourceWriteOptions.PRECOMBINE_FIELD.key(), "ts"). - option(HoodieWriteConfig.TABLE_NAME.key(), "stock_ticks_derived_mor"). + option(HoodieWriteConfig.TBL_NAME.key(), "stock_ticks_derived_mor"). option(DataSourceWriteOptions.HIVE_TABLE.key(), "stock_ticks_derived_mor"). option(DataSourceWriteOptions.HIVE_DATABASE.key(), "default"). option(DataSourceWriteOptions.HIVE_URL.key(), "jdbc:hive2://hiveserver:10000"). @@ -74,7 +74,7 @@ spark.sql("select key, `_hoodie_partition_path` as datestr, symbol, ts, open, cl option(DataSourceWriteOptions.RECORDKEY_FIELD.key(), "key"). option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key(), "datestr"). option(DataSourceWriteOptions.PRECOMBINE_FIELD.key(), "ts"). - option(HoodieWriteConfig.TABLE_NAME.key(), "stock_ticks_derived_mor_bs"). + option(HoodieWriteConfig.TBL_NAME.key(), "stock_ticks_derived_mor_bs"). option(DataSourceWriteOptions.HIVE_TABLE.key(), "stock_ticks_derived_mor_bs"). option(DataSourceWriteOptions.HIVE_DATABASE.key(), "default"). option(DataSourceWriteOptions.HIVE_URL.key(), "jdbc:hive2://hiveserver:10000"). diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java index 1c747ab97..27bcd81fa 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java @@ -204,10 +204,10 @@ public class HoodieLogFileCommand implements CommandMarker { .getCommitTimeline().lastInstant().get().getTimestamp()) .withReadBlocksLazily( Boolean.parseBoolean( - HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLED.defaultValue())) + HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLE.defaultValue())) .withReverseReader( Boolean.parseBoolean( - HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLED.defaultValue())) + HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue())) .withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue()) .withMaxMemorySizeInBytes( HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java index ccf03273c..f86937d23 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java @@ -18,8 +18,8 @@ package org.apache.hudi.cli.commands; -import org.apache.hudi.cli.DeDupeType; import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.cli.DeDupeType; import org.apache.hudi.cli.DedupeSparkJob; import org.apache.hudi.cli.utils.SparkUtil; import org.apache.hudi.client.SparkRDDWriteClient; @@ -361,17 +361,17 @@ public class SparkMain { TypedProperties properties = propsFilePath == null ? UtilHelpers.buildProperties(configs) : UtilHelpers.readConfig(FSUtils.getFs(propsFilePath, jsc.hadoopConfiguration()), new Path(propsFilePath), configs).getConfig(); - properties.setProperty(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH.key(), sourcePath); + properties.setProperty(HoodieBootstrapConfig.BASE_PATH.key(), sourcePath); - if (!StringUtils.isNullOrEmpty(keyGenerator) && KeyGeneratorType.getNames().contains(keyGenerator.toUpperCase(Locale.ROOT))) { - properties.setProperty(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_TYPE.key(), keyGenerator.toUpperCase(Locale.ROOT)); + if (!StringUtils.isNullOrEmpty(keyGenerator) && KeyGeneratorType.getNames().contains(keyGenerator.toUpperCase(Locale.ROOT))) { + properties.setProperty(HoodieBootstrapConfig.KEYGEN_TYPE.key(), keyGenerator.toUpperCase(Locale.ROOT)); } else { - properties.setProperty(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS.key(), keyGenerator); + properties.setProperty(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key(), keyGenerator); } - properties.setProperty(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER.key(), fullBootstrapInputProvider); - properties.setProperty(HoodieBootstrapConfig.BOOTSTRAP_PARALLELISM.key(), parallelism); - properties.setProperty(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR.key(), selectorClass); + properties.setProperty(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.key(), fullBootstrapInputProvider); + properties.setProperty(HoodieBootstrapConfig.PARALLELISM_VALUE.key(), parallelism); + properties.setProperty(HoodieBootstrapConfig.MODE_SELECTOR_CLASS_NAME.key(), selectorClass); properties.setProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key(), recordKeyCols); properties.setProperty(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), partitionFields); diff --git a/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala b/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala index 33d014520..3802bb46a 100644 --- a/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala +++ b/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala @@ -25,8 +25,7 @@ import org.apache.hudi.avro.HoodieAvroWriteSupport import org.apache.hudi.client.SparkTaskContextSupplier import org.apache.hudi.common.HoodieJsonPayload import org.apache.hudi.common.bloom.{BloomFilter, BloomFilterFactory} -import org.apache.hudi.common.model.HoodieFileFormat -import org.apache.hudi.common.model.HoodieRecord +import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord} import org.apache.hudi.common.util.BaseFileUtils import org.apache.hudi.config.{HoodieIndexConfig, HoodieStorageConfig} import org.apache.hudi.io.storage.{HoodieAvroParquetConfig, HoodieParquetWriter} @@ -43,10 +42,10 @@ object SparkHelpers { def skipKeysAndWriteNewFile(instantTime: String, fs: FileSystem, sourceFile: Path, destinationFile: Path, keysToSkip: Set[String]) { val sourceRecords = BaseFileUtils.getInstance(HoodieFileFormat.PARQUET).readAvroRecords(fs.getConf, sourceFile) val schema: Schema = sourceRecords.get(0).getSchema - val filter: BloomFilter = BloomFilterFactory.createBloomFilter(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES.defaultValue.toInt, HoodieIndexConfig.BLOOM_FILTER_FPP.defaultValue.toDouble, - HoodieIndexConfig.HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.defaultValue.toInt, HoodieIndexConfig.BLOOM_INDEX_FILTER_TYPE.defaultValue); + val filter: BloomFilter = BloomFilterFactory.createBloomFilter(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES_VALUE.defaultValue.toInt, HoodieIndexConfig.BLOOM_FILTER_FPP_VALUE.defaultValue.toDouble, + HoodieIndexConfig.BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.defaultValue.toInt, HoodieIndexConfig.BLOOM_FILTER_TYPE.defaultValue); val writeSupport: HoodieAvroWriteSupport = new HoodieAvroWriteSupport(new AvroSchemaConverter(fs.getConf).convert(schema), schema, org.apache.hudi.common.util.Option.of(filter)) - val parquetConfig: HoodieAvroParquetConfig = new HoodieAvroParquetConfig(writeSupport, CompressionCodecName.GZIP, HoodieStorageConfig.PARQUET_BLOCK_SIZE_BYTES.defaultValue.toInt, HoodieStorageConfig.PARQUET_PAGE_SIZE_BYTES.defaultValue.toInt, HoodieStorageConfig.PARQUET_FILE_MAX_BYTES.defaultValue.toInt, fs.getConf, HoodieStorageConfig.PARQUET_COMPRESSION_RATIO.defaultValue.toDouble) + val parquetConfig: HoodieAvroParquetConfig = new HoodieAvroParquetConfig(writeSupport, CompressionCodecName.GZIP, HoodieStorageConfig.PARQUET_BLOCK_SIZE.defaultValue.toInt, HoodieStorageConfig.PARQUET_PAGE_SIZE.defaultValue.toInt, HoodieStorageConfig.PARQUET_MAX_FILE_SIZE.defaultValue.toInt, fs.getConf, HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue.toDouble) // Add current classLoad for config, if not will throw classNotFound of 'HoodieWrapperFileSystem'. parquetConfig.getHadoopConf().setClassLoader(Thread.currentThread.getContextClassLoader) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index a44601d4d..c03077a66 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -208,10 +208,10 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest { HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES) .withReadBlocksLazily( Boolean.parseBoolean( - HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLED.defaultValue())) + HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLE.defaultValue())) .withReverseReader( Boolean.parseBoolean( - HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLED.defaultValue())) + HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue())) .withBufferSize(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.defaultValue()) .withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue()) .withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue()) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java index f500f3075..4cc76914c 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java @@ -67,7 +67,7 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { // Create table and connect new TableCommand().createTable( tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), - HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue(), TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); + HoodieTableConfig.ARCHIVELOG_FOLDER.defaultValue(), TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); } /** diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java index 5fb919146..b3a31dc19 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java @@ -112,6 +112,6 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest { HoodieConfig hoodieConfig = HoodieConfig.create(fsDataInputStream); fsDataInputStream.close(); assertEquals(Integer.toString(HoodieTableVersion.ZERO.versionCode()), hoodieConfig - .getString(HoodieTableConfig.HOODIE_TABLE_VERSION_PROP)); + .getString(HoodieTableConfig.VERSION)); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java index 08ed28363..6d1059cb9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java @@ -79,7 +79,7 @@ public class HoodieWriteCommitHttpCallbackClient implements Closeable { } private String getApiKey() { - return writeConfig.getString(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_API_KEY); + return writeConfig.getString(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_API_KEY_VALUE); } private String getUrl() { @@ -97,7 +97,7 @@ public class HoodieWriteCommitHttpCallbackClient implements Closeable { } private Integer getHttpTimeoutSeconds() { - return writeConfig.getInt(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_TIMEOUT_SECONDS); + return writeConfig.getInt(HoodieWriteCommitCallbackConfig.CALLBACK_HTTP_TIMEOUT_IN_SECONDS); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java index 66166d4c6..aaa5eed28 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java @@ -39,7 +39,7 @@ public class HoodieCommitCallbackFactory { return (HoodieWriteCommitCallback) instance; } else { throw new HoodieCommitCallbackException(String.format("The value of the config option %s can not be null or " - + "empty", HoodieWriteCommitCallbackConfig.CALLBACK_CLASS.key())); + + "empty", HoodieWriteCommitCallbackConfig.CALLBACK_CLASS_NAME.key())); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java index 836b0c158..94bb7830c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java @@ -44,57 +44,55 @@ import java.util.Properties; + " writers and new hudi writers in parallel, to validate the migration.") public class HoodieBootstrapConfig extends HoodieConfig { - public static final ConfigProperty BOOTSTRAP_BASE_PATH = ConfigProperty + public static final ConfigProperty BASE_PATH = ConfigProperty .key("hoodie.bootstrap.base.path") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Base path of the dataset that needs to be bootstrapped as a Hudi table"); - @Deprecated - public static final String BOOTSTRAP_BASE_PATH_PROP = BOOTSTRAP_BASE_PATH.key(); - public static final ConfigProperty BOOTSTRAP_MODE_SELECTOR = ConfigProperty + public static final ConfigProperty MODE_SELECTOR_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.mode.selector") .defaultValue(MetadataOnlyBootstrapModeSelector.class.getCanonicalName()) .sinceVersion("0.6.0") .withDocumentation("Selects the mode in which each file/partition in the bootstrapped dataset gets bootstrapped"); - public static final ConfigProperty FULL_BOOTSTRAP_INPUT_PROVIDER = ConfigProperty + public static final ConfigProperty FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.full.input.provider") .defaultValue("org.apache.hudi.bootstrap.SparkParquetBootstrapDataProvider") .sinceVersion("0.6.0") .withDocumentation("Class to use for reading the bootstrap dataset partitions/files, for Bootstrap mode FULL_RECORD"); - public static final ConfigProperty BOOTSTRAP_KEYGEN_CLASS = ConfigProperty + public static final ConfigProperty KEYGEN_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.keygen.class") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Key generator implementation to be used for generating keys from the bootstrapped dataset"); - public static final ConfigProperty BOOTSTRAP_KEYGEN_TYPE = ConfigProperty + public static final ConfigProperty KEYGEN_TYPE = ConfigProperty .key("hoodie.bootstrap.keygen.type") .defaultValue(KeyGeneratorType.SIMPLE.name()) .sinceVersion("0.9.0") .withDocumentation("Type of build-in key generator, currently support SIMPLE, COMPLEX, TIMESTAMP, CUSTOM, NON_PARTITION, GLOBAL_DELETE"); - public static final ConfigProperty BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = ConfigProperty + public static final ConfigProperty PARTITION_PATH_TRANSLATOR_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.partitionpath.translator.class") .defaultValue(IdentityBootstrapPartitionPathTranslator.class.getName()) .sinceVersion("0.6.0") .withDocumentation("Translates the partition paths from the bootstrapped data into how is laid out as a Hudi table."); - public static final ConfigProperty BOOTSTRAP_PARALLELISM = ConfigProperty + public static final ConfigProperty PARALLELISM_VALUE = ConfigProperty .key("hoodie.bootstrap.parallelism") .defaultValue("1500") .sinceVersion("0.6.0") .withDocumentation("Parallelism value to be used to bootstrap data into hudi"); - public static final ConfigProperty BOOTSTRAP_MODE_SELECTOR_REGEX = ConfigProperty + public static final ConfigProperty PARTITION_SELECTOR_REGEX_PATTERN = ConfigProperty .key("hoodie.bootstrap.mode.selector.regex") .defaultValue(".*") .sinceVersion("0.6.0") .withDocumentation("Matches each bootstrap dataset partition against this regex and applies the mode below to it."); - public static final ConfigProperty BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = ConfigProperty + public static final ConfigProperty PARTITION_SELECTOR_REGEX_MODE = ConfigProperty .key("hoodie.bootstrap.mode.selector.regex.mode") .defaultValue(BootstrapMode.METADATA_ONLY.name()) .sinceVersion("0.6.0") @@ -102,13 +100,87 @@ public class HoodieBootstrapConfig extends HoodieConfig { + "METADATA_ONLY will generate just skeleton base files with keys/footers, avoiding full cost of rewriting the dataset. " + "FULL_RECORD will perform a full copy/rewrite of the data as a Hudi table."); - public static final ConfigProperty BOOTSTRAP_INDEX_CLASS = ConfigProperty + public static final ConfigProperty INDEX_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.index.class") .defaultValue(HFileBootstrapIndex.class.getName()) .sinceVersion("0.6.0") .withDocumentation("Implementation to use, for mapping a skeleton base file to a boostrap base file."); + + /** + * @deprecated Use {@link #BASE_PATH} and its methods instead + */ @Deprecated - public static final String BOOTSTRAP_INDEX_CLASS_PROP = BOOTSTRAP_INDEX_CLASS.key(); + public static final String BOOTSTRAP_BASE_PATH_PROP = BASE_PATH.key(); + /** + * @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_INDEX_CLASS_PROP = INDEX_CLASS_NAME.key(); + /** + * @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = INDEX_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #MODE_SELECTOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_MODE_SELECTOR = MODE_SELECTOR_CLASS_NAME.key(); + /** + * @deprecated Use {@link #FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String FULL_BOOTSTRAP_INPUT_PROVIDER = FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.key(); + /** + * @deprecated Use {@link #FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_FULL_BOOTSTRAP_INPUT_PROVIDER = FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #KEYGEN_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_KEYGEN_CLASS = KEYGEN_CLASS_NAME.key(); + /** + * @deprecated Use {@link #PARTITION_PATH_TRANSLATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = PARTITION_PATH_TRANSLATOR_CLASS_NAME.key(); + /** + * @deprecated Use {@link #PARTITION_PATH_TRANSLATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS = PARTITION_PATH_TRANSLATOR_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_PARALLELISM = PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_PARALLELISM = PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #PARTITION_SELECTOR_REGEX_PATTERN} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_MODE_SELECTOR_REGEX = PARTITION_SELECTOR_REGEX_PATTERN.key(); + /** + * @deprecated Use {@link #PARTITION_SELECTOR_REGEX_MODE} and its methods instead + */ + @Deprecated + public static final String BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = PARTITION_SELECTOR_REGEX_MODE.key(); + /** + * @deprecated Use {@link #PARTITION_SELECTOR_REGEX_PATTERN} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_MODE_SELECTOR_REGEX = PARTITION_SELECTOR_REGEX_PATTERN.defaultValue(); + /** + * @deprecated Use {@link #PARTITION_SELECTOR_REGEX_MODE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_MODE_SELECTOR_REGEX_MODE = PARTITION_SELECTOR_REGEX_MODE.defaultValue(); private HoodieBootstrapConfig() { super(); @@ -130,48 +202,48 @@ public class HoodieBootstrapConfig extends HoodieConfig { } public Builder withBootstrapBasePath(String basePath) { - bootstrapConfig.setValue(BOOTSTRAP_BASE_PATH, basePath); + bootstrapConfig.setValue(BASE_PATH, basePath); return this; } public Builder withBootstrapModeSelector(String partitionSelectorClass) { - bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR, partitionSelectorClass); + bootstrapConfig.setValue(MODE_SELECTOR_CLASS_NAME, partitionSelectorClass); return this; } public Builder withFullBootstrapInputProvider(String partitionSelectorClass) { - bootstrapConfig.setValue(FULL_BOOTSTRAP_INPUT_PROVIDER, partitionSelectorClass); + bootstrapConfig.setValue(FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME, partitionSelectorClass); return this; } public Builder withBootstrapKeyGenClass(String keyGenClass) { - bootstrapConfig.setValue(BOOTSTRAP_KEYGEN_CLASS, keyGenClass); + bootstrapConfig.setValue(KEYGEN_CLASS_NAME, keyGenClass); return this; } public Builder withBootstrapKeyGenType(String keyGenType) { - bootstrapConfig.setValue(BOOTSTRAP_KEYGEN_TYPE, keyGenType); + bootstrapConfig.setValue(KEYGEN_TYPE, keyGenType); return this; } public Builder withBootstrapPartitionPathTranslatorClass(String partitionPathTranslatorClass) { bootstrapConfig - .setValue(BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS, partitionPathTranslatorClass); + .setValue(PARTITION_PATH_TRANSLATOR_CLASS_NAME, partitionPathTranslatorClass); return this; } public Builder withBootstrapParallelism(int parallelism) { - bootstrapConfig.setValue(BOOTSTRAP_PARALLELISM, String.valueOf(parallelism)); + bootstrapConfig.setValue(PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } public Builder withBootstrapModeSelectorRegex(String regex) { - bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR_REGEX, regex); + bootstrapConfig.setValue(PARTITION_SELECTOR_REGEX_PATTERN, regex); return this; } public Builder withBootstrapModeForRegexMatch(BootstrapMode modeForRegexMatch) { - bootstrapConfig.setValue(BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, modeForRegexMatch.name()); + bootstrapConfig.setValue(PARTITION_SELECTOR_REGEX_MODE, modeForRegexMatch.name()); return this; } @@ -182,7 +254,7 @@ public class HoodieBootstrapConfig extends HoodieConfig { public HoodieBootstrapConfig build() { // TODO: use infer function instead - bootstrapConfig.setDefaultValue(BOOTSTRAP_INDEX_CLASS, HoodieTableConfig.getDefaultBootstrapIndexClass( + bootstrapConfig.setDefaultValue(INDEX_CLASS_NAME, HoodieTableConfig.getDefaultBootstrapIndexClass( bootstrapConfig.getProps())); bootstrapConfig.setDefaults(HoodieBootstrapConfig.class.getName()); return bootstrapConfig; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java index 4c68ec43e..f1e930b12 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java @@ -40,21 +40,21 @@ public class HoodieClusteringConfig extends HoodieConfig { // Any strategy specific params can be saved with this prefix public static final String CLUSTERING_STRATEGY_PARAM_PREFIX = "hoodie.clustering.plan.strategy."; - public static final ConfigProperty CLUSTERING_TARGET_PARTITIONS = ConfigProperty + public static final ConfigProperty DAYBASED_LOOKBACK_PARTITIONS = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.lookback.partitions") .defaultValue("2") .sinceVersion("0.7.0") .withDocumentation("Number of partitions to list to create ClusteringPlan"); - public static final ConfigProperty CLUSTERING_PLAN_STRATEGY_CLASS = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_CLASS_NAME = ConfigProperty .key("hoodie.clustering.plan.strategy.class") .defaultValue("org.apache.hudi.client.clustering.plan.strategy.SparkRecentDaysClusteringPlanStrategy") .sinceVersion("0.7.0") .withDocumentation("Config to provide a strategy class (subclass of ClusteringPlanStrategy) to create clustering plan " + "i.e select what file groups are being clustered. Default strategy, looks at the last N (determined by " - + CLUSTERING_TARGET_PARTITIONS.key() + ") day based partitions picks the small file slices within those partitions."); + + DAYBASED_LOOKBACK_PARTITIONS.key() + ") day based partitions picks the small file slices within those partitions."); - public static final ConfigProperty CLUSTERING_EXECUTION_STRATEGY_CLASS = ConfigProperty + public static final ConfigProperty EXECUTION_STRATEGY_CLASS_NAME = ConfigProperty .key("hoodie.clustering.execution.strategy.class") .defaultValue("org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy") .sinceVersion("0.7.0") @@ -67,36 +67,32 @@ public class HoodieClusteringConfig extends HoodieConfig { .defaultValue("false") .sinceVersion("0.7.0") .withDocumentation("Turn on inline clustering - clustering will be run after each write operation is complete"); - @Deprecated - public static final String INLINE_CLUSTERING_PROP = INLINE_CLUSTERING.key(); - public static final ConfigProperty INLINE_CLUSTERING_MAX_COMMIT = ConfigProperty + public static final ConfigProperty INLINE_CLUSTERING_MAX_COMMITS = ConfigProperty .key("hoodie.clustering.inline.max.commits") .defaultValue("4") .sinceVersion("0.7.0") .withDocumentation("Config to control frequency of clustering planning"); - @Deprecated - public static final String INLINE_CLUSTERING_MAX_COMMIT_PROP = INLINE_CLUSTERING_MAX_COMMIT.key(); - public static final ConfigProperty ASYNC_CLUSTERING_MAX_COMMIT_PROP = ConfigProperty + public static final ConfigProperty ASYNC_CLUSTERING_MAX_COMMITS = ConfigProperty .key("hoodie.clustering.async.max.commits") .defaultValue("4") .sinceVersion("0.9.0") .withDocumentation("Config to control frequency of async clustering"); - public static final ConfigProperty CLUSTERING_SKIP_PARTITIONS_FROM_LATEST = ConfigProperty - .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.skipfromlatest.partitions") - .defaultValue("0") - .sinceVersion("0.9.0") - .withDocumentation("Number of partitions to skip from latest when choosing partitions to create ClusteringPlan"); + public static final ConfigProperty PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST = ConfigProperty + .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.skipfromlatest.partitions") + .defaultValue("0") + .sinceVersion("0.9.0") + .withDocumentation("Number of partitions to skip from latest when choosing partitions to create ClusteringPlan"); - public static final ConfigProperty CLUSTERING_PLAN_SMALL_FILE_LIMIT = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_SMALL_FILE_LIMIT = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "small.file.limit") .defaultValue(String.valueOf(600 * 1024 * 1024L)) .sinceVersion("0.7.0") .withDocumentation("Files smaller than the size specified here are candidates for clustering"); - public static final ConfigProperty CLUSTERING_MAX_BYTES_PER_GROUP = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "max.bytes.per.group") .defaultValue(String.valueOf(2 * 1024 * 1024 * 1024L)) .sinceVersion("0.7.0") @@ -104,46 +100,156 @@ public class HoodieClusteringConfig extends HoodieConfig { + " is defined by below two properties (CLUSTERING_MAX_BYTES_PER_GROUP * CLUSTERING_MAX_NUM_GROUPS)." + " Max amount of data to be included in one group"); - public static final ConfigProperty CLUSTERING_MAX_NUM_GROUPS = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_MAX_GROUPS = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "max.num.groups") .defaultValue("30") .sinceVersion("0.7.0") .withDocumentation("Maximum number of groups to create as part of ClusteringPlan. Increasing groups will increase parallelism"); - public static final ConfigProperty CLUSTERING_TARGET_FILE_MAX_BYTES = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_TARGET_FILE_MAX_BYTES = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "target.file.max.bytes") .defaultValue(String.valueOf(1024 * 1024 * 1024L)) .sinceVersion("0.7.0") .withDocumentation("Each group can produce 'N' (CLUSTERING_MAX_GROUP_SIZE/CLUSTERING_TARGET_FILE_SIZE) output file groups"); - public static final ConfigProperty CLUSTERING_SORT_COLUMNS_PROPERTY = ConfigProperty + public static final ConfigProperty PLAN_STRATEGY_SORT_COLUMNS = ConfigProperty .key(CLUSTERING_STRATEGY_PARAM_PREFIX + "sort.columns") .noDefaultValue() .sinceVersion("0.7.0") .withDocumentation("Columns to sort the data by when clustering"); - public static final ConfigProperty CLUSTERING_UPDATES_STRATEGY = ConfigProperty + public static final ConfigProperty UPDATES_STRATEGY = ConfigProperty .key("hoodie.clustering.updates.strategy") .defaultValue("org.apache.hudi.client.clustering.update.strategy.SparkRejectUpdateStrategy") .sinceVersion("0.7.0") .withDocumentation("Determines how to handle updates, deletes to file groups that are under clustering." + " Default strategy just rejects the update"); - @Deprecated - public static final String CLUSTERING_UPDATES_STRATEGY_PROP = CLUSTERING_UPDATES_STRATEGY.key(); public static final ConfigProperty ASYNC_CLUSTERING_ENABLE = ConfigProperty .key("hoodie.clustering.async.enabled") .defaultValue("false") .sinceVersion("0.7.0") .withDocumentation("Enable running of clustering service, asynchronously as inserts happen on the table."); - @Deprecated - public static final String ASYNC_CLUSTERING_ENABLE_OPT_KEY = "hoodie.clustering.async.enabled"; - public static final ConfigProperty CLUSTERING_PRESERVE_HOODIE_COMMIT_METADATA = ConfigProperty + public static final ConfigProperty PRESERVE_COMMIT_METADATA = ConfigProperty .key("hoodie.clustering.preserve.commit.metadata") .defaultValue(false) .sinceVersion("0.9.0") .withDocumentation("When rewriting data, preserves existing hoodie_commit_time"); + + /** + * @deprecated Use {@link #PLAN_STRATEGY_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_PLAN_STRATEGY_CLASS = PLAN_STRATEGY_CLASS_NAME.key(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_PLAN_STRATEGY_CLASS = PLAN_STRATEGY_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #EXECUTION_STRATEGY_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_EXECUTION_STRATEGY_CLASS = EXECUTION_STRATEGY_CLASS_NAME.key(); + /** + * @deprecated Use {@link #EXECUTION_STRATEGY_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_EXECUTION_STRATEGY_CLASS = EXECUTION_STRATEGY_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #INLINE_CLUSTERING} and its methods instead + */ + @Deprecated + public static final String INLINE_CLUSTERING_PROP = INLINE_CLUSTERING.key(); + /** + * @deprecated Use {@link #INLINE_CLUSTERING} and its methods instead + */ + @Deprecated + private static final String DEFAULT_INLINE_CLUSTERING = INLINE_CLUSTERING.defaultValue(); + /** + * @deprecated Use {@link #INLINE_CLUSTERING_MAX_COMMITS} and its methods instead + */ + @Deprecated + public static final String INLINE_CLUSTERING_MAX_COMMIT_PROP = INLINE_CLUSTERING_MAX_COMMITS.key(); + /** + * @deprecated Use {@link #INLINE_CLUSTERING_MAX_COMMITS} and its methods instead + */ + @Deprecated + private static final String DEFAULT_INLINE_CLUSTERING_NUM_COMMITS = INLINE_CLUSTERING_MAX_COMMITS.defaultValue(); + /** + * @deprecated Use {@link #DAYBASED_LOOKBACK_PARTITIONS} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_TARGET_PARTITIONS = DAYBASED_LOOKBACK_PARTITIONS.key(); + /** + * @deprecated Use {@link #DAYBASED_LOOKBACK_PARTITIONS} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_TARGET_PARTITIONS = DAYBASED_LOOKBACK_PARTITIONS.defaultValue(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_SMALL_FILE_LIMIT} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_PLAN_SMALL_FILE_LIMIT = PLAN_STRATEGY_SMALL_FILE_LIMIT.key(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_SMALL_FILE_LIMIT} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_PLAN_SMALL_FILE_LIMIT = PLAN_STRATEGY_SMALL_FILE_LIMIT.defaultValue(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_MAX_BYTES_PER_GROUP = PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP.key(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_MAX_GROUP_SIZE = PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP.defaultValue(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_MAX_GROUPS} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_MAX_NUM_GROUPS = PLAN_STRATEGY_MAX_GROUPS.key(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_MAX_GROUPS} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_MAX_NUM_GROUPS = PLAN_STRATEGY_MAX_GROUPS.defaultValue(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_TARGET_FILE_MAX_BYTES} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_TARGET_FILE_MAX_BYTES = PLAN_STRATEGY_TARGET_FILE_MAX_BYTES.key(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_TARGET_FILE_MAX_BYTES} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_TARGET_FILE_MAX_BYTES = PLAN_STRATEGY_TARGET_FILE_MAX_BYTES.defaultValue(); + /** + * @deprecated Use {@link #PLAN_STRATEGY_SORT_COLUMNS} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_SORT_COLUMNS_PROPERTY = PLAN_STRATEGY_SORT_COLUMNS.key(); + /** + * @deprecated Use {@link #UPDATES_STRATEGY} and its methods instead + */ + @Deprecated + public static final String CLUSTERING_UPDATES_STRATEGY_PROP = UPDATES_STRATEGY.key(); + /** + * @deprecated Use {@link #UPDATES_STRATEGY} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLUSTERING_UPDATES_STRATEGY = UPDATES_STRATEGY.defaultValue(); + /** + * @deprecated Use {@link #ASYNC_CLUSTERING_ENABLE} and its methods instead + */ + @Deprecated + public static final String ASYNC_CLUSTERING_ENABLE_OPT_KEY = ASYNC_CLUSTERING_ENABLE.key(); + /** @deprecated Use {@link #ASYNC_CLUSTERING_ENABLE} and its methods instead */ + @Deprecated + public static final String DEFAULT_ASYNC_CLUSTERING_ENABLE_OPT_VAL = ASYNC_CLUSTERING_ENABLE.defaultValue(); public HoodieClusteringConfig() { super(); @@ -165,47 +271,47 @@ public class HoodieClusteringConfig extends HoodieConfig { } public Builder withClusteringPlanStrategyClass(String clusteringStrategyClass) { - clusteringConfig.setValue(CLUSTERING_PLAN_STRATEGY_CLASS, clusteringStrategyClass); + clusteringConfig.setValue(PLAN_STRATEGY_CLASS_NAME, clusteringStrategyClass); return this; } public Builder withClusteringExecutionStrategyClass(String runClusteringStrategyClass) { - clusteringConfig.setValue(CLUSTERING_EXECUTION_STRATEGY_CLASS, runClusteringStrategyClass); + clusteringConfig.setValue(EXECUTION_STRATEGY_CLASS_NAME, runClusteringStrategyClass); return this; } public Builder withClusteringTargetPartitions(int clusteringTargetPartitions) { - clusteringConfig.setValue(CLUSTERING_TARGET_PARTITIONS, String.valueOf(clusteringTargetPartitions)); + clusteringConfig.setValue(DAYBASED_LOOKBACK_PARTITIONS, String.valueOf(clusteringTargetPartitions)); return this; } public Builder withClusteringSkipPartitionsFromLatest(int clusteringSkipPartitionsFromLatest) { - clusteringConfig.setValue(CLUSTERING_SKIP_PARTITIONS_FROM_LATEST, String.valueOf(clusteringSkipPartitionsFromLatest)); + clusteringConfig.setValue(PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST, String.valueOf(clusteringSkipPartitionsFromLatest)); return this; } public Builder withClusteringPlanSmallFileLimit(long clusteringSmallFileLimit) { - clusteringConfig.setValue(CLUSTERING_PLAN_SMALL_FILE_LIMIT, String.valueOf(clusteringSmallFileLimit)); + clusteringConfig.setValue(PLAN_STRATEGY_SMALL_FILE_LIMIT, String.valueOf(clusteringSmallFileLimit)); return this; } public Builder withClusteringSortColumns(String sortColumns) { - clusteringConfig.setValue(CLUSTERING_SORT_COLUMNS_PROPERTY, sortColumns); + clusteringConfig.setValue(PLAN_STRATEGY_SORT_COLUMNS, sortColumns); return this; } public Builder withClusteringMaxBytesInGroup(long clusteringMaxGroupSize) { - clusteringConfig.setValue(CLUSTERING_MAX_BYTES_PER_GROUP, String.valueOf(clusteringMaxGroupSize)); + clusteringConfig.setValue(PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP, String.valueOf(clusteringMaxGroupSize)); return this; } public Builder withClusteringMaxNumGroups(int maxNumGroups) { - clusteringConfig.setValue(CLUSTERING_MAX_NUM_GROUPS, String.valueOf(maxNumGroups)); + clusteringConfig.setValue(PLAN_STRATEGY_MAX_GROUPS, String.valueOf(maxNumGroups)); return this; } public Builder withClusteringTargetFileMaxBytes(long targetFileSize) { - clusteringConfig.setValue(CLUSTERING_TARGET_FILE_MAX_BYTES, String.valueOf(targetFileSize)); + clusteringConfig.setValue(PLAN_STRATEGY_TARGET_FILE_MAX_BYTES, String.valueOf(targetFileSize)); return this; } @@ -215,12 +321,12 @@ public class HoodieClusteringConfig extends HoodieConfig { } public Builder withInlineClusteringNumCommits(int numCommits) { - clusteringConfig.setValue(INLINE_CLUSTERING_MAX_COMMIT, String.valueOf(numCommits)); + clusteringConfig.setValue(INLINE_CLUSTERING_MAX_COMMITS, String.valueOf(numCommits)); return this; } public Builder withAsyncClusteringMaxCommits(int numCommits) { - clusteringConfig.setValue(ASYNC_CLUSTERING_MAX_COMMIT_PROP, String.valueOf(numCommits)); + clusteringConfig.setValue(ASYNC_CLUSTERING_MAX_COMMITS, String.valueOf(numCommits)); return this; } @@ -230,7 +336,7 @@ public class HoodieClusteringConfig extends HoodieConfig { } public Builder withClusteringUpdatesStrategy(String updatesStrategyClass) { - clusteringConfig.setValue(CLUSTERING_UPDATES_STRATEGY, updatesStrategyClass); + clusteringConfig.setValue(UPDATES_STRATEGY, updatesStrategyClass); return this; } @@ -240,7 +346,7 @@ public class HoodieClusteringConfig extends HoodieConfig { } public Builder withPreserveHoodieCommitMetadata(Boolean preserveHoodieCommitMetadata) { - clusteringConfig.setValue(CLUSTERING_PRESERVE_HOODIE_COMMIT_METADATA, String.valueOf(preserveHoodieCommitMetadata)); + clusteringConfig.setValue(PRESERVE_COMMIT_METADATA, String.valueOf(preserveHoodieCommitMetadata)); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index 1ae84954c..ce74aad6b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -56,24 +56,18 @@ public class HoodieCompactionConfig extends HoodieConfig { .withDocumentation("When enabled, the cleaner table service is invoked immediately after each commit," + " to delete older file slices. It's recommended to enable this, to ensure metadata and data storage" + " growth is bounded."); - @Deprecated - public static final String AUTO_CLEAN_PROP = AUTO_CLEAN.key(); public static final ConfigProperty ASYNC_CLEAN = ConfigProperty .key("hoodie.clean.async") .defaultValue("false") .withDocumentation("Only applies when " + AUTO_CLEAN.key() + " is turned on. " + "When turned on runs cleaner async with writing, which can speed up overall write performance."); - @Deprecated - public static final String ASYNC_CLEAN_PROP = ASYNC_CLEAN.key();; public static final ConfigProperty CLEANER_COMMITS_RETAINED = ConfigProperty .key("hoodie.cleaner.commits.retained") .defaultValue("10") .withDocumentation("Number of commits to retain, without cleaning. This will be retained for num_of_commits * time_between_commits " + "(scheduled). This also directly translates into how much data retention the table supports for incremental queries."); - @Deprecated - public static final String CLEANER_COMMITS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key();; public static final ConfigProperty CLEANER_POLICY = ConfigProperty .key("hoodie.cleaner.policy") @@ -82,48 +76,36 @@ public class HoodieCompactionConfig extends HoodieConfig { + " By default, cleaner spares the file slices written by the last N commits, determined by " + CLEANER_COMMITS_RETAINED.key() + " Long running query plans may often refer to older file slices and will break if those are cleaned, before the query has had" + " a chance to run. So, it is good to make sure that the data is retained for more than the maximum query execution time"); - @Deprecated - public static final String CLEANER_POLICY_PROP = CLEANER_POLICY.key(); public static final ConfigProperty INLINE_COMPACT = ConfigProperty .key("hoodie.compact.inline") .defaultValue("false") .withDocumentation("When set to true, compaction service is triggered after each write. While being " + " simpler operationally, this adds extra latency on the write path."); - @Deprecated - public static final String INLINE_COMPACT_PROP = INLINE_COMPACT.key(); public static final ConfigProperty INLINE_COMPACT_NUM_DELTA_COMMITS = ConfigProperty .key("hoodie.compact.inline.max.delta.commits") .defaultValue("5") .withDocumentation("Number of delta commits after the last compaction, before scheduling of a new compaction is attempted."); - @Deprecated - public static final String INLINE_COMPACT_NUM_DELTA_COMMITS_PROP = INLINE_COMPACT_NUM_DELTA_COMMITS.key(); public static final ConfigProperty INLINE_COMPACT_TIME_DELTA_SECONDS = ConfigProperty .key("hoodie.compact.inline.max.delta.seconds") .defaultValue(String.valueOf(60 * 60)) .withDocumentation("Number of elapsed seconds after the last compaction, before scheduling a new one."); - @Deprecated - public static final String INLINE_COMPACT_TIME_DELTA_SECONDS_PROP = INLINE_COMPACT_TIME_DELTA_SECONDS.key(); public static final ConfigProperty INLINE_COMPACT_TRIGGER_STRATEGY = ConfigProperty .key("hoodie.compact.inline.trigger.strategy") .defaultValue(CompactionTriggerStrategy.NUM_COMMITS.name()) .withDocumentation("Controls how compaction scheduling is triggered, by time or num delta commits or combination of both. " + "Valid options: " + Arrays.stream(CompactionTriggerStrategy.values()).map(Enum::name).collect(Collectors.joining(","))); - @Deprecated - public static final String INLINE_COMPACT_TRIGGER_STRATEGY_PROP = INLINE_COMPACT_TRIGGER_STRATEGY.key(); public static final ConfigProperty CLEANER_FILE_VERSIONS_RETAINED = ConfigProperty .key("hoodie.cleaner.fileversions.retained") .defaultValue("3") .withDocumentation("When " + HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS.name() + " cleaning policy is used, " + " the minimum number of file slices to retain in each file group, during cleaning."); - @Deprecated - public static final String CLEANER_FILE_VERSIONS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key(); - public static final ConfigProperty CLEANER_INCREMENTAL_MODE = ConfigProperty + public static final ConfigProperty CLEANER_INCREMENTAL_MODE_ENABLE = ConfigProperty .key("hoodie.cleaner.incremental.mode") .defaultValue("true") .withDocumentation("When enabled, the plans for each cleaner service run is computed incrementally off the events " @@ -136,26 +118,20 @@ public class HoodieCompactionConfig extends HoodieConfig { .withDocumentation("Archiving service moves older entries from timeline into an archived log after each write, to " + " keep the metadata overhead constant, even as the table size grows." + "This config controls the maximum number of instants to retain in the active timeline. "); - @Deprecated - public static final String MAX_COMMITS_TO_KEEP_PROP = MAX_COMMITS_TO_KEEP.key(); public static final ConfigProperty MIN_COMMITS_TO_KEEP = ConfigProperty .key("hoodie.keep.min.commits") .defaultValue("20") .withDocumentation("Similar to " + MAX_COMMITS_TO_KEEP.key() + ", but controls the minimum number of" + "instants to retain in the active timeline."); - @Deprecated - public static final String MIN_COMMITS_TO_KEEP_PROP = MIN_COMMITS_TO_KEEP.key(); public static final ConfigProperty COMMITS_ARCHIVAL_BATCH_SIZE = ConfigProperty .key("hoodie.commits.archival.batch") .defaultValue(String.valueOf(10)) .withDocumentation("Archiving of instants is batched in best-effort manner, to pack more instants into a single" + " archive log. This config controls such archival batch size."); - @Deprecated - public static final String COMMITS_ARCHIVAL_BATCH_SIZE_PROP = COMMITS_ARCHIVAL_BATCH_SIZE.key(); - public static final ConfigProperty CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = ConfigProperty + public static final ConfigProperty CLEANER_BOOTSTRAP_BASE_FILE_ENABLE = ConfigProperty .key("hoodie.cleaner.delete.bootstrap.base.file") .defaultValue("false") .withDocumentation("When set to true, cleaner also deletes the bootstrap base file when it's skeleton base file is " @@ -163,7 +139,7 @@ public class HoodieCompactionConfig extends HoodieConfig { + " table receives updates/deletes. Another reason to turn this on, would be to ensure data residing in bootstrap " + " base files are also physically deleted, to comply with data privacy enforcement processes."); - public static final ConfigProperty PARQUET_SMALL_FILE_LIMIT_BYTES = ConfigProperty + public static final ConfigProperty PARQUET_SMALL_FILE_LIMIT = ConfigProperty .key("hoodie.parquet.small.file.limit") .defaultValue(String.valueOf(104857600)) .withDocumentation("During upsert operation, we opportunistically expand existing small files on storage, instead of writing" @@ -177,10 +153,8 @@ public class HoodieCompactionConfig extends HoodieConfig { + " to bin pack records into partitions. If the previous commit is too small to make an accurate estimation, " + " Hudi will search commits in the reverse order, until we find a commit that has totalBytesWritten " + " larger than (PARQUET_SMALL_FILE_LIMIT_BYTES * this_threshold)"); - @Deprecated - public static final String RECORD_SIZE_ESTIMATION_THRESHOLD_PROP = RECORD_SIZE_ESTIMATION_THRESHOLD.key(); - public static final ConfigProperty CLEANER_PARALLELISM = ConfigProperty + public static final ConfigProperty CLEANER_PARALLELISM_VALUE = ConfigProperty .key("hoodie.cleaner.parallelism") .defaultValue("200") .withDocumentation("Parallelism for the cleaning operation. Increase this if cleaning becomes slow."); @@ -191,8 +165,6 @@ public class HoodieCompactionConfig extends HoodieConfig { .defaultValue(String.valueOf(500 * 1024)) .withDocumentation("Amount of MBs to spend during compaction run for the LogFileSizeBasedCompactionStrategy. " + "This value helps bound ingestion latency while compaction is run inline mode."); - @Deprecated - public static final String TARGET_IO_PER_COMPACTION_IN_MB_PROP = TARGET_IO_PER_COMPACTION_IN_MB.key(); public static final ConfigProperty COMPACTION_STRATEGY = ConfigProperty .key("hoodie.compaction.strategy") @@ -200,34 +172,26 @@ public class HoodieCompactionConfig extends HoodieConfig { .withDocumentation("Compaction strategy decides which file groups are picked up for " + "compaction during each compaction run. By default. Hudi picks the log file " + "with most accumulated unmerged data"); - @Deprecated - public static final String COMPACTION_STRATEGY_PROP = COMPACTION_STRATEGY.key(); - public static final ConfigProperty PAYLOAD_CLASS = ConfigProperty + public static final ConfigProperty PAYLOAD_CLASS_NAME = ConfigProperty .key("hoodie.compaction.payload.class") .defaultValue(OverwriteWithLatestAvroPayload.class.getName()) .withDocumentation("This needs to be same as class used during insert/upserts. Just like writing, compaction also uses " + "the record payload class to merge records in the log against each other, merge again with the base file and " + "produce the final record to be written after compaction."); - @Deprecated - public static final String PAYLOAD_CLASS_PROP = PAYLOAD_CLASS.key(); - public static final ConfigProperty COMPACTION_LAZY_BLOCK_READ_ENABLED = ConfigProperty + public static final ConfigProperty COMPACTION_LAZY_BLOCK_READ_ENABLE = ConfigProperty .key("hoodie.compaction.lazy.block.read") .defaultValue("false") .withDocumentation("When merging the delta log files, this config helps to choose whether the log blocks " + "should be read lazily or not. Choose true to use lazy block reading (low memory usage, but incurs seeks to each block" + " header) or false for immediate block read (higher memory usage)"); - @Deprecated - public static final String COMPACTION_LAZY_BLOCK_READ_ENABLED_PROP = COMPACTION_LAZY_BLOCK_READ_ENABLED.key(); - public static final ConfigProperty COMPACTION_REVERSE_LOG_READ_ENABLED = ConfigProperty + public static final ConfigProperty COMPACTION_REVERSE_LOG_READ_ENABLE = ConfigProperty .key("hoodie.compaction.reverse.log.read") .defaultValue("false") .withDocumentation("HoodieLogFormatReader reads a logfile in the forward direction starting from pos=0 to pos=file_length. " + "If this config is set to true, the reader reads the logfile in reverse direction, from pos=file_length to pos=0"); - @Deprecated - public static final String COMPACTION_REVERSE_LOG_READ_ENABLED_PROP = COMPACTION_REVERSE_LOG_READ_ENABLED.key(); public static final ConfigProperty FAILED_WRITES_CLEANER_POLICY = ConfigProperty .key("hoodie.cleaner.policy.failed.writes") @@ -235,21 +199,17 @@ public class HoodieCompactionConfig extends HoodieConfig { .withDocumentation("Cleaning policy for failed writes to be used. Hudi will delete any files written by " + "failed writes to re-claim space. Choose to perform this rollback of failed writes eagerly before " + "every writer starts (only supported for single writer) or lazily by the cleaner (required for multi-writers)"); - @Deprecated - public static final String FAILED_WRITES_CLEANER_POLICY_PROP = FAILED_WRITES_CLEANER_POLICY.key(); public static final ConfigProperty TARGET_PARTITIONS_PER_DAYBASED_COMPACTION = ConfigProperty .key("hoodie.compaction.daybased.target.partitions") .defaultValue("10") .withDocumentation("Used by org.apache.hudi.io.compact.strategy.DayBasedCompactionStrategy to denote the number of " + "latest partitions to compact during a compaction run."); - @Deprecated - public static final String TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.key(); /** * Configs related to specific table types. */ - public static final ConfigProperty COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = ConfigProperty + public static final ConfigProperty COPY_ON_WRITE_INSERT_SPLIT_SIZE = ConfigProperty .key("hoodie.copyonwrite.insert.split.size") .defaultValue(String.valueOf(500000)) .withDocumentation("Number of inserts assigned for each partition/bucket for writing. " @@ -257,20 +217,237 @@ public class HoodieCompactionConfig extends HoodieConfig { + " over provision to 500K. As long as auto-tuning of splits is turned on, this only affects the first " + " write, where there is no history to learn record sizes from."); - public static final ConfigProperty COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = ConfigProperty + public static final ConfigProperty COPY_ON_WRITE_AUTO_SPLIT_INSERTS = ConfigProperty .key("hoodie.copyonwrite.insert.auto.split") .defaultValue("true") .withDocumentation("Config to control whether we control insert split sizes automatically based on average" + " record sizes. It's recommended to keep this turned on, since hand tuning is otherwise extremely" + " cumbersome."); - public static final ConfigProperty COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = ConfigProperty + public static final ConfigProperty COPY_ON_WRITE_RECORD_SIZE_ESTIMATE = ConfigProperty .key("hoodie.copyonwrite.record.size.estimate") .defaultValue(String.valueOf(1024)) .withDocumentation("The average record size. If not explicitly specified, hudi will compute the " + "record size estimate compute dynamically based on commit metadata. " + " This is critical in computing the insert parallelism and bin-packing inserts into small files."); + /** @deprecated Use {@link #CLEANER_POLICY} and its methods instead */ + @Deprecated + public static final String CLEANER_POLICY_PROP = CLEANER_POLICY.key(); + /** @deprecated Use {@link #AUTO_CLEAN} and its methods instead */ + @Deprecated + public static final String AUTO_CLEAN_PROP = AUTO_CLEAN.key(); + /** @deprecated Use {@link #ASYNC_CLEAN} and its methods instead */ + @Deprecated + public static final String ASYNC_CLEAN_PROP = ASYNC_CLEAN.key(); + /** @deprecated Use {@link #INLINE_COMPACT} and its methods instead */ + @Deprecated + public static final String INLINE_COMPACT_PROP = INLINE_COMPACT.key(); + /** @deprecated Use {@link #INLINE_COMPACT_NUM_DELTA_COMMITS} and its methods instead */ + @Deprecated + public static final String INLINE_COMPACT_NUM_DELTA_COMMITS_PROP = INLINE_COMPACT_NUM_DELTA_COMMITS.key(); + /** @deprecated Use {@link #INLINE_COMPACT_TIME_DELTA_SECONDS} and its methods instead */ + @Deprecated + public static final String INLINE_COMPACT_TIME_DELTA_SECONDS_PROP = INLINE_COMPACT_TIME_DELTA_SECONDS.key(); + /** @deprecated Use {@link #INLINE_COMPACT_TRIGGER_STRATEGY} and its methods instead */ + @Deprecated + public static final String INLINE_COMPACT_TRIGGER_STRATEGY_PROP = INLINE_COMPACT_TRIGGER_STRATEGY.key(); + /** @deprecated Use {@link #CLEANER_FILE_VERSIONS_RETAINED} and its methods instead */ + @Deprecated + public static final String CLEANER_FILE_VERSIONS_RETAINED_PROP = CLEANER_FILE_VERSIONS_RETAINED.key(); + /** + * @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods instead + */ + @Deprecated + public static final String CLEANER_COMMITS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key(); + /** + * @deprecated Use {@link #CLEANER_INCREMENTAL_MODE_ENABLE} and its methods instead + */ + @Deprecated + public static final String CLEANER_INCREMENTAL_MODE = CLEANER_INCREMENTAL_MODE_ENABLE.key(); + /** + * @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods instead + */ + @Deprecated + public static final String MAX_COMMITS_TO_KEEP_PROP = MAX_COMMITS_TO_KEEP.key(); + /** + * @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods instead + */ + @Deprecated + public static final String MIN_COMMITS_TO_KEEP_PROP = MIN_COMMITS_TO_KEEP.key(); + /** + * @deprecated Use {@link #COMMITS_ARCHIVAL_BATCH_SIZE} and its methods instead + */ + @Deprecated + public static final String COMMITS_ARCHIVAL_BATCH_SIZE_PROP = COMMITS_ARCHIVAL_BATCH_SIZE.key(); + /** + * @deprecated Use {@link #CLEANER_BOOTSTRAP_BASE_FILE_ENABLE} and its methods instead + */ + @Deprecated + public static final String CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = CLEANER_BOOTSTRAP_BASE_FILE_ENABLE.key(); + /** + * @deprecated Use {@link #PARQUET_SMALL_FILE_LIMIT} and its methods instead + */ + @Deprecated + public static final String PARQUET_SMALL_FILE_LIMIT_BYTES = PARQUET_SMALL_FILE_LIMIT.key(); + /** + * @deprecated Use {@link #PARQUET_SMALL_FILE_LIMIT} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES = PARQUET_SMALL_FILE_LIMIT.defaultValue(); + /** + * @deprecated Use {@link #RECORD_SIZE_ESTIMATION_THRESHOLD} and its methods instead + */ + @Deprecated + public static final String RECORD_SIZE_ESTIMATION_THRESHOLD_PROP = RECORD_SIZE_ESTIMATION_THRESHOLD.key(); + /** + * @deprecated Use {@link #RECORD_SIZE_ESTIMATION_THRESHOLD} and its methods instead + */ + @Deprecated + public static final String DEFAULT_RECORD_SIZE_ESTIMATION_THRESHOLD = RECORD_SIZE_ESTIMATION_THRESHOLD.defaultValue(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_INSERT_SPLIT_SIZE} and its methods instead + */ + @Deprecated + public static final String COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_INSERT_SPLIT_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE = COPY_ON_WRITE_INSERT_SPLIT_SIZE.defaultValue(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_AUTO_SPLIT_INSERTS} and its methods instead + */ + @Deprecated + public static final String COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = COPY_ON_WRITE_AUTO_SPLIT_INSERTS.key(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_AUTO_SPLIT_INSERTS} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS = COPY_ON_WRITE_AUTO_SPLIT_INSERTS.defaultValue(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_RECORD_SIZE_ESTIMATE} and its methods instead + */ + @Deprecated + public static final String COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(); + /** + * @deprecated Use {@link #COPY_ON_WRITE_RECORD_SIZE_ESTIMATE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE = COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.defaultValue(); + /** + * @deprecated Use {@link #CLEANER_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String CLEANER_PARALLELISM = CLEANER_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #CLEANER_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CLEANER_PARALLELISM = CLEANER_PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #TARGET_IO_PER_COMPACTION_IN_MB} and its methods instead + */ + @Deprecated + public static final String TARGET_IO_PER_COMPACTION_IN_MB_PROP = TARGET_IO_PER_COMPACTION_IN_MB.key(); + /** + * @deprecated Use {@link #TARGET_IO_PER_COMPACTION_IN_MB} and its methods instead + */ + @Deprecated + public static final String DEFAULT_TARGET_IO_PER_COMPACTION_IN_MB = TARGET_IO_PER_COMPACTION_IN_MB.defaultValue(); + /** + * @deprecated Use {@link #COMPACTION_STRATEGY} and its methods instead + */ + @Deprecated + public static final String COMPACTION_STRATEGY_PROP = COMPACTION_STRATEGY.key(); + /** @deprecated Use {@link #COMPACTION_STRATEGY} and its methods instead */ + @Deprecated + public static final String DEFAULT_COMPACTION_STRATEGY = COMPACTION_STRATEGY.defaultValue(); + /** @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods instead */ + @Deprecated + public static final String DEFAULT_PAYLOAD_CLASS = PAYLOAD_CLASS_NAME.defaultValue(); + /** @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods instead */ + @Deprecated + public static final String PAYLOAD_CLASS_PROP = PAYLOAD_CLASS_NAME.key(); + /** @deprecated Use {@link #COMPACTION_LAZY_BLOCK_READ_ENABLE} and its methods instead */ + @Deprecated + public static final String COMPACTION_LAZY_BLOCK_READ_ENABLED_PROP = COMPACTION_LAZY_BLOCK_READ_ENABLE.key(); + /** @deprecated Use {@link #COMPACTION_LAZY_BLOCK_READ_ENABLE} and its methods instead */ + @Deprecated + public static final String DEFAULT_COMPACTION_LAZY_BLOCK_READ_ENABLED = COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue(); + /** @deprecated Use {@link #COMPACTION_REVERSE_LOG_READ_ENABLE} and its methods instead */ + @Deprecated + public static final String COMPACTION_REVERSE_LOG_READ_ENABLED_PROP = COMPACTION_REVERSE_LOG_READ_ENABLE.key(); + /** @deprecated Use {@link #COMPACTION_REVERSE_LOG_READ_ENABLE} and its methods instead */ + @Deprecated + public static final String DEFAULT_COMPACTION_REVERSE_LOG_READ_ENABLED = COMPACTION_REVERSE_LOG_READ_ENABLE.defaultValue(); + /** @deprecated Use {@link #CLEANER_POLICY} and its methods instead */ + @Deprecated + private static final String DEFAULT_CLEANER_POLICY = CLEANER_POLICY.defaultValue(); + /** @deprecated Use {@link #FAILED_WRITES_CLEANER_POLICY} and its methods instead */ + @Deprecated + public static final String FAILED_WRITES_CLEANER_POLICY_PROP = FAILED_WRITES_CLEANER_POLICY.key(); + /** @deprecated Use {@link #FAILED_WRITES_CLEANER_POLICY} and its methods instead */ + @Deprecated + private static final String DEFAULT_FAILED_WRITES_CLEANER_POLICY = FAILED_WRITES_CLEANER_POLICY.defaultValue(); + /** @deprecated Use {@link #AUTO_CLEAN} and its methods instead */ + @Deprecated + private static final String DEFAULT_AUTO_CLEAN = AUTO_CLEAN.defaultValue(); + /** + * @deprecated Use {@link #ASYNC_CLEAN} and its methods instead + */ + @Deprecated + private static final String DEFAULT_ASYNC_CLEAN = ASYNC_CLEAN.defaultValue(); + /** + * @deprecated Use {@link #INLINE_COMPACT} and its methods instead + */ + @Deprecated + private static final String DEFAULT_INLINE_COMPACT = INLINE_COMPACT.defaultValue(); + /** + * @deprecated Use {@link #CLEANER_INCREMENTAL_MODE_ENABLE} and its methods instead + */ + @Deprecated + private static final String DEFAULT_INCREMENTAL_CLEANER = CLEANER_INCREMENTAL_MODE_ENABLE.defaultValue(); + /** @deprecated Use {@link #INLINE_COMPACT_NUM_DELTA_COMMITS} and its methods instead */ + @Deprecated + private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = INLINE_COMPACT_NUM_DELTA_COMMITS.defaultValue(); + /** @deprecated Use {@link #INLINE_COMPACT_TIME_DELTA_SECONDS} and its methods instead */ + @Deprecated + private static final String DEFAULT_INLINE_COMPACT_TIME_DELTA_SECONDS = INLINE_COMPACT_TIME_DELTA_SECONDS.defaultValue(); + /** @deprecated Use {@link #INLINE_COMPACT_TRIGGER_STRATEGY} and its methods instead */ + @Deprecated + private static final String DEFAULT_INLINE_COMPACT_TRIGGER_STRATEGY = INLINE_COMPACT_TRIGGER_STRATEGY.defaultValue(); + /** @deprecated Use {@link #CLEANER_FILE_VERSIONS_RETAINED} and its methods instead */ + @Deprecated + private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = CLEANER_FILE_VERSIONS_RETAINED.defaultValue(); + /** @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods instead */ + @Deprecated + private static final String DEFAULT_CLEANER_COMMITS_RETAINED = CLEANER_COMMITS_RETAINED.defaultValue(); + /** @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods instead */ + @Deprecated + private static final String DEFAULT_MAX_COMMITS_TO_KEEP = MAX_COMMITS_TO_KEEP.defaultValue(); + /** + * @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods instead + */ + @Deprecated + private static final String DEFAULT_MIN_COMMITS_TO_KEEP = MIN_COMMITS_TO_KEEP.defaultValue(); + /** + * @deprecated Use {@link #COMMITS_ARCHIVAL_BATCH_SIZE} and its methods instead + */ + @Deprecated + private static final String DEFAULT_COMMITS_ARCHIVAL_BATCH_SIZE = COMMITS_ARCHIVAL_BATCH_SIZE.defaultValue(); + /** + * @deprecated Use {@link #CLEANER_BOOTSTRAP_BASE_FILE_ENABLE} and its methods instead + */ + @Deprecated + private static final String DEFAULT_CLEANER_BOOTSTRAP_BASE_FILE_ENABLED = CLEANER_BOOTSTRAP_BASE_FILE_ENABLE.defaultValue(); + /** @deprecated Use {@link #TARGET_PARTITIONS_PER_DAYBASED_COMPACTION} and its methods instead */ + @Deprecated + public static final String TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.key(); + /** @deprecated Use {@link #TARGET_PARTITIONS_PER_DAYBASED_COMPACTION} and its methods instead */ + @Deprecated + public static final String DEFAULT_TARGET_PARTITIONS_PER_DAYBASED_COMPACTION = TARGET_PARTITIONS_PER_DAYBASED_COMPACTION.defaultValue(); + private HoodieCompactionConfig() { super(); } @@ -306,7 +483,7 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder withIncrementalCleaningMode(Boolean incrementalCleaningMode) { - compactionConfig.setValue(CLEANER_INCREMENTAL_MODE, String.valueOf(incrementalCleaningMode)); + compactionConfig.setValue(CLEANER_INCREMENTAL_MODE_ENABLE, String.valueOf(incrementalCleaningMode)); return this; } @@ -342,7 +519,7 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder compactionSmallFileSize(long smallFileLimitBytes) { - compactionConfig.setValue(PARQUET_SMALL_FILE_LIMIT_BYTES, String.valueOf(smallFileLimitBytes)); + compactionConfig.setValue(PARQUET_SMALL_FILE_LIMIT, String.valueOf(smallFileLimitBytes)); return this; } @@ -352,22 +529,22 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder insertSplitSize(int insertSplitSize) { - compactionConfig.setValue(COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE, String.valueOf(insertSplitSize)); + compactionConfig.setValue(COPY_ON_WRITE_INSERT_SPLIT_SIZE, String.valueOf(insertSplitSize)); return this; } public Builder autoTuneInsertSplits(boolean autoTuneInsertSplits) { - compactionConfig.setValue(COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS, String.valueOf(autoTuneInsertSplits)); + compactionConfig.setValue(COPY_ON_WRITE_AUTO_SPLIT_INSERTS, String.valueOf(autoTuneInsertSplits)); return this; } public Builder approxRecordSize(int recordSizeEstimate) { - compactionConfig.setValue(COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE, String.valueOf(recordSizeEstimate)); + compactionConfig.setValue(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE, String.valueOf(recordSizeEstimate)); return this; } public Builder withCleanerParallelism(int cleanerParallelism) { - compactionConfig.setValue(CLEANER_PARALLELISM, String.valueOf(cleanerParallelism)); + compactionConfig.setValue(CLEANER_PARALLELISM_VALUE, String.valueOf(cleanerParallelism)); return this; } @@ -377,7 +554,7 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder withPayloadClass(String payloadClassName) { - compactionConfig.setValue(PAYLOAD_CLASS, payloadClassName); + compactionConfig.setValue(PAYLOAD_CLASS_NAME, payloadClassName); return this; } @@ -397,12 +574,12 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder withCompactionLazyBlockReadEnabled(Boolean compactionLazyBlockReadEnabled) { - compactionConfig.setValue(COMPACTION_LAZY_BLOCK_READ_ENABLED, String.valueOf(compactionLazyBlockReadEnabled)); + compactionConfig.setValue(COMPACTION_LAZY_BLOCK_READ_ENABLE, String.valueOf(compactionLazyBlockReadEnabled)); return this; } public Builder withCompactionReverseLogReadEnabled(Boolean compactionReverseLogReadEnabled) { - compactionConfig.setValue(COMPACTION_REVERSE_LOG_READ_ENABLED, String.valueOf(compactionReverseLogReadEnabled)); + compactionConfig.setValue(COMPACTION_REVERSE_LOG_READ_ENABLE, String.valueOf(compactionReverseLogReadEnabled)); return this; } @@ -417,7 +594,7 @@ public class HoodieCompactionConfig extends HoodieConfig { } public Builder withCleanBootstrapBaseFileEnabled(Boolean cleanBootstrapSourceFileEnabled) { - compactionConfig.setValue(CLEANER_BOOTSTRAP_BASE_FILE_ENABLED, String.valueOf(cleanBootstrapSourceFileEnabled)); + compactionConfig.setValue(CLEANER_BOOTSTRAP_BASE_FILE_ENABLE, String.valueOf(cleanBootstrapSourceFileEnabled)); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java index 2c8da8bfd..7d048d53b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java @@ -36,138 +36,114 @@ import java.util.Properties; + "records as either inserts or updates to older records.") public class HoodieHBaseIndexConfig extends HoodieConfig { - public static final ConfigProperty HBASE_ZKQUORUM = ConfigProperty + public static final ConfigProperty ZKQUORUM = ConfigProperty .key("hoodie.index.hbase.zkquorum") .noDefaultValue() .withDocumentation("Only applies if index type is HBASE. HBase ZK Quorum url to connect to"); - @Deprecated - public static final String HBASE_ZKQUORUM_PROP = HBASE_ZKQUORUM.key(); - public static final ConfigProperty HBASE_ZKPORT = ConfigProperty + public static final ConfigProperty ZKPORT = ConfigProperty .key("hoodie.index.hbase.zkport") .noDefaultValue() .withDocumentation("Only applies if index type is HBASE. HBase ZK Quorum port to connect to"); - @Deprecated - public static final String HBASE_ZKPORT_PROP = HBASE_ZKPORT.key(); - public static final ConfigProperty HBASE_TABLENAME = ConfigProperty + public static final ConfigProperty TABLENAME = ConfigProperty .key("hoodie.index.hbase.table") .noDefaultValue() .withDocumentation("Only applies if index type is HBASE. HBase Table name to use as the index. " + "Hudi stores the row_key and [partition_path, fileID, commitTime] mapping in the table"); - @Deprecated - public static final String HBASE_TABLENAME_PROP = HBASE_TABLENAME.key(); - public static final ConfigProperty HBASE_GET_BATCH_SIZE = ConfigProperty + public static final ConfigProperty GET_BATCH_SIZE = ConfigProperty .key("hoodie.index.hbase.get.batch.size") .defaultValue(100) .withDocumentation("Controls the batch size for performing gets against HBase. " + "Batching improves throughput, by saving round trips."); - @Deprecated - public static final String HBASE_GET_BATCH_SIZE_PROP = HBASE_GET_BATCH_SIZE.key(); - public static final ConfigProperty HBASE_ZK_ZNODEPARENT = ConfigProperty + public static final ConfigProperty ZK_NODE_PATH = ConfigProperty .key("hoodie.index.hbase.zknode.path") .noDefaultValue() .withDocumentation("Only applies if index type is HBASE. This is the root znode that will contain " + "all the znodes created/used by HBase"); - public static final ConfigProperty HBASE_PUT_BATCH_SIZE = ConfigProperty + public static final ConfigProperty PUT_BATCH_SIZE = ConfigProperty .key("hoodie.index.hbase.put.batch.size") .defaultValue(100) .withDocumentation("Controls the batch size for performing puts against HBase. " + "Batching improves throughput, by saving round trips."); - @Deprecated - public static final String HBASE_PUT_BATCH_SIZE_PROP = HBASE_PUT_BATCH_SIZE.key(); - public static final ConfigProperty HBASE_INDEX_QPS_ALLOCATOR_CLASS = ConfigProperty + public static final ConfigProperty QPS_ALLOCATOR_CLASS_NAME = ConfigProperty .key("hoodie.index.hbase.qps.allocator.class") .defaultValue(DefaultHBaseQPSResourceAllocator.class.getName()) .withDocumentation("Property to set which implementation of HBase QPS resource allocator to be used, which" + "controls the batching rate dynamically."); - public static final ConfigProperty HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE = ConfigProperty + public static final ConfigProperty PUT_BATCH_SIZE_AUTO_COMPUTE = ConfigProperty .key("hoodie.index.hbase.put.batch.size.autocompute") .defaultValue("false") .withDocumentation("Property to set to enable auto computation of put batch size"); - @Deprecated - public static final String HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP = HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE.key(); - public static final ConfigProperty HBASE_QPS_FRACTION = ConfigProperty + public static final ConfigProperty QPS_FRACTION = ConfigProperty .key("hoodie.index.hbase.qps.fraction") .defaultValue(0.5f) .withDocumentation("Property to set the fraction of the global share of QPS that should be allocated to this job. Let's say there are 3" + " jobs which have input size in terms of number of rows required for HbaseIndexing as x, 2x, 3x respectively. Then" + " this fraction for the jobs would be (0.17) 1/6, 0.33 (2/6) and 0.5 (3/6) respectively." + " Default is 50%, which means a total of 2 jobs can run using HbaseIndex without overwhelming Region Servers."); - @Deprecated - public static final String HBASE_QPS_FRACTION_PROP = HBASE_QPS_FRACTION.key(); - public static final ConfigProperty HBASE_MAX_QPS_PER_REGION_SERVER = ConfigProperty + public static final ConfigProperty MAX_QPS_PER_REGION_SERVER = ConfigProperty .key("hoodie.index.hbase.max.qps.per.region.server") .defaultValue(1000) .withDocumentation("Property to set maximum QPS allowed per Region Server. This should be same across various jobs. This is intended to\n" + " limit the aggregate QPS generated across various jobs to an Hbase Region Server. It is recommended to set this\n" + " value based on global indexing throughput needs and most importantly, how much the HBase installation in use is\n" + " able to tolerate without Region Servers going down."); - @Deprecated - public static final String HBASE_MAX_QPS_PER_REGION_SERVER_PROP = HBASE_MAX_QPS_PER_REGION_SERVER.key(); - public static final ConfigProperty HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = ConfigProperty + public static final ConfigProperty COMPUTE_QPS_DYNAMICALLY = ConfigProperty .key("hoodie.index.hbase.dynamic_qps") .defaultValue(false) .withDocumentation("Property to decide if HBASE_QPS_FRACTION_PROP is dynamically calculated based on write volume."); - public static final ConfigProperty HBASE_MIN_QPS_FRACTION = ConfigProperty + public static final ConfigProperty MIN_QPS_FRACTION = ConfigProperty .key("hoodie.index.hbase.min.qps.fraction") .noDefaultValue() .withDocumentation("Minimum for HBASE_QPS_FRACTION_PROP to stabilize skewed write workloads"); - @Deprecated - public static final String HBASE_MIN_QPS_FRACTION_PROP = HBASE_MIN_QPS_FRACTION.key(); - public static final ConfigProperty HBASE_MAX_QPS_FRACTION = ConfigProperty + public static final ConfigProperty MAX_QPS_FRACTION = ConfigProperty .key("hoodie.index.hbase.max.qps.fraction") .noDefaultValue() .withDocumentation("Maximum for HBASE_QPS_FRACTION_PROP to stabilize skewed write workloads"); - @Deprecated - public static final String HBASE_MAX_QPS_FRACTION_PROP = HBASE_MAX_QPS_FRACTION.key(); - public static final ConfigProperty HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = ConfigProperty + public static final ConfigProperty DESIRED_PUTS_TIME_IN_SECONDS = ConfigProperty .key("hoodie.index.hbase.desired_puts_time_in_secs") .defaultValue(600) .withDocumentation(""); - public static final ConfigProperty HBASE_SLEEP_MS_PUT_BATCH = ConfigProperty + public static final ConfigProperty SLEEP_MS_FOR_PUT_BATCH = ConfigProperty .key("hoodie.index.hbase.sleep.ms.for.put.batch") .noDefaultValue() .withDocumentation(""); - @Deprecated - public static final String HBASE_SLEEP_MS_PUT_BATCH_PROP = HBASE_SLEEP_MS_PUT_BATCH.key(); - public static final ConfigProperty HBASE_SLEEP_MS_GET_BATCH = ConfigProperty + public static final ConfigProperty SLEEP_MS_FOR_GET_BATCH = ConfigProperty .key("hoodie.index.hbase.sleep.ms.for.get.batch") .noDefaultValue() .withDocumentation(""); - @Deprecated - public static final String HBASE_SLEEP_MS_GET_BATCH_PROP = HBASE_SLEEP_MS_GET_BATCH.key();; - public static final ConfigProperty HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS = ConfigProperty + public static final ConfigProperty ZK_SESSION_TIMEOUT_MS = ConfigProperty .key("hoodie.index.hbase.zk.session_timeout_ms") .defaultValue(60 * 1000) .withDocumentation("Session timeout value to use for Zookeeper failure detection, for the HBase client." + "Lower this value, if you want to fail faster."); - public static final ConfigProperty HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS = ConfigProperty + public static final ConfigProperty ZK_CONNECTION_TIMEOUT_MS = ConfigProperty .key("hoodie.index.hbase.zk.connection_timeout_ms") .defaultValue(15 * 1000) .withDocumentation("Timeout to use for establishing connection with zookeeper, from HBase client."); - public static final ConfigProperty HBASE_ZK_PATH_QPS_ROOT = ConfigProperty + public static final ConfigProperty ZKPATH_QPS_ROOT = ConfigProperty .key("hoodie.index.hbase.zkpath.qps_root") .defaultValue("/QPS_ROOT") .withDocumentation("chroot in zookeeper, to use for all qps allocation co-ordination."); - public static final ConfigProperty HBASE_INDEX_UPDATE_PARTITION_PATH = ConfigProperty + public static final ConfigProperty UPDATE_PARTITION_PATH_ENABLE = ConfigProperty .key("hoodie.hbase.index.update.partition.path") .defaultValue(false) .withDocumentation("Only applies if index type is HBASE. " @@ -175,12 +151,175 @@ public class HoodieHBaseIndexConfig extends HoodieConfig { + "this config when set, will delete old record in old paritition " + "and will insert it as new record in new partition."); - public static final ConfigProperty HBASE_INDEX_ROLLBACK_SYNC = ConfigProperty + public static final ConfigProperty ROLLBACK_SYNC_ENABLE = ConfigProperty .key("hoodie.index.hbase.rollback.sync") .defaultValue(false) .withDocumentation("When set to true, the rollback method will delete the last failed task index. " + "The default value is false. Because deleting the index will add extra load on the Hbase cluster for each rollback"); + /** + * @deprecated Use {@link #ZKQUORUM} and its methods instead + */ + @Deprecated + public static final String HBASE_ZKQUORUM_PROP = ZKQUORUM.key(); + /** + * @deprecated Use {@link #ZKPORT} and its methods instead + */ + @Deprecated + public static final String HBASE_ZKPORT_PROP = ZKPORT.key(); + /** + * @deprecated Use {@link #TABLENAME} and its methods instead + */ + @Deprecated + public static final String HBASE_TABLENAME_PROP = TABLENAME.key(); + /** + * @deprecated Use {@link #GET_BATCH_SIZE} and its methods instead + */ + @Deprecated + public static final String HBASE_GET_BATCH_SIZE_PROP = GET_BATCH_SIZE.key(); + /** + * @deprecated Use {@link #ZK_NODE_PATH} and its methods instead + */ + @Deprecated + public static final String HBASE_ZK_ZNODEPARENT = ZK_NODE_PATH.key(); + /** + * @deprecated Use {@link #PUT_BATCH_SIZE} and its methods instead + */ + @Deprecated + public static final String HBASE_PUT_BATCH_SIZE_PROP = PUT_BATCH_SIZE.key(); + /** + * @deprecated Use {@link #QPS_ALLOCATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String HBASE_INDEX_QPS_ALLOCATOR_CLASS = QPS_ALLOCATOR_CLASS_NAME.key(); + /** + * @deprecated Use {@link #QPS_ALLOCATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HBASE_INDEX_QPS_ALLOCATOR_CLASS = QPS_ALLOCATOR_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #PUT_BATCH_SIZE_AUTO_COMPUTE} and its methods instead + */ + @Deprecated + public static final String HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP = PUT_BATCH_SIZE_AUTO_COMPUTE.key(); + /** + * @deprecated Use {@link #PUT_BATCH_SIZE_AUTO_COMPUTE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE = PUT_BATCH_SIZE_AUTO_COMPUTE.defaultValue(); + /** + * @deprecated Use {@link #MAX_QPS_FRACTION} and its methods instead + */ + @Deprecated + public static final String HBASE_QPS_FRACTION_PROP = QPS_FRACTION.key(); + /** + * @deprecated Use {@link #MAX_QPS_PER_REGION_SERVER} and its methods instead + */ + @Deprecated + public static final String HBASE_MAX_QPS_PER_REGION_SERVER_PROP = MAX_QPS_PER_REGION_SERVER.key(); + @Deprecated + public static final int DEFAULT_HBASE_BATCH_SIZE = 100; + /** + * @deprecated Use {@link #MAX_QPS_PER_REGION_SERVER} and its methods instead + */ + @Deprecated + public static final int DEFAULT_HBASE_MAX_QPS_PER_REGION_SERVER = MAX_QPS_PER_REGION_SERVER.defaultValue(); + /** + * @deprecated Use {@link #QPS_FRACTION} and its methods instead + */ + @Deprecated + public static final float DEFAULT_HBASE_QPS_FRACTION = QPS_FRACTION.defaultValue(); + /** + * @deprecated Use {@link #COMPUTE_QPS_DYNAMICALLY} and its methods instead + */ + @Deprecated + public static final String HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = COMPUTE_QPS_DYNAMICALLY.key(); + /** + * @deprecated Use {@link #COMPUTE_QPS_DYNAMICALLY} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY = COMPUTE_QPS_DYNAMICALLY.defaultValue(); + /** + * @deprecated Use {@link #MIN_QPS_FRACTION} and its methods instead + */ + @Deprecated + public static final String HBASE_MIN_QPS_FRACTION_PROP = MIN_QPS_FRACTION.key(); + /** + * @deprecated Use {@link #MAX_QPS_FRACTION} and its methods instead + */ + @Deprecated + public static final String HBASE_MAX_QPS_FRACTION_PROP = MAX_QPS_FRACTION.key(); + /** + * @deprecated Use {@link #DESIRED_PUTS_TIME_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final String HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = DESIRED_PUTS_TIME_IN_SECONDS.key(); + /** + * @deprecated Use {@link #DESIRED_PUTS_TIME_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS = DESIRED_PUTS_TIME_IN_SECONDS.defaultValue(); + /** + * @deprecated Use {@link #SLEEP_MS_FOR_PUT_BATCH} and its methods instead + */ + @Deprecated + public static final String HBASE_SLEEP_MS_PUT_BATCH_PROP = SLEEP_MS_FOR_PUT_BATCH.key(); + /** + * @deprecated Use {@link #SLEEP_MS_FOR_GET_BATCH} and its methods instead + */ + @Deprecated + public static final String HBASE_SLEEP_MS_GET_BATCH_PROP = SLEEP_MS_FOR_GET_BATCH.key(); + /** + * @deprecated Use {@link #ZK_SESSION_TIMEOUT_MS} and its methods instead + */ + @Deprecated + public static final String HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS = ZK_SESSION_TIMEOUT_MS.key(); + /** + * @deprecated Use {@link #ZK_SESSION_TIMEOUT_MS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_ZK_SESSION_TIMEOUT_MS = ZK_SESSION_TIMEOUT_MS.defaultValue(); + /** + * @deprecated Use {@link #ZK_CONNECTION_TIMEOUT_MS} and its methods instead + */ + @Deprecated + public static final String HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS = ZK_CONNECTION_TIMEOUT_MS.key(); + /** + * @deprecated Use {@link #ZK_CONNECTION_TIMEOUT_MS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_ZK_CONNECTION_TIMEOUT_MS = ZK_CONNECTION_TIMEOUT_MS.defaultValue(); + /** + * @deprecated Use {@link #ZKPATH_QPS_ROOT} and its methods instead + */ + @Deprecated + public static final String HBASE_ZK_PATH_QPS_ROOT = ZKPATH_QPS_ROOT.key(); + /** + * @deprecated Use {@link #ZKPATH_QPS_ROOT} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HBASE_ZK_PATH_QPS_ROOT = ZKPATH_QPS_ROOT.defaultValue(); + /** + * @deprecated Use {@link #UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final String HBASE_INDEX_UPDATE_PARTITION_PATH = UPDATE_PARTITION_PATH_ENABLE.key(); + /** + * @deprecated Use {@link #UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final Boolean DEFAULT_HBASE_INDEX_UPDATE_PARTITION_PATH = UPDATE_PARTITION_PATH_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #ROLLBACK_SYNC_ENABLE} and its methods instead + */ + @Deprecated + public static final String HBASE_INDEX_ROLLBACK_SYNC = ROLLBACK_SYNC_ENABLE.key(); + /** + * @deprecated Use {@link #ROLLBACK_SYNC_ENABLE} and its methods instead + */ + @Deprecated + public static final Boolean DEFAULT_HBASE_INDEX_ROLLBACK_SYNC = ROLLBACK_SYNC_ENABLE.defaultValue(); + private HoodieHBaseIndexConfig() { super(); } @@ -206,102 +345,102 @@ public class HoodieHBaseIndexConfig extends HoodieConfig { } public HoodieHBaseIndexConfig.Builder hbaseZkQuorum(String zkString) { - hBaseIndexConfig.setValue(HBASE_ZKQUORUM, zkString); + hBaseIndexConfig.setValue(ZKQUORUM, zkString); return this; } public HoodieHBaseIndexConfig.Builder hbaseZkPort(int port) { - hBaseIndexConfig.setValue(HBASE_ZKPORT, String.valueOf(port)); + hBaseIndexConfig.setValue(ZKPORT, String.valueOf(port)); return this; } public HoodieHBaseIndexConfig.Builder hbaseTableName(String tableName) { - hBaseIndexConfig.setValue(HBASE_TABLENAME, tableName); + hBaseIndexConfig.setValue(TABLENAME, tableName); return this; } public Builder hbaseZkZnodeQPSPath(String zkZnodeQPSPath) { - hBaseIndexConfig.setValue(HBASE_ZK_PATH_QPS_ROOT, zkZnodeQPSPath); + hBaseIndexConfig.setValue(ZKPATH_QPS_ROOT, zkZnodeQPSPath); return this; } public Builder hbaseIndexGetBatchSize(int getBatchSize) { - hBaseIndexConfig.setValue(HBASE_GET_BATCH_SIZE, String.valueOf(getBatchSize)); + hBaseIndexConfig.setValue(GET_BATCH_SIZE, String.valueOf(getBatchSize)); return this; } public Builder hbaseIndexPutBatchSize(int putBatchSize) { - hBaseIndexConfig.setValue(HBASE_PUT_BATCH_SIZE, String.valueOf(putBatchSize)); + hBaseIndexConfig.setValue(PUT_BATCH_SIZE, String.valueOf(putBatchSize)); return this; } public Builder hbaseIndexPutBatchSizeAutoCompute(boolean putBatchSizeAutoCompute) { - hBaseIndexConfig.setValue(HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE, String.valueOf(putBatchSizeAutoCompute)); + hBaseIndexConfig.setValue(PUT_BATCH_SIZE_AUTO_COMPUTE, String.valueOf(putBatchSizeAutoCompute)); return this; } public Builder hbaseIndexDesiredPutsTime(int desiredPutsTime) { - hBaseIndexConfig.setValue(HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS, String.valueOf(desiredPutsTime)); + hBaseIndexConfig.setValue(DESIRED_PUTS_TIME_IN_SECONDS, String.valueOf(desiredPutsTime)); return this; } public Builder hbaseIndexShouldComputeQPSDynamically(boolean shouldComputeQPsDynamically) { - hBaseIndexConfig.setValue(HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY, String.valueOf(shouldComputeQPsDynamically)); + hBaseIndexConfig.setValue(COMPUTE_QPS_DYNAMICALLY, String.valueOf(shouldComputeQPsDynamically)); return this; } public Builder hbaseIndexQPSFraction(float qpsFraction) { - hBaseIndexConfig.setValue(HBASE_QPS_FRACTION, String.valueOf(qpsFraction)); + hBaseIndexConfig.setValue(QPS_FRACTION, String.valueOf(qpsFraction)); return this; } public Builder hbaseIndexMinQPSFraction(float minQPSFraction) { - hBaseIndexConfig.setValue(HBASE_MIN_QPS_FRACTION, String.valueOf(minQPSFraction)); + hBaseIndexConfig.setValue(MIN_QPS_FRACTION, String.valueOf(minQPSFraction)); return this; } public Builder hbaseIndexMaxQPSFraction(float maxQPSFraction) { - hBaseIndexConfig.setValue(HBASE_MAX_QPS_FRACTION, String.valueOf(maxQPSFraction)); + hBaseIndexConfig.setValue(MAX_QPS_FRACTION, String.valueOf(maxQPSFraction)); return this; } public Builder hbaseIndexSleepMsBetweenPutBatch(int sleepMsBetweenPutBatch) { - hBaseIndexConfig.setValue(HBASE_SLEEP_MS_PUT_BATCH, String.valueOf(sleepMsBetweenPutBatch)); + hBaseIndexConfig.setValue(SLEEP_MS_FOR_PUT_BATCH, String.valueOf(sleepMsBetweenPutBatch)); return this; } public Builder hbaseIndexSleepMsBetweenGetBatch(int sleepMsBetweenGetBatch) { - hBaseIndexConfig.setValue(HBASE_SLEEP_MS_GET_BATCH, String.valueOf(sleepMsBetweenGetBatch)); + hBaseIndexConfig.setValue(SLEEP_MS_FOR_GET_BATCH, String.valueOf(sleepMsBetweenGetBatch)); return this; } public Builder hbaseIndexUpdatePartitionPath(boolean updatePartitionPath) { - hBaseIndexConfig.setValue(HBASE_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath)); + hBaseIndexConfig.setValue(UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath)); return this; } public Builder hbaseIndexRollbackSync(boolean rollbackSync) { - hBaseIndexConfig.setValue(HBASE_INDEX_ROLLBACK_SYNC, String.valueOf(rollbackSync)); + hBaseIndexConfig.setValue(ROLLBACK_SYNC_ENABLE, String.valueOf(rollbackSync)); return this; } public Builder withQPSResourceAllocatorType(String qpsResourceAllocatorClass) { - hBaseIndexConfig.setValue(HBASE_INDEX_QPS_ALLOCATOR_CLASS, qpsResourceAllocatorClass); + hBaseIndexConfig.setValue(QPS_ALLOCATOR_CLASS_NAME, qpsResourceAllocatorClass); return this; } public Builder hbaseIndexZkSessionTimeout(int zkSessionTimeout) { - hBaseIndexConfig.setValue(HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS, String.valueOf(zkSessionTimeout)); + hBaseIndexConfig.setValue(ZK_SESSION_TIMEOUT_MS, String.valueOf(zkSessionTimeout)); return this; } public Builder hbaseIndexZkConnectionTimeout(int zkConnectionTimeout) { - hBaseIndexConfig.setValue(HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS, String.valueOf(zkConnectionTimeout)); + hBaseIndexConfig.setValue(ZK_CONNECTION_TIMEOUT_MS, String.valueOf(zkConnectionTimeout)); return this; } public Builder hbaseZkZnodeParent(String zkZnodeParent) { - hBaseIndexConfig.setValue(HBASE_ZK_ZNODEPARENT, zkZnodeParent); + hBaseIndexConfig.setValue(ZK_NODE_PATH, zkZnodeParent); return this; } @@ -317,7 +456,7 @@ public class HoodieHBaseIndexConfig extends HoodieConfig { */ public HoodieHBaseIndexConfig.Builder hbaseIndexMaxQPSPerRegionServer(int maxQPSPerRegionServer) { // This should be same across various jobs - hBaseIndexConfig.setValue(HoodieHBaseIndexConfig.HBASE_MAX_QPS_PER_REGION_SERVER, + hBaseIndexConfig.setValue(HoodieHBaseIndexConfig.MAX_QPS_PER_REGION_SERVER, String.valueOf(maxQPSPerRegionServer)); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index ca3e58e33..4f9b9fcd5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -34,11 +34,11 @@ import java.io.FileReader; import java.io.IOException; import java.util.Properties; -import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_ZKQUORUM; -import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_ZKPORT; -import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_TABLENAME; -import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_GET_BATCH_SIZE; -import static org.apache.hudi.config.HoodieHBaseIndexConfig.HBASE_PUT_BATCH_SIZE; +import static org.apache.hudi.config.HoodieHBaseIndexConfig.GET_BATCH_SIZE; +import static org.apache.hudi.config.HoodieHBaseIndexConfig.PUT_BATCH_SIZE; +import static org.apache.hudi.config.HoodieHBaseIndexConfig.TABLENAME; +import static org.apache.hudi.config.HoodieHBaseIndexConfig.ZKPORT; +import static org.apache.hudi.config.HoodieHBaseIndexConfig.ZKQUORUM; /** * Indexing related config. @@ -57,19 +57,15 @@ public class HoodieIndexConfig extends HoodieConfig { + "Possible options are [BLOOM | GLOBAL_BLOOM |SIMPLE | GLOBAL_SIMPLE | INMEMORY | HBASE]. " + "Bloom filters removes the dependency on a external system " + "and is stored in the footer of the Parquet Data Files"); - @Deprecated - public static final String INDEX_TYPE_PROP = INDEX_TYPE.key(); - public static final ConfigProperty INDEX_CLASS = ConfigProperty + public static final ConfigProperty INDEX_CLASS_NAME = ConfigProperty .key("hoodie.index.class") .defaultValue("") .withDocumentation("Full path of user-defined index class and must be a subclass of HoodieIndex class. " + "It will take precedence over the hoodie.index.type configuration if specified"); - @Deprecated - public static final String INDEX_CLASS_PROP = INDEX_CLASS.key(); // ***** Bloom Index configs ***** - public static final ConfigProperty BLOOM_FILTER_NUM_ENTRIES = ConfigProperty + public static final ConfigProperty BLOOM_FILTER_NUM_ENTRIES_VALUE = ConfigProperty .key("hoodie.index.bloom.num_entries") .defaultValue("60000") .withDocumentation("Only applies if index type is BLOOM. " @@ -81,7 +77,7 @@ public class HoodieIndexConfig extends HoodieConfig { + "increase the size every base file linearly (roughly 4KB for every 50000 entries). " + "This config is also used with DYNAMIC bloom filter which determines the initial size for the bloom."); - public static final ConfigProperty BLOOM_FILTER_FPP = ConfigProperty + public static final ConfigProperty BLOOM_FILTER_FPP_VALUE = ConfigProperty .key("hoodie.index.bloom.fpp") .defaultValue("0.000000001") .withDocumentation("Only applies if index type is BLOOM. " @@ -97,8 +93,6 @@ public class HoodieIndexConfig extends HoodieConfig { .withDocumentation("Only applies if index type is BLOOM. " + "This is the amount of parallelism for index lookup, which involves a shuffle. " + "By default, this is auto computed based on input workload characteristics."); - @Deprecated - public static final String BLOOM_INDEX_PARALLELISM_PROP = BLOOM_INDEX_PARALLELISM.key(); public static final ConfigProperty BLOOM_INDEX_PRUNE_BY_RANGES = ConfigProperty .key("hoodie.bloom.index.prune.by.ranges") @@ -108,8 +102,6 @@ public class HoodieIndexConfig extends HoodieConfig { + "if the key has a monotonously increasing prefix, such as timestamp. " + "If the record key is completely random, it is better to turn this off, since range pruning will only " + " add extra overhead to the index lookup."); - @Deprecated - public static final String BLOOM_INDEX_PRUNE_BY_RANGES_PROP = BLOOM_INDEX_PRUNE_BY_RANGES.key(); public static final ConfigProperty BLOOM_INDEX_USE_CACHING = ConfigProperty .key("hoodie.bloom.index.use.caching") @@ -117,8 +109,6 @@ public class HoodieIndexConfig extends HoodieConfig { .withDocumentation("Only applies if index type is BLOOM." + "When true, the input RDD will cached to speed up index lookup by reducing IO " + "for computing parallelism or affected partitions"); - @Deprecated - public static final String BLOOM_INDEX_USE_CACHING_PROP = BLOOM_INDEX_USE_CACHING.key(); public static final ConfigProperty BLOOM_INDEX_TREE_BASED_FILTER = ConfigProperty .key("hoodie.bloom.index.use.treebased.filter") @@ -126,8 +116,6 @@ public class HoodieIndexConfig extends HoodieConfig { .withDocumentation("Only applies if index type is BLOOM. " + "When true, interval tree based file pruning optimization is enabled. " + "This mode speeds-up file-pruning based on key ranges when compared with the brute-force mode"); - @Deprecated - public static final String BLOOM_INDEX_TREE_BASED_FILTER_PROP = BLOOM_INDEX_TREE_BASED_FILTER.key(); // TODO: On by default. Once stable, we will remove the other mode. public static final ConfigProperty BLOOM_INDEX_BUCKETIZED_CHECKING = ConfigProperty @@ -136,17 +124,15 @@ public class HoodieIndexConfig extends HoodieConfig { .withDocumentation("Only applies if index type is BLOOM. " + "When true, bucketized bloom filtering is enabled. " + "This reduces skew seen in sort based bloom index lookup"); - @Deprecated - public static final String BLOOM_INDEX_BUCKETIZED_CHECKING_PROP = BLOOM_INDEX_BUCKETIZED_CHECKING.key(); - public static final ConfigProperty BLOOM_INDEX_FILTER_TYPE = ConfigProperty + public static final ConfigProperty BLOOM_FILTER_TYPE = ConfigProperty .key("hoodie.bloom.index.filter.type") .defaultValue(BloomFilterTypeCode.DYNAMIC_V0.name()) .withDocumentation("Filter type used. Default is BloomFilterTypeCode.DYNAMIC_V0. " + "Available values are [BloomFilterTypeCode.SIMPLE , BloomFilterTypeCode.DYNAMIC_V0]. " + "Dynamic bloom filters auto size themselves based on number of keys."); - public static final ConfigProperty HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = ConfigProperty + public static final ConfigProperty BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = ConfigProperty .key("hoodie.bloom.index.filter.dynamic.max.entries") .defaultValue("100000") .withDocumentation("The threshold for the maximum number of keys to record in a dynamic Bloom filter row. " @@ -158,24 +144,18 @@ public class HoodieIndexConfig extends HoodieConfig { .withDocumentation("Only applies if index type is SIMPLE. " + "When true, the incoming writes will cached to speed up index lookup by reducing IO " + "for computing parallelism or affected partitions"); - @Deprecated - public static final String SIMPLE_INDEX_USE_CACHING_PROP = SIMPLE_INDEX_USE_CACHING.key(); public static final ConfigProperty SIMPLE_INDEX_PARALLELISM = ConfigProperty .key("hoodie.simple.index.parallelism") .defaultValue("50") .withDocumentation("Only applies if index type is SIMPLE. " + "This is the amount of parallelism for index lookup, which involves a Spark Shuffle"); - @Deprecated - public static final String SIMPLE_INDEX_PARALLELISM_PROP = SIMPLE_INDEX_PARALLELISM.key(); public static final ConfigProperty GLOBAL_SIMPLE_INDEX_PARALLELISM = ConfigProperty .key("hoodie.global.simple.index.parallelism") .defaultValue("100") .withDocumentation("Only applies if index type is GLOBAL_SIMPLE. " + "This is the amount of parallelism for index lookup, which involves a Spark Shuffle"); - @Deprecated - public static final String GLOBAL_SIMPLE_INDEX_PARALLELISM_PROP = GLOBAL_SIMPLE_INDEX_PARALLELISM.key(); // 1B bloom filter checks happen in 250 seconds. 500ms to read a bloom filter. // 10M checks in 2500ms, thus amortizing the cost of reading bloom filter across partitions. @@ -186,16 +166,14 @@ public class HoodieIndexConfig extends HoodieConfig { + "This configuration controls the “bucket” size which tracks the number of record-key checks made against " + "a single file and is the unit of work allocated to each partition performing bloom filter lookup. " + "A higher value would amortize the fixed cost of reading a bloom filter to memory."); - @Deprecated - public static final String BLOOM_INDEX_KEYS_PER_BUCKET_PROP = BLOOM_INDEX_KEYS_PER_BUCKET.key(); - public static final ConfigProperty BLOOM_INDEX_INPUT_STORAGE_LEVEL = ConfigProperty + public static final ConfigProperty BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE = ConfigProperty .key("hoodie.bloom.index.input.storage.level") .defaultValue("MEMORY_AND_DISK_SER") .withDocumentation("Only applies when #bloomIndexUseCaching is set. Determine what level of persistence is used to cache input RDDs. " + "Refer to org.apache.spark.storage.StorageLevel for different values"); - public static final ConfigProperty SIMPLE_INDEX_INPUT_STORAGE_LEVEL = ConfigProperty + public static final ConfigProperty SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE = ConfigProperty .key("hoodie.simple.index.input.storage.level") .defaultValue("MEMORY_AND_DISK_SER") .withDocumentation("Only applies when #simpleIndexUseCaching is set. Determine what level of persistence is used to cache input RDDs. " @@ -209,7 +187,7 @@ public class HoodieIndexConfig extends HoodieConfig { *

* When set to false, a record will be updated to the old partition. */ - public static final ConfigProperty BLOOM_INDEX_UPDATE_PARTITION_PATH = ConfigProperty + public static final ConfigProperty BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE = ConfigProperty .key("hoodie.bloom.index.update.partition.path") .defaultValue("true") .withDocumentation("Only applies if index type is GLOBAL_BLOOM. " @@ -217,26 +195,195 @@ public class HoodieIndexConfig extends HoodieConfig { + "inserting the incoming record into the new partition and deleting the original record in the old partition. " + "When set to false, the original record will only be updated in the old partition"); - public static final ConfigProperty SIMPLE_INDEX_UPDATE_PARTITION_PATH = ConfigProperty + public static final ConfigProperty SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE = ConfigProperty .key("hoodie.simple.index.update.partition.path") .defaultValue("true") - .withDocumentation("Similar to " + BLOOM_INDEX_UPDATE_PARTITION_PATH + ", but for simple index."); + .withDocumentation("Similar to " + BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE + ", but for simple index."); /** * Deprecated configs. These are now part of {@link HoodieHBaseIndexConfig}. */ @Deprecated - public static final String HBASE_ZKQUORUM_PROP = HBASE_ZKQUORUM.key(); + public static final String HBASE_ZKQUORUM_PROP = ZKQUORUM.key(); @Deprecated - public static final String HBASE_ZKPORT_PROP = HBASE_ZKPORT.key(); + public static final String HBASE_ZKPORT_PROP = ZKPORT.key(); @Deprecated - public static final String HBASE_ZK_ZNODEPARENT = HoodieHBaseIndexConfig.HBASE_ZK_ZNODEPARENT.key(); + public static final String HBASE_ZK_ZNODEPARENT = HoodieHBaseIndexConfig.ZK_NODE_PATH.key(); @Deprecated - public static final String HBASE_TABLENAME_PROP = HBASE_TABLENAME.key(); + public static final String HBASE_TABLENAME_PROP = TABLENAME.key(); @Deprecated - public static final String HBASE_GET_BATCH_SIZE_PROP = HBASE_GET_BATCH_SIZE.key(); + public static final String HBASE_GET_BATCH_SIZE_PROP = GET_BATCH_SIZE.key(); @Deprecated - public static final String HBASE_PUT_BATCH_SIZE_PROP = HBASE_PUT_BATCH_SIZE.key(); + public static final String HBASE_PUT_BATCH_SIZE_PROP = PUT_BATCH_SIZE.key(); + @Deprecated + public static final String DEFAULT_HBASE_BATCH_SIZE = "100"; + /** @deprecated Use {@link #INDEX_TYPE} and its methods instead */ + @Deprecated + public static final String INDEX_TYPE_PROP = INDEX_TYPE.key(); + /** + * @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String INDEX_CLASS_PROP = INDEX_CLASS_NAME.key(); + /** + * @deprecated Use {@link #INDEX_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_INDEX_CLASS = INDEX_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_FILTER_NUM_ENTRIES_VALUE} and its methods instead + */ + @Deprecated + public static final String BLOOM_FILTER_NUM_ENTRIES = BLOOM_FILTER_NUM_ENTRIES_VALUE.key(); + /** + * @deprecated Use {@link #BLOOM_FILTER_NUM_ENTRIES_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_FILTER_NUM_ENTRIES = BLOOM_FILTER_NUM_ENTRIES_VALUE.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_FILTER_FPP_VALUE} and its methods instead + */ + @Deprecated + public static final String BLOOM_FILTER_FPP = BLOOM_FILTER_FPP_VALUE.key(); + /** + * @deprecated Use {@link #BLOOM_FILTER_FPP_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_FILTER_FPP = BLOOM_FILTER_FPP_VALUE.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_PARALLELISM} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_PARALLELISM_PROP = BLOOM_INDEX_PARALLELISM.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_PARALLELISM} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_PARALLELISM = BLOOM_INDEX_PARALLELISM.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_PRUNE_BY_RANGES} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_PRUNE_BY_RANGES_PROP = BLOOM_INDEX_PRUNE_BY_RANGES.key(); + /** @deprecated Use {@link #BLOOM_INDEX_PRUNE_BY_RANGES} and its methods instead */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_PRUNE_BY_RANGES = BLOOM_INDEX_PRUNE_BY_RANGES.defaultValue(); + /** @deprecated Use {@link #BLOOM_INDEX_USE_CACHING} and its methods instead */ + @Deprecated + public static final String BLOOM_INDEX_USE_CACHING_PROP = BLOOM_INDEX_USE_CACHING.key(); + /** @deprecated Use {@link #BLOOM_INDEX_USE_CACHING} and its methods instead */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_USE_CACHING = BLOOM_INDEX_USE_CACHING.defaultValue(); + /** @deprecated Use {@link #BLOOM_INDEX_TREE_BASED_FILTER} and its methods instead */ + @Deprecated + public static final String BLOOM_INDEX_TREE_BASED_FILTER_PROP = BLOOM_INDEX_TREE_BASED_FILTER.key(); + /** @deprecated Use {@link #BLOOM_INDEX_TREE_BASED_FILTER} and its methods instead */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_TREE_BASED_FILTER = BLOOM_INDEX_TREE_BASED_FILTER.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_BUCKETIZED_CHECKING} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_BUCKETIZED_CHECKING_PROP = BLOOM_INDEX_BUCKETIZED_CHECKING.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_BUCKETIZED_CHECKING} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_BUCKETIZED_CHECKING = BLOOM_INDEX_BUCKETIZED_CHECKING.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_FILTER_TYPE} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_FILTER_TYPE = BLOOM_FILTER_TYPE.key(); + /** + * @deprecated Use {@link #BLOOM_FILTER_TYPE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_FILTER_TYPE = BLOOM_FILTER_TYPE.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES} and its methods instead + */ + @Deprecated + public static final String HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES = BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES.defaultValue(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_USE_CACHING} and its methods instead + */ + @Deprecated + public static final String SIMPLE_INDEX_USE_CACHING_PROP = SIMPLE_INDEX_USE_CACHING.key(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_USE_CACHING} and its methods instead + */ + @Deprecated + public static final String DEFAULT_SIMPLE_INDEX_USE_CACHING = SIMPLE_INDEX_USE_CACHING.defaultValue(); + /** @deprecated Use {@link #SIMPLE_INDEX_PARALLELISM} and its methods instead */ + @Deprecated + public static final String SIMPLE_INDEX_PARALLELISM_PROP = SIMPLE_INDEX_PARALLELISM.key(); + /** @deprecated Use {@link #SIMPLE_INDEX_PARALLELISM} and its methods instead */ + @Deprecated + public static final String DEFAULT_SIMPLE_INDEX_PARALLELISM = SIMPLE_INDEX_PARALLELISM.defaultValue(); + /** @deprecated Use {@link #GLOBAL_SIMPLE_INDEX_PARALLELISM} and its methods instead */ + @Deprecated + public static final String GLOBAL_SIMPLE_INDEX_PARALLELISM_PROP = GLOBAL_SIMPLE_INDEX_PARALLELISM.key(); + /** + * @deprecated Use {@link #GLOBAL_SIMPLE_INDEX_PARALLELISM} and its methods instead + */ + @Deprecated + public static final String DEFAULT_GLOBAL_SIMPLE_INDEX_PARALLELISM = GLOBAL_SIMPLE_INDEX_PARALLELISM.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_KEYS_PER_BUCKET} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_KEYS_PER_BUCKET_PROP = BLOOM_INDEX_KEYS_PER_BUCKET.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_KEYS_PER_BUCKET} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_KEYS_PER_BUCKET = BLOOM_INDEX_KEYS_PER_BUCKET.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_INPUT_STORAGE_LEVEL = BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_INPUT_STORAGE_LEVEL = BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE.defaultValue(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String SIMPLE_INDEX_INPUT_STORAGE_LEVEL = SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE.key(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_SIMPLE_INDEX_INPUT_STORAGE_LEVEL = SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE.defaultValue(); + /** + * @deprecated Use {@link #BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final String BLOOM_INDEX_UPDATE_PARTITION_PATH = BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE.key(); + /** + * @deprecated Use {@link #BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BLOOM_INDEX_UPDATE_PARTITION_PATH = BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final String SIMPLE_INDEX_UPDATE_PARTITION_PATH = SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE.key(); + /** + * @deprecated Use {@link #SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_SIMPLE_INDEX_UPDATE_PARTITION_PATH = SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE.defaultValue(); private EngineType engineType; @@ -280,7 +427,7 @@ public class HoodieIndexConfig extends HoodieConfig { } public Builder withIndexClass(String indexClass) { - hoodieIndexConfig.setValue(INDEX_CLASS, indexClass); + hoodieIndexConfig.setValue(INDEX_CLASS_NAME, indexClass); return this; } @@ -290,12 +437,12 @@ public class HoodieIndexConfig extends HoodieConfig { } public Builder bloomFilterNumEntries(int numEntries) { - hoodieIndexConfig.setValue(BLOOM_FILTER_NUM_ENTRIES, String.valueOf(numEntries)); + hoodieIndexConfig.setValue(BLOOM_FILTER_NUM_ENTRIES_VALUE, String.valueOf(numEntries)); return this; } public Builder bloomFilterFPP(double fpp) { - hoodieIndexConfig.setValue(BLOOM_FILTER_FPP, String.valueOf(fpp)); + hoodieIndexConfig.setValue(BLOOM_FILTER_FPP_VALUE, String.valueOf(fpp)); return this; } @@ -330,12 +477,12 @@ public class HoodieIndexConfig extends HoodieConfig { } public Builder withBloomIndexInputStorageLevel(String level) { - hoodieIndexConfig.setValue(BLOOM_INDEX_INPUT_STORAGE_LEVEL, level); + hoodieIndexConfig.setValue(BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE, level); return this; } public Builder withBloomIndexUpdatePartitionPath(boolean updatePartitionPath) { - hoodieIndexConfig.setValue(BLOOM_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath)); + hoodieIndexConfig.setValue(BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath)); return this; } @@ -350,7 +497,7 @@ public class HoodieIndexConfig extends HoodieConfig { } public Builder withSimpleIndexInputStorageLevel(String level) { - hoodieIndexConfig.setValue(SIMPLE_INDEX_INPUT_STORAGE_LEVEL, level); + hoodieIndexConfig.setValue(SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE, level); return this; } @@ -360,7 +507,7 @@ public class HoodieIndexConfig extends HoodieConfig { } public Builder withGlobalSimpleIndexUpdatePartitionPath(boolean updatePartitionPath) { - hoodieIndexConfig.setValue(SIMPLE_INDEX_UPDATE_PARTITION_PATH, String.valueOf(updatePartitionPath)); + hoodieIndexConfig.setValue(SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE, String.valueOf(updatePartitionPath)); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java index 41ad4e55c..fd7b310a5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java @@ -70,8 +70,6 @@ public class HoodieLockConfig extends HoodieConfig { .sinceVersion("0.8.0") .withDocumentation("Initial amount of time to wait between retries to acquire locks, " + " subsequent retries will exponentially backoff."); - @Deprecated - public static final String LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS.key(); public static final ConfigProperty LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS = ConfigProperty .key(LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY) @@ -79,72 +77,54 @@ public class HoodieLockConfig extends HoodieConfig { .sinceVersion("0.8.0") .withDocumentation("Maximum amount of time to wait between retries by lock provider client. This bounds" + " the maximum delay from the exponential backoff. Currently used by ZK based lock provider only."); - @Deprecated - public static final String LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS.key(); public static final ConfigProperty LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS = ConfigProperty .key(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY) .defaultValue(String.valueOf(10000L)) .sinceVersion("0.8.0") .withDocumentation("Amount of time to wait between retries on the lock provider by the lock manager"); - @Deprecated - public static final String LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS.key(); public static final ConfigProperty LOCK_ACQUIRE_NUM_RETRIES = ConfigProperty .key(LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY) .defaultValue(DEFAULT_LOCK_ACQUIRE_NUM_RETRIES) .sinceVersion("0.8.0") .withDocumentation("Maximum number of times to retry lock acquire, at each lock provider"); - @Deprecated - public static final String LOCK_ACQUIRE_NUM_RETRIES_PROP = LOCK_ACQUIRE_NUM_RETRIES.key(); public static final ConfigProperty LOCK_ACQUIRE_CLIENT_NUM_RETRIES = ConfigProperty .key(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY) .defaultValue(String.valueOf(0)) .sinceVersion("0.8.0") .withDocumentation("Maximum number of times to retry to acquire lock additionally from the lock manager."); - @Deprecated - public static final String LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP = LOCK_ACQUIRE_CLIENT_NUM_RETRIES.key(); public static final ConfigProperty LOCK_ACQUIRE_WAIT_TIMEOUT_MS = ConfigProperty .key(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY) .defaultValue(60 * 1000) .sinceVersion("0.8.0") .withDocumentation("Timeout in ms, to wait on an individual lock acquire() call, at the lock provider."); - @Deprecated - public static final String LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP = LOCK_ACQUIRE_WAIT_TIMEOUT_MS.key(); public static final ConfigProperty FILESYSTEM_LOCK_PATH = ConfigProperty .key(FILESYSTEM_LOCK_PATH_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("For DFS based lock providers, path to store the locks under."); - @Deprecated - public static final String FILESYSTEM_LOCK_PATH_PROP = FILESYSTEM_LOCK_PATH.key(); public static final ConfigProperty HIVE_DATABASE_NAME = ConfigProperty .key(HIVE_DATABASE_NAME_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("For Hive based lock provider, the Hive database to acquire lock against"); - @Deprecated - public static final String HIVE_DATABASE_NAME_PROP = HIVE_DATABASE_NAME.key(); public static final ConfigProperty HIVE_TABLE_NAME = ConfigProperty .key(HIVE_TABLE_NAME_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("For Hive based lock provider, the Hive table to acquire lock against"); - @Deprecated - public static final String HIVE_TABLE_NAME_PROP = HIVE_TABLE_NAME.key(); public static final ConfigProperty HIVE_METASTORE_URI = ConfigProperty .key(HIVE_METASTORE_URI_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("For Hive based lock provider, the Hive metastore URI to acquire locks against."); - @Deprecated - public static final String HIVE_METASTORE_URI_PROP = HIVE_METASTORE_URI.key(); public static final ConfigProperty ZK_BASE_PATH = ConfigProperty .key(ZK_BASE_PATH_PROP_KEY) @@ -152,40 +132,30 @@ public class HoodieLockConfig extends HoodieConfig { .sinceVersion("0.8.0") .withDocumentation("The base path on Zookeeper under which to create lock related ZNodes. " + "This should be same for all concurrent writers to the same table"); - @Deprecated - public static final String ZK_BASE_PATH_PROP = ZK_BASE_PATH.key(); public static final ConfigProperty ZK_SESSION_TIMEOUT_MS = ConfigProperty .key(ZK_SESSION_TIMEOUT_MS_PROP_KEY) .defaultValue(DEFAULT_ZK_SESSION_TIMEOUT_MS) .sinceVersion("0.8.0") .withDocumentation("Timeout in ms, to wait after losing connection to ZooKeeper, before the session is expired"); - @Deprecated - public static final String ZK_SESSION_TIMEOUT_MS_PROP = ZK_SESSION_TIMEOUT_MS.key(); public static final ConfigProperty ZK_CONNECTION_TIMEOUT_MS = ConfigProperty .key(ZK_CONNECTION_TIMEOUT_MS_PROP_KEY) .defaultValue(DEFAULT_ZK_CONNECTION_TIMEOUT_MS) .sinceVersion("0.8.0") .withDocumentation("Timeout in ms, to wait for establishing connection with Zookeeper."); - @Deprecated - public static final String ZK_CONNECTION_TIMEOUT_MS_PROP = ZK_CONNECTION_TIMEOUT_MS.key(); public static final ConfigProperty ZK_CONNECT_URL = ConfigProperty .key(ZK_CONNECT_URL_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("Zookeeper URL to connect to."); - @Deprecated - public static final String ZK_CONNECT_URL_PROP = ZK_CONNECT_URL.key(); public static final ConfigProperty ZK_PORT = ConfigProperty .key(ZK_PORT_PROP_KEY) .noDefaultValue() .sinceVersion("0.8.0") .withDocumentation("Zookeeper port to connect to."); - @Deprecated - public static final String ZK_PORT_PROP = ZK_PORT.key(); public static final ConfigProperty ZK_LOCK_KEY = ConfigProperty .key(ZK_LOCK_KEY_PROP_KEY) @@ -193,28 +163,35 @@ public class HoodieLockConfig extends HoodieConfig { .sinceVersion("0.8.0") .withDocumentation("Key name under base_path at which to create a ZNode and acquire lock. " + "Final path on zk will look like base_path/lock_key. We recommend setting this to the table name"); - @Deprecated - public static final String ZK_LOCK_KEY_PROP = ZK_LOCK_KEY.key(); // Pluggable type of lock provider - public static final ConfigProperty LOCK_PROVIDER_CLASS = ConfigProperty + public static final ConfigProperty LOCK_PROVIDER_CLASS_NAME = ConfigProperty .key(LOCK_PREFIX + "provider") .defaultValue(ZookeeperBasedLockProvider.class.getName()) .sinceVersion("0.8.0") .withDocumentation("Lock provider class name, user can provide their own implementation of LockProvider " + "which should be subclass of org.apache.hudi.common.lock.LockProvider"); - @Deprecated - public static final String LOCK_PROVIDER_CLASS_PROP = LOCK_PROVIDER_CLASS.key(); // Pluggable strategies to use when resolving conflicts - public static final ConfigProperty WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS = ConfigProperty + public static final ConfigProperty WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME = ConfigProperty .key(LOCK_PREFIX + "conflict.resolution.strategy") .defaultValue(SimpleConcurrentFileWritesConflictResolutionStrategy.class.getName()) .sinceVersion("0.8.0") .withDocumentation("Lock provider class name, this should be subclass of " + "org.apache.hudi.client.transaction.ConflictResolutionStrategy"); + + /** @deprecated Use {@link #WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME} and its methods instead */ @Deprecated - public static final String WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_PROP = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS.key(); + public static final String WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_PROP = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.key(); + /** @deprecated Use {@link #WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME} and its methods instead */ + @Deprecated + public static final String DEFAULT_WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS = WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.defaultValue(); + /** @deprecated Use {@link #LOCK_PROVIDER_CLASS_NAME} and its methods instead */ + @Deprecated + public static final String LOCK_PROVIDER_CLASS_PROP = LOCK_PROVIDER_CLASS_NAME.key(); + /** @deprecated Use {@link #LOCK_PROVIDER_CLASS_NAME} and its methods instead */ + @Deprecated + public static final String DEFAULT_LOCK_PROVIDER_CLASS = LOCK_PROVIDER_CLASS_NAME.defaultValue(); private HoodieLockConfig() { super(); @@ -241,7 +218,7 @@ public class HoodieLockConfig extends HoodieConfig { } public HoodieLockConfig.Builder withLockProvider(Class lockProvider) { - lockConfig.setValue(LOCK_PROVIDER_CLASS, lockProvider.getName()); + lockConfig.setValue(LOCK_PROVIDER_CLASS_NAME, lockProvider.getName()); return this; } @@ -321,7 +298,7 @@ public class HoodieLockConfig extends HoodieConfig { } public HoodieLockConfig.Builder withConflictResolutionStrategy(ConflictResolutionStrategy conflictResolutionStrategy) { - lockConfig.setValue(WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS, conflictResolutionStrategy.getClass().getName()); + lockConfig.setValue(WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME, conflictResolutionStrategy.getClass().getName()); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java index b8f3410aa..8845ccbee 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java @@ -46,8 +46,6 @@ public class HoodieMemoryConfig extends HoodieConfig { .defaultValue(String.valueOf(0.6)) .withDocumentation("This fraction is multiplied with the user memory fraction (1 - spark.memory.fraction) " + "to get a final fraction of heap space to use during merge"); - @Deprecated - public static final String MAX_MEMORY_FRACTION_FOR_MERGE_PROP = MAX_MEMORY_FRACTION_FOR_MERGE.key(); // Default max memory fraction during compaction, excess spills to disk public static final ConfigProperty MAX_MEMORY_FRACTION_FOR_COMPACTION = ConfigProperty @@ -58,8 +56,6 @@ public class HoodieMemoryConfig extends HoodieConfig { + "less than or equal to the number of entries in the corresponding parquet file. This can lead to " + "OOM in the Scanner. Hence, a spillable map helps alleviate the memory pressure. Use this config to " + "set the max allowable inMemory footprint of the spillable map"); - @Deprecated - public static final String MAX_MEMORY_FRACTION_FOR_COMPACTION_PROP = MAX_MEMORY_FRACTION_FOR_COMPACTION.key(); // Default memory size (1GB) per compaction (used if SparkEnv is absent), excess spills to disk public static final long DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES = 1024 * 1024 * 1024L; @@ -70,29 +66,21 @@ public class HoodieMemoryConfig extends HoodieConfig { .key("hoodie.memory.merge.max.size") .defaultValue(DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES) .withDocumentation("Maximum amount of memory used for merge operations, before spilling to local storage."); - @Deprecated - public static final String MAX_MEMORY_FOR_MERGE_PROP = MAX_MEMORY_FOR_MERGE.key(); public static final ConfigProperty MAX_MEMORY_FOR_COMPACTION = ConfigProperty .key("hoodie.memory.compaction.max.size") .noDefaultValue() .withDocumentation("Maximum amount of memory used for compaction operations, before spilling to local storage."); - @Deprecated - public static final String MAX_MEMORY_FOR_COMPACTION_PROP = MAX_MEMORY_FOR_COMPACTION.key(); public static final ConfigProperty MAX_DFS_STREAM_BUFFER_SIZE = ConfigProperty .key("hoodie.memory.dfs.buffer.max.size") .defaultValue(16 * 1024 * 1024) .withDocumentation("Property to control the max memory for dfs input stream buffer size"); - @Deprecated - public static final String MAX_DFS_STREAM_BUFFER_SIZE_PROP = MAX_DFS_STREAM_BUFFER_SIZE.key(); public static final ConfigProperty SPILLABLE_MAP_BASE_PATH = ConfigProperty .key("hoodie.memory.spillable.map.path") .defaultValue("/tmp/") .withDocumentation("Default file path prefix for spillable map"); - @Deprecated - public static final String SPILLABLE_MAP_BASE_PATH_PROP = SPILLABLE_MAP_BASE_PATH.key(); public static final ConfigProperty WRITESTATUS_FAILURE_FRACTION = ConfigProperty .key("hoodie.memory.writestatus.failure.fraction") @@ -100,8 +88,43 @@ public class HoodieMemoryConfig extends HoodieConfig { .withDocumentation("Property to control how what fraction of the failed record, exceptions we report back to driver. " + "Default is 10%. If set to 100%, with lot of failures, this can cause memory pressure, cause OOMs and " + "mask actual data errors."); + + /** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_MERGE} and its methods instead */ + @Deprecated + public static final String MAX_MEMORY_FRACTION_FOR_MERGE_PROP = MAX_MEMORY_FRACTION_FOR_MERGE.key(); + /** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_MERGE} and its methods instead */ + @Deprecated + public static final String DEFAULT_MAX_MEMORY_FRACTION_FOR_MERGE = MAX_MEMORY_FRACTION_FOR_MERGE.defaultValue(); + /** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_COMPACTION} and its methods instead */ + @Deprecated + public static final String MAX_MEMORY_FRACTION_FOR_COMPACTION_PROP = MAX_MEMORY_FRACTION_FOR_COMPACTION.key(); + /** @deprecated Use {@link #MAX_MEMORY_FRACTION_FOR_COMPACTION} and its methods instead */ + @Deprecated + public static final String DEFAULT_MAX_MEMORY_FRACTION_FOR_COMPACTION = MAX_MEMORY_FRACTION_FOR_COMPACTION.defaultValue(); + /** @deprecated Use {@link #MAX_MEMORY_FOR_MERGE} and its methods instead */ + @Deprecated + public static final String MAX_MEMORY_FOR_MERGE_PROP = MAX_MEMORY_FOR_MERGE.key(); + /** @deprecated Use {@link #MAX_MEMORY_FOR_COMPACTION} and its methods instead */ + @Deprecated + public static final String MAX_MEMORY_FOR_COMPACTION_PROP = MAX_MEMORY_FOR_COMPACTION.key(); + /** @deprecated Use {@link #MAX_DFS_STREAM_BUFFER_SIZE} and its methods instead */ + @Deprecated + public static final String MAX_DFS_STREAM_BUFFER_SIZE_PROP = MAX_DFS_STREAM_BUFFER_SIZE.key(); + /** @deprecated Use {@link #MAX_DFS_STREAM_BUFFER_SIZE} and its methods instead */ + @Deprecated + public static final int DEFAULT_MAX_DFS_STREAM_BUFFER_SIZE = MAX_DFS_STREAM_BUFFER_SIZE.defaultValue(); + /** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead */ + @Deprecated + public static final String SPILLABLE_MAP_BASE_PATH_PROP = SPILLABLE_MAP_BASE_PATH.key(); + /** @deprecated Use {@link #SPILLABLE_MAP_BASE_PATH} and its methods instead */ + @Deprecated + public static final String DEFAULT_SPILLABLE_MAP_BASE_PATH = SPILLABLE_MAP_BASE_PATH.defaultValue(); + /** @deprecated Use {@link #WRITESTATUS_FAILURE_FRACTION} and its methods instead */ @Deprecated public static final String WRITESTATUS_FAILURE_FRACTION_PROP = WRITESTATUS_FAILURE_FRACTION.key(); + /** @deprecated Use {@link #WRITESTATUS_FAILURE_FRACTION} and its methods instead */ + @Deprecated + public static final double DEFAULT_WRITESTATUS_FAILURE_FRACTION = WRITESTATUS_FAILURE_FRACTION.defaultValue(); private HoodieMemoryConfig() { super(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java index b5434b0cc..e1d0b63ef 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java @@ -43,13 +43,13 @@ public class HoodieMetricsConfig extends HoodieConfig { public static final String METRIC_PREFIX = "hoodie.metrics"; - public static final ConfigProperty METRICS_ON = ConfigProperty + public static final ConfigProperty TURN_METRICS_ON = ConfigProperty .key(METRIC_PREFIX + ".on") .defaultValue(false) .sinceVersion("0.5.0") .withDocumentation("Turn on/off metrics reporting. off by default."); - public static final ConfigProperty METRICS_REPORTER_TYPE = ConfigProperty + public static final ConfigProperty METRICS_REPORTER_TYPE_VALUE = ConfigProperty .key(METRIC_PREFIX + ".reporter.type") .defaultValue(MetricsReporterType.GRAPHITE) .sinceVersion("0.5.0") @@ -58,13 +58,13 @@ public class HoodieMetricsConfig extends HoodieConfig { // Graphite public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite"; - public static final ConfigProperty GRAPHITE_SERVER_HOST = ConfigProperty + public static final ConfigProperty GRAPHITE_SERVER_HOST_NAME = ConfigProperty .key(GRAPHITE_PREFIX + ".host") .defaultValue("localhost") .sinceVersion("0.5.0") .withDocumentation("Graphite host to connect to"); - public static final ConfigProperty GRAPHITE_SERVER_PORT = ConfigProperty + public static final ConfigProperty GRAPHITE_SERVER_PORT_NUM = ConfigProperty .key(GRAPHITE_PREFIX + ".port") .defaultValue(4756) .sinceVersion("0.5.0") @@ -73,38 +73,119 @@ public class HoodieMetricsConfig extends HoodieConfig { // Jmx public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx"; - public static final ConfigProperty JMX_HOST = ConfigProperty + public static final ConfigProperty JMX_HOST_NAME = ConfigProperty .key(JMX_PREFIX + ".host") .defaultValue("localhost") .sinceVersion("0.5.1") .withDocumentation("Jmx host to connect to"); - public static final ConfigProperty JMX_PORT = ConfigProperty + public static final ConfigProperty JMX_PORT_NUM = ConfigProperty .key(JMX_PREFIX + ".port") .defaultValue(9889) .sinceVersion("0.5.1") .withDocumentation("Jmx port to connect to"); - public static final ConfigProperty GRAPHITE_METRIC_PREFIX = ConfigProperty + public static final ConfigProperty GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty .key(GRAPHITE_PREFIX + ".metric.prefix") .noDefaultValue() .sinceVersion("0.5.1") .withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g"); // User defined - public static final ConfigProperty METRICS_REPORTER_CLASS = ConfigProperty + public static final ConfigProperty METRICS_REPORTER_CLASS_NAME = ConfigProperty .key(METRIC_PREFIX + ".reporter.class") .defaultValue("") .sinceVersion("0.6.0") .withDocumentation(""); // Enable metrics collection from executors - public static final ConfigProperty ENABLE_EXECUTOR_METRICS = ConfigProperty + public static final ConfigProperty EXECUTOR_METRICS_ENABLE = ConfigProperty .key(METRIC_PREFIX + ".executor.enable") .noDefaultValue() .sinceVersion("0.7.0") .withDocumentation(""); + /** + * @deprecated Use {@link #TURN_METRICS_ON} and its methods instead + */ + @Deprecated + public static final String METRICS_ON = TURN_METRICS_ON.key(); + /** + * @deprecated Use {@link #TURN_METRICS_ON} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_METRICS_ON = TURN_METRICS_ON.defaultValue(); + /** + * @deprecated Use {@link #METRICS_REPORTER_TYPE_VALUE} and its methods instead + */ + @Deprecated + public static final String METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.key(); + /** + * @deprecated Use {@link #METRICS_REPORTER_TYPE_VALUE} and its methods instead + */ + @Deprecated + public static final MetricsReporterType DEFAULT_METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.defaultValue(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key(); + /** + * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String JMX_HOST = JMX_HOST_NAME.key(); + /** + * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String JMX_PORT = JMX_PORT_NUM.key(); + /** + * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead + */ + @Deprecated + public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key(); + /** + * @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String METRICS_REPORTER_CLASS = METRICS_REPORTER_CLASS_NAME.key(); + /** + * @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_METRICS_REPORTER_CLASS = METRICS_REPORTER_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #EXECUTOR_METRICS_ENABLE} and its methods instead + */ + @Deprecated + public static final String ENABLE_EXECUTOR_METRICS = EXECUTOR_METRICS_ENABLE.key(); + private HoodieMetricsConfig() { super(); } @@ -130,47 +211,47 @@ public class HoodieMetricsConfig extends HoodieConfig { } public Builder on(boolean metricsOn) { - hoodieMetricsConfig.setValue(METRICS_ON, String.valueOf(metricsOn)); + hoodieMetricsConfig.setValue(TURN_METRICS_ON, String.valueOf(metricsOn)); return this; } public Builder withReporterType(String reporterType) { - hoodieMetricsConfig.setValue(METRICS_REPORTER_TYPE, reporterType); + hoodieMetricsConfig.setValue(METRICS_REPORTER_TYPE_VALUE, reporterType); return this; } public Builder toGraphiteHost(String host) { - hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST, host); + hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host); return this; } public Builder onGraphitePort(int port) { - hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT, String.valueOf(port)); + hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port)); return this; } public Builder toJmxHost(String host) { - hoodieMetricsConfig.setValue(JMX_HOST, host); + hoodieMetricsConfig.setValue(JMX_HOST_NAME, host); return this; } public Builder onJmxPort(String port) { - hoodieMetricsConfig.setValue(JMX_PORT, port); + hoodieMetricsConfig.setValue(JMX_PORT_NUM, port); return this; } public Builder usePrefix(String prefix) { - hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX, prefix); + hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix); return this; } public Builder withReporterClass(String className) { - hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS, className); + hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS_NAME, className); return this; } public Builder withExecutorMetrics(boolean enable) { - hoodieMetricsConfig.setValue(ENABLE_EXECUTOR_METRICS, String.valueOf(enable)); + hoodieMetricsConfig.setValue(EXECUTOR_METRICS_ENABLE, String.valueOf(enable)); return this; } @@ -178,7 +259,7 @@ public class HoodieMetricsConfig extends HoodieConfig { hoodieMetricsConfig.setDefaults(HoodieMetricsConfig.class.getName()); - MetricsReporterType reporterType = MetricsReporterType.valueOf(hoodieMetricsConfig.getString(METRICS_REPORTER_TYPE)); + MetricsReporterType reporterType = MetricsReporterType.valueOf(hoodieMetricsConfig.getString(METRICS_REPORTER_TYPE_VALUE)); hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.DATADOG, HoodieMetricsDatadogConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build()); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java index df70ce39b..d63cb0fef 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java @@ -43,63 +43,124 @@ public class HoodieMetricsDatadogConfig extends HoodieConfig { public static final String DATADOG_PREFIX = METRIC_PREFIX + ".datadog"; - public static final ConfigProperty DATADOG_REPORT_PERIOD_SECONDS = ConfigProperty + public static final ConfigProperty REPORT_PERIOD_IN_SECONDS = ConfigProperty .key(DATADOG_PREFIX + ".report.period.seconds") .defaultValue(30) .sinceVersion("0.6.0") .withDocumentation("Datadog reporting period in seconds. Default to 30."); - public static final ConfigProperty DATADOG_API_SITE = ConfigProperty + public static final ConfigProperty API_SITE_VALUE = ConfigProperty .key(DATADOG_PREFIX + ".api.site") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog API site: EU or US"); - public static final ConfigProperty DATADOG_API_KEY = ConfigProperty + public static final ConfigProperty API_KEY = ConfigProperty .key(DATADOG_PREFIX + ".api.key") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog API key"); - public static final ConfigProperty DATADOG_API_KEY_SKIP_VALIDATION = ConfigProperty + public static final ConfigProperty API_KEY_SKIP_VALIDATION = ConfigProperty .key(DATADOG_PREFIX + ".api.key.skip.validation") .defaultValue(false) .sinceVersion("0.6.0") .withDocumentation("Before sending metrics via Datadog API, whether to skip validating Datadog API key or not. " + "Default to false."); - public static final ConfigProperty DATADOG_API_KEY_SUPPLIER = ConfigProperty + public static final ConfigProperty API_KEY_SUPPLIER = ConfigProperty .key(DATADOG_PREFIX + ".api.key.supplier") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog API key supplier to supply the API key at runtime. " + "This will take effect if hoodie.metrics.datadog.api.key is not set."); - public static final ConfigProperty DATADOG_API_TIMEOUT_SECONDS = ConfigProperty + public static final ConfigProperty API_TIMEOUT_IN_SECONDS = ConfigProperty .key(DATADOG_PREFIX + ".api.timeout.seconds") .defaultValue(3) .sinceVersion("0.6.0") .withDocumentation("Datadog API timeout in seconds. Default to 3."); - public static final ConfigProperty DATADOG_METRIC_PREFIX = ConfigProperty + public static final ConfigProperty METRIC_PREFIX_VALUE = ConfigProperty .key(DATADOG_PREFIX + ".metric.prefix") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog metric prefix to be prepended to each metric name with a dot as delimiter. " + "For example, if it is set to foo, foo. will be prepended."); - public static final ConfigProperty DATADOG_METRIC_HOST = ConfigProperty + public static final ConfigProperty METRIC_HOST_NAME = ConfigProperty .key(DATADOG_PREFIX + ".metric.host") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog metric host to be sent along with metrics data."); - public static final ConfigProperty DATADOG_METRIC_TAGS = ConfigProperty + public static final ConfigProperty METRIC_TAG_VALUES = ConfigProperty .key(DATADOG_PREFIX + ".metric.tags") .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Datadog metric tags (comma-delimited) to be sent along with metrics data."); + /** + * @deprecated Use {@link #REPORT_PERIOD_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final String DATADOG_REPORT_PERIOD_SECONDS = REPORT_PERIOD_IN_SECONDS.key(); + /** + * @deprecated Use {@link #REPORT_PERIOD_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_DATADOG_REPORT_PERIOD_SECONDS = REPORT_PERIOD_IN_SECONDS.defaultValue(); + /** + * @deprecated Use {@link #API_SITE_VALUE} and its methods instead + */ + @Deprecated + public static final String DATADOG_API_SITE = API_SITE_VALUE.key(); + /** + * @deprecated Use {@link #API_KEY} and its methods instead + */ + @Deprecated + public static final String DATADOG_API_KEY = API_KEY.key(); + /** + * @deprecated Use {@link #API_KEY_SKIP_VALIDATION} and its methods instead + */ + @Deprecated + public static final String DATADOG_API_KEY_SKIP_VALIDATION = API_KEY_SKIP_VALIDATION.key(); + /** + * @deprecated Use {@link #API_KEY_SKIP_VALIDATION} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_DATADOG_API_KEY_SKIP_VALIDATION = API_KEY_SKIP_VALIDATION.defaultValue(); + /** + * @deprecated Use {@link #API_KEY_SUPPLIER} and its methods instead + */ + @Deprecated + public static final String DATADOG_API_KEY_SUPPLIER = API_KEY_SUPPLIER.key(); + /** + * @deprecated Use {@link #API_TIMEOUT_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final String DATADOG_API_TIMEOUT_SECONDS = API_TIMEOUT_IN_SECONDS.key(); + /** + * @deprecated Use {@link #API_TIMEOUT_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_DATADOG_API_TIMEOUT_SECONDS = API_TIMEOUT_IN_SECONDS.defaultValue(); + /** + * @deprecated Use {@link #METRIC_PREFIX_VALUE} and its methods instead + */ + @Deprecated + public static final String DATADOG_METRIC_PREFIX = METRIC_PREFIX_VALUE.key(); + /** + * @deprecated Use {@link #METRIC_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DATADOG_METRIC_HOST = METRIC_HOST_NAME.key(); + /** + * @deprecated Use {@link #METRIC_TAG_VALUES} and its methods instead + */ + @Deprecated + public static final String DATADOG_METRIC_TAGS = METRIC_TAG_VALUES.key(); + private HoodieMetricsDatadogConfig() { super(); } @@ -118,47 +179,47 @@ public class HoodieMetricsDatadogConfig extends HoodieConfig { } public Builder withDatadogReportPeriodSeconds(int period) { - metricsDatadogConfig.setValue(DATADOG_REPORT_PERIOD_SECONDS, String.valueOf(period)); + metricsDatadogConfig.setValue(REPORT_PERIOD_IN_SECONDS, String.valueOf(period)); return this; } public Builder withDatadogApiSite(String apiSite) { - metricsDatadogConfig.setValue(DATADOG_API_SITE, apiSite); + metricsDatadogConfig.setValue(API_SITE_VALUE, apiSite); return this; } public Builder withDatadogApiKey(String apiKey) { - metricsDatadogConfig.setValue(DATADOG_API_KEY, apiKey); + metricsDatadogConfig.setValue(API_KEY, apiKey); return this; } public Builder withDatadogApiKeySkipValidation(boolean skip) { - metricsDatadogConfig.setValue(DATADOG_API_KEY_SKIP_VALIDATION, String.valueOf(skip)); + metricsDatadogConfig.setValue(API_KEY_SKIP_VALIDATION, String.valueOf(skip)); return this; } public Builder withDatadogApiKeySupplier(String apiKeySupplier) { - metricsDatadogConfig.setValue(DATADOG_API_KEY_SUPPLIER, apiKeySupplier); + metricsDatadogConfig.setValue(API_KEY_SUPPLIER, apiKeySupplier); return this; } public Builder withDatadogApiTimeoutSeconds(int timeout) { - metricsDatadogConfig.setValue(DATADOG_API_TIMEOUT_SECONDS, String.valueOf(timeout)); + metricsDatadogConfig.setValue(API_TIMEOUT_IN_SECONDS, String.valueOf(timeout)); return this; } public Builder withDatadogPrefix(String prefix) { - metricsDatadogConfig.setValue(DATADOG_METRIC_PREFIX, prefix); + metricsDatadogConfig.setValue(METRIC_PREFIX_VALUE, prefix); return this; } public Builder withDatadogHost(String host) { - metricsDatadogConfig.setValue(DATADOG_METRIC_HOST, host); + metricsDatadogConfig.setValue(METRIC_HOST_NAME, host); return this; } public Builder withDatadogTags(String tags) { - metricsDatadogConfig.setValue(DATADOG_METRIC_TAGS, tags); + metricsDatadogConfig.setValue(METRIC_TAG_VALUES, tags); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java index a8eac6218..615222f4a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java @@ -36,37 +36,37 @@ public class HoodieMetricsPrometheusConfig extends HoodieConfig { // Prometheus PushGateWay public static final String PUSHGATEWAY_PREFIX = METRIC_PREFIX + ".pushgateway"; - public static final ConfigProperty PUSHGATEWAY_HOST = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_HOST_NAME = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".host") .defaultValue("localhost") .sinceVersion("0.6.0") .withDocumentation("Hostname of the prometheus push gateway"); - public static final ConfigProperty PUSHGATEWAY_PORT = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_PORT_NUM = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".port") .defaultValue(9091) .sinceVersion("0.6.0") .withDocumentation("Port for the push gateway."); - public static final ConfigProperty PUSHGATEWAY_REPORT_PERIOD_SECONDS = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".report.period.seconds") .defaultValue(30) .sinceVersion("0.6.0") .withDocumentation("Reporting interval in seconds."); - public static final ConfigProperty PUSHGATEWAY_DELETE_ON_SHUTDOWN = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".delete.on.shutdown") .defaultValue(true) .sinceVersion("0.6.0") .withDocumentation(""); - public static final ConfigProperty PUSHGATEWAY_JOB_NAME = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_JOBNAME = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".job.name") .defaultValue("") .sinceVersion("0.6.0") .withDocumentation("Name of the push gateway job."); - public static final ConfigProperty PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = ConfigProperty + public static final ConfigProperty PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX = ConfigProperty .key(PUSHGATEWAY_PREFIX + ".random.job.name.suffix") .defaultValue(true) .sinceVersion("0.6.0") @@ -75,12 +75,83 @@ public class HoodieMetricsPrometheusConfig extends HoodieConfig { // Prometheus HttpServer public static final String PROMETHEUS_PREFIX = METRIC_PREFIX + ".prometheus"; - public static final ConfigProperty PROMETHEUS_PORT = ConfigProperty + public static final ConfigProperty PROMETHEUS_PORT_NUM = ConfigProperty .key(PROMETHEUS_PREFIX + ".port") .defaultValue(9090) .sinceVersion("0.6.0") .withDocumentation("Port for prometheus server."); + /** + * @deprecated Use {@link #PUSHGATEWAY_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_HOST = PUSHGATEWAY_HOST_NAME.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_HOST_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PUSHGATEWAY_HOST = PUSHGATEWAY_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #PUSHGATEWAY_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_PORT = PUSHGATEWAY_PORT_NUM.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_PUSHGATEWAY_PORT = PUSHGATEWAY_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_REPORT_PERIOD_SECONDS = PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_PUSHGATEWAY_REPORT_PERIOD_SECONDS = PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS.defaultValue(); + /** + * @deprecated Use {@link #PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_DELETE_ON_SHUTDOWN = PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_PUSHGATEWAY_DELETE_ON_SHUTDOWN = PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #PUSHGATEWAY_JOBNAME} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_JOB_NAME = PUSHGATEWAY_JOBNAME.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_JOBNAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PUSHGATEWAY_JOB_NAME = PUSHGATEWAY_JOBNAME.defaultValue(); + /** + * @deprecated Use {@link #PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX} and its methods instead + */ + @Deprecated + public static final String PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX.key(); + /** + * @deprecated Use {@link #PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX = PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX.defaultValue(); + /** + * @deprecated Use {@link #PROMETHEUS_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String PROMETHEUS_PORT = PROMETHEUS_PORT_NUM.key(); + /** + * @deprecated Use {@link #PROMETHEUS_PORT_NUM} and its methods instead + */ + @Deprecated + public static final int DEFAULT_PROMETHEUS_PORT = PROMETHEUS_PORT_NUM.defaultValue(); + private HoodieMetricsPrometheusConfig() { super(); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java index 95d4fe3f8..2989d8c2b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java @@ -40,21 +40,17 @@ import static org.apache.hudi.common.model.HoodiePayloadProps.PAYLOAD_ORDERING_F + "control merges based on specific business fields in the data.") public class HoodiePayloadConfig extends HoodieConfig { - public static final ConfigProperty PAYLOAD_ORDERING_FIELD = ConfigProperty + public static final ConfigProperty ORDERING_FIELD = ConfigProperty .key(PAYLOAD_ORDERING_FIELD_PROP_KEY) .defaultValue("ts") .withDocumentation("Table column/field name to order records that have the same key, before " + "merging and writing to storage."); - @Deprecated - public static final String PAYLOAD_ORDERING_FIELD_PROP = PAYLOAD_ORDERING_FIELD.key(); - public static final ConfigProperty PAYLOAD_EVENT_TIME_FIELD = ConfigProperty + public static final ConfigProperty EVENT_TIME_FIELD = ConfigProperty .key(PAYLOAD_EVENT_TIME_FIELD_PROP_KEY) .defaultValue("ts") .withDocumentation("Table column/field name to derive timestamp associated with the records. This can" + "be useful for e.g, determining the freshness of the table."); - @Deprecated - public static final String PAYLOAD_EVENT_TIME_FIELD_PROP = PAYLOAD_EVENT_TIME_FIELD.key(); private HoodiePayloadConfig() { super(); @@ -81,12 +77,12 @@ public class HoodiePayloadConfig extends HoodieConfig { } public Builder withPayloadOrderingField(String payloadOrderingField) { - payloadConfig.setValue(PAYLOAD_ORDERING_FIELD, String.valueOf(payloadOrderingField)); + payloadConfig.setValue(ORDERING_FIELD, String.valueOf(payloadOrderingField)); return this; } public Builder withPayloadEventTimeField(String payloadEventTimeField) { - payloadConfig.setValue(PAYLOAD_EVENT_TIME_FIELD, String.valueOf(payloadEventTimeField)); + payloadConfig.setValue(EVENT_TIME_FIELD, String.valueOf(payloadEventTimeField)); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java index 1373c1368..8ea085835 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePreCommitValidatorConfig.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import javax.annotation.concurrent.Immutable; + import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -33,26 +34,26 @@ import java.util.Properties; @Immutable public class HoodiePreCommitValidatorConfig extends HoodieConfig { - public static final ConfigProperty PRE_COMMIT_VALIDATORS = ConfigProperty + public static final ConfigProperty VALIDATOR_CLASS_NAMES = ConfigProperty .key("hoodie.precommit.validators") .defaultValue("") .withDocumentation("Comma separated list of class names that can be invoked to validate commit"); public static final String VALIDATOR_TABLE_VARIABLE = ""; - public static final ConfigProperty PRE_COMMIT_VALIDATORS_EQUALITY_SQL_QUERIES = ConfigProperty + public static final ConfigProperty EQUALITY_SQL_QUERIES = ConfigProperty .key("hoodie.precommit.validators.equality.sql.queries") .defaultValue("") .withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit." - + " Multiple queries separated by ';' delimiter are supported." - + " Example: \"select count(*) from \\" + + " Multiple queries separated by ';' delimiter are supported." + + " Example: \"select count(*) from \\" + " Note \\ is replaced by table state before and after commit."); - - public static final ConfigProperty PRE_COMMIT_VALIDATORS_SINGLE_VALUE_SQL_QUERIES = ConfigProperty + + public static final ConfigProperty SINGLE_VALUE_SQL_QUERIES = ConfigProperty .key("hoodie.precommit.validators.single.value.sql.queries") .defaultValue("") - .withDocumentation("Spark SQL queries to run on table before committing new data to validate state after commit." - + "Multiple queries separated by ';' delimiter are supported." - + "Expected result is included as part of query separated by '#'. Example query: 'query1#result1:query2#result2'" + .withDocumentation("Spark SQL queries to run on table before committing new data to validate state after commit." + + "Multiple queries separated by ';' delimiter are supported." + + "Expected result is included as part of query separated by '#'. Example query: 'query1#result1:query2#result2'" + "Note \\ variable is expected to be present in query."); /** @@ -61,11 +62,11 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig { * Example query: 'select count(*) from \ where col=null' * Note \ variable is expected to be present in query. */ - public static final ConfigProperty PRE_COMMIT_VALIDATORS_INEQUALITY_SQL_QUERIES = ConfigProperty + public static final ConfigProperty INEQUALITY_SQL_QUERIES = ConfigProperty .key("hoodie.precommit.validators.inequality.sql.queries") .defaultValue("") - .withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit." - + "Multiple queries separated by ';' delimiter are supported." + .withDocumentation("Spark SQL queries to run on table before committing new data to validate state before and after commit." + + "Multiple queries separated by ';' delimiter are supported." + "Example query: 'select count(*) from \\ where col=null'" + "Note \\ variable is expected to be present in query."); @@ -94,22 +95,22 @@ public class HoodiePreCommitValidatorConfig extends HoodieConfig { } public Builder withPreCommitValidator(String preCommitValidators) { - preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS, preCommitValidators); + preCommitValidatorConfig.setValue(VALIDATOR_CLASS_NAMES, preCommitValidators); return this; } public Builder withPrecommitValidatorEqualitySqlQueries(String preCommitValidators) { - preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_EQUALITY_SQL_QUERIES, preCommitValidators); + preCommitValidatorConfig.setValue(EQUALITY_SQL_QUERIES, preCommitValidators); return this; } public Builder withPrecommitValidatorSingleResultSqlQueries(String preCommitValidators) { - preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_SINGLE_VALUE_SQL_QUERIES, preCommitValidators); + preCommitValidatorConfig.setValue(SINGLE_VALUE_SQL_QUERIES, preCommitValidators); return this; } public Builder withPrecommitValidatorInequalitySqlQueries(String preCommitValidators) { - preCommitValidatorConfig.setValue(PRE_COMMIT_VALIDATORS_INEQUALITY_SQL_QUERIES, preCommitValidators); + preCommitValidatorConfig.setValue(INEQUALITY_SQL_QUERIES, preCommitValidators); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java index 401537154..9dcde926c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java @@ -39,25 +39,25 @@ import java.util.Properties; description = "Configurations that control aspects around writing, sizing, reading base and log files.") public class HoodieStorageConfig extends HoodieConfig { - public static final ConfigProperty PARQUET_FILE_MAX_BYTES = ConfigProperty + public static final ConfigProperty PARQUET_MAX_FILE_SIZE = ConfigProperty .key("hoodie.parquet.max.file.size") .defaultValue(String.valueOf(120 * 1024 * 1024)) .withDocumentation("Target size for parquet files produced by Hudi write phases. " + "For DFS, this needs to be aligned with the underlying filesystem block size for optimal performance."); - public static final ConfigProperty PARQUET_BLOCK_SIZE_BYTES = ConfigProperty + public static final ConfigProperty PARQUET_BLOCK_SIZE = ConfigProperty .key("hoodie.parquet.block.size") .defaultValue(String.valueOf(120 * 1024 * 1024)) .withDocumentation("Parquet RowGroup size. It's recommended to make this large enough that scan costs can be" + " amortized by packing enough column values into a single row group."); - public static final ConfigProperty PARQUET_PAGE_SIZE_BYTES = ConfigProperty + public static final ConfigProperty PARQUET_PAGE_SIZE = ConfigProperty .key("hoodie.parquet.page.size") .defaultValue(String.valueOf(1 * 1024 * 1024)) .withDocumentation("Parquet page size. Page is the unit of read within a parquet file. " + "Within a block, pages are compressed separately."); - public static final ConfigProperty ORC_FILE_MAX_BYTES = ConfigProperty + public static final ConfigProperty ORC_FILE_MAX_SIZE = ConfigProperty .key("hoodie.orc.max.file.size") .defaultValue(String.valueOf(120 * 1024 * 1024)) .withDocumentation("Target file size for ORC base files."); @@ -69,64 +69,175 @@ public class HoodieStorageConfig extends HoodieConfig { public static final ConfigProperty ORC_BLOCK_SIZE = ConfigProperty .key("hoodie.orc.block.size") - .defaultValue(ORC_FILE_MAX_BYTES.defaultValue()) + .defaultValue(ORC_FILE_MAX_SIZE.defaultValue()) .withDocumentation("ORC block size, recommended to be aligned with the target file size."); - public static final ConfigProperty HFILE_FILE_MAX_BYTES = ConfigProperty + public static final ConfigProperty HFILE_MAX_FILE_SIZE = ConfigProperty .key("hoodie.hfile.max.file.size") .defaultValue(String.valueOf(120 * 1024 * 1024)) .withDocumentation("Target file size for HFile base files."); - public static final ConfigProperty HFILE_BLOCK_SIZE_BYTES = ConfigProperty + public static final ConfigProperty HFILE_BLOCK_SIZE = ConfigProperty .key("hoodie.hfile.block.size") .defaultValue(String.valueOf(1024 * 1024)) .withDocumentation("Lower values increase the size of metadata tracked within HFile, but can offer potentially " + "faster lookup times."); // used to size log files - public static final ConfigProperty LOGFILE_SIZE_MAX_BYTES = ConfigProperty + public static final ConfigProperty LOGFILE_MAX_SIZE = ConfigProperty .key("hoodie.logfile.max.size") .defaultValue(String.valueOf(1024 * 1024 * 1024)) // 1 GB .withDocumentation("LogFile max size. This is the maximum size allowed for a log file " + "before it is rolled over to the next version."); // used to size data blocks in log file - public static final ConfigProperty LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = ConfigProperty + public static final ConfigProperty LOGFILE_DATA_BLOCK_MAX_SIZE = ConfigProperty .key("hoodie.logfile.data.block.max.size") .defaultValue(String.valueOf(256 * 1024 * 1024)) .withDocumentation("LogFile Data block max size. This is the maximum size allowed for a single data block " + "to be appended to a log file. This helps to make sure the data appended to the log file is broken up " + "into sizable blocks to prevent from OOM errors. This size should be greater than the JVM memory."); - public static final ConfigProperty PARQUET_COMPRESSION_RATIO = ConfigProperty + public static final ConfigProperty PARQUET_COMPRESSION_RATIO_FRACTION = ConfigProperty .key("hoodie.parquet.compression.ratio") .defaultValue(String.valueOf(0.1)) .withDocumentation("Expected compression of parquet data used by Hudi, when it tries to size new parquet files. " + "Increase this value, if bulk_insert is producing smaller than expected sized files"); // Default compression codec for parquet - public static final ConfigProperty PARQUET_COMPRESSION_CODEC = ConfigProperty + public static final ConfigProperty PARQUET_COMPRESSION_CODEC_NAME = ConfigProperty .key("hoodie.parquet.compression.codec") .defaultValue("gzip") .withDocumentation("Compression Codec for parquet files"); - public static final ConfigProperty HFILE_COMPRESSION_ALGORITHM = ConfigProperty + public static final ConfigProperty HFILE_COMPRESSION_ALGORITHM_NAME = ConfigProperty .key("hoodie.hfile.compression.algorithm") .defaultValue("GZ") .withDocumentation("Compression codec to use for hfile base files."); - public static final ConfigProperty ORC_COMPRESSION_CODEC = ConfigProperty + public static final ConfigProperty ORC_COMPRESSION_CODEC_NAME = ConfigProperty .key("hoodie.orc.compression.codec") .defaultValue("ZLIB") .withDocumentation("Compression codec to use for ORC base files."); // Default compression ratio for log file to parquet, general 3x - public static final ConfigProperty LOGFILE_TO_PARQUET_COMPRESSION_RATIO = ConfigProperty + public static final ConfigProperty LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION = ConfigProperty .key("hoodie.logfile.to.parquet.compression.ratio") .defaultValue(String.valueOf(0.35)) .withDocumentation("Expected additional compression as records move from log files to parquet. Used for merge_on_read " + "table to send inserts into log files & control the size of compacted parquet file."); + /** + * @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead + */ + @Deprecated + public static final String PARQUET_FILE_MAX_BYTES = PARQUET_MAX_FILE_SIZE.key(); + /** + * @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARQUET_FILE_MAX_BYTES = PARQUET_MAX_FILE_SIZE.defaultValue(); + /** + * @deprecated Use {@link #PARQUET_BLOCK_SIZE} and its methods instead + */ + @Deprecated + public static final String PARQUET_BLOCK_SIZE_BYTES = PARQUET_BLOCK_SIZE.key(); + /** + * @deprecated Use {@link #PARQUET_BLOCK_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARQUET_BLOCK_SIZE_BYTES = PARQUET_BLOCK_SIZE.defaultValue(); + /** + * @deprecated Use {@link #PARQUET_PAGE_SIZE} and its methods instead + */ + @Deprecated + public static final String PARQUET_PAGE_SIZE_BYTES = PARQUET_PAGE_SIZE.key(); + /** + * @deprecated Use {@link #PARQUET_PAGE_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARQUET_PAGE_SIZE_BYTES = PARQUET_PAGE_SIZE.defaultValue(); + /** + * @deprecated Use {@link #HFILE_MAX_FILE_SIZE} and its methods instead + */ + @Deprecated + public static final String HFILE_FILE_MAX_BYTES = HFILE_MAX_FILE_SIZE.key(); + /** + * @deprecated Use {@link #HFILE_MAX_FILE_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HFILE_FILE_MAX_BYTES = HFILE_MAX_FILE_SIZE.defaultValue(); + /** + * @deprecated Use {@link #HFILE_BLOCK_SIZE} and its methods instead + */ + @Deprecated + public static final String HFILE_BLOCK_SIZE_BYTES = HFILE_BLOCK_SIZE.defaultValue(); + /** + * @deprecated Use {@link #HFILE_BLOCK_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HFILE_BLOCK_SIZE_BYTES = HFILE_BLOCK_SIZE.defaultValue(); + /** + * @deprecated Use {@link #LOGFILE_MAX_SIZE} and its methods instead + */ + @Deprecated + public static final String LOGFILE_SIZE_MAX_BYTES = LOGFILE_MAX_SIZE.key(); + /** + * @deprecated Use {@link #LOGFILE_MAX_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_LOGFILE_SIZE_MAX_BYTES = LOGFILE_MAX_SIZE.defaultValue(); + /** + * @deprecated Use {@link #LOGFILE_DATA_BLOCK_MAX_SIZE} and its methods instead + */ + @Deprecated + public static final String LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = LOGFILE_DATA_BLOCK_MAX_SIZE.key(); + /** + * @deprecated Use {@link #LOGFILE_DATA_BLOCK_MAX_SIZE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES = LOGFILE_DATA_BLOCK_MAX_SIZE.defaultValue(); + /** + * @deprecated Use {@link #PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead + */ + @Deprecated + public static final String PARQUET_COMPRESSION_RATIO = PARQUET_COMPRESSION_RATIO_FRACTION.key(); + /** + * @deprecated Use {@link #PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead + */ + @Deprecated + public static final String DEFAULT_STREAM_COMPRESSION_RATIO = PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue(); + /** + * @deprecated Use {@link #PARQUET_COMPRESSION_CODEC_NAME} and its methods instead + */ + @Deprecated + public static final String PARQUET_COMPRESSION_CODEC = PARQUET_COMPRESSION_CODEC_NAME.key(); + /** + * @deprecated Use {@link #HFILE_COMPRESSION_ALGORITHM_NAME} and its methods instead + */ + @Deprecated + public static final String HFILE_COMPRESSION_ALGORITHM = HFILE_COMPRESSION_ALGORITHM_NAME.key(); + /** + * @deprecated Use {@link #PARQUET_COMPRESSION_CODEC_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARQUET_COMPRESSION_CODEC = PARQUET_COMPRESSION_CODEC_NAME.defaultValue(); + /** + * @deprecated Use {@link #HFILE_COMPRESSION_ALGORITHM_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HFILE_COMPRESSION_ALGORITHM = HFILE_COMPRESSION_ALGORITHM_NAME.defaultValue(); + /** + * @deprecated Use {@link #LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead + */ + @Deprecated + public static final String LOGFILE_TO_PARQUET_COMPRESSION_RATIO = LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(); + /** + * @deprecated Use {@link #LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION} and its methods instead + */ + @Deprecated + public static final String DEFAULT_LOGFILE_TO_PARQUET_COMPRESSION_RATIO = LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue(); + private HoodieStorageConfig() { super(); } @@ -152,62 +263,62 @@ public class HoodieStorageConfig extends HoodieConfig { } public Builder parquetMaxFileSize(long maxFileSize) { - storageConfig.setValue(PARQUET_FILE_MAX_BYTES, String.valueOf(maxFileSize)); + storageConfig.setValue(PARQUET_MAX_FILE_SIZE, String.valueOf(maxFileSize)); return this; } public Builder parquetBlockSize(int blockSize) { - storageConfig.setValue(PARQUET_BLOCK_SIZE_BYTES, String.valueOf(blockSize)); + storageConfig.setValue(PARQUET_BLOCK_SIZE, String.valueOf(blockSize)); return this; } public Builder parquetPageSize(int pageSize) { - storageConfig.setValue(PARQUET_PAGE_SIZE_BYTES, String.valueOf(pageSize)); + storageConfig.setValue(PARQUET_PAGE_SIZE, String.valueOf(pageSize)); return this; } public Builder hfileMaxFileSize(long maxFileSize) { - storageConfig.setValue(HFILE_FILE_MAX_BYTES, String.valueOf(maxFileSize)); + storageConfig.setValue(HFILE_MAX_FILE_SIZE, String.valueOf(maxFileSize)); return this; } public Builder hfileBlockSize(int blockSize) { - storageConfig.setValue(HFILE_BLOCK_SIZE_BYTES, String.valueOf(blockSize)); + storageConfig.setValue(HFILE_BLOCK_SIZE, String.valueOf(blockSize)); return this; } public Builder logFileDataBlockMaxSize(int dataBlockSize) { - storageConfig.setValue(LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES, String.valueOf(dataBlockSize)); + storageConfig.setValue(LOGFILE_DATA_BLOCK_MAX_SIZE, String.valueOf(dataBlockSize)); return this; } public Builder logFileMaxSize(int logFileSize) { - storageConfig.setValue(LOGFILE_SIZE_MAX_BYTES, String.valueOf(logFileSize)); + storageConfig.setValue(LOGFILE_MAX_SIZE, String.valueOf(logFileSize)); return this; } public Builder parquetCompressionRatio(double parquetCompressionRatio) { - storageConfig.setValue(PARQUET_COMPRESSION_RATIO, String.valueOf(parquetCompressionRatio)); + storageConfig.setValue(PARQUET_COMPRESSION_RATIO_FRACTION, String.valueOf(parquetCompressionRatio)); return this; } public Builder parquetCompressionCodec(String parquetCompressionCodec) { - storageConfig.setValue(PARQUET_COMPRESSION_CODEC, parquetCompressionCodec); + storageConfig.setValue(PARQUET_COMPRESSION_CODEC_NAME, parquetCompressionCodec); return this; } public Builder hfileCompressionAlgorithm(String hfileCompressionAlgorithm) { - storageConfig.setValue(HFILE_COMPRESSION_ALGORITHM, hfileCompressionAlgorithm); + storageConfig.setValue(HFILE_COMPRESSION_ALGORITHM_NAME, hfileCompressionAlgorithm); return this; } public Builder logFileToParquetCompressionRatio(double logFileToParquetCompressionRatio) { - storageConfig.setValue(LOGFILE_TO_PARQUET_COMPRESSION_RATIO, String.valueOf(logFileToParquetCompressionRatio)); + storageConfig.setValue(LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION, String.valueOf(logFileToParquetCompressionRatio)); return this; } public Builder orcMaxFileSize(long maxFileSize) { - storageConfig.setValue(ORC_FILE_MAX_BYTES, String.valueOf(maxFileSize)); + storageConfig.setValue(ORC_FILE_MAX_SIZE, String.valueOf(maxFileSize)); return this; } @@ -222,7 +333,7 @@ public class HoodieStorageConfig extends HoodieConfig { } public Builder orcCompressionCodec(String orcCompressionCodec) { - storageConfig.setValue(ORC_COMPRESSION_CODEC, orcCompressionCodec); + storageConfig.setValue(ORC_COMPRESSION_CODEC_NAME, orcCompressionCodec); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java index d313ea35b..09c2b09be 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java @@ -38,20 +38,18 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig { public static final String CALLBACK_PREFIX = "hoodie.write.commit.callback."; - public static final ConfigProperty CALLBACK_ON = ConfigProperty + public static final ConfigProperty TURN_CALLBACK_ON = ConfigProperty .key(CALLBACK_PREFIX + "on") .defaultValue(false) .sinceVersion("0.6.0") .withDocumentation("Turn commit callback on/off. off by default."); - public static final ConfigProperty CALLBACK_CLASS = ConfigProperty + public static final ConfigProperty CALLBACK_CLASS_NAME = ConfigProperty .key(CALLBACK_PREFIX + "class") .defaultValue("org.apache.hudi.callback.impl.HoodieWriteCommitHttpCallback") .sinceVersion("0.6.0") .withDocumentation("Full path of callback class and must be a subclass of HoodieWriteCommitCallback class, " + "org.apache.hudi.callback.impl.HoodieWriteCommitHttpCallback by default"); - @Deprecated - public static final String CALLBACK_CLASS_PROP = CALLBACK_CLASS.key(); // ***** HTTP callback configs ***** public static final ConfigProperty CALLBACK_HTTP_URL = ConfigProperty @@ -59,21 +57,65 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig { .noDefaultValue() .sinceVersion("0.6.0") .withDocumentation("Callback host to be sent along with callback messages"); - @Deprecated - public static final String CALLBACK_HTTP_URL_PROP = CALLBACK_HTTP_URL.key(); - public static final ConfigProperty CALLBACK_HTTP_API_KEY = ConfigProperty + public static final ConfigProperty CALLBACK_HTTP_API_KEY_VALUE = ConfigProperty .key(CALLBACK_PREFIX + "http.api.key") .defaultValue("hudi_write_commit_http_callback") .sinceVersion("0.6.0") .withDocumentation("Http callback API key. hudi_write_commit_http_callback by default"); - public static final ConfigProperty CALLBACK_HTTP_TIMEOUT_SECONDS = ConfigProperty + public static final ConfigProperty CALLBACK_HTTP_TIMEOUT_IN_SECONDS = ConfigProperty .key(CALLBACK_PREFIX + "http.timeout.seconds") .defaultValue(3) .sinceVersion("0.6.0") .withDocumentation("Callback timeout in seconds. 3 by default"); + /** + * @deprecated Use {@link #TURN_CALLBACK_ON} and its methods instead + */ + @Deprecated + public static final String CALLBACK_ON = TURN_CALLBACK_ON.key(); + /** + * @deprecated Use {@link #TURN_CALLBACK_ON} and its methods instead + */ + @Deprecated + public static final boolean DEFAULT_CALLBACK_ON = TURN_CALLBACK_ON.defaultValue(); + /** + * @deprecated Use {@link #CALLBACK_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String CALLBACK_CLASS_PROP = CALLBACK_CLASS_NAME.key(); + /** + * @deprecated Use {@link #CALLBACK_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CALLBACK_CLASS_PROP = CALLBACK_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #CALLBACK_HTTP_URL} and its methods instead + */ + @Deprecated + public static final String CALLBACK_HTTP_URL_PROP = CALLBACK_HTTP_URL.key(); + /** + * @deprecated Use {@link #CALLBACK_HTTP_API_KEY_VALUE} and its methods instead + */ + @Deprecated + public static final String CALLBACK_HTTP_API_KEY = CALLBACK_HTTP_API_KEY_VALUE.key(); + /** + * @deprecated Use {@link #CALLBACK_HTTP_API_KEY_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_CALLBACK_HTTP_API_KEY = CALLBACK_HTTP_API_KEY_VALUE.defaultValue(); + /** + * @deprecated Use {@link #CALLBACK_HTTP_TIMEOUT_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final String CALLBACK_HTTP_TIMEOUT_SECONDS = CALLBACK_HTTP_TIMEOUT_IN_SECONDS.key(); + /** + * @deprecated Use {@link #CALLBACK_HTTP_TIMEOUT_IN_SECONDS} and its methods instead + */ + @Deprecated + public static final int DEFAULT_CALLBACK_HTTP_TIMEOUT_SECONDS = CALLBACK_HTTP_TIMEOUT_IN_SECONDS.defaultValue(); + private HoodieWriteCommitCallbackConfig() { super(); } @@ -99,12 +141,12 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig { } public HoodieWriteCommitCallbackConfig.Builder writeCommitCallbackOn(String callbackOn) { - writeCommitCallbackConfig.setValue(CALLBACK_ON, callbackOn); + writeCommitCallbackConfig.setValue(TURN_CALLBACK_ON, callbackOn); return this; } public HoodieWriteCommitCallbackConfig.Builder withCallbackClass(String callbackClass) { - writeCommitCallbackConfig.setValue(CALLBACK_CLASS, callbackClass); + writeCommitCallbackConfig.setValue(CALLBACK_CLASS_NAME, callbackClass); return this; } @@ -114,12 +156,12 @@ public class HoodieWriteCommitCallbackConfig extends HoodieConfig { } public Builder withCallbackHttpTimeoutSeconds(String timeoutSeconds) { - writeCommitCallbackConfig.setValue(CALLBACK_HTTP_TIMEOUT_SECONDS, timeoutSeconds); + writeCommitCallbackConfig.setValue(CALLBACK_HTTP_TIMEOUT_IN_SECONDS, timeoutSeconds); return this; } public Builder withCallbackHttpApiKey(String apiKey) { - writeCommitCallbackConfig.setValue(CALLBACK_HTTP_API_KEY, apiKey); + writeCommitCallbackConfig.setValue(CALLBACK_HTTP_API_KEY_VALUE, apiKey); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 122b90f80..448ce9f7b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -42,6 +42,7 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode; import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.keygen.SimpleAvroKeyGenerator; import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.hudi.metrics.MetricsReporterType; import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite; @@ -78,7 +79,7 @@ public class HoodieWriteConfig extends HoodieConfig { private static final long serialVersionUID = 0L; - public static final ConfigProperty TABLE_NAME = ConfigProperty + public static final ConfigProperty TBL_NAME = ConfigProperty .key("hoodie.table.name") .noDefaultValue() .withDocumentation("Table name that will be used for registering with metastores like HMS. Needs to be same across runs."); @@ -88,22 +89,18 @@ public class HoodieWriteConfig extends HoodieConfig { .defaultValue("ts") .withDocumentation("Field used in preCombining before actual write. When two records have the same key value, " + "we will pick the one with the largest value for the precombine field, determined by Object.compareTo(..)"); - @Deprecated - public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD.key(); - public static final ConfigProperty WRITE_PAYLOAD_CLASS = ConfigProperty + public static final ConfigProperty WRITE_PAYLOAD_CLASS_NAME = ConfigProperty .key("hoodie.datasource.write.payload.class") .defaultValue(OverwriteWithLatestAvroPayload.class.getName()) .withDocumentation("Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. " + "This will render any value set for PRECOMBINE_FIELD_OPT_VAL in-effective"); - public static final ConfigProperty KEYGENERATOR_CLASS = ConfigProperty + public static final ConfigProperty KEYGENERATOR_CLASS_NAME = ConfigProperty .key("hoodie.datasource.write.keygenerator.class") .noDefaultValue() .withDocumentation("Key generator class, that implements `org.apache.hudi.keygen.KeyGenerator` " + "extract a key out of incoming records."); - @Deprecated - public static final String KEYGENERATOR_CLASS_PROP = KEYGENERATOR_CLASS.key(); public static final ConfigProperty KEYGENERATOR_TYPE = ConfigProperty .key("hoodie.datasource.write.keygenerator.type") @@ -111,13 +108,13 @@ public class HoodieWriteConfig extends HoodieConfig { .withDocumentation("Easily configure one the built-in key generators, instead of specifying the key generator class." + "Currently supports SIMPLE, COMPLEX, TIMESTAMP, CUSTOM, NON_PARTITION, GLOBAL_DELETE"); - public static final ConfigProperty ROLLBACK_USING_MARKERS = ConfigProperty + public static final ConfigProperty ROLLBACK_USING_MARKERS_ENABLE = ConfigProperty .key("hoodie.rollback.using.markers") .defaultValue("false") .withDocumentation("Enables a more efficient mechanism for rollbacks based on the marker files generated " + "during the writes. Turned off by default."); - public static final ConfigProperty TIMELINE_LAYOUT_VERSION = ConfigProperty + public static final ConfigProperty TIMELINE_LAYOUT_VERSION_NUM = ConfigProperty .key("hoodie.timeline.layout.version") .defaultValue(Integer.toString(TimelineLayoutVersion.VERSION_1)) .sinceVersion("0.5.1") @@ -137,55 +134,53 @@ public class HoodieWriteConfig extends HoodieConfig { + "Always prefix it explicitly with the storage scheme (e.g hdfs://, s3:// etc). " + "Hudi stores all the main meta-data about commits, savepoints, cleaning audit logs " + "etc in .hoodie directory under this base path directory."); - @Deprecated - public static final String BASE_PATH_PROP = BASE_PATH.key(); - public static final ConfigProperty AVRO_SCHEMA = ConfigProperty + public static final ConfigProperty AVRO_SCHEMA_STRING = ConfigProperty .key("hoodie.avro.schema") .noDefaultValue() .withDocumentation("Schema string representing the current write schema of the table. Hudi passes this to " + "implementations of HoodieRecordPayload to convert incoming records to avro. This is also used as the write schema " + "evolving records during an update."); - public static final ConfigProperty AVRO_SCHEMA_VALIDATE = ConfigProperty + public static final ConfigProperty AVRO_SCHEMA_VALIDATE_ENABLE = ConfigProperty .key("hoodie.avro.schema.validate") .defaultValue("false") .withDocumentation("Validate the schema used for the write against the latest schema, for backwards compatibility."); - public static final ConfigProperty INSERT_PARALLELISM = ConfigProperty + public static final ConfigProperty INSERT_PARALLELISM_VALUE = ConfigProperty .key("hoodie.insert.shuffle.parallelism") .defaultValue("1500") .withDocumentation("Parallelism for inserting records into the table. Inserts can shuffle data before writing to tune file sizes and optimize the storage layout."); - public static final ConfigProperty BULKINSERT_PARALLELISM = ConfigProperty + public static final ConfigProperty BULKINSERT_PARALLELISM_VALUE = ConfigProperty .key("hoodie.bulkinsert.shuffle.parallelism") .defaultValue("1500") .withDocumentation("For large initial imports using bulk_insert operation, controls the parallelism to use for sort modes or custom partitioning done" + "before writing records to the table."); - public static final ConfigProperty BULKINSERT_USER_DEFINED_PARTITIONER_CLASS = ConfigProperty + public static final ConfigProperty BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME = ConfigProperty .key("hoodie.bulkinsert.user.defined.partitioner.class") .noDefaultValue() .withDocumentation("If specified, this class will be used to re-partition records before they are bulk inserted. This can be used to sort, pack, cluster data" + " optimally for common query patterns."); - public static final ConfigProperty UPSERT_PARALLELISM = ConfigProperty + public static final ConfigProperty UPSERT_PARALLELISM_VALUE = ConfigProperty .key("hoodie.upsert.shuffle.parallelism") .defaultValue("1500") .withDocumentation("Parallelism to use for upsert operation on the table. Upserts can shuffle data to perform index lookups, file sizing, bin packing records optimally" + "into file groups."); - public static final ConfigProperty DELETE_PARALLELISM = ConfigProperty + public static final ConfigProperty DELETE_PARALLELISM_VALUE = ConfigProperty .key("hoodie.delete.shuffle.parallelism") .defaultValue("1500") .withDocumentation("Parallelism used for “delete” operation. Delete operations also performs shuffles, similar to upsert operation."); - public static final ConfigProperty ROLLBACK_PARALLELISM = ConfigProperty + public static final ConfigProperty ROLLBACK_PARALLELISM_VALUE = ConfigProperty .key("hoodie.rollback.parallelism") .defaultValue("100") .withDocumentation("Parallelism for rollback of commits. Rollbacks perform delete of files or logging delete blocks to file groups on storage in parallel."); - public static final ConfigProperty WRITE_BUFFER_LIMIT_BYTES = ConfigProperty + public static final ConfigProperty WRITE_BUFFER_LIMIT_BYTES_VALUE = ConfigProperty .key("hoodie.write.buffer.limit.bytes") .defaultValue(String.valueOf(4 * 1024 * 1024)) .withDocumentation("Size of in-memory buffer used for parallelizing network reads and lake storage writes."); @@ -195,8 +190,6 @@ public class HoodieWriteConfig extends HoodieConfig { .defaultValue("false") .withDocumentation("When inserted records share same key, controls whether they should be first combined (i.e de-duplicated) before" + " writing to storage."); - @Deprecated - public static final String COMBINE_BEFORE_INSERT_PROP = COMBINE_BEFORE_INSERT.key(); public static final ConfigProperty COMBINE_BEFORE_UPSERT = ConfigProperty .key("hoodie.combine.before.upsert") @@ -204,40 +197,32 @@ public class HoodieWriteConfig extends HoodieConfig { .withDocumentation("When upserted records share same key, controls whether they should be first combined (i.e de-duplicated) before" + " writing to storage. This should be turned off only if you are absolutely certain that there are no duplicates incoming, " + " otherwise it can lead to duplicate keys and violate the uniqueness guarantees."); - @Deprecated - public static final String COMBINE_BEFORE_UPSERT_PROP = COMBINE_BEFORE_UPSERT.key(); public static final ConfigProperty COMBINE_BEFORE_DELETE = ConfigProperty .key("hoodie.combine.before.delete") .defaultValue("true") .withDocumentation("During delete operations, controls whether we should combine deletes (and potentially also upserts) before " + " writing to storage."); - @Deprecated - public static final String COMBINE_BEFORE_DELETE_PROP = COMBINE_BEFORE_DELETE.key(); - public static final ConfigProperty WRITE_STATUS_STORAGE_LEVEL = ConfigProperty + public static final ConfigProperty WRITE_STATUS_STORAGE_LEVEL_VALUE = ConfigProperty .key("hoodie.write.status.storage.level") .defaultValue("MEMORY_AND_DISK_SER") .withDocumentation("Write status objects hold metadata about a write (stats, errors), that is not yet committed to storage. " + "This controls the how that information is cached for inspection by clients. We rarely expect this to be changed."); - public static final ConfigProperty HOODIE_AUTO_COMMIT = ConfigProperty + public static final ConfigProperty AUTO_COMMIT_ENABLE = ConfigProperty .key("hoodie.auto.commit") .defaultValue("true") .withDocumentation("Controls whether a write operation should auto commit. This can be turned off to perform inspection" + " of the uncommitted write before deciding to commit."); - @Deprecated - public static final String HOODIE_AUTO_COMMIT_PROP = HOODIE_AUTO_COMMIT.key(); - public static final ConfigProperty HOODIE_WRITE_STATUS_CLASS = ConfigProperty + public static final ConfigProperty WRITE_STATUS_CLASS_NAME = ConfigProperty .key("hoodie.writestatus.class") .defaultValue(WriteStatus.class.getName()) .withDocumentation("Subclass of " + WriteStatus.class.getName() + " to be used to collect information about a write. Can be " + "overridden to collection additional metrics/statistics about the data if needed."); - @Deprecated - public static final String HOODIE_WRITE_STATUS_CLASS_PROP = HOODIE_WRITE_STATUS_CLASS.key(); - public static final ConfigProperty FINALIZE_WRITE_PARALLELISM = ConfigProperty + public static final ConfigProperty FINALIZE_WRITE_PARALLELISM_VALUE = ConfigProperty .key("hoodie.finalize.write.parallelism") .defaultValue("1500") .withDocumentation("Parallelism for the write finalization internal operation, which involves removing any partially written " @@ -268,23 +253,23 @@ public class HoodieWriteConfig extends HoodieConfig { .sinceVersion("0.9.0") .withDocumentation("The batch interval in milliseconds for marker creation batch processing"); - public static final ConfigProperty MARKERS_DELETE_PARALLELISM = ConfigProperty + public static final ConfigProperty MARKERS_DELETE_PARALLELISM_VALUE = ConfigProperty .key("hoodie.markers.delete.parallelism") .defaultValue("100") .withDocumentation("Determines the parallelism for deleting marker files, which are used to track all files (valid or invalid/partial) written during " + "a write operation. Increase this value if delays are observed, with large batch writes."); - public static final ConfigProperty BULKINSERT_SORT_MODE = ConfigProperty + public static final ConfigProperty BULK_INSERT_SORT_MODE = ConfigProperty .key("hoodie.bulkinsert.sort.mode") .defaultValue(BulkInsertSortMode.GLOBAL_SORT.toString()) .withDocumentation("Sorting modes to use for sorting records for bulk insert. This is user when user " - + BULKINSERT_USER_DEFINED_PARTITIONER_CLASS.key() + "is not configured. Available values are - " + + BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME.key() + "is not configured. Available values are - " + "GLOBAL_SORT: this ensures best file sizes, with lowest memory overhead at cost of sorting. " + "PARTITION_SORT: Strikes a balance by only sorting within a partition, still keeping the memory overhead of writing " + "lowest and best effort file sizing. " + "NONE: No sorting. Fastest and matches `spark.write.parquet()` in terms of number of files, overheads"); - public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_ENABLED = ConfigProperty + public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_ENABLE = ConfigProperty .key("hoodie.embed.timeline.server") .defaultValue("true") .withDocumentation("When true, spins up an instance of the timeline server (meta server that serves cached file listings, statistics)," @@ -296,65 +281,57 @@ public class HoodieWriteConfig extends HoodieConfig { .withDocumentation("Controls whether the timeline server instance should be cached and reused across the JVM (across task lifecycles)" + "to avoid startup costs. This should rarely be changed."); - public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_PORT = ConfigProperty + public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_PORT_NUM = ConfigProperty .key("hoodie.embed.timeline.server.port") .defaultValue("0") .withDocumentation("Port at which the timeline server listens for requests. When running embedded in each writer, it picks " + "a free port and communicates to all the executors. This should rarely be changed."); - public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_THREADS = ConfigProperty + public static final ConfigProperty EMBEDDED_TIMELINE_NUM_SERVER_THREADS = ConfigProperty .key("hoodie.embed.timeline.server.threads") .defaultValue("-1") .withDocumentation("Number of threads to serve requests in the timeline server. By default, auto configured based on the number of underlying cores."); - public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_COMPRESS_OUTPUT = ConfigProperty + public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE = ConfigProperty .key("hoodie.embed.timeline.server.gzip") .defaultValue("true") .withDocumentation("Controls whether gzip compression is used, for large responses from the timeline server, to improve latency."); - public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_USE_ASYNC = ConfigProperty + public static final ConfigProperty EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE = ConfigProperty .key("hoodie.embed.timeline.server.async") .defaultValue("false") .withDocumentation("Controls whether or not, the requests to the timeline server are processed in asynchronous fashion, " + "potentially improving throughput."); - public static final ConfigProperty FAIL_ON_TIMELINE_ARCHIVING_ENABLED = ConfigProperty + public static final ConfigProperty FAIL_ON_TIMELINE_ARCHIVING_ENABLE = ConfigProperty .key("hoodie.fail.on.timeline.archiving") .defaultValue("true") .withDocumentation("Timeline archiving removes older instants from the timeline, after each write operation, to minimize metadata overhead. " + "Controls whether or not, the write should be failed as well, if such archiving fails."); - @Deprecated - public static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP = FAIL_ON_TIMELINE_ARCHIVING_ENABLED.key(); public static final ConfigProperty INITIAL_CONSISTENCY_CHECK_INTERVAL_MS = ConfigProperty .key("hoodie.consistency.check.initial_interval_ms") .defaultValue(2000L) .withDocumentation("Initial time between successive attempts to ensure written data's metadata is consistent on storage. Grows with exponential" + " backoff after the initial value."); - @Deprecated - public static final String INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP = INITIAL_CONSISTENCY_CHECK_INTERVAL_MS.key(); public static final ConfigProperty MAX_CONSISTENCY_CHECK_INTERVAL_MS = ConfigProperty .key("hoodie.consistency.check.max_interval_ms") .defaultValue(300000L) .withDocumentation("Max time to wait between successive attempts at performing consistency checks"); - @Deprecated - public static final String MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP = MAX_CONSISTENCY_CHECK_INTERVAL_MS.key(); public static final ConfigProperty MAX_CONSISTENCY_CHECKS = ConfigProperty .key("hoodie.consistency.check.max_checks") .defaultValue(7) .withDocumentation("Maximum number of checks, for consistency of written data."); - @Deprecated - public static final String MAX_CONSISTENCY_CHECKS_PROP = MAX_CONSISTENCY_CHECKS.key(); - public static final ConfigProperty MERGE_DATA_VALIDATION_CHECK_ENABLED = ConfigProperty + public static final ConfigProperty MERGE_DATA_VALIDATION_CHECK_ENABLE = ConfigProperty .key("hoodie.merge.data.validation.enabled") .defaultValue("false") .withDocumentation("When enabled, data validation checks are performed during merges to ensure expected " + "number of records after merge operation."); - public static final ConfigProperty MERGE_ALLOW_DUPLICATE_ON_INSERTS = ConfigProperty + public static final ConfigProperty MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE = ConfigProperty .key("hoodie.merge.allow.duplicate.on.inserts") .defaultValue("false") .withDocumentation("When enabled, we allow duplicate keys even if inserts are routed to merge with an existing file (for ensuring file sizing)." @@ -364,33 +341,25 @@ public class HoodieWriteConfig extends HoodieConfig { .key("hoodie.client.heartbeat.interval_in_ms") .defaultValue(60 * 1000) .withDocumentation("Writers perform heartbeats to indicate liveness. Controls how often (in ms), such heartbeats are registered to lake storage."); - @Deprecated - public static final String CLIENT_HEARTBEAT_INTERVAL_IN_MS_PROP = CLIENT_HEARTBEAT_INTERVAL_IN_MS.key(); public static final ConfigProperty CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES = ConfigProperty .key("hoodie.client.heartbeat.tolerable.misses") .defaultValue(2) .withDocumentation("Number of heartbeat misses, before a writer is deemed not alive and all pending writes are aborted."); - @Deprecated - public static final String CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES_PROP = CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES.key(); - public static final ConfigProperty WRITE_CONCURRENCY_MOD = ConfigProperty + public static final ConfigProperty WRITE_CONCURRENCY_MODE = ConfigProperty .key("hoodie.write.concurrency.mode") .defaultValue(WriteConcurrencyMode.SINGLE_WRITER.name()) .withDocumentation("Enable different concurrency modes. Options are " + "SINGLE_WRITER: Only one active writer to the table. Maximizes throughput" + "OPTIMISTIC_CONCURRENCY_CONTROL: Multiple writers can operate on the table and exactly one of them succeed " + "if a conflict (writes affect the same file group) is detected."); - @Deprecated - public static final String WRITE_CONCURRENCY_MODE_PROP = WRITE_CONCURRENCY_MOD.key(); public static final ConfigProperty WRITE_META_KEY_PREFIXES = ConfigProperty .key("hoodie.write.meta.key.prefixes") .defaultValue("") .withDocumentation("Comma separated metadata key prefixes to override from latest commit " + "during overlapping commits via multi writing"); - @Deprecated - public static final String WRITE_META_KEY_PREFIXES_PROP = WRITE_META_KEY_PREFIXES.key(); /** * Currently the use this to specify the write schema. @@ -412,22 +381,22 @@ public class HoodieWriteConfig extends HoodieConfig { * Given the importance of supporting such cases for the user's migration to 0.5.x, we are proposing a safety flag * (disabled by default) which will allow this old behavior. */ - public static final ConfigProperty ALLOW_MULTI_WRITE_ON_SAME_INSTANT = ConfigProperty + public static final ConfigProperty ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE = ConfigProperty .key("_.hoodie.allow.multi.write.on.same.instant") .defaultValue("false") .withDocumentation(""); - public static final ConfigProperty EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = ConfigProperty - .key(AVRO_SCHEMA.key() + ".external.transformation") + public static final ConfigProperty AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE = ConfigProperty + .key(AVRO_SCHEMA_STRING.key() + ".external.transformation") .defaultValue("false") - .withAlternatives(AVRO_SCHEMA.key() + ".externalTransformation") + .withAlternatives(AVRO_SCHEMA_STRING.key() + ".externalTransformation") .withDocumentation("When enabled, records in older schema are rewritten into newer schema during upsert,delete and background" + " compaction,clustering operations."); public static final ConfigProperty ALLOW_EMPTY_COMMIT = ConfigProperty - .key("hoodie.allow.empty.commit") - .defaultValue(true) - .withDocumentation("Whether to allow generation of empty commits, even if no data was written in the commit. " + .key("hoodie.allow.empty.commit") + .defaultValue(true) + .withDocumentation("Whether to allow generation of empty commits, even if no data was written in the commit. " + "It's useful in cases where extra metadata needs to be published regardless e.g tracking source offsets when ingesting data"); public static final ConfigProperty ALLOW_OPERATION_METADATA_FIELD = ConfigProperty @@ -448,6 +417,384 @@ public class HoodieWriteConfig extends HoodieConfig { private HoodieCommonConfig commonConfig; private EngineType engineType; + /** + * @deprecated Use {@link #TBL_NAME} and its methods instead + */ + @Deprecated + public static final String TABLE_NAME = TBL_NAME.key(); + /** + * @deprecated Use {@link #PRECOMBINE_FIELD} and its methods instead + */ + @Deprecated + public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD.key(); + /** + * @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String WRITE_PAYLOAD_CLASS = WRITE_PAYLOAD_CLASS_NAME.key(); + /** + * @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_WRITE_PAYLOAD_CLASS = WRITE_PAYLOAD_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #KEYGENERATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String KEYGENERATOR_CLASS_PROP = KEYGENERATOR_CLASS_NAME.key(); + /** + * @deprecated Use {@link #KEYGENERATOR_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_KEYGENERATOR_CLASS = SimpleAvroKeyGenerator.class.getName(); + /** + * @deprecated Use {@link #ROLLBACK_USING_MARKERS_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_ROLLBACK_USING_MARKERS = ROLLBACK_USING_MARKERS_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #ROLLBACK_USING_MARKERS_ENABLE} and its methods instead + */ + @Deprecated + public static final String ROLLBACK_USING_MARKERS = ROLLBACK_USING_MARKERS_ENABLE.key(); + /** + * @deprecated Use {@link #TIMELINE_LAYOUT_VERSION_NUM} and its methods instead + */ + @Deprecated + public static final String TIMELINE_LAYOUT_VERSION = TIMELINE_LAYOUT_VERSION_NUM.key(); + /** + * @deprecated Use {@link #BASE_PATH} and its methods instead + */ + @Deprecated + public static final String BASE_PATH_PROP = BASE_PATH.key(); + /** + * @deprecated Use {@link #AVRO_SCHEMA_STRING} and its methods instead + */ + @Deprecated + public static final String AVRO_SCHEMA = AVRO_SCHEMA_STRING.key(); + /** + * @deprecated Use {@link #AVRO_SCHEMA_VALIDATE_ENABLE} and its methods instead + */ + @Deprecated + public static final String AVRO_SCHEMA_VALIDATE = AVRO_SCHEMA_VALIDATE_ENABLE.key(); + /** + * @deprecated Use {@link #AVRO_SCHEMA_VALIDATE_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_AVRO_SCHEMA_VALIDATE = AVRO_SCHEMA_VALIDATE_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #INSERT_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_PARALLELISM = INSERT_PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #INSERT_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String INSERT_PARALLELISM = INSERT_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #BULKINSERT_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String BULKINSERT_PARALLELISM = BULKINSERT_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String BULKINSERT_USER_DEFINED_PARTITIONER_CLASS = BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME.key(); + @Deprecated + public static final String BULKINSERT_INPUT_DATA_SCHEMA_DDL = "hoodie.bulkinsert.schema.ddl"; + /** + * @deprecated Use {@link #UPSERT_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String UPSERT_PARALLELISM = UPSERT_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #DELETE_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DELETE_PARALLELISM = DELETE_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #ROLLBACK_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_ROLLBACK_PARALLELISM = ROLLBACK_PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #ROLLBACK_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String ROLLBACK_PARALLELISM = ROLLBACK_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #WRITE_BUFFER_LIMIT_BYTES_VALUE} and its methods instead + */ + @Deprecated + public static final String WRITE_BUFFER_LIMIT_BYTES = WRITE_BUFFER_LIMIT_BYTES_VALUE.key(); + /** + * @deprecated Use {@link #WRITE_BUFFER_LIMIT_BYTES_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_WRITE_BUFFER_LIMIT_BYTES = WRITE_BUFFER_LIMIT_BYTES_VALUE.defaultValue(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_INSERT} and its methods instead + */ + @Deprecated + public static final String COMBINE_BEFORE_INSERT_PROP = COMBINE_BEFORE_INSERT.key(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_INSERT} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COMBINE_BEFORE_INSERT = COMBINE_BEFORE_INSERT.defaultValue(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_UPSERT} and its methods instead + */ + @Deprecated + public static final String COMBINE_BEFORE_UPSERT_PROP = COMBINE_BEFORE_UPSERT.key(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_UPSERT} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COMBINE_BEFORE_UPSERT = COMBINE_BEFORE_UPSERT.defaultValue(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_DELETE} and its methods instead + */ + @Deprecated + public static final String COMBINE_BEFORE_DELETE_PROP = COMBINE_BEFORE_DELETE.key(); + /** + * @deprecated Use {@link #COMBINE_BEFORE_DELETE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_COMBINE_BEFORE_DELETE = COMBINE_BEFORE_DELETE.defaultValue(); + /** + * @deprecated Use {@link #WRITE_STATUS_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String WRITE_STATUS_STORAGE_LEVEL = WRITE_STATUS_STORAGE_LEVEL_VALUE.key(); + /** + * @deprecated Use {@link #WRITE_STATUS_STORAGE_LEVEL_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_WRITE_STATUS_STORAGE_LEVEL = WRITE_STATUS_STORAGE_LEVEL_VALUE.defaultValue(); + /** + * @deprecated Use {@link #AUTO_COMMIT_ENABLE} and its methods instead + */ + @Deprecated + public static final String HOODIE_AUTO_COMMIT_PROP = AUTO_COMMIT_ENABLE.key(); + /** + * @deprecated Use {@link #AUTO_COMMIT_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HOODIE_AUTO_COMMIT = AUTO_COMMIT_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #WRITE_STATUS_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String HOODIE_WRITE_STATUS_CLASS_PROP = WRITE_STATUS_CLASS_NAME.key(); + /** + * @deprecated Use {@link #WRITE_STATUS_CLASS_NAME} and its methods instead + */ + @Deprecated + public static final String DEFAULT_HOODIE_WRITE_STATUS_CLASS = WRITE_STATUS_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #FINALIZE_WRITE_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String FINALIZE_WRITE_PARALLELISM = FINALIZE_WRITE_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #FINALIZE_WRITE_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_FINALIZE_WRITE_PARALLELISM = FINALIZE_WRITE_PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #MARKERS_DELETE_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String MARKERS_DELETE_PARALLELISM = MARKERS_DELETE_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #MARKERS_DELETE_PARALLELISM_VALUE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_MARKERS_DELETE_PARALLELISM = MARKERS_DELETE_PARALLELISM_VALUE.defaultValue(); + /** + * @deprecated Use {@link #BULK_INSERT_SORT_MODE} and its methods instead + */ + @Deprecated + public static final String BULKINSERT_SORT_MODE = BULK_INSERT_SORT_MODE.key(); + /** + * @deprecated Use {@link #BULK_INSERT_SORT_MODE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_BULKINSERT_SORT_MODE = BULK_INSERT_SORT_MODE.defaultValue(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_ENABLE} and its methods instead + */ + @Deprecated + public static final String EMBEDDED_TIMELINE_SERVER_ENABLED = EMBEDDED_TIMELINE_SERVER_ENABLE.key(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = EMBEDDED_TIMELINE_SERVER_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String EMBEDDED_TIMELINE_SERVER_PORT = EMBEDDED_TIMELINE_SERVER_PORT_NUM.key(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_PORT_NUM} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT = EMBEDDED_TIMELINE_SERVER_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_NUM_SERVER_THREADS} and its methods instead + */ + @Deprecated + public static final String EMBEDDED_TIMELINE_SERVER_THREADS = EMBEDDED_TIMELINE_NUM_SERVER_THREADS.key(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_NUM_SERVER_THREADS} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_THREADS = EMBEDDED_TIMELINE_NUM_SERVER_THREADS.defaultValue(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE} and its methods instead + */ + @Deprecated + public static final String EMBEDDED_TIMELINE_SERVER_COMPRESS_OUTPUT = EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE.key(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EMBEDDED_TIMELINE_COMPRESS_OUTPUT = EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE} and its methods instead + */ + @Deprecated + public static final String EMBEDDED_TIMELINE_SERVER_USE_ASYNC = EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE.key(); + /** + * @deprecated Use {@link #EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ASYNC = EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #FAIL_ON_TIMELINE_ARCHIVING_ENABLE} and its methods instead + */ + @Deprecated + public static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP = FAIL_ON_TIMELINE_ARCHIVING_ENABLE.key(); + /** + * @deprecated Use {@link #FAIL_ON_TIMELINE_ARCHIVING_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_FAIL_ON_TIMELINE_ARCHIVING_ENABLED = FAIL_ON_TIMELINE_ARCHIVING_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #INITIAL_CONSISTENCY_CHECK_INTERVAL_MS} and its methods instead + */ + @Deprecated + public static final String INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP = INITIAL_CONSISTENCY_CHECK_INTERVAL_MS.key(); + /** + * @deprecated Use {@link #INITIAL_CONSISTENCY_CHECK_INTERVAL_MS} and its methods instead + */ + @Deprecated + public static long DEFAULT_INITIAL_CONSISTENCY_CHECK_INTERVAL_MS = INITIAL_CONSISTENCY_CHECK_INTERVAL_MS.defaultValue(); + /** + * @deprecated Use {@link #MAX_CONSISTENCY_CHECK_INTERVAL_MS} and its methods instead + */ + @Deprecated + public static final String MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP = MAX_CONSISTENCY_CHECK_INTERVAL_MS.key(); + /** + * @deprecated Use {@link #MAX_CONSISTENCY_CHECK_INTERVAL_MS} and its methods instead + */ + @Deprecated + public static long DEFAULT_MAX_CONSISTENCY_CHECK_INTERVAL_MS = MAX_CONSISTENCY_CHECK_INTERVAL_MS.defaultValue(); + /** + * @deprecated Use {@link #MAX_CONSISTENCY_CHECKS} and its methods instead + */ + @Deprecated + public static final String MAX_CONSISTENCY_CHECKS_PROP = MAX_CONSISTENCY_CHECKS.key(); + /** + * @deprecated Use {@link #MAX_CONSISTENCY_CHECKS} and its methods instead + */ + @Deprecated + public static int DEFAULT_MAX_CONSISTENCY_CHECKS = MAX_CONSISTENCY_CHECKS.defaultValue(); + /** + * @deprecated Use {@link #MERGE_DATA_VALIDATION_CHECK_ENABLE} and its methods instead + */ + @Deprecated + private static final String MERGE_DATA_VALIDATION_CHECK_ENABLED = MERGE_DATA_VALIDATION_CHECK_ENABLE.key(); + /** + * @deprecated Use {@link #MERGE_DATA_VALIDATION_CHECK_ENABLE} and its methods instead + */ + @Deprecated + private static final String DEFAULT_MERGE_DATA_VALIDATION_CHECK_ENABLED = MERGE_DATA_VALIDATION_CHECK_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE} and its methods instead + */ + @Deprecated + private static final String MERGE_ALLOW_DUPLICATE_ON_INSERTS = MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE.key(); + /** + * @deprecated Use {@link #MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE} and its methods instead + */ + @Deprecated + private static final String DEFAULT_MERGE_ALLOW_DUPLICATE_ON_INSERTS = MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #CLIENT_HEARTBEAT_INTERVAL_IN_MS} and its methods instead + */ + @Deprecated + public static final String CLIENT_HEARTBEAT_INTERVAL_IN_MS_PROP = CLIENT_HEARTBEAT_INTERVAL_IN_MS.key(); + /** + * @deprecated Use {@link #CLIENT_HEARTBEAT_INTERVAL_IN_MS} and its methods instead + */ + @Deprecated + public static final Integer DEFAULT_CLIENT_HEARTBEAT_INTERVAL_IN_MS = CLIENT_HEARTBEAT_INTERVAL_IN_MS.defaultValue(); + /** + * @deprecated Use {@link #CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES} and its methods instead + */ + @Deprecated + public static final String CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES_PROP = CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES.key(); + /** + * @deprecated Use {@link #CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES} and its methods instead + */ + @Deprecated + public static final Integer DEFAULT_CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES = CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES.defaultValue(); + /** + * @deprecated Use {@link #WRITE_CONCURRENCY_MODE} and its methods instead + */ + @Deprecated + public static final String WRITE_CONCURRENCY_MODE_PROP = WRITE_CONCURRENCY_MODE.key(); + /** + * @deprecated Use {@link #WRITE_CONCURRENCY_MODE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_WRITE_CONCURRENCY_MODE = WRITE_CONCURRENCY_MODE.defaultValue(); + /** + * @deprecated Use {@link #WRITE_META_KEY_PREFIXES} and its methods instead + */ + @Deprecated + public static final String WRITE_META_KEY_PREFIXES_PROP = WRITE_META_KEY_PREFIXES.key(); + /** + * @deprecated Use {@link #WRITE_META_KEY_PREFIXES} and its methods instead + */ + @Deprecated + public static final String DEFAULT_WRITE_META_KEY_PREFIXES = WRITE_META_KEY_PREFIXES.defaultValue(); + /** + * @deprecated Use {@link #ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE} and its methods instead + */ + @Deprecated + public static final String ALLOW_MULTI_WRITE_ON_SAME_INSTANT = ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE.key(); + /** + * @deprecated Use {@link #ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_ALLOW_MULTI_WRITE_ON_SAME_INSTANT = ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE} and its methods instead + */ + @Deprecated + public static final String EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE.key(); + /** + * @deprecated Use {@link #AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE} and its methods instead + */ + @Deprecated + public static final String DEFAULT_EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION = AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE.defaultValue(); + /** * Use Spark engine by default. */ @@ -482,11 +829,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public String getSchema() { - return getString(AVRO_SCHEMA); + return getString(AVRO_SCHEMA_STRING); } public void setSchema(String schemaStr) { - setValue(AVRO_SCHEMA, schemaStr); + setValue(AVRO_SCHEMA_STRING, schemaStr); } /** @@ -504,11 +851,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean getAvroSchemaValidate() { - return getBoolean(AVRO_SCHEMA_VALIDATE); + return getBoolean(AVRO_SCHEMA_VALIDATE_ENABLE); } public String getTableName() { - return getString(TABLE_NAME); + return getString(TBL_NAME); } public String getPreCombineField() { @@ -516,15 +863,15 @@ public class HoodieWriteConfig extends HoodieConfig { } public String getWritePayloadClass() { - return getString(WRITE_PAYLOAD_CLASS); + return getString(WRITE_PAYLOAD_CLASS_NAME); } public String getKeyGeneratorClass() { - return getString(KEYGENERATOR_CLASS); + return getString(KEYGENERATOR_CLASS_NAME); } public Boolean shouldAutoCommit() { - return getBoolean(HOODIE_AUTO_COMMIT); + return getBoolean(AUTO_COMMIT_ENABLE); } public Boolean shouldAssumeDatePartitioning() { @@ -532,35 +879,35 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean shouldUseExternalSchemaTransformation() { - return getBoolean(EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION); + return getBoolean(AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE); } public Integer getTimelineLayoutVersion() { - return getInt(TIMELINE_LAYOUT_VERSION); + return getInt(TIMELINE_LAYOUT_VERSION_NUM); } public int getBulkInsertShuffleParallelism() { - return getInt(BULKINSERT_PARALLELISM); + return getInt(BULKINSERT_PARALLELISM_VALUE); } public String getUserDefinedBulkInsertPartitionerClass() { - return getString(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS); + return getString(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME); } public int getInsertShuffleParallelism() { - return getInt(INSERT_PARALLELISM); + return getInt(INSERT_PARALLELISM_VALUE); } public int getUpsertShuffleParallelism() { - return getInt(UPSERT_PARALLELISM); + return getInt(UPSERT_PARALLELISM_VALUE); } public int getDeleteShuffleParallelism() { - return Math.max(getInt(DELETE_PARALLELISM), 1); + return Math.max(getInt(DELETE_PARALLELISM_VALUE), 1); } public int getRollbackParallelism() { - return getInt(ROLLBACK_PARALLELISM); + return getInt(ROLLBACK_PARALLELISM_VALUE); } public int getFileListingParallelism() { @@ -568,11 +915,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean shouldRollbackUsingMarkers() { - return getBoolean(ROLLBACK_USING_MARKERS); + return getBoolean(ROLLBACK_USING_MARKERS_ENABLE); } public int getWriteBufferLimitBytes() { - return Integer.parseInt(getStringOrDefault(WRITE_BUFFER_LIMIT_BYTES)); + return Integer.parseInt(getStringOrDefault(WRITE_BUFFER_LIMIT_BYTES_VALUE)); } public boolean shouldCombineBeforeInsert() { @@ -588,15 +935,15 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean shouldAllowMultiWriteOnSameInstant() { - return getBoolean(ALLOW_MULTI_WRITE_ON_SAME_INSTANT); + return getBoolean(ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE); } public String getWriteStatusClassName() { - return getString(HOODIE_WRITE_STATUS_CLASS); + return getString(WRITE_STATUS_CLASS_NAME); } public int getFinalizeWriteParallelism() { - return getInt(FINALIZE_WRITE_PARALLELISM); + return getInt(FINALIZE_WRITE_PARALLELISM_VALUE); } public MarkerType getMarkersType() { @@ -613,11 +960,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getMarkersDeleteParallelism() { - return getInt(MARKERS_DELETE_PARALLELISM); + return getInt(MARKERS_DELETE_PARALLELISM_VALUE); } public boolean isEmbeddedTimelineServerEnabled() { - return getBoolean(EMBEDDED_TIMELINE_SERVER_ENABLED); + return getBoolean(EMBEDDED_TIMELINE_SERVER_ENABLE); } public boolean isEmbeddedTimelineServerReuseEnabled() { @@ -625,23 +972,23 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getEmbeddedTimelineServerPort() { - return Integer.parseInt(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_PORT)); + return Integer.parseInt(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_PORT_NUM)); } public int getEmbeddedTimelineServerThreads() { - return Integer.parseInt(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_THREADS)); + return Integer.parseInt(getStringOrDefault(EMBEDDED_TIMELINE_NUM_SERVER_THREADS)); } public boolean getEmbeddedTimelineServerCompressOutput() { - return Boolean.parseBoolean(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_COMPRESS_OUTPUT)); + return Boolean.parseBoolean(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_COMPRESS_ENABLE)); } public boolean getEmbeddedTimelineServerUseAsync() { - return Boolean.parseBoolean(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_USE_ASYNC)); + return Boolean.parseBoolean(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_USE_ASYNC_ENABLE)); } public boolean isFailOnTimelineArchivingEnabled() { - return getBoolean(FAIL_ON_TIMELINE_ARCHIVING_ENABLED); + return getBoolean(FAIL_ON_TIMELINE_ARCHIVING_ENABLE); } public int getMaxConsistencyChecks() { @@ -657,16 +1004,16 @@ public class HoodieWriteConfig extends HoodieConfig { } public BulkInsertSortMode getBulkInsertSortMode() { - String sortMode = getString(BULKINSERT_SORT_MODE); + String sortMode = getString(BULK_INSERT_SORT_MODE); return BulkInsertSortMode.valueOf(sortMode.toUpperCase()); } public boolean isMergeDataValidationCheckEnabled() { - return getBoolean(MERGE_DATA_VALIDATION_CHECK_ENABLED); + return getBoolean(MERGE_DATA_VALIDATION_CHECK_ENABLE); } public boolean allowDuplicateInserts() { - return getBoolean(MERGE_ALLOW_DUPLICATE_ON_INSERTS); + return getBoolean(MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE); } public EngineType getEngineType() { @@ -674,8 +1021,8 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean populateMetaFields() { - return Boolean.parseBoolean(getStringOrDefault(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS, - HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue())); + return Boolean.parseBoolean(getStringOrDefault(HoodieTableConfig.POPULATE_META_FIELDS, + HoodieTableConfig.POPULATE_META_FIELDS.defaultValue())); } /** @@ -702,7 +1049,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getParquetSmallFileLimit() { - return getInt(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT_BYTES); + return getInt(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT); } public double getRecordSizeEstimationThreshold() { @@ -710,19 +1057,19 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getCopyOnWriteInsertSplitSize() { - return getInt(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE); + return getInt(HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE); } public int getCopyOnWriteRecordSizeEstimate() { - return getInt(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE); + return getInt(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE); } public boolean shouldAutoTuneInsertSplits() { - return getBoolean(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS); + return getBoolean(HoodieCompactionConfig.COPY_ON_WRITE_AUTO_SPLIT_INSERTS); } public int getCleanerParallelism() { - return getInt(HoodieCompactionConfig.CLEANER_PARALLELISM); + return getInt(HoodieCompactionConfig.CLEANER_PARALLELISM_VALUE); } public boolean isAutoClean() { @@ -734,7 +1081,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean incrementalCleanerModeEnabled() { - return getBoolean(HoodieCompactionConfig.CLEANER_INCREMENTAL_MODE); + return getBoolean(HoodieCompactionConfig.CLEANER_INCREMENTAL_MODE_ENABLE); } public boolean inlineCompactionEnabled() { @@ -762,11 +1109,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public Boolean getCompactionLazyBlockReadEnabled() { - return getBoolean(HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLED); + return getBoolean(HoodieCompactionConfig.COMPACTION_LAZY_BLOCK_READ_ENABLE); } public Boolean getCompactionReverseLogReadEnabled() { - return getBoolean(HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLED); + return getBoolean(HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLE); } public boolean inlineClusteringEnabled() { @@ -778,7 +1125,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean isPreserveHoodieCommitMetadata() { - return getBoolean(HoodieClusteringConfig.CLUSTERING_PRESERVE_HOODIE_COMMIT_METADATA); + return getBoolean(HoodieClusteringConfig.PRESERVE_COMMIT_METADATA); } public boolean isClusteringEnabled() { @@ -787,15 +1134,15 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getInlineClusterMaxCommits() { - return getInt(HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT); + return getInt(HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMITS); } public int getAsyncClusterMaxCommits() { - return getInt(HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMIT_PROP); + return getInt(HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS); } public String getPayloadClass() { - return getString(HoodieCompactionConfig.PAYLOAD_CLASS); + return getString(HoodieCompactionConfig.PAYLOAD_CLASS_NAME); } public int getTargetPartitionsPerDayBasedCompaction() { @@ -807,11 +1154,11 @@ public class HoodieWriteConfig extends HoodieConfig { } public Boolean shouldCleanBootstrapBaseFile() { - return getBoolean(HoodieCompactionConfig.CLEANER_BOOTSTRAP_BASE_FILE_ENABLED); + return getBoolean(HoodieCompactionConfig.CLEANER_BOOTSTRAP_BASE_FILE_ENABLE); } public String getClusteringUpdatesStrategyClass() { - return getString(HoodieClusteringConfig.CLUSTERING_UPDATES_STRATEGY); + return getString(HoodieClusteringConfig.UPDATES_STRATEGY); } public HoodieFailedWritesCleaningPolicy getFailedWritesCleanPolicy() { @@ -823,39 +1170,39 @@ public class HoodieWriteConfig extends HoodieConfig { * Clustering properties. */ public String getClusteringPlanStrategyClass() { - return getString(HoodieClusteringConfig.CLUSTERING_PLAN_STRATEGY_CLASS); + return getString(HoodieClusteringConfig.PLAN_STRATEGY_CLASS_NAME); } public String getClusteringExecutionStrategyClass() { - return getString(HoodieClusteringConfig.CLUSTERING_EXECUTION_STRATEGY_CLASS); + return getString(HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME); } public long getClusteringMaxBytesInGroup() { - return getLong(HoodieClusteringConfig.CLUSTERING_MAX_BYTES_PER_GROUP); + return getLong(HoodieClusteringConfig.PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP); } public long getClusteringSmallFileLimit() { - return getLong(HoodieClusteringConfig.CLUSTERING_PLAN_SMALL_FILE_LIMIT); + return getLong(HoodieClusteringConfig.PLAN_STRATEGY_SMALL_FILE_LIMIT); } public int getClusteringMaxNumGroups() { - return getInt(HoodieClusteringConfig.CLUSTERING_MAX_NUM_GROUPS); + return getInt(HoodieClusteringConfig.PLAN_STRATEGY_MAX_GROUPS); } public long getClusteringTargetFileMaxBytes() { - return getLong(HoodieClusteringConfig.CLUSTERING_TARGET_FILE_MAX_BYTES); + return getLong(HoodieClusteringConfig.PLAN_STRATEGY_TARGET_FILE_MAX_BYTES); } public int getTargetPartitionsForClustering() { - return getInt(HoodieClusteringConfig.CLUSTERING_TARGET_PARTITIONS); + return getInt(HoodieClusteringConfig.DAYBASED_LOOKBACK_PARTITIONS); } public int getSkipPartitionsFromLatestForClustering() { - return getInt(HoodieClusteringConfig.CLUSTERING_SKIP_PARTITIONS_FROM_LATEST); + return getInt(HoodieClusteringConfig.PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST); } public String getClusteringSortColumns() { - return getString(HoodieClusteringConfig.CLUSTERING_SORT_COLUMNS_PROPERTY); + return getString(HoodieClusteringConfig.PLAN_STRATEGY_SORT_COLUMNS); } /** @@ -866,79 +1213,79 @@ public class HoodieWriteConfig extends HoodieConfig { } public String getIndexClass() { - return getString(HoodieIndexConfig.INDEX_CLASS); + return getString(HoodieIndexConfig.INDEX_CLASS_NAME); } public int getBloomFilterNumEntries() { - return getInt(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES); + return getInt(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES_VALUE); } public double getBloomFilterFPP() { - return getDouble(HoodieIndexConfig.BLOOM_FILTER_FPP); + return getDouble(HoodieIndexConfig.BLOOM_FILTER_FPP_VALUE); } public String getHbaseZkQuorum() { - return getString(HoodieHBaseIndexConfig.HBASE_ZKQUORUM); + return getString(HoodieHBaseIndexConfig.ZKQUORUM); } public int getHbaseZkPort() { - return getInt(HoodieHBaseIndexConfig.HBASE_ZKPORT); + return getInt(HoodieHBaseIndexConfig.ZKPORT); } public String getHBaseZkZnodeParent() { - return getString(HoodieHBaseIndexConfig.HBASE_ZK_ZNODEPARENT); + return getString(HoodieHBaseIndexConfig.ZK_NODE_PATH); } public String getHbaseTableName() { - return getString(HoodieHBaseIndexConfig.HBASE_TABLENAME); + return getString(HoodieHBaseIndexConfig.TABLENAME); } public int getHbaseIndexGetBatchSize() { - return getInt(HoodieHBaseIndexConfig.HBASE_GET_BATCH_SIZE); + return getInt(HoodieHBaseIndexConfig.GET_BATCH_SIZE); } public Boolean getHBaseIndexRollbackSync() { - return getBoolean(HoodieHBaseIndexConfig.HBASE_INDEX_ROLLBACK_SYNC); + return getBoolean(HoodieHBaseIndexConfig.ROLLBACK_SYNC_ENABLE); } public int getHbaseIndexPutBatchSize() { - return getInt(HoodieHBaseIndexConfig.HBASE_PUT_BATCH_SIZE); + return getInt(HoodieHBaseIndexConfig.PUT_BATCH_SIZE); } public Boolean getHbaseIndexPutBatchSizeAutoCompute() { - return getBoolean(HoodieHBaseIndexConfig.HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE); + return getBoolean(HoodieHBaseIndexConfig.PUT_BATCH_SIZE_AUTO_COMPUTE); } public String getHBaseQPSResourceAllocatorClass() { - return getString(HoodieHBaseIndexConfig.HBASE_INDEX_QPS_ALLOCATOR_CLASS); + return getString(HoodieHBaseIndexConfig.QPS_ALLOCATOR_CLASS_NAME); } public String getHBaseQPSZKnodePath() { - return getString(HoodieHBaseIndexConfig.HBASE_ZK_PATH_QPS_ROOT); + return getString(HoodieHBaseIndexConfig.ZKPATH_QPS_ROOT); } public String getHBaseZkZnodeSessionTimeout() { - return getString(HoodieHBaseIndexConfig.HOODIE_INDEX_HBASE_ZK_SESSION_TIMEOUT_MS); + return getString(HoodieHBaseIndexConfig.ZK_SESSION_TIMEOUT_MS); } public String getHBaseZkZnodeConnectionTimeout() { - return getString(HoodieHBaseIndexConfig.HOODIE_INDEX_HBASE_ZK_CONNECTION_TIMEOUT_MS); + return getString(HoodieHBaseIndexConfig.ZK_CONNECTION_TIMEOUT_MS); } public boolean getHBaseIndexShouldComputeQPSDynamically() { - return getBoolean(HoodieHBaseIndexConfig.HOODIE_INDEX_COMPUTE_QPS_DYNAMICALLY); + return getBoolean(HoodieHBaseIndexConfig.COMPUTE_QPS_DYNAMICALLY); } public int getHBaseIndexDesiredPutsTime() { - return getInt(HoodieHBaseIndexConfig.HOODIE_INDEX_DESIRED_PUTS_TIME_IN_SECS); + return getInt(HoodieHBaseIndexConfig.DESIRED_PUTS_TIME_IN_SECONDS); } public String getBloomFilterType() { - return getString(HoodieIndexConfig.BLOOM_INDEX_FILTER_TYPE); + return getString(HoodieIndexConfig.BLOOM_FILTER_TYPE); } public int getDynamicBloomFilterMaxNumEntries() { - return getInt(HoodieIndexConfig.HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES); + return getInt(HoodieIndexConfig.BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES); } /** @@ -947,15 +1294,15 @@ public class HoodieWriteConfig extends HoodieConfig { * the jobs would be (0.17) 1/6, 0.33 (2/6) and 0.5 (3/6) respectively. */ public float getHbaseIndexQPSFraction() { - return getFloat(HoodieHBaseIndexConfig.HBASE_QPS_FRACTION); + return getFloat(HoodieHBaseIndexConfig.QPS_FRACTION); } public float getHBaseIndexMinQPSFraction() { - return getFloat(HoodieHBaseIndexConfig.HBASE_MIN_QPS_FRACTION); + return getFloat(HoodieHBaseIndexConfig.MIN_QPS_FRACTION); } public float getHBaseIndexMaxQPSFraction() { - return getFloat(HoodieHBaseIndexConfig.HBASE_MAX_QPS_FRACTION); + return getFloat(HoodieHBaseIndexConfig.MAX_QPS_FRACTION); } /** @@ -963,11 +1310,11 @@ public class HoodieWriteConfig extends HoodieConfig { * Hoodie jobs to an Hbase Region Server */ public int getHbaseIndexMaxQPSPerRegionServer() { - return getInt(HoodieHBaseIndexConfig.HBASE_MAX_QPS_PER_REGION_SERVER); + return getInt(HoodieHBaseIndexConfig.MAX_QPS_PER_REGION_SERVER); } public boolean getHbaseIndexUpdatePartitionPath() { - return getBoolean(HoodieHBaseIndexConfig.HBASE_INDEX_UPDATE_PARTITION_PATH); + return getBoolean(HoodieHBaseIndexConfig.UPDATE_PARTITION_PATH_ENABLE); } public int getBloomIndexParallelism() { @@ -995,7 +1342,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean getBloomIndexUpdatePartitionPath() { - return getBoolean(HoodieIndexConfig.BLOOM_INDEX_UPDATE_PARTITION_PATH); + return getBoolean(HoodieIndexConfig.BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE); } public int getSimpleIndexParallelism() { @@ -1011,58 +1358,58 @@ public class HoodieWriteConfig extends HoodieConfig { } public boolean getGlobalSimpleIndexUpdatePartitionPath() { - return getBoolean(HoodieIndexConfig.SIMPLE_INDEX_UPDATE_PARTITION_PATH); + return getBoolean(HoodieIndexConfig.SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE); } /** * storage properties. */ public long getParquetMaxFileSize() { - return getLong(HoodieStorageConfig.PARQUET_FILE_MAX_BYTES); + return getLong(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE); } public int getParquetBlockSize() { - return getInt(HoodieStorageConfig.PARQUET_BLOCK_SIZE_BYTES); + return getInt(HoodieStorageConfig.PARQUET_BLOCK_SIZE); } public int getParquetPageSize() { - return getInt(HoodieStorageConfig.PARQUET_PAGE_SIZE_BYTES); + return getInt(HoodieStorageConfig.PARQUET_PAGE_SIZE); } public int getLogFileDataBlockMaxSize() { - return getInt(HoodieStorageConfig.LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES); + return getInt(HoodieStorageConfig.LOGFILE_DATA_BLOCK_MAX_SIZE); } public int getLogFileMaxSize() { - return getInt(HoodieStorageConfig.LOGFILE_SIZE_MAX_BYTES); + return getInt(HoodieStorageConfig.LOGFILE_MAX_SIZE); } public double getParquetCompressionRatio() { - return getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO); + return getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION); } public CompressionCodecName getParquetCompressionCodec() { - return CompressionCodecName.fromConf(getString(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC)); + return CompressionCodecName.fromConf(getString(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC_NAME)); } public double getLogFileToParquetCompressionRatio() { - return getDouble(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO); + return getDouble(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION); } public long getHFileMaxFileSize() { - return getLong(HoodieStorageConfig.HFILE_FILE_MAX_BYTES); + return getLong(HoodieStorageConfig.HFILE_MAX_FILE_SIZE); } public int getHFileBlockSize() { - return getInt(HoodieStorageConfig.HFILE_BLOCK_SIZE_BYTES); + return getInt(HoodieStorageConfig.HFILE_BLOCK_SIZE); } public Compression.Algorithm getHFileCompressionAlgorithm() { - return Compression.Algorithm.valueOf(getString(HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM)); + return Compression.Algorithm.valueOf(getString(HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME)); } public long getOrcMaxFileSize() { - return getLong(HoodieStorageConfig.ORC_FILE_MAX_BYTES); + return getLong(HoodieStorageConfig.ORC_FILE_MAX_SIZE); } public int getOrcStripeSize() { @@ -1074,114 +1421,114 @@ public class HoodieWriteConfig extends HoodieConfig { } public CompressionKind getOrcCompressionCodec() { - return CompressionKind.valueOf(getString(HoodieStorageConfig.ORC_COMPRESSION_CODEC)); + return CompressionKind.valueOf(getString(HoodieStorageConfig.ORC_COMPRESSION_CODEC_NAME)); } /** * metrics properties. */ public boolean isMetricsOn() { - return getBoolean(HoodieMetricsConfig.METRICS_ON); + return getBoolean(HoodieMetricsConfig.TURN_METRICS_ON); } public boolean isExecutorMetricsEnabled() { return Boolean.parseBoolean( - getStringOrDefault(HoodieMetricsConfig.ENABLE_EXECUTOR_METRICS, "false")); + getStringOrDefault(HoodieMetricsConfig.EXECUTOR_METRICS_ENABLE, "false")); } public MetricsReporterType getMetricsReporterType() { - return MetricsReporterType.valueOf(getString(HoodieMetricsConfig.METRICS_REPORTER_TYPE)); + return MetricsReporterType.valueOf(getString(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE)); } public String getGraphiteServerHost() { - return getString(HoodieMetricsConfig.GRAPHITE_SERVER_HOST); + return getString(HoodieMetricsConfig.GRAPHITE_SERVER_HOST_NAME); } public int getGraphiteServerPort() { - return getInt(HoodieMetricsConfig.GRAPHITE_SERVER_PORT); + return getInt(HoodieMetricsConfig.GRAPHITE_SERVER_PORT_NUM); } public String getGraphiteMetricPrefix() { - return getString(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX); + return getString(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX_VALUE); } public String getJmxHost() { - return getString(HoodieMetricsConfig.JMX_HOST); + return getString(HoodieMetricsConfig.JMX_HOST_NAME); } public String getJmxPort() { - return getString(HoodieMetricsConfig.JMX_PORT); + return getString(HoodieMetricsConfig.JMX_PORT_NUM); } public int getDatadogReportPeriodSeconds() { - return getInt(HoodieMetricsDatadogConfig.DATADOG_REPORT_PERIOD_SECONDS); + return getInt(HoodieMetricsDatadogConfig.REPORT_PERIOD_IN_SECONDS); } public ApiSite getDatadogApiSite() { - return ApiSite.valueOf(getString(HoodieMetricsDatadogConfig.DATADOG_API_SITE)); + return ApiSite.valueOf(getString(HoodieMetricsDatadogConfig.API_SITE_VALUE)); } public String getDatadogApiKey() { - if (props.containsKey(HoodieMetricsDatadogConfig.DATADOG_API_KEY.key())) { - return getString(HoodieMetricsDatadogConfig.DATADOG_API_KEY); + if (props.containsKey(HoodieMetricsDatadogConfig.API_KEY.key())) { + return getString(HoodieMetricsDatadogConfig.API_KEY); } else { Supplier apiKeySupplier = ReflectionUtils.loadClass( - getString(HoodieMetricsDatadogConfig.DATADOG_API_KEY_SUPPLIER)); + getString(HoodieMetricsDatadogConfig.API_KEY_SUPPLIER)); return apiKeySupplier.get(); } } public boolean getDatadogApiKeySkipValidation() { - return getBoolean(HoodieMetricsDatadogConfig.DATADOG_API_KEY_SKIP_VALIDATION); + return getBoolean(HoodieMetricsDatadogConfig.API_KEY_SKIP_VALIDATION); } public int getDatadogApiTimeoutSeconds() { - return getInt(HoodieMetricsDatadogConfig.DATADOG_API_TIMEOUT_SECONDS); + return getInt(HoodieMetricsDatadogConfig.API_TIMEOUT_IN_SECONDS); } public String getDatadogMetricPrefix() { - return getString(HoodieMetricsDatadogConfig.DATADOG_METRIC_PREFIX); + return getString(HoodieMetricsDatadogConfig.METRIC_PREFIX_VALUE); } public String getDatadogMetricHost() { - return getString(HoodieMetricsDatadogConfig.DATADOG_METRIC_HOST); + return getString(HoodieMetricsDatadogConfig.METRIC_HOST_NAME); } public List getDatadogMetricTags() { return Arrays.stream(getStringOrDefault( - HoodieMetricsDatadogConfig.DATADOG_METRIC_TAGS, ",").split("\\s*,\\s*")).collect(Collectors.toList()); + HoodieMetricsDatadogConfig.METRIC_TAG_VALUES, ",").split("\\s*,\\s*")).collect(Collectors.toList()); } public String getMetricReporterClassName() { - return getString(HoodieMetricsConfig.METRICS_REPORTER_CLASS); + return getString(HoodieMetricsConfig.METRICS_REPORTER_CLASS_NAME); } public int getPrometheusPort() { - return getInt(HoodieMetricsPrometheusConfig.PROMETHEUS_PORT); + return getInt(HoodieMetricsPrometheusConfig.PROMETHEUS_PORT_NUM); } public String getPushGatewayHost() { - return getString(HoodieMetricsPrometheusConfig.PUSHGATEWAY_HOST); + return getString(HoodieMetricsPrometheusConfig.PUSHGATEWAY_HOST_NAME); } public int getPushGatewayPort() { - return getInt(HoodieMetricsPrometheusConfig.PUSHGATEWAY_PORT); + return getInt(HoodieMetricsPrometheusConfig.PUSHGATEWAY_PORT_NUM); } public int getPushGatewayReportPeriodSeconds() { - return getInt(HoodieMetricsPrometheusConfig.PUSHGATEWAY_REPORT_PERIOD_SECONDS); + return getInt(HoodieMetricsPrometheusConfig.PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS); } public boolean getPushGatewayDeleteOnShutdown() { - return getBoolean(HoodieMetricsPrometheusConfig.PUSHGATEWAY_DELETE_ON_SHUTDOWN); + return getBoolean(HoodieMetricsPrometheusConfig.PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE); } public String getPushGatewayJobName() { - return getString(HoodieMetricsPrometheusConfig.PUSHGATEWAY_JOB_NAME); + return getString(HoodieMetricsPrometheusConfig.PUSHGATEWAY_JOBNAME); } public boolean getPushGatewayRandomJobNameSuffix() { - return getBoolean(HoodieMetricsPrometheusConfig.PUSHGATEWAY_RANDOM_JOB_NAME_SUFFIX); + return getBoolean(HoodieMetricsPrometheusConfig.PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX); } /** @@ -1239,47 +1586,47 @@ public class HoodieWriteConfig extends HoodieConfig { * Commit call back configs. */ public boolean writeCommitCallbackOn() { - return getBoolean(HoodieWriteCommitCallbackConfig.CALLBACK_ON); + return getBoolean(HoodieWriteCommitCallbackConfig.TURN_CALLBACK_ON); } public String getCallbackClass() { - return getString(HoodieWriteCommitCallbackConfig.CALLBACK_CLASS); + return getString(HoodieWriteCommitCallbackConfig.CALLBACK_CLASS_NAME); } public String getBootstrapSourceBasePath() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH); + return getString(HoodieBootstrapConfig.BASE_PATH); } public String getBootstrapModeSelectorClass() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR); + return getString(HoodieBootstrapConfig.MODE_SELECTOR_CLASS_NAME); } public String getFullBootstrapInputProvider() { - return getString(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER); + return getString(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME); } public String getBootstrapKeyGeneratorClass() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS); + return getString(HoodieBootstrapConfig.KEYGEN_CLASS_NAME); } public String getBootstrapKeyGeneratorType() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_TYPE); + return getString(HoodieBootstrapConfig.KEYGEN_TYPE); } public String getBootstrapModeSelectorRegex() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX); + return getString(HoodieBootstrapConfig.PARTITION_SELECTOR_REGEX_PATTERN); } public BootstrapMode getBootstrapModeForRegexMatch() { - return BootstrapMode.valueOf(getString(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX_MODE)); + return BootstrapMode.valueOf(getString(HoodieBootstrapConfig.PARTITION_SELECTOR_REGEX_MODE)); } public String getBootstrapPartitionPathTranslatorClass() { - return getString(HoodieBootstrapConfig.BOOTSTRAP_PARTITION_PATH_TRANSLATOR_CLASS); + return getString(HoodieBootstrapConfig.PARTITION_PATH_TRANSLATOR_CLASS_NAME); } public int getBootstrapParallelism() { - return getInt(HoodieBootstrapConfig.BOOTSTRAP_PARALLELISM); + return getInt(HoodieBootstrapConfig.PARALLELISM_VALUE); } public Long getMaxMemoryPerPartitionMerge() { @@ -1306,27 +1653,27 @@ public class HoodieWriteConfig extends HoodieConfig { } public int getMetadataInsertParallelism() { - return getInt(HoodieMetadataConfig.METADATA_INSERT_PARALLELISM_PROP); + return getInt(HoodieMetadataConfig.INSERT_PARALLELISM_VALUE); } public int getMetadataCompactDeltaCommitMax() { - return getInt(HoodieMetadataConfig.METADATA_COMPACT_NUM_DELTA_COMMITS_PROP); + return getInt(HoodieMetadataConfig.COMPACT_NUM_DELTA_COMMITS); } public boolean isMetadataAsyncClean() { - return getBoolean(HoodieMetadataConfig.METADATA_ASYNC_CLEAN_PROP); + return getBoolean(HoodieMetadataConfig.ASYNC_CLEAN_ENABLE); } public int getMetadataMaxCommitsToKeep() { - return getInt(HoodieMetadataConfig.MAX_COMMITS_TO_KEEP_PROP); + return getInt(HoodieMetadataConfig.MAX_COMMITS_TO_KEEP); } public int getMetadataMinCommitsToKeep() { - return getInt(HoodieMetadataConfig.MIN_COMMITS_TO_KEEP_PROP); + return getInt(HoodieMetadataConfig.MIN_COMMITS_TO_KEEP); } public int getMetadataCleanerCommitsRetained() { - return getInt(HoodieMetadataConfig.CLEANER_COMMITS_RETAINED_PROP); + return getInt(HoodieMetadataConfig.CLEANER_COMMITS_RETAINED); } /** @@ -1335,7 +1682,7 @@ public class HoodieWriteConfig extends HoodieConfig { */ public String getLockProviderClass() { - return getString(HoodieLockConfig.LOCK_PROVIDER_CLASS); + return getString(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME); } public String getLockHiveDatabaseName() { @@ -1347,7 +1694,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public ConflictResolutionStrategy getWriteConflictResolutionStrategy() { - return ReflectionUtils.loadClass(getString(HoodieLockConfig.WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS)); + return ReflectionUtils.loadClass(getString(HoodieLockConfig.WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME)); } public Long getLockAcquireWaitTimeoutInMs() { @@ -1355,7 +1702,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public WriteConcurrencyMode getWriteConcurrencyMode() { - return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MOD)); + return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MODE)); } public Boolean inlineTableServices() { @@ -1365,21 +1712,21 @@ public class HoodieWriteConfig extends HoodieConfig { public String getWriteMetaKeyPrefixes() { return getString(WRITE_META_KEY_PREFIXES); } - + public String getPreCommitValidators() { - return getString(HoodiePreCommitValidatorConfig.PRE_COMMIT_VALIDATORS); + return getString(HoodiePreCommitValidatorConfig.VALIDATOR_CLASS_NAMES); } public String getPreCommitValidatorEqualitySqlQueries() { - return getString(HoodiePreCommitValidatorConfig.PRE_COMMIT_VALIDATORS_EQUALITY_SQL_QUERIES); + return getString(HoodiePreCommitValidatorConfig.EQUALITY_SQL_QUERIES); } public String getPreCommitValidatorSingleResultSqlQueries() { - return getString(HoodiePreCommitValidatorConfig.PRE_COMMIT_VALIDATORS_SINGLE_VALUE_SQL_QUERIES); + return getString(HoodiePreCommitValidatorConfig.SINGLE_VALUE_SQL_QUERIES); } public String getPreCommitValidatorInequalitySqlQueries() { - return getString(HoodiePreCommitValidatorConfig.PRE_COMMIT_VALIDATORS_INEQUALITY_SQL_QUERIES); + return getString(HoodiePreCommitValidatorConfig.INEQUALITY_SQL_QUERIES); } public boolean allowEmptyCommit() { @@ -1441,17 +1788,17 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withSchema(String schemaStr) { - writeConfig.setValue(AVRO_SCHEMA, schemaStr); + writeConfig.setValue(AVRO_SCHEMA_STRING, schemaStr); return this; } public Builder withAvroSchemaValidate(boolean enable) { - writeConfig.setValue(AVRO_SCHEMA_VALIDATE, String.valueOf(enable)); + writeConfig.setValue(AVRO_SCHEMA_VALIDATE_ENABLE, String.valueOf(enable)); return this; } public Builder forTable(String tableName) { - writeConfig.setValue(TABLE_NAME, tableName); + writeConfig.setValue(TBL_NAME, tableName); return this; } @@ -1461,53 +1808,53 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withWritePayLoad(String payload) { - writeConfig.setValue(WRITE_PAYLOAD_CLASS, payload); + writeConfig.setValue(WRITE_PAYLOAD_CLASS_NAME, payload); return this; } public Builder withKeyGenerator(String keyGeneratorClass) { - writeConfig.setValue(KEYGENERATOR_CLASS, keyGeneratorClass); + writeConfig.setValue(KEYGENERATOR_CLASS_NAME, keyGeneratorClass); return this; } public Builder withTimelineLayoutVersion(int version) { - writeConfig.setValue(TIMELINE_LAYOUT_VERSION, String.valueOf(version)); + writeConfig.setValue(TIMELINE_LAYOUT_VERSION_NUM, String.valueOf(version)); return this; } public Builder withBulkInsertParallelism(int bulkInsertParallelism) { - writeConfig.setValue(BULKINSERT_PARALLELISM, String.valueOf(bulkInsertParallelism)); + writeConfig.setValue(BULKINSERT_PARALLELISM_VALUE, String.valueOf(bulkInsertParallelism)); return this; } public Builder withUserDefinedBulkInsertPartitionerClass(String className) { - writeConfig.setValue(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS, className); + writeConfig.setValue(BULKINSERT_USER_DEFINED_PARTITIONER_CLASS_NAME, className); return this; } public Builder withDeleteParallelism(int parallelism) { - writeConfig.setValue(DELETE_PARALLELISM, String.valueOf(parallelism)); + writeConfig.setValue(DELETE_PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } public Builder withParallelism(int insertShuffleParallelism, int upsertShuffleParallelism) { - writeConfig.setValue(INSERT_PARALLELISM, String.valueOf(insertShuffleParallelism)); - writeConfig.setValue(UPSERT_PARALLELISM, String.valueOf(upsertShuffleParallelism)); + writeConfig.setValue(INSERT_PARALLELISM_VALUE, String.valueOf(insertShuffleParallelism)); + writeConfig.setValue(UPSERT_PARALLELISM_VALUE, String.valueOf(upsertShuffleParallelism)); return this; } public Builder withRollbackParallelism(int rollbackParallelism) { - writeConfig.setValue(ROLLBACK_PARALLELISM, String.valueOf(rollbackParallelism)); + writeConfig.setValue(ROLLBACK_PARALLELISM_VALUE, String.valueOf(rollbackParallelism)); return this; } public Builder withRollbackUsingMarkers(boolean rollbackUsingMarkers) { - writeConfig.setValue(ROLLBACK_USING_MARKERS, String.valueOf(rollbackUsingMarkers)); + writeConfig.setValue(ROLLBACK_USING_MARKERS_ENABLE, String.valueOf(rollbackUsingMarkers)); return this; } public Builder withWriteBufferLimitBytes(int writeBufferLimit) { - writeConfig.setValue(WRITE_BUFFER_LIMIT_BYTES, String.valueOf(writeBufferLimit)); + writeConfig.setValue(WRITE_BUFFER_LIMIT_BYTES_VALUE, String.valueOf(writeBufferLimit)); return this; } @@ -1523,7 +1870,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withWriteStatusStorageLevel(String level) { - writeConfig.setValue(WRITE_STATUS_STORAGE_LEVEL, level); + writeConfig.setValue(WRITE_STATUS_STORAGE_LEVEL_VALUE, level); return this; } @@ -1594,12 +1941,12 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withAutoCommit(boolean autoCommit) { - writeConfig.setValue(HOODIE_AUTO_COMMIT, String.valueOf(autoCommit)); + writeConfig.setValue(AUTO_COMMIT_ENABLE, String.valueOf(autoCommit)); return this; } public Builder withWriteStatusClass(Class writeStatusClass) { - writeConfig.setValue(HOODIE_WRITE_STATUS_CLASS, writeStatusClass.getName()); + writeConfig.setValue(WRITE_STATUS_CLASS_NAME, writeStatusClass.getName()); return this; } @@ -1622,7 +1969,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withFinalizeWriteParallelism(int parallelism) { - writeConfig.setValue(FINALIZE_WRITE_PARALLELISM, String.valueOf(parallelism)); + writeConfig.setValue(FINALIZE_WRITE_PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } @@ -1642,12 +1989,12 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withMarkersDeleteParallelism(int parallelism) { - writeConfig.setValue(MARKERS_DELETE_PARALLELISM, String.valueOf(parallelism)); + writeConfig.setValue(MARKERS_DELETE_PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } public Builder withEmbeddedTimelineServerEnabled(boolean enabled) { - writeConfig.setValue(EMBEDDED_TIMELINE_SERVER_ENABLED, String.valueOf(enabled)); + writeConfig.setValue(EMBEDDED_TIMELINE_SERVER_ENABLE, String.valueOf(enabled)); return this; } @@ -1657,32 +2004,32 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withEmbeddedTimelineServerPort(int port) { - writeConfig.setValue(EMBEDDED_TIMELINE_SERVER_PORT, String.valueOf(port)); + writeConfig.setValue(EMBEDDED_TIMELINE_SERVER_PORT_NUM, String.valueOf(port)); return this; } public Builder withBulkInsertSortMode(String mode) { - writeConfig.setValue(BULKINSERT_SORT_MODE, mode); + writeConfig.setValue(BULK_INSERT_SORT_MODE, mode); return this; } public Builder withAllowMultiWriteOnSameInstant(boolean allow) { - writeConfig.setValue(ALLOW_MULTI_WRITE_ON_SAME_INSTANT, String.valueOf(allow)); + writeConfig.setValue(ALLOW_MULTI_WRITE_ON_SAME_INSTANT_ENABLE, String.valueOf(allow)); return this; } public Builder withExternalSchemaTrasformation(boolean enabled) { - writeConfig.setValue(EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION, String.valueOf(enabled)); + writeConfig.setValue(AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE, String.valueOf(enabled)); return this; } public Builder withMergeDataValidationCheckEnabled(boolean enabled) { - writeConfig.setValue(MERGE_DATA_VALIDATION_CHECK_ENABLED, String.valueOf(enabled)); + writeConfig.setValue(MERGE_DATA_VALIDATION_CHECK_ENABLE, String.valueOf(enabled)); return this; } public Builder withMergeAllowDuplicateOnInserts(boolean routeInsertsToNewFiles) { - writeConfig.setValue(MERGE_ALLOW_DUPLICATE_ON_INSERTS, String.valueOf(routeInsertsToNewFiles)); + writeConfig.setValue(MERGE_ALLOW_DUPLICATE_ON_INSERTS_ENABLE, String.valueOf(routeInsertsToNewFiles)); return this; } @@ -1697,7 +2044,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withWriteConcurrencyMode(WriteConcurrencyMode concurrencyMode) { - writeConfig.setValue(WRITE_CONCURRENCY_MOD, concurrencyMode.value()); + writeConfig.setValue(WRITE_CONCURRENCY_MODE, concurrencyMode.value()); return this; } @@ -1707,7 +2054,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withPopulateMetaFields(boolean populateMetaFields) { - writeConfig.setValue(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); + writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); return this; } @@ -1754,15 +2101,15 @@ public class HoodieWriteConfig extends HoodieConfig { HoodieLockConfig.newBuilder().fromProperties(writeConfig.getProps()).build()); writeConfig.setDefaultOnCondition(!isPreCommitValidationConfigSet, HoodiePreCommitValidatorConfig.newBuilder().fromProperties(writeConfig.getProps()).build()); - writeConfig.setDefaultValue(TIMELINE_LAYOUT_VERSION, String.valueOf(TimelineLayoutVersion.CURR_VERSION)); + writeConfig.setDefaultValue(TIMELINE_LAYOUT_VERSION_NUM, String.valueOf(TimelineLayoutVersion.CURR_VERSION)); } private void validate() { - String layoutVersion = writeConfig.getString(TIMELINE_LAYOUT_VERSION); + String layoutVersion = writeConfig.getString(TIMELINE_LAYOUT_VERSION_NUM); // Ensure Layout Version is good new TimelineLayoutVersion(Integer.parseInt(layoutVersion)); Objects.requireNonNull(writeConfig.getString(BASE_PATH)); - if (writeConfig.getString(WRITE_CONCURRENCY_MOD) + if (writeConfig.getString(WRITE_CONCURRENCY_MODE) .equalsIgnoreCase(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL.name())) { ValidationUtils.checkArgument(writeConfig.getString(HoodieCompactionConfig.FAILED_WRITES_CLEANER_POLICY) != HoodieFailedWritesCleaningPolicy.EAGER.name(), "To enable optimistic concurrency control, set hoodie.cleaner.policy.failed.writes=LAZY"); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java index 0046c1802..4923d980c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java @@ -186,7 +186,7 @@ public class KeyGenUtils { */ public static KeyGenerator createKeyGeneratorByClassName(TypedProperties props) throws IOException { KeyGenerator keyGenerator = null; - String keyGeneratorClass = props.getString(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), null); + String keyGeneratorClass = props.getString(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), null); if (!StringUtils.isNullOrEmpty(keyGeneratorClass)) { try { keyGenerator = (KeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass, props); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java index 6010ab9d5..0e17aff24 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java @@ -41,7 +41,7 @@ import java.util.Objects; /** * Factory help to create {@link org.apache.hudi.keygen.KeyGenerator}. *

- * This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS} firstly, this ensures the class prop + * This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS_NAME} firstly, this ensures the class prop * will not be overwritten by {@link KeyGeneratorType} */ public class HoodieAvroKeyGeneratorFactory { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 65c51f0bd..ab5fa994d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -67,7 +67,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER; import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX; import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP; @@ -299,9 +299,9 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta LOG.info("Creating a new metadata table in " + metadataWriteConfig.getBasePath() + " at instant " + createInstantTime); HoodieTableMetaClient.withPropertyBuilder() - .setTableType(HoodieTableType.MERGE_ON_READ) - .setTableName(tableName) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setTableType(HoodieTableType.MERGE_ON_READ) + .setTableName(tableName) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .setPayloadClassName(HoodieMetadataPayload.class.getName()) .setBaseFileFormat(HoodieFileFormat.HFILE.toString()) .initTable(hadoopConf.get(), metadataWriteConfig.getBasePath()); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java index 87dbd06ac..ad40c8ec7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -18,10 +18,6 @@ package org.apache.hudi.table; -import org.apache.avro.Schema; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanerPlan; @@ -71,6 +67,10 @@ import org.apache.hudi.table.action.bootstrap.HoodieBootstrapWriteMetadata; import org.apache.hudi.table.marker.WriteMarkers; import org.apache.hudi.table.marker.WriteMarkersFactory; +import org.apache.avro.Schema; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -114,7 +114,7 @@ public abstract class HoodieTable implem HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(config.getMetadataConfig().getProps()) .build(); this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath(), - FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue()); + FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue()); this.viewManager = FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig(), () -> metadata); this.metaClient = metaClient; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java index c63a68347..317512f76 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java @@ -18,9 +18,6 @@ package org.apache.hudi.table; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; import org.apache.hudi.client.utils.MetadataConversionUtils; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -49,6 +46,9 @@ import org.apache.hudi.metadata.HoodieTableMetadata; import org.apache.hudi.table.marker.WriteMarkers; import org.apache.hudi.table.marker.WriteMarkersFactory; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -204,7 +204,7 @@ public class HoodieTimelineArchiveLog { // instant on the metadata table. This is required for metadata table sync. if (config.isMetadataTableEnabled()) { try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(), - config.getBasePath(), FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue())) { + config.getBasePath(), FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue())) { Option lastSyncedInstantTime = tableMetadata.getUpdateTime(); if (lastSyncedInstantTime.isPresent()) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java index a83261564..6a43f61c5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java @@ -32,9 +32,9 @@ public abstract class BaseOneToTwoUpgradeHandler implements UpgradeHandler { @Override public Map upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) { Map tablePropsToAdd = new HashMap<>(); - tablePropsToAdd.put(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP, getPartitionColumns(config)); - tablePropsToAdd.put(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); - tablePropsToAdd.put(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP, config.getString(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP)); + tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config)); + tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); + tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT)); return tablePropsToAdd; } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java index 9fbe6d933..68143a215 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java @@ -69,7 +69,7 @@ public class TestHoodieOrcReaderWriter { Configuration conf = new Configuration(); int orcStripSize = Integer.parseInt(HoodieStorageConfig.ORC_STRIPE_SIZE.defaultValue()); int orcBlockSize = Integer.parseInt(HoodieStorageConfig.ORC_BLOCK_SIZE.defaultValue()); - int maxFileSize = Integer.parseInt(HoodieStorageConfig.ORC_FILE_MAX_BYTES.defaultValue()); + int maxFileSize = Integer.parseInt(HoodieStorageConfig.ORC_FILE_MAX_SIZE.defaultValue()); HoodieOrcConfig config = new HoodieOrcConfig(conf, CompressionKind.ZLIB, orcStripSize, orcBlockSize, maxFileSize, filter); TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class); String instantTime = "000"; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java index 11771c378..c17bc1bde 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java @@ -44,7 +44,7 @@ public class TestHoodieAvroKeyGeneratorFactory { // set KeyGenerator class only props = getCommonProps(); - props.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), SimpleAvroKeyGenerator.class.getName()); + props.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), SimpleAvroKeyGenerator.class.getName()); KeyGenerator keyGenerator2 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props); Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator2.getClass().getName()); @@ -56,7 +56,7 @@ public class TestHoodieAvroKeyGeneratorFactory { // set wrong class name final TypedProperties props2 = getCommonProps(); - props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), TestHoodieAvroKeyGeneratorFactory.class.getName()); + props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), TestHoodieAvroKeyGeneratorFactory.class.getName()); assertThrows(IOException.class, () -> HoodieAvroKeyGeneratorFactory.createKeyGenerator(props2)); // set wrong keyGenerator type diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java index f4ecf42bb..016e070d7 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java @@ -33,7 +33,7 @@ public class TestHoodieMetricsDatadogConfig { @Test public void getDatadogMetricTagsShouldReturnEmptyListWhenNotSet() { HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").build(); - writeConfig.getProps().remove(HoodieMetricsDatadogConfig.DATADOG_METRIC_TAGS.key()); + writeConfig.getProps().remove(HoodieMetricsDatadogConfig.METRIC_TAG_VALUES.key()); assertIterableEquals(Collections.emptyList(), writeConfig.getDatadogMetricTags()); } } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java index 353690d4c..0c4c7712a 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java @@ -92,12 +92,12 @@ public class HoodieWriteableTestTable extends HoodieTestTable { FileCreateUtils.createPartitionMetaFile(basePath, partition); String fileName = baseFileName(currentInstantTime, fileId); - if (HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().equals(HoodieFileFormat.PARQUET)) { + if (HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().equals(HoodieFileFormat.PARQUET)) { HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport( new AvroSchemaConverter().convert(schema), schema, Option.of(filter)); HoodieAvroParquetConfig config = new HoodieAvroParquetConfig(writeSupport, CompressionCodecName.GZIP, ParquetWriter.DEFAULT_BLOCK_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE, 120 * 1024 * 1024, - new Configuration(), Double.parseDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO.defaultValue())); + new Configuration(), Double.parseDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue())); try (HoodieParquetWriter writer = new HoodieParquetWriter( currentInstantTime, new Path(Paths.get(basePath, partition, fileName).toString()), @@ -115,11 +115,11 @@ public class HoodieWriteableTestTable extends HoodieTestTable { } } } - } else if (HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().equals(HoodieFileFormat.ORC)) { + } else if (HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().equals(HoodieFileFormat.ORC)) { Configuration conf = new Configuration(); int orcStripSize = Integer.parseInt(HoodieStorageConfig.ORC_STRIPE_SIZE.defaultValue()); int orcBlockSize = Integer.parseInt(HoodieStorageConfig.ORC_BLOCK_SIZE.defaultValue()); - int maxFileSize = Integer.parseInt(HoodieStorageConfig.ORC_FILE_MAX_BYTES.defaultValue()); + int maxFileSize = Integer.parseInt(HoodieStorageConfig.ORC_FILE_MAX_SIZE.defaultValue()); HoodieOrcConfig config = new HoodieOrcConfig(conf, CompressionKind.ZLIB, orcStripSize, orcBlockSize, maxFileSize, filter); try (HoodieOrcWriter writer = new HoodieOrcWriter( currentInstantTime, diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java index aa68f3022..d58619b2b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkSizeBasedClusteringPlanStrategy.java @@ -32,6 +32,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieSparkCopyOnWriteTable; import org.apache.hudi.table.HoodieSparkMergeOnReadTable; import org.apache.hudi.table.action.cluster.strategy.PartitionAwareClusteringPlanStrategy; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -42,7 +43,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; -import static org.apache.hudi.config.HoodieClusteringConfig.CLUSTERING_SORT_COLUMNS_PROPERTY; +import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_STRATEGY_SORT_COLUMNS; /** * Clustering Strategy based on following. @@ -102,7 +103,7 @@ public class SparkSizeBasedClusteringPlanStrategy getStrategyParams() { Map params = new HashMap<>(); if (!StringUtils.isNullOrEmpty(getWriteConfig().getClusteringSortColumns())) { - params.put(CLUSTERING_SORT_COLUMNS_PROPERTY.key(), getWriteConfig().getClusteringSortColumns()); + params.put(PLAN_STRATEGY_SORT_COLUMNS.key(), getWriteConfig().getClusteringSortColumns()); } return params; } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkSortAndSizeExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkSortAndSizeExecutionStrategy.java index 4dd5cd0b5..1df6eee90 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkSortAndSizeExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkSortAndSizeExecutionStrategy.java @@ -18,7 +18,6 @@ package org.apache.hudi.client.clustering.run.strategy; -import org.apache.avro.Schema; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -32,6 +31,8 @@ import org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner; import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.commit.SparkBulkInsertHelper; + +import org.apache.avro.Schema; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -40,7 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import static org.apache.hudi.config.HoodieClusteringConfig.CLUSTERING_SORT_COLUMNS_PROPERTY; +import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_STRATEGY_SORT_COLUMNS; /** * Clustering Strategy based on following. @@ -63,11 +64,11 @@ public class SparkSortAndSizeExecutionStrategy> final List fileGroupIdList, final boolean preserveHoodieMetadata) { LOG.info("Starting clustering for a group, parallelism:" + numOutputGroups + " commit:" + instantTime); Properties props = getWriteConfig().getProps(); - props.put(HoodieWriteConfig.BULKINSERT_PARALLELISM.key(), String.valueOf(numOutputGroups)); + props.put(HoodieWriteConfig.BULKINSERT_PARALLELISM_VALUE.key(), String.valueOf(numOutputGroups)); // We are calling another action executor - disable auto commit. Strategy is only expected to write data in new files. - props.put(HoodieWriteConfig.HOODIE_AUTO_COMMIT.key(), Boolean.FALSE.toString()); - props.put(HoodieStorageConfig.PARQUET_FILE_MAX_BYTES.key(), String.valueOf(getWriteConfig().getClusteringTargetFileMaxBytes())); - HoodieWriteConfig newConfig = HoodieWriteConfig.newBuilder().withProps(props).build(); + props.put(HoodieWriteConfig.AUTO_COMMIT_ENABLE.key(), Boolean.FALSE.toString()); + props.put(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE.key(), String.valueOf(getWriteConfig().getClusteringTargetFileMaxBytes())); + HoodieWriteConfig newConfig = HoodieWriteConfig.newBuilder().withProps(props).build(); return (JavaRDD) SparkBulkInsertHelper.newInstance().bulkInsert(inputRecords, instantTime, getHoodieTable(), newConfig, false, getPartitioner(strategyParams, schema), true, numOutputGroups, preserveHoodieMetadata); } @@ -76,8 +77,8 @@ public class SparkSortAndSizeExecutionStrategy> * Create BulkInsertPartitioner based on strategy params. */ protected Option> getPartitioner(Map strategyParams, Schema schema) { - if (strategyParams.containsKey(CLUSTERING_SORT_COLUMNS_PROPERTY.key())) { - return Option.of(new RDDCustomColumnsSortPartitioner(strategyParams.get(CLUSTERING_SORT_COLUMNS_PROPERTY.key()).split(","), + if (strategyParams.containsKey(PLAN_STRATEGY_SORT_COLUMNS.key())) { + return Option.of(new RDDCustomColumnsSortPartitioner(strategyParams.get(PLAN_STRATEGY_SORT_COLUMNS.key()).split(","), HoodieAvroUtils.addMetadataFields(schema))); } else { return Option.empty(); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMemoryUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMemoryUtils.java index 9ca5a0ce6..58d822a23 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMemoryUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMemoryUtils.java @@ -25,21 +25,21 @@ import org.apache.spark.storage.StorageLevel; import java.util.Properties; -import static org.apache.hudi.config.HoodieWriteConfig.WRITE_STATUS_STORAGE_LEVEL; +import static org.apache.hudi.config.HoodieWriteConfig.WRITE_STATUS_STORAGE_LEVEL_VALUE; /** * Spark config utils. */ public class SparkMemoryUtils { public static StorageLevel getWriteStatusStorageLevel(Properties properties) { - return StorageLevel.fromString(new HoodieConfig(properties).getString(WRITE_STATUS_STORAGE_LEVEL)); + return StorageLevel.fromString(new HoodieConfig(properties).getString(WRITE_STATUS_STORAGE_LEVEL_VALUE)); } public static StorageLevel getBloomIndexInputStorageLevel(Properties properties) { - return StorageLevel.fromString(new HoodieConfig(properties).getString(HoodieIndexConfig.BLOOM_INDEX_INPUT_STORAGE_LEVEL)); + return StorageLevel.fromString(new HoodieConfig(properties).getString(HoodieIndexConfig.BLOOM_INDEX_INPUT_STORAGE_LEVEL_VALUE)); } public static StorageLevel getSimpleIndexInputStorageLevel(Properties properties) { - return StorageLevel.fromString(new HoodieConfig(properties).getString(HoodieIndexConfig.SIMPLE_INDEX_INPUT_STORAGE_LEVEL)); + return StorageLevel.fromString(new HoodieConfig(properties).getString(HoodieIndexConfig.SIMPLE_INDEX_INPUT_STORAGE_LEVEL_VALUE)); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java index 9432a3cb5..b27f84e41 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java @@ -25,6 +25,7 @@ import org.apache.hudi.config.HoodiePreCommitValidatorConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieValidationException; import org.apache.hudi.table.HoodieSparkTable; + import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -49,7 +50,7 @@ public class SqlQueryEqualityPreCommitValidator extends SparkH /** * multiPutBatchSize will be computed and re-set in updateLocation if - * {@link HoodieHBaseIndexConfig#HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE} is set to true. + * {@link HoodieHBaseIndexConfig#PUT_BATCH_SIZE_AUTO_COMPUTE} is set to true. */ private Integer multiPutBatchSize; private Integer numRegionServersForTable; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java index 0fd601110..d4e99f7dd 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java @@ -43,7 +43,7 @@ import java.util.Objects; /** * Factory help to create {@link org.apache.hudi.keygen.KeyGenerator}. *

- * This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS} firstly, this ensures the class prop + * This factory will try {@link HoodieWriteConfig#KEYGENERATOR_CLASS_NAME} firstly, this ensures the class prop * will not be overwritten by {@link KeyGeneratorType} */ public class HoodieSparkKeyGeneratorFactory { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index e78c2c8ea..bec7ee4a8 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -18,24 +18,6 @@ package org.apache.hudi.client.functional; -import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.client.HoodieWriteResult; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.WriteStatus; @@ -81,6 +63,9 @@ import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter; import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -93,6 +78,22 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + @Tag("functional") public class TestHoodieBackedMetadata extends HoodieClientTestHarness { @@ -800,8 +801,8 @@ public class TestHoodieBackedMetadata extends HoodieClientTestHarness { assertEquals(writer.getMetadataReader().getUpdateTime().get(), beforeInflightActionTimestamp); // Reader should sync to all the completed instants - HoodieTableMetadata metadata = HoodieTableMetadata.create(context, client.getConfig().getMetadataConfig(), - client.getConfig().getBasePath(), FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue()); + HoodieTableMetadata metadata = HoodieTableMetadata.create(context, client.getConfig().getMetadataConfig(), + client.getConfig().getBasePath(), FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue()); assertEquals(((HoodieBackedTableMetadata)metadata).getReaderTime().get(), newCommitTime); // Remove the inflight instance holding back table sync @@ -813,8 +814,8 @@ public class TestHoodieBackedMetadata extends HoodieClientTestHarness { assertEquals(writer.getMetadataReader().getUpdateTime().get(), newCommitTime); // Reader should sync to all the completed instants - metadata = HoodieTableMetadata.create(context, client.getConfig().getMetadataConfig(), - client.getConfig().getBasePath(), FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue()); + metadata = HoodieTableMetadata.create(context, client.getConfig().getMetadataConfig(), + client.getConfig().getBasePath(), FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue()); assertEquals(writer.getMetadataReader().getUpdateTime().get(), newCommitTime); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 45dedd6ec..de4f42177 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -18,21 +18,18 @@ package org.apache.hudi.client.functional; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; -import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.client.AbstractHoodieWriteClient; import org.apache.hudi.client.HoodieWriteResult; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.SparkTaskContextSupplier; import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.client.validator.SparkPreCommitValidator; import org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator; import org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FSUtils; @@ -58,9 +55,9 @@ import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestTable; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.RawTripTestPayload; +import org.apache.hudi.common.util.BaseFileUtils; import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.CollectionUtils; -import org.apache.hudi.common.util.BaseFileUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; @@ -93,6 +90,10 @@ import org.apache.hudi.table.marker.WriteMarkersFactory; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; import org.apache.hudi.testutils.HoodieSparkWriteableTestTable; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -136,7 +137,7 @@ import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAM import static org.apache.hudi.common.testutils.Transformations.randomSelectAsHoodieKeys; import static org.apache.hudi.common.testutils.Transformations.recordsToRecordKeySet; import static org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE; -import static org.apache.hudi.config.HoodieClusteringConfig.CLUSTERING_EXECUTION_STRATEGY_CLASS; +import static org.apache.hudi.config.HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -2353,7 +2354,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { protected HoodieInstant createRequestedReplaceInstant(HoodieTableMetaClient metaClient, String clusterTime, List[] fileSlices) throws IOException { HoodieClusteringPlan clusteringPlan = - ClusteringUtils.createClusteringPlan(CLUSTERING_EXECUTION_STRATEGY_CLASS.defaultValue(), STRATEGY_PARAMS, fileSlices, Collections.emptyMap()); + ClusteringUtils.createClusteringPlan(EXECUTION_STRATEGY_CLASS_NAME.defaultValue(), STRATEGY_PARAMS, fileSlices, Collections.emptyMap()); HoodieInstant clusteringInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, clusterTime); HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder() diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java index 9f52a0110..be80e9d37 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java @@ -46,7 +46,7 @@ public class TestHBaseQPSResourceAllocator { @Test public void testsExplicitDefaultQPSResourceAllocator() { - HoodieWriteConfig config = getConfig(Option.of(HoodieHBaseIndexConfig.HBASE_INDEX_QPS_ALLOCATOR_CLASS.defaultValue())); + HoodieWriteConfig config = getConfig(Option.of(HoodieHBaseIndexConfig.QPS_ALLOCATOR_CLASS_NAME.defaultValue())); SparkHoodieHBaseIndex index = new SparkHoodieHBaseIndex(config); HBaseIndexQPSResourceAllocator hBaseIndexQPSResourceAllocator = index.createQPSResourceAllocator(config); assertEquals(hBaseIndexQPSResourceAllocator.getClass().getName(), diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java index d47992354..9dbb3a72c 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java @@ -53,7 +53,7 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities { properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); } if (useKeyGeneratorClassName) { - properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), CustomKeyGenerator.class.getName()); + properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); } else { properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name()); } @@ -96,7 +96,7 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities { TypedProperties properties = new TypedProperties(); properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); if (useKeyGeneratorClassName) { - properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), CustomKeyGenerator.class.getName()); + properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); } else { properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name()); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java index 8aaba7643..682329a76 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java @@ -51,7 +51,7 @@ public class TestHoodieSparkKeyGeneratorFactory { // set KeyGenerator class only props = getCommonProps(); - props.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), SimpleKeyGenerator.class.getName()); + props.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), SimpleKeyGenerator.class.getName()); KeyGenerator keyGenerator2 = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props); Assertions.assertEquals(SimpleKeyGenerator.class.getName(), keyGenerator2.getClass().getName()); @@ -63,7 +63,7 @@ public class TestHoodieSparkKeyGeneratorFactory { // set wrong class name final TypedProperties props2 = getCommonProps(); - props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key(), TestHoodieSparkKeyGeneratorFactory.class.getName()); + props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), TestHoodieSparkKeyGeneratorFactory.class.getName()); assertThrows(IOException.class, () -> HoodieSparkKeyGeneratorFactory.createKeyGenerator(props2)); // set wrong keyGenerator type diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java index 32e0b4382..afbe94937 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java @@ -45,7 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; */ public class TestConsistencyGuard extends HoodieClientTestHarness { - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); // multiple parameters, uses Collection public static List consistencyGuardType() { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 5bc293a46..023913813 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -128,7 +128,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { dfs.mkdirs(new Path(basePath)); Properties properties = populateMetaFields ? new Properties() : getPropertiesForKeyGen(); - properties.setProperty(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.key(), baseFileFormat.toString()); + properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), baseFileFormat.toString()); metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ, properties); initTestDataGenerator(); @@ -140,7 +140,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @BeforeEach public void init() throws IOException { - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), true); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), true); } @AfterEach @@ -165,7 +165,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testSimpleInsertAndUpdate(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(true); addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); @@ -266,7 +266,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsAndPreserveMetadataParams") public void testSimpleClusteringNoUpdates(boolean populateMetaFields, boolean preserveCommitMetadata) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); testClustering(false, populateMetaFields, preserveCommitMetadata); } @@ -274,7 +274,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsAndPreserveMetadataParams") public void testSimpleClusteringWithUpdates(boolean populateMetaFields, boolean preserveCommitMetadata) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); testClustering(true, populateMetaFields, preserveCommitMetadata); } @@ -478,7 +478,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testSimpleInsertUpdateAndDelete(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(true); addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); HoodieWriteConfig cfg = cfgBuilder.build(); @@ -777,7 +777,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testRollbackWithDeltaAndCompactionCommitUsingFileList(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); testRollbackWithDeltaAndCompactionCommit(false, populateMetaFields); } @@ -785,7 +785,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testRollbackWithDeltaAndCompactionCommitUsingMarkers(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); testRollbackWithDeltaAndCompactionCommit(true, populateMetaFields); } @@ -793,7 +793,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testMultiRollbackWithDeltaAndCompactionCommit(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(false); addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); HoodieWriteConfig cfg = cfgBuilder.build(); @@ -966,7 +966,7 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness { @MethodSource("populateMetaFieldsParams") public void testUpsertPartitioner(boolean populateMetaFields) throws Exception { clean(); - init(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue(), populateMetaFields); + init(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue(), populateMetaFields); HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(true); addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); HoodieWriteConfig cfg = cfgBuilder.build(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java index fa5155fa7..823eac407 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; + import org.junit.jupiter.api.Test; import java.text.SimpleDateFormat; @@ -275,7 +276,7 @@ public class TestHoodieCompactionStrategy { private final long size; public TestHoodieBaseFile(long size) { - super("/tmp/XYXYXYXYXYYX_11_20180918020003" + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension()); + super("/tmp/XYXYXYXYXYYX_11_20180918020003" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()); this.size = size; } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java index 62623b1cd..f03d9f396 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java @@ -18,15 +18,16 @@ package org.apache.hudi.table.action.rollback; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hudi.common.HoodieRollbackStat; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.CollectionUtils; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.junit.jupiter.api.Test; import java.util.HashMap; @@ -34,11 +35,11 @@ import java.util.Map; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class TestRollbackUtils { - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); private FileStatus generateFileStatus(String filePath) { Path dataFile1Path = new Path(filePath); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java index 68876d79b..123a33c3d 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java @@ -75,8 +75,8 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_TABLE_TYPE_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.BASE_FILE_FORMAT; +import static org.apache.hudi.common.table.HoodieTableConfig.TYPE; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; import static org.apache.hudi.common.util.MarkerUtils.MARKERS_FILENAME_PREFIX; @@ -145,7 +145,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { // init config, table and client. Map params = new HashMap<>(); if (tableType == HoodieTableType.MERGE_ON_READ) { - params.put(HOODIE_TABLE_TYPE_PROP.key(), HoodieTableType.MERGE_ON_READ.name()); + params.put(TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); } HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(false).withProps(params).build(); @@ -206,7 +206,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { Map params = new HashMap<>(); addNewTableParamsToProps(params); if (tableType == HoodieTableType.MERGE_ON_READ) { - params.put(HOODIE_TABLE_TYPE_PROP.key(), HoodieTableType.MERGE_ON_READ.name()); + params.put(TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); } HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(false).withProps(params).build(); @@ -233,8 +233,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { private void addNewTableParamsToProps(Map params) { params.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "uuid"); params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "partition_path"); - params.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP.key(), metaClient.getTableConfig().getTableName()); - params.put(HOODIE_BASE_FILE_FORMAT_PROP.key(), HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().name()); + params.put(HoodieTableConfig.NAME.key(), metaClient.getTableConfig().getTableName()); + params.put(BASE_FILE_FORMAT.key(), BASE_FILE_FORMAT.defaultValue().name()); } private void doInsert(SparkRDDWriteClient client) { @@ -248,11 +248,11 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { private void downgradeTableConfigsFromTwoToOne(HoodieWriteConfig cfg) throws IOException { Properties properties = new Properties(cfg.getProps()); - properties.remove(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS.key()); - properties.remove(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key()); - properties.remove(HoodieTableConfig.HOODIE_TABLE_NAME_PROP.key()); - properties.remove(HOODIE_BASE_FILE_FORMAT_PROP.key()); - properties.setProperty(HoodieTableConfig.HOODIE_TABLE_VERSION_PROP.key(), "1"); + properties.remove(HoodieTableConfig.RECORDKEY_FIELDS.key()); + properties.remove(HoodieTableConfig.PARTITION_FIELDS.key()); + properties.remove(HoodieTableConfig.NAME.key()); + properties.remove(BASE_FILE_FORMAT.key()); + properties.setProperty(HoodieTableConfig.VERSION.key(), "1"); metaClient = HoodieTestUtils.init(hadoopConf, basePath, getTableType(), properties); // set hoodie.table.version to 1 in hoodie.properties file @@ -265,7 +265,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key())); assertEquals(tableConfig.getTableName(), cfg.getTableName()); - assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(HOODIE_BASE_FILE_FORMAT_PROP.key())); + assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(BASE_FILE_FORMAT.key())); } @ParameterizedTest(name = TEST_NAME_WITH_DOWNGRADE_PARAMS) @@ -278,7 +278,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { addNewTableParamsToProps(params); } if (tableType == HoodieTableType.MERGE_ON_READ) { - params.put(HOODIE_TABLE_TYPE_PROP.key(), HoodieTableType.MERGE_ON_READ.name()); + params.put(TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); } HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(true) @@ -288,10 +288,10 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { if (fromVersion == HoodieTableVersion.TWO) { // set table configs HoodieTableConfig tableConfig = metaClient.getTableConfig(); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_NAME_PROP, cfg.getTableName()); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); - tableConfig.setValue(HOODIE_BASE_FILE_FORMAT_PROP, cfg.getString(HOODIE_BASE_FILE_FORMAT_PROP)); + tableConfig.setValue(HoodieTableConfig.NAME, cfg.getTableName()); + tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); + tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); + tableConfig.setValue(BASE_FILE_FORMAT, cfg.getString(BASE_FILE_FORMAT)); } // prepare data. Make 2 commits, in which 2nd is not committed. @@ -433,7 +433,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { private List triggerCommit(String newCommitTime, HoodieTableType tableType, boolean enableMarkedBasedRollback) { Map params = new HashMap<>(); if (tableType == HoodieTableType.MERGE_ON_READ) { - params.put(HOODIE_TABLE_TYPE_PROP.key(), HoodieTableType.MERGE_ON_READ.name()); + params.put(TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); } HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(enableMarkedBasedRollback).withProps(params).build(); SparkRDDWriteClient client = getHoodieWriteClient(cfg); @@ -571,6 +571,6 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { HoodieConfig hoodieConfig = HoodieConfig.create(fsDataInputStream); fsDataInputStream.close(); assertEquals(Integer.toString(expectedVersion.versionCode()), hoodieConfig - .getString(HoodieTableConfig.HOODIE_TABLE_VERSION_PROP)); + .getString(HoodieTableConfig.VERSION)); } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java index 233b679d8..a6ac67b85 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java @@ -17,13 +17,6 @@ package org.apache.hudi.testutils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.SparkTaskContextSupplier; @@ -46,6 +39,14 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.SimpleKeyGenerator; import org.apache.hudi.table.WorkloadStat; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -54,7 +55,6 @@ import org.apache.spark.sql.SQLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; -import scala.Tuple2; import java.io.IOException; import java.io.Serializable; @@ -64,13 +64,15 @@ import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import scala.Tuple2; + /** * The test harness for resource initialization and cleanup. */ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness implements Serializable { private static final Logger LOG = LogManager.getLogger(HoodieClientTestHarness.class); - + private String testMethodName; protected transient JavaSparkContext jsc = null; protected transient HoodieSparkEngineContext context = null; @@ -232,12 +234,12 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im protected Properties getPropertiesForKeyGen() { Properties properties = new Properties(); - properties.put(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), "false"); - properties.put("hoodie.datasource.write.recordkey.field","_row_key"); - properties.put("hoodie.datasource.write.partitionpath.field","partition_path"); - properties.put(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS.key(), "_row_key"); - properties.put(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key(), "partition_path"); - properties.put(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS.key(), SimpleKeyGenerator.class.getName()); + properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + properties.put("hoodie.datasource.write.recordkey.field", "_row_key"); + properties.put("hoodie.datasource.write.partitionpath.field", "partition_path"); + properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key"); + properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path"); + properties.put(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key(), SimpleKeyGenerator.class.getName()); return properties; } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java index 15fb50c26..59172c55a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java @@ -121,9 +121,9 @@ public class HoodieClientTestUtils { getLatestFileIDsToFullPath(basePath, commitTimeline, Arrays.asList(commitInstant)); LOG.info("Path :" + paths.values()); Dataset unFilteredRows = null; - if (HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().equals(HoodieFileFormat.PARQUET)) { + if (HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().equals(HoodieFileFormat.PARQUET)) { unFilteredRows = sqlContext.read().parquet(paths.values().toArray(new String[paths.size()])); - } else if (HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().equals(HoodieFileFormat.ORC)) { + } else if (HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().equals(HoodieFileFormat.ORC)) { unFilteredRows = sqlContext.read().orc(paths.values().toArray(new String[paths.size()])); } if (unFilteredRows != null) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index 842d8fb43..f55f46bb6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.config; import javax.annotation.concurrent.Immutable; + import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -38,21 +39,21 @@ public final class HoodieMetadataConfig extends HoodieConfig { public static final String METADATA_PREFIX = "hoodie.metadata"; // Enable the internal Metadata Table which saves file listings - public static final ConfigProperty METADATA_ENABLE_PROP = ConfigProperty + public static final ConfigProperty ENABLE = ConfigProperty .key(METADATA_PREFIX + ".enable") .defaultValue(false) .sinceVersion("0.7.0") .withDocumentation("Enable the internal metadata table which serves table metadata like level file listings"); // Enable syncing the Metadata Table - public static final ConfigProperty METADATA_SYNC_ENABLE_PROP = ConfigProperty + public static final ConfigProperty SYNC_ENABLE = ConfigProperty .key(METADATA_PREFIX + ".sync.enable") .defaultValue(true) .sinceVersion("0.9.0") .withDocumentation("Enable syncing of metadata table from actions on the dataset"); // Validate contents of Metadata Table on each access against the actual filesystem - public static final ConfigProperty METADATA_VALIDATE_PROP = ConfigProperty + public static final ConfigProperty VALIDATE_ENABLE = ConfigProperty .key(METADATA_PREFIX + ".validate") .defaultValue(false) .sinceVersion("0.7.0") @@ -61,68 +62,68 @@ public final class HoodieMetadataConfig extends HoodieConfig { public static final boolean DEFAULT_METADATA_ENABLE_FOR_READERS = false; // Enable metrics for internal Metadata Table - public static final ConfigProperty METADATA_METRICS_ENABLE_PROP = ConfigProperty + public static final ConfigProperty METRICS_ENABLE = ConfigProperty .key(METADATA_PREFIX + ".metrics.enable") .defaultValue(false) .sinceVersion("0.7.0") .withDocumentation("Enable publishing of metrics around metadata table."); // Parallelism for inserts - public static final ConfigProperty METADATA_INSERT_PARALLELISM_PROP = ConfigProperty + public static final ConfigProperty INSERT_PARALLELISM_VALUE = ConfigProperty .key(METADATA_PREFIX + ".insert.parallelism") .defaultValue(1) .sinceVersion("0.7.0") .withDocumentation("Parallelism to use when inserting to the metadata table"); // Async clean - public static final ConfigProperty METADATA_ASYNC_CLEAN_PROP = ConfigProperty + public static final ConfigProperty ASYNC_CLEAN_ENABLE = ConfigProperty .key(METADATA_PREFIX + ".clean.async") .defaultValue(false) .sinceVersion("0.7.0") .withDocumentation("Enable asynchronous cleaning for metadata table"); // Maximum delta commits before compaction occurs - public static final ConfigProperty METADATA_COMPACT_NUM_DELTA_COMMITS_PROP = ConfigProperty + public static final ConfigProperty COMPACT_NUM_DELTA_COMMITS = ConfigProperty .key(METADATA_PREFIX + ".compact.max.delta.commits") .defaultValue(24) .sinceVersion("0.7.0") .withDocumentation("Controls how often the metadata table is compacted."); // Archival settings - public static final ConfigProperty MIN_COMMITS_TO_KEEP_PROP = ConfigProperty + public static final ConfigProperty MIN_COMMITS_TO_KEEP = ConfigProperty .key(METADATA_PREFIX + ".keep.min.commits") .defaultValue(20) .sinceVersion("0.7.0") .withDocumentation("Controls the archival of the metadata table’s timeline."); - public static final ConfigProperty MAX_COMMITS_TO_KEEP_PROP = ConfigProperty + public static final ConfigProperty MAX_COMMITS_TO_KEEP = ConfigProperty .key(METADATA_PREFIX + ".keep.max.commits") .defaultValue(30) .sinceVersion("0.7.0") .withDocumentation("Controls the archival of the metadata table’s timeline."); // Cleaner commits retained - public static final ConfigProperty CLEANER_COMMITS_RETAINED_PROP = ConfigProperty + public static final ConfigProperty CLEANER_COMMITS_RETAINED = ConfigProperty .key(METADATA_PREFIX + ".cleaner.commits.retained") .defaultValue(3) .sinceVersion("0.7.0") .withDocumentation("Controls retention/history for metadata table."); // Regex to filter out matching directories during bootstrap - public static final ConfigProperty DIRECTORY_FILTER_REGEX = ConfigProperty + public static final ConfigProperty DIR_FILTER_REGEX = ConfigProperty .key(METADATA_PREFIX + ".dir.filter.regex") .defaultValue("") .sinceVersion("0.7.0") .withDocumentation("Directories matching this regex, will be filtered out when initializing metadata table from lake storage for the first time."); - public static final ConfigProperty HOODIE_ASSUME_DATE_PARTITIONING_PROP = ConfigProperty + public static final ConfigProperty ASSUME_DATE_PARTITIONING = ConfigProperty .key("hoodie.assume.date.partitioning") .defaultValue("false") .sinceVersion("0.3.0") .withDocumentation("Should HoodieWriteClient assume the data is partitioned by dates, i.e three levels from base path. " + "This is a stop-gap to support tables created by versions < 0.3.1. Will be removed eventually"); - public static final ConfigProperty FILE_LISTING_PARALLELISM_PROP = ConfigProperty + public static final ConfigProperty FILE_LISTING_PARALLELISM_VALUE = ConfigProperty .key("hoodie.file.listing.parallelism") .defaultValue(1500) .sinceVersion("0.7.0") @@ -137,31 +138,31 @@ public final class HoodieMetadataConfig extends HoodieConfig { } public int getFileListingParallelism() { - return Math.max(getInt(HoodieMetadataConfig.FILE_LISTING_PARALLELISM_PROP), 1); + return Math.max(getInt(HoodieMetadataConfig.FILE_LISTING_PARALLELISM_VALUE), 1); } public Boolean shouldAssumeDatePartitioning() { - return getBoolean(HoodieMetadataConfig.HOODIE_ASSUME_DATE_PARTITIONING_PROP); + return getBoolean(HoodieMetadataConfig.ASSUME_DATE_PARTITIONING); } public boolean enabled() { - return getBoolean(METADATA_ENABLE_PROP); + return getBoolean(ENABLE); } public boolean enableSync() { - return enabled() && getBoolean(HoodieMetadataConfig.METADATA_SYNC_ENABLE_PROP); + return enabled() && getBoolean(HoodieMetadataConfig.SYNC_ENABLE); } public boolean validateFileListingMetadata() { - return getBoolean(METADATA_VALIDATE_PROP); + return getBoolean(VALIDATE_ENABLE); } public boolean enableMetrics() { - return getBoolean(METADATA_METRICS_ENABLE_PROP); + return getBoolean(METRICS_ENABLE); } public String getDirectoryFilterRegex() { - return getString(DIRECTORY_FILTER_REGEX); + return getString(DIR_FILTER_REGEX); } public static class Builder { @@ -181,63 +182,63 @@ public final class HoodieMetadataConfig extends HoodieConfig { } public Builder enable(boolean enable) { - metadataConfig.setValue(METADATA_ENABLE_PROP, String.valueOf(enable)); + metadataConfig.setValue(ENABLE, String.valueOf(enable)); return this; } public Builder enableSync(boolean enable) { - metadataConfig.setValue(METADATA_SYNC_ENABLE_PROP, String.valueOf(enable)); + metadataConfig.setValue(SYNC_ENABLE, String.valueOf(enable)); return this; } public Builder enableMetrics(boolean enableMetrics) { - metadataConfig.setValue(METADATA_METRICS_ENABLE_PROP, String.valueOf(enableMetrics)); + metadataConfig.setValue(METRICS_ENABLE, String.valueOf(enableMetrics)); return this; } public Builder validate(boolean validate) { - metadataConfig.setValue(METADATA_VALIDATE_PROP, String.valueOf(validate)); + metadataConfig.setValue(VALIDATE_ENABLE, String.valueOf(validate)); return this; } public Builder withInsertParallelism(int parallelism) { - metadataConfig.setValue(METADATA_INSERT_PARALLELISM_PROP, String.valueOf(parallelism)); + metadataConfig.setValue(INSERT_PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } public Builder withAsyncClean(boolean asyncClean) { - metadataConfig.setValue(METADATA_ASYNC_CLEAN_PROP, String.valueOf(asyncClean)); + metadataConfig.setValue(ASYNC_CLEAN_ENABLE, String.valueOf(asyncClean)); return this; } public Builder withMaxNumDeltaCommitsBeforeCompaction(int maxNumDeltaCommitsBeforeCompaction) { - metadataConfig.setValue(METADATA_COMPACT_NUM_DELTA_COMMITS_PROP, String.valueOf(maxNumDeltaCommitsBeforeCompaction)); + metadataConfig.setValue(COMPACT_NUM_DELTA_COMMITS, String.valueOf(maxNumDeltaCommitsBeforeCompaction)); return this; } public Builder archiveCommitsWith(int minToKeep, int maxToKeep) { - metadataConfig.setValue(MIN_COMMITS_TO_KEEP_PROP, String.valueOf(minToKeep)); - metadataConfig.setValue(MAX_COMMITS_TO_KEEP_PROP, String.valueOf(maxToKeep)); + metadataConfig.setValue(MIN_COMMITS_TO_KEEP, String.valueOf(minToKeep)); + metadataConfig.setValue(MAX_COMMITS_TO_KEEP, String.valueOf(maxToKeep)); return this; } public Builder retainCommits(int commitsRetained) { - metadataConfig.setValue(CLEANER_COMMITS_RETAINED_PROP, String.valueOf(commitsRetained)); + metadataConfig.setValue(CLEANER_COMMITS_RETAINED, String.valueOf(commitsRetained)); return this; } public Builder withFileListingParallelism(int parallelism) { - metadataConfig.setValue(FILE_LISTING_PARALLELISM_PROP, String.valueOf(parallelism)); + metadataConfig.setValue(FILE_LISTING_PARALLELISM_VALUE, String.valueOf(parallelism)); return this; } public Builder withAssumeDatePartitioning(boolean assumeDatePartitioning) { - metadataConfig.setValue(HOODIE_ASSUME_DATE_PARTITIONING_PROP, String.valueOf(assumeDatePartitioning)); + metadataConfig.setValue(ASSUME_DATE_PARTITIONING, String.valueOf(assumeDatePartitioning)); return this; } public Builder withDirectoryFilterRegex(String regex) { - metadataConfig.setValue(DIRECTORY_FILTER_REGEX, regex); + metadataConfig.setValue(DIR_FILTER_REGEX, regex); return this; } @@ -246,4 +247,140 @@ public final class HoodieMetadataConfig extends HoodieConfig { return metadataConfig; } } + + /** + * @deprecated Use {@link #ENABLE} and its methods. + */ + @Deprecated + public static final String METADATA_ENABLE_PROP = ENABLE.key(); + /** + * @deprecated Use {@link #ENABLE} and its methods. + */ + @Deprecated + public static final boolean DEFAULT_METADATA_ENABLE = ENABLE.defaultValue(); + /** + * @deprecated Use {@link #VALIDATE_ENABLE} and its methods. + */ + @Deprecated + public static final String METADATA_VALIDATE_PROP = VALIDATE_ENABLE.key(); + /** + * @deprecated Use {@link #VALIDATE_ENABLE} and its methods. + */ + @Deprecated + public static final boolean DEFAULT_METADATA_VALIDATE = VALIDATE_ENABLE.defaultValue(); + + /** + * @deprecated Use {@link #METRICS_ENABLE} and its methods. + */ + @Deprecated + public static final String METADATA_METRICS_ENABLE_PROP = METRICS_ENABLE.key(); + /** + * @deprecated Use {@link #METRICS_ENABLE} and its methods. + */ + @Deprecated + public static final boolean DEFAULT_METADATA_METRICS_ENABLE = METRICS_ENABLE.defaultValue(); + + /** + * @deprecated Use {@link #INSERT_PARALLELISM_VALUE} and its methods. + */ + @Deprecated + public static final String METADATA_INSERT_PARALLELISM_PROP = INSERT_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #INSERT_PARALLELISM_VALUE} and its methods. + */ + @Deprecated + public static final int DEFAULT_METADATA_INSERT_PARALLELISM = INSERT_PARALLELISM_VALUE.defaultValue(); + + /** + * @deprecated Use {@link #ASYNC_CLEAN_ENABLE} and its methods. + */ + @Deprecated + public static final String METADATA_ASYNC_CLEAN_PROP = ASYNC_CLEAN_ENABLE.key(); + /** + * @deprecated Use {@link #ASYNC_CLEAN_ENABLE} and its methods. + */ + @Deprecated + public static final boolean DEFAULT_METADATA_ASYNC_CLEAN = ASYNC_CLEAN_ENABLE.defaultValue(); + + /** + * @deprecated Use {@link #COMPACT_NUM_DELTA_COMMITS} and its methods. + */ + @Deprecated + public static final String METADATA_COMPACT_NUM_DELTA_COMMITS_PROP = COMPACT_NUM_DELTA_COMMITS.key(); + /** + * @deprecated Use {@link #COMPACT_NUM_DELTA_COMMITS} and its methods. + */ + @Deprecated + public static final int DEFAULT_METADATA_COMPACT_NUM_DELTA_COMMITS = COMPACT_NUM_DELTA_COMMITS.defaultValue(); + + /** + * @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods. + */ + @Deprecated + public static final String MIN_COMMITS_TO_KEEP_PROP = MIN_COMMITS_TO_KEEP.key(); + /** + * @deprecated Use {@link #MIN_COMMITS_TO_KEEP} and its methods. + */ + @Deprecated + public static final int DEFAULT_MIN_COMMITS_TO_KEEP = MIN_COMMITS_TO_KEEP.defaultValue(); + /** + * @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods. + */ + @Deprecated + public static final String MAX_COMMITS_TO_KEEP_PROP = MAX_COMMITS_TO_KEEP.key(); + /** + * @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods. + */ + @Deprecated + public static final int DEFAULT_MAX_COMMITS_TO_KEEP = MAX_COMMITS_TO_KEEP.defaultValue(); + /** + * @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods. + */ + @Deprecated + public static final String CLEANER_COMMITS_RETAINED_PROP = CLEANER_COMMITS_RETAINED.key(); + /** + * @deprecated Use {@link #CLEANER_COMMITS_RETAINED} and its methods. + */ + @Deprecated + public static final int DEFAULT_CLEANER_COMMITS_RETAINED = CLEANER_COMMITS_RETAINED.defaultValue(); + /** + * @deprecated No longer takes any effect. + */ + @Deprecated + public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX + ".fallback.enable"; + /** + * @deprecated No longer takes any effect. + */ + @Deprecated + public static final String DEFAULT_ENABLE_FALLBACK = "true"; + /** + * @deprecated Use {@link #DIR_FILTER_REGEX} and its methods. + */ + @Deprecated + public static final String DIRECTORY_FILTER_REGEX = DIR_FILTER_REGEX.key(); + /** + * @deprecated Use {@link #DIR_FILTER_REGEX} and its methods. + */ + @Deprecated + public static final String DEFAULT_DIRECTORY_FILTER_REGEX = DIR_FILTER_REGEX.defaultValue(); + /** + * @deprecated Use {@link #ASSUME_DATE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = ASSUME_DATE_PARTITIONING.key(); + /** + * @deprecated Use {@link #ASSUME_DATE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String DEFAULT_ASSUME_DATE_PARTITIONING = ASSUME_DATE_PARTITIONING.defaultValue(); + /** + * @deprecated Use {@link #FILE_LISTING_PARALLELISM_VALUE} and its methods. + */ + @Deprecated + public static final String FILE_LISTING_PARALLELISM_PROP = FILE_LISTING_PARALLELISM_VALUE.key(); + /** + * @deprecated Use {@link #FILE_LISTING_PARALLELISM_VALUE} and its methods. + */ + @Deprecated + public static final int DEFAULT_FILE_LISTING_PARALLELISM = FILE_LISTING_PARALLELISM_VALUE.defaultValue(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java index 8f1614996..41d8cd775 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java @@ -74,6 +74,60 @@ public class LockConfiguration implements Serializable { public static final String ZK_LOCK_KEY_PROP_KEY = ZOOKEEPER_BASED_LOCK_PROPERTY_PREFIX + "lock_key"; + /** @deprecated Use {@link #LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY; + /** @deprecated Use {@link #LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP_KEY; + @Deprecated + public static final String DEFAULT_LOCK_ACQUIRE_MAX_RETRY_WAIT_TIME_IN_MILLIS = String.valueOf(5000L); + /** @deprecated Use {@link #LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY; + @Deprecated + public static final String DEFAULT_LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS = String.valueOf(10000L); + /** @deprecated Use {@link #LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_NUM_RETRIES_PROP = LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY; + /** @deprecated Use {@link #LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP = LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY; + @Deprecated + public static final String DEFAULT_LOCK_ACQUIRE_CLIENT_NUM_RETRIES = String.valueOf(0); + /** @deprecated Use {@link #LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY} */ + @Deprecated + public static final String LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP = LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY; + @Deprecated + public static final int DEFAULT_ACQUIRE_LOCK_WAIT_TIMEOUT_MS = 60 * 1000; + /** @deprecated Use {@link #HIVE_DATABASE_NAME_PROP_KEY} */ + @Deprecated + public static final String HIVE_DATABASE_NAME_PROP = HIVE_DATABASE_NAME_PROP_KEY; + /** @deprecated Use {@link #HIVE_TABLE_NAME_PROP_KEY} */ + @Deprecated + public static final String HIVE_TABLE_NAME_PROP = HIVE_TABLE_NAME_PROP_KEY; + /** @deprecated Use {@link #HIVE_METASTORE_URI_PROP_KEY} */ + @Deprecated + public static final String HIVE_METASTORE_URI_PROP = HIVE_METASTORE_URI_PROP_KEY; + /** @deprecated Use {@link #ZK_BASE_PATH_PROP_KEY} */ + @Deprecated + public static final String ZK_BASE_PATH_PROP = ZK_BASE_PATH_PROP_KEY; + /** @deprecated Use {@link #ZK_SESSION_TIMEOUT_MS_PROP_KEY} */ + @Deprecated + public static final String ZK_SESSION_TIMEOUT_MS_PROP = ZK_SESSION_TIMEOUT_MS_PROP_KEY; + /** @deprecated Use {@link #ZK_CONNECTION_TIMEOUT_MS_PROP_KEY} */ + @Deprecated + public static final String ZK_CONNECTION_TIMEOUT_MS_PROP = ZK_CONNECTION_TIMEOUT_MS_PROP_KEY; + /** @deprecated Use {@link #ZK_CONNECT_URL_PROP_KEY} */ + @Deprecated + public static final String ZK_CONNECT_URL_PROP = ZK_CONNECT_URL_PROP_KEY; + /** @deprecated Use {@link #ZK_PORT_PROP_KEY} */ + @Deprecated + public static final String ZK_PORT_PROP = ZK_PORT_PROP_KEY; + /** @deprecated Use {@link #ZK_LOCK_KEY_PROP_KEY} */ + @Deprecated + public static final String ZK_LOCK_KEY_PROP = ZK_LOCK_KEY_PROP_KEY; + private final TypedProperties props; public LockConfiguration(Properties props) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java index 74830a4b2..323e41f07 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java @@ -37,7 +37,7 @@ import java.util.Properties; + "(Tip: S3 is NOT eventually consistent anymore!)") public class ConsistencyGuardConfig extends HoodieConfig { - public static final ConfigProperty CONSISTENCY_CHECK_ENABLED_PROP = ConfigProperty + public static final ConfigProperty ENABLE = ConfigProperty .key("hoodie.consistency.check.enabled") .defaultValue("false") .sinceVersion("0.5.0") @@ -45,14 +45,14 @@ public class ConsistencyGuardConfig extends HoodieConfig { .withDocumentation("Enabled to handle S3 eventual consistency issue. This property is no longer required " + "since S3 is now strongly consistent. Will be removed in the future releases."); - public static final ConfigProperty INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP = ConfigProperty + public static final ConfigProperty INITIAL_CHECK_INTERVAL_MS = ConfigProperty .key("hoodie.consistency.check.initial_interval_ms") .defaultValue(400L) .sinceVersion("0.5.0") .deprecatedAfter("0.7.0") .withDocumentation("Amount of time (in ms) to wait, before checking for consistency after an operation on storage."); - public static final ConfigProperty MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP = ConfigProperty + public static final ConfigProperty MAX_CHECK_INTERVAL_MS = ConfigProperty .key("hoodie.consistency.check.max_interval_ms") .defaultValue(20000L) .sinceVersion("0.5.0") @@ -60,7 +60,7 @@ public class ConsistencyGuardConfig extends HoodieConfig { .withDocumentation("Maximum amount of time (in ms), to wait for consistency checking."); // maximum number of checks, for consistency of written data. Will wait upto 140 Secs - public static final ConfigProperty MAX_CONSISTENCY_CHECKS_PROP = ConfigProperty + public static final ConfigProperty MAX_CHECKS = ConfigProperty .key("hoodie.consistency.check.max_checks") .defaultValue(6) .sinceVersion("0.5.0") @@ -68,14 +68,14 @@ public class ConsistencyGuardConfig extends HoodieConfig { .withDocumentation("Maximum number of consistency checks to perform, with exponential backoff."); // sleep time for OptimisticConsistencyGuard - public static final ConfigProperty OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS_PROP = ConfigProperty + public static final ConfigProperty OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS = ConfigProperty .key("hoodie.optimistic.consistency.guard.sleep_time_ms") .defaultValue(500L) .sinceVersion("0.6.0") .withDocumentation("Amount of time (in ms), to wait after which we assume storage is consistent."); // config to enable OptimisticConsistencyGuard in finalizeWrite instead of FailSafeConsistencyGuard - public static final ConfigProperty ENABLE_OPTIMISTIC_CONSISTENCY_GUARD_PROP = ConfigProperty + public static final ConfigProperty OPTIMISTIC_CONSISTENCY_GUARD_ENABLE = ConfigProperty .key("_hoodie.optimistic.consistency.guard.enable") .defaultValue(false) .sinceVersion("0.6.0") @@ -90,27 +90,27 @@ public class ConsistencyGuardConfig extends HoodieConfig { } public boolean isConsistencyCheckEnabled() { - return getBoolean(CONSISTENCY_CHECK_ENABLED_PROP); + return getBoolean(ENABLE); } public int getMaxConsistencyChecks() { - return getInt(MAX_CONSISTENCY_CHECKS_PROP); + return getInt(MAX_CHECKS); } public int getInitialConsistencyCheckIntervalMs() { - return getInt(INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP); + return getInt(INITIAL_CHECK_INTERVAL_MS); } public int getMaxConsistencyCheckIntervalMs() { - return getInt(MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP); + return getInt(MAX_CHECK_INTERVAL_MS); } public long getOptimisticConsistencyGuardSleepTimeMs() { - return getLong(OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS_PROP); + return getLong(OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS); } public boolean shouldEnableOptimisticConsistencyGuard() { - return getBoolean(ENABLE_OPTIMISTIC_CONSISTENCY_GUARD_PROP); + return getBoolean(OPTIMISTIC_CONSISTENCY_GUARD_ENABLE); } /** @@ -133,32 +133,32 @@ public class ConsistencyGuardConfig extends HoodieConfig { } public Builder withConsistencyCheckEnabled(boolean enabled) { - consistencyGuardConfig.setValue(CONSISTENCY_CHECK_ENABLED_PROP, String.valueOf(enabled)); + consistencyGuardConfig.setValue(ENABLE, String.valueOf(enabled)); return this; } public Builder withInitialConsistencyCheckIntervalMs(int initialIntevalMs) { - consistencyGuardConfig.setValue(INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP, String.valueOf(initialIntevalMs)); + consistencyGuardConfig.setValue(INITIAL_CHECK_INTERVAL_MS, String.valueOf(initialIntevalMs)); return this; } public Builder withMaxConsistencyCheckIntervalMs(int maxIntervalMs) { - consistencyGuardConfig.setValue(MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP, String.valueOf(maxIntervalMs)); + consistencyGuardConfig.setValue(MAX_CHECK_INTERVAL_MS, String.valueOf(maxIntervalMs)); return this; } public Builder withMaxConsistencyChecks(int maxConsistencyChecks) { - consistencyGuardConfig.setValue(MAX_CONSISTENCY_CHECKS_PROP, String.valueOf(maxConsistencyChecks)); + consistencyGuardConfig.setValue(MAX_CHECKS, String.valueOf(maxConsistencyChecks)); return this; } public Builder withOptimisticConsistencyGuardSleepTimeMs(long sleepTimeMs) { - consistencyGuardConfig.setValue(OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS_PROP, String.valueOf(sleepTimeMs)); + consistencyGuardConfig.setValue(OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS, String.valueOf(sleepTimeMs)); return this; } public Builder withEnableOptimisticConsistencyGuard(boolean enableOptimisticConsistencyGuard) { - consistencyGuardConfig.setValue(ENABLE_OPTIMISTIC_CONSISTENCY_GUARD_PROP, String.valueOf(enableOptimisticConsistencyGuard)); + consistencyGuardConfig.setValue(OPTIMISTIC_CONSISTENCY_GUARD_ENABLE, String.valueOf(enableOptimisticConsistencyGuard)); return this; } @@ -167,4 +167,65 @@ public class ConsistencyGuardConfig extends HoodieConfig { return consistencyGuardConfig; } } + + /** + * @deprecated use {@link #ENABLE} and its methods. + */ + @Deprecated + private static final String CONSISTENCY_CHECK_ENABLED_PROP = ENABLE.key(); + /** + * @deprecated use {@link #ENABLE} and its methods. + */ + @Deprecated + private static final String DEFAULT_CONSISTENCY_CHECK_ENABLED = ENABLE.defaultValue(); + /** + * @deprecated use {@link #INITIAL_CHECK_INTERVAL_MS} and its methods. + */ + @Deprecated + private static final String INITIAL_CONSISTENCY_CHECK_INTERVAL_MS_PROP = INITIAL_CHECK_INTERVAL_MS.key(); + /** + * @deprecated use {@link #INITIAL_CHECK_INTERVAL_MS} and its methods. + */ + @Deprecated + private static long DEFAULT_INITIAL_CONSISTENCY_CHECK_INTERVAL_MS = INITIAL_CHECK_INTERVAL_MS.defaultValue(); + /** + * @deprecated use {@link #MAX_CHECK_INTERVAL_MS} and its methods. + */ + @Deprecated + private static final String MAX_CONSISTENCY_CHECK_INTERVAL_MS_PROP = MAX_CHECK_INTERVAL_MS.key(); + /** + * @deprecated use {@link #MAX_CHECK_INTERVAL_MS} and its methods. + */ + @Deprecated + private static long DEFAULT_MAX_CONSISTENCY_CHECK_INTERVAL_MS = MAX_CHECK_INTERVAL_MS.defaultValue(); + /** + * @deprecated use {@link #MAX_CHECKS} and its methods. + */ + @Deprecated + private static final String MAX_CONSISTENCY_CHECKS_PROP = MAX_CHECKS.key(); + /** + * @deprecated use {@link #MAX_CHECKS} and its methods. + */ + @Deprecated + private static int DEFAULT_MAX_CONSISTENCY_CHECKS = MAX_CHECKS.defaultValue(); + /** + * @deprecated use {@link #OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS} and its methods. + */ + @Deprecated + private static final String OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS_PROP = OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS.key(); + /** + * @deprecated use {@link #OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS} and its methods. + */ + @Deprecated + private static long DEFAULT_OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS_PROP = OPTIMISTIC_CONSISTENCY_GUARD_SLEEP_TIME_MS.defaultValue(); + /** + * @deprecated use {@link #OPTIMISTIC_CONSISTENCY_GUARD_ENABLE} and its methods. + */ + @Deprecated + private static final String ENABLE_OPTIMISTIC_CONSISTENCY_GUARD = OPTIMISTIC_CONSISTENCY_GUARD_ENABLE.key(); + /** + * @deprecated use {@link #OPTIMISTIC_CONSISTENCY_GUARD_ENABLE} and its methods. + */ + @Deprecated + private static boolean DEFAULT_ENABLE_OPTIMISTIC_CONSISTENCY_GUARD = OPTIMISTIC_CONSISTENCY_GUARD_ENABLE.defaultValue(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 80883965d..7e39053e1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -18,14 +18,6 @@ package org.apache.hudi.common.fs; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -42,6 +34,15 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.InvalidHoodiePathException; import org.apache.hudi.metadata.HoodieTableMetadata; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -141,7 +142,7 @@ public class FSUtils { // TODO: this should be removed public static String makeDataFileName(String instantTime, String writeToken, String fileId) { return String.format("%s_%s_%s%s", fileId, writeToken, instantTime, - HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension()); + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()); } public static String makeDataFileName(String instantTime, String writeToken, String fileId, String fileExtension) { @@ -153,7 +154,7 @@ public class FSUtils { } public static String maskWithoutFileId(String instantTime, int taskPartitionId) { - return String.format("*_%s_%s%s", taskPartitionId, instantTime, HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP + return String.format("*_%s_%s%s", taskPartitionId, instantTime, HoodieTableConfig.BASE_FILE_FORMAT .defaultValue().getFileExtension()); } @@ -271,7 +272,7 @@ public class FSUtils { .withAssumeDatePartitioning(assumeDatePartitioning) .build(); try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePathStr, - FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue())) { + FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue())) { return tableMetadata.getAllPartitionPaths(); } catch (Exception e) { throw new HoodieException("Error fetching partition paths from metadata table", e); @@ -281,7 +282,7 @@ public class FSUtils { public static List getAllPartitionPaths(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String basePathStr) { try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePathStr, - FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue())) { + FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue())) { return tableMetadata.getAllPartitionPaths(); } catch (Exception e) { throw new HoodieException("Error fetching partition paths from metadata table", e); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePayloadProps.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePayloadProps.java index 24bf3d774..b04b8dfdd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePayloadProps.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePayloadProps.java @@ -48,4 +48,15 @@ public class HoodiePayloadProps { * to tell if the current record is a update record or insert record for mor table. */ public static final String PAYLOAD_IS_UPDATE_RECORD_FOR_MOR = "hoodie.is.update.record.for.mor"; + + /** @deprecated Use {@link #PAYLOAD_ORDERING_FIELD_PROP_KEY} */ + @Deprecated + public static final String PAYLOAD_ORDERING_FIELD_PROP = PAYLOAD_ORDERING_FIELD_PROP_KEY; + @Deprecated + public static String DEFAULT_PAYLOAD_ORDERING_FIELD_VAL = "ts"; + /** @deprecated Use {@link #PAYLOAD_EVENT_TIME_FIELD_PROP_KEY} */ + @Deprecated + public static final String PAYLOAD_EVENT_TIME_FIELD_PROP = PAYLOAD_EVENT_TIME_FIELD_PROP_KEY; + @Deprecated + public static String DEFAULT_PAYLOAD_EVENT_TIME_FIELD_VAL = "ts"; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 853fffdca..6d3a7d001 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -69,95 +69,95 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties"; - public static final ConfigProperty HOODIE_TABLE_NAME_PROP = ConfigProperty + public static final ConfigProperty NAME = ConfigProperty .key("hoodie.table.name") .noDefaultValue() .withDocumentation("Table name that will be used for registering with Hive. Needs to be same across runs."); - public static final ConfigProperty HOODIE_TABLE_TYPE_PROP = ConfigProperty + public static final ConfigProperty TYPE = ConfigProperty .key("hoodie.table.type") .defaultValue(HoodieTableType.COPY_ON_WRITE) .withDocumentation("The table type for the underlying data, for this write. This can’t change between writes."); - public static final ConfigProperty HOODIE_TABLE_VERSION_PROP = ConfigProperty + public static final ConfigProperty VERSION = ConfigProperty .key("hoodie.table.version") .defaultValue(HoodieTableVersion.ZERO) .withDocumentation("Version of table, used for running upgrade/downgrade steps between releases with potentially " + "breaking/backwards compatible changes."); - public static final ConfigProperty HOODIE_TABLE_PRECOMBINE_FIELD_PROP = ConfigProperty + public static final ConfigProperty PRECOMBINE_FIELD = ConfigProperty .key("hoodie.table.precombine.field") .noDefaultValue() .withDocumentation("Field used in preCombining before actual write. By default, when two records have the same key value, " + "the largest value for the precombine field determined by Object.compareTo(..), is picked."); - public static final ConfigProperty HOODIE_TABLE_PARTITION_FIELDS_PROP = ConfigProperty + public static final ConfigProperty PARTITION_FIELDS = ConfigProperty .key("hoodie.table.partition.fields") .noDefaultValue() .withDocumentation("Fields used to partition the table. Concatenated values of these fields are used as " + "the partition path, by invoking toString()"); - public static final ConfigProperty HOODIE_TABLE_RECORDKEY_FIELDS = ConfigProperty + public static final ConfigProperty RECORDKEY_FIELDS = ConfigProperty .key("hoodie.table.recordkey.fields") .noDefaultValue() .withDocumentation("Columns used to uniquely identify the table. Concatenated values of these fields are used as " + " the record key component of HoodieKey."); - public static final ConfigProperty HOODIE_TABLE_CREATE_SCHEMA = ConfigProperty + public static final ConfigProperty CREATE_SCHEMA = ConfigProperty .key("hoodie.table.create.schema") .noDefaultValue() .withDocumentation("Schema used when creating the table, for the first time."); - public static final ConfigProperty HOODIE_BASE_FILE_FORMAT_PROP = ConfigProperty + public static final ConfigProperty BASE_FILE_FORMAT = ConfigProperty .key("hoodie.table.base.file.format") .defaultValue(HoodieFileFormat.PARQUET) .withAlternatives("hoodie.table.ro.file.format") .withDocumentation("Base file format to store all the base file data."); - public static final ConfigProperty HOODIE_LOG_FILE_FORMAT_PROP = ConfigProperty + public static final ConfigProperty LOG_FILE_FORMAT = ConfigProperty .key("hoodie.table.log.file.format") .defaultValue(HoodieFileFormat.HOODIE_LOG) .withAlternatives("hoodie.table.rt.file.format") .withDocumentation("Log format used for the delta logs."); - public static final ConfigProperty HOODIE_TIMELINE_LAYOUT_VERSION_PROP = ConfigProperty + public static final ConfigProperty TIMELINE_LAYOUT_VERSION = ConfigProperty .key("hoodie.timeline.layout.version") .noDefaultValue() .withDocumentation("Version of timeline used, by the table."); - public static final ConfigProperty HOODIE_PAYLOAD_CLASS_PROP = ConfigProperty + public static final ConfigProperty PAYLOAD_CLASS_NAME = ConfigProperty .key("hoodie.compaction.payload.class") .defaultValue(OverwriteWithLatestAvroPayload.class.getName()) .withDocumentation("Payload class to use for performing compactions, i.e merge delta logs with current base file and then " + " produce a new base file."); - public static final ConfigProperty HOODIE_ARCHIVELOG_FOLDER_PROP = ConfigProperty + public static final ConfigProperty ARCHIVELOG_FOLDER = ConfigProperty .key("hoodie.archivelog.folder") .defaultValue("archived") .withDocumentation("path under the meta folder, to store archived timeline instants at."); - public static final ConfigProperty HOODIE_BOOTSTRAP_INDEX_ENABLE_PROP = ConfigProperty + public static final ConfigProperty BOOTSTRAP_INDEX_ENABLE = ConfigProperty .key("hoodie.bootstrap.index.enable") .noDefaultValue() .withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined."); - public static final ConfigProperty HOODIE_BOOTSTRAP_INDEX_CLASS_PROP = ConfigProperty + public static final ConfigProperty BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty .key("hoodie.bootstrap.index.class") .defaultValue(HFileBootstrapIndex.class.getName()) .withDocumentation("Implementation to use, for mapping base files to bootstrap base file, that contain actual data."); - public static final ConfigProperty HOODIE_BOOTSTRAP_BASE_PATH_PROP = ConfigProperty + public static final ConfigProperty BOOTSTRAP_BASE_PATH = ConfigProperty .key("hoodie.bootstrap.base.path") .noDefaultValue() .withDocumentation("Base path of the dataset that needs to be bootstrapped as a Hudi table"); - public static final ConfigProperty HOODIE_POPULATE_META_FIELDS = ConfigProperty + public static final ConfigProperty POPULATE_META_FIELDS = ConfigProperty .key("hoodie.populate.meta.fields") .defaultValue("true") .withDocumentation("When enabled, populates all meta fields. When disabled, no meta fields are populated " + "and incremental queries will not be functional. This is only meant to be used for append only/immutable data for batch processing"); - public static final ConfigProperty HOODIE_TABLE_KEY_GENERATOR_CLASS = ConfigProperty + public static final ConfigProperty KEY_GENERATOR_CLASS_NAME = ConfigProperty .key("hoodie.table.keygenerator.class") .noDefaultValue() .withDocumentation("Key Generator class property for the hoodie table"); @@ -172,9 +172,9 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { try (FSDataInputStream inputStream = fs.open(propertyPath)) { props.load(inputStream); } - if (contains(HOODIE_PAYLOAD_CLASS_PROP) && payloadClassName != null - && !getString(HOODIE_PAYLOAD_CLASS_PROP).equals(payloadClassName)) { - setValue(HOODIE_PAYLOAD_CLASS_PROP, payloadClassName); + if (contains(PAYLOAD_CLASS_NAME) && payloadClassName != null + && !getString(PAYLOAD_CLASS_NAME).equals(payloadClassName)) { + setValue(PAYLOAD_CLASS_NAME, payloadClassName); try (FSDataOutputStream outputStream = fs.create(propertyPath)) { props.store(outputStream, "Properties saved on " + new Date(System.currentTimeMillis())); } @@ -182,7 +182,7 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { } catch (IOException e) { throw new HoodieIOException("Could not load Hoodie properties from " + propertyPath, e); } - ValidationUtils.checkArgument(contains(HOODIE_TABLE_TYPE_PROP) && contains(HOODIE_TABLE_NAME_PROP), + ValidationUtils.checkArgument(contains(TYPE) && contains(NAME), "hoodie.properties file seems invalid. Please check for left over `.updated` files if any, manually copy it to hoodie.properties and retry"); } @@ -205,21 +205,21 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { HoodieConfig hoodieConfig = new HoodieConfig(properties); Path propertyPath = new Path(metadataFolder, HOODIE_PROPERTIES_FILE); try (FSDataOutputStream outputStream = fs.create(propertyPath)) { - if (!hoodieConfig.contains(HOODIE_TABLE_NAME_PROP)) { - throw new IllegalArgumentException(HOODIE_TABLE_NAME_PROP.key() + " property needs to be specified"); + if (!hoodieConfig.contains(NAME)) { + throw new IllegalArgumentException(NAME.key() + " property needs to be specified"); } - hoodieConfig.setDefaultValue(HOODIE_TABLE_TYPE_PROP); - if (hoodieConfig.getString(HOODIE_TABLE_TYPE_PROP).equals(HoodieTableType.MERGE_ON_READ.name())) { - hoodieConfig.setDefaultValue(HOODIE_PAYLOAD_CLASS_PROP); + hoodieConfig.setDefaultValue(TYPE); + if (hoodieConfig.getString(TYPE).equals(HoodieTableType.MERGE_ON_READ.name())) { + hoodieConfig.setDefaultValue(PAYLOAD_CLASS_NAME); } - hoodieConfig.setDefaultValue(HOODIE_ARCHIVELOG_FOLDER_PROP); - if (!hoodieConfig.contains(HOODIE_TIMELINE_LAYOUT_VERSION_PROP)) { + hoodieConfig.setDefaultValue(ARCHIVELOG_FOLDER); + if (!hoodieConfig.contains(TIMELINE_LAYOUT_VERSION)) { // Use latest Version as default unless forced by client - hoodieConfig.setValue(HOODIE_TIMELINE_LAYOUT_VERSION_PROP, TimelineLayoutVersion.CURR_VERSION.toString()); + hoodieConfig.setValue(TIMELINE_LAYOUT_VERSION, TimelineLayoutVersion.CURR_VERSION.toString()); } - if (hoodieConfig.contains(HOODIE_BOOTSTRAP_BASE_PATH_PROP)) { + if (hoodieConfig.contains(BOOTSTRAP_BASE_PATH)) { // Use the default bootstrap index class. - hoodieConfig.setDefaultValue(HOODIE_BOOTSTRAP_INDEX_CLASS_PROP, getDefaultBootstrapIndexClass(properties)); + hoodieConfig.setDefaultValue(BOOTSTRAP_INDEX_CLASS_NAME, getDefaultBootstrapIndexClass(properties)); } hoodieConfig.getProps().store(outputStream, "Properties saved on " + new Date(System.currentTimeMillis())); } @@ -229,12 +229,12 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * Read the table type from the table properties and if not found, return the default. */ public HoodieTableType getTableType() { - return HoodieTableType.valueOf(getStringOrDefault(HOODIE_TABLE_TYPE_PROP)); + return HoodieTableType.valueOf(getStringOrDefault(TYPE)); } public Option getTimelineLayoutVersion() { - return contains(HOODIE_TIMELINE_LAYOUT_VERSION_PROP) - ? Option.of(new TimelineLayoutVersion(getInt(HOODIE_TIMELINE_LAYOUT_VERSION_PROP))) + return contains(TIMELINE_LAYOUT_VERSION) + ? Option.of(new TimelineLayoutVersion(getInt(TIMELINE_LAYOUT_VERSION))) : Option.empty(); } @@ -242,13 +242,13 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * @return the hoodie.table.version from hoodie.properties file. */ public HoodieTableVersion getTableVersion() { - return contains(HOODIE_TABLE_VERSION_PROP) - ? HoodieTableVersion.versionFromCode(getInt(HOODIE_TABLE_VERSION_PROP)) - : HOODIE_TABLE_VERSION_PROP.defaultValue(); + return contains(VERSION) + ? HoodieTableVersion.versionFromCode(getInt(VERSION)) + : VERSION.defaultValue(); } public void setTableVersion(HoodieTableVersion tableVersion) { - setValue(HOODIE_TABLE_VERSION_PROP, Integer.toString(tableVersion.versionCode())); + setValue(VERSION, Integer.toString(tableVersion.versionCode())); } /** @@ -257,26 +257,26 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { public String getPayloadClass() { // There could be tables written with payload class from com.uber.hoodie. Need to transparently // change to org.apache.hudi - return getStringOrDefault(HOODIE_PAYLOAD_CLASS_PROP).replace("com.uber.hoodie", + return getStringOrDefault(PAYLOAD_CLASS_NAME).replace("com.uber.hoodie", "org.apache.hudi"); } public String getPreCombineField() { - return getString(HOODIE_TABLE_PRECOMBINE_FIELD_PROP); + return getString(PRECOMBINE_FIELD); } public Option getRecordKeyFields() { - if (contains(HOODIE_TABLE_RECORDKEY_FIELDS)) { - return Option.of(Arrays.stream(getString(HOODIE_TABLE_RECORDKEY_FIELDS).split(",")) - .filter(p -> p.length() > 0).collect(Collectors.toList()).toArray(new String[]{})); + if (contains(RECORDKEY_FIELDS)) { + return Option.of(Arrays.stream(getString(RECORDKEY_FIELDS).split(",")) + .filter(p -> p.length() > 0).collect(Collectors.toList()).toArray(new String[] {})); } return Option.empty(); } public Option getPartitionFields() { - if (contains(HOODIE_TABLE_PARTITION_FIELDS_PROP)) { - return Option.of(Arrays.stream(getString(HOODIE_TABLE_PARTITION_FIELDS_PROP).split(",")) - .filter(p -> p.length() > 0).collect(Collectors.toList()).toArray(new String[]{})); + if (contains(PARTITION_FIELDS)) { + return Option.of(Arrays.stream(getString(PARTITION_FIELDS).split(",")) + .filter(p -> p.length() > 0).collect(Collectors.toList()).toArray(new String[] {})); } return Option.empty(); } @@ -285,7 +285,7 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * @returns the partition field prop. */ public String getPartitionFieldProp() { - return getString(HOODIE_TABLE_PARTITION_FIELDS_PROP); + return getString(PARTITION_FIELDS); } /** @@ -294,24 +294,24 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { public String getBootstrapIndexClass() { // There could be tables written with payload class from com.uber.hoodie. Need to transparently // change to org.apache.hudi - return getStringOrDefault(HOODIE_BOOTSTRAP_INDEX_CLASS_PROP, getDefaultBootstrapIndexClass(props)); + return getStringOrDefault(BOOTSTRAP_INDEX_CLASS_NAME, getDefaultBootstrapIndexClass(props)); } public static String getDefaultBootstrapIndexClass(Properties props) { - String defaultClass = HOODIE_BOOTSTRAP_INDEX_CLASS_PROP.defaultValue(); - if ("false".equalsIgnoreCase(props.getProperty(HOODIE_BOOTSTRAP_INDEX_ENABLE_PROP.key()))) { + String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue(); + if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) { defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS; } return defaultClass; } public Option getBootstrapBasePath() { - return Option.ofNullable(getString(HOODIE_BOOTSTRAP_BASE_PATH_PROP)); + return Option.ofNullable(getString(BOOTSTRAP_BASE_PATH)); } public Option getTableCreateSchema() { - if (contains(HOODIE_TABLE_CREATE_SCHEMA)) { - return Option.of(new Schema.Parser().parse(getString(HOODIE_TABLE_CREATE_SCHEMA))); + if (contains(CREATE_SCHEMA)) { + return Option.of(new Schema.Parser().parse(getString(CREATE_SCHEMA))); } else { return Option.empty(); } @@ -321,7 +321,7 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * Read the table name. */ public String getTableName() { - return getString(HOODIE_TABLE_NAME_PROP); + return getString(NAME); } /** @@ -330,7 +330,7 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * @return HoodieFileFormat for the base file Storage format */ public HoodieFileFormat getBaseFileFormat() { - return HoodieFileFormat.valueOf(getStringOrDefault(HOODIE_BASE_FILE_FORMAT_PROP)); + return HoodieFileFormat.valueOf(getStringOrDefault(BASE_FILE_FORMAT)); } /** @@ -339,32 +339,133 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable { * @return HoodieFileFormat for the log Storage format */ public HoodieFileFormat getLogFileFormat() { - return HoodieFileFormat.valueOf(getStringOrDefault(HOODIE_LOG_FILE_FORMAT_PROP)); + return HoodieFileFormat.valueOf(getStringOrDefault(LOG_FILE_FORMAT)); } /** * Get the relative path of archive log folder under metafolder, for this table. */ public String getArchivelogFolder() { - return getStringOrDefault(HOODIE_ARCHIVELOG_FOLDER_PROP); + return getStringOrDefault(ARCHIVELOG_FOLDER); } /** * @returns true is meta fields need to be populated. else returns false. */ public boolean populateMetaFields() { - return Boolean.parseBoolean(getStringOrDefault(HOODIE_POPULATE_META_FIELDS)); + return Boolean.parseBoolean(getStringOrDefault(POPULATE_META_FIELDS)); } /** * @returns the record key field prop. */ public String getRecordKeyFieldProp() { - return getString(HOODIE_TABLE_RECORDKEY_FIELDS); + return getString(RECORDKEY_FIELDS); } public Map propsMap() { return props.entrySet().stream() .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue()))); } + + /** + * @deprecated Use {@link #BASE_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final String HOODIE_RO_FILE_FORMAT_PROP_NAME = "hoodie.table.ro.file.format"; + /** + * @deprecated Use {@link #LOG_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final String HOODIE_RT_FILE_FORMAT_PROP_NAME = "hoodie.table.rt.file.format"; + /** + * @deprecated Use {@link #NAME} and its methods. + */ + @Deprecated + public static final String HOODIE_TABLE_NAME_PROP_NAME = NAME.key(); + /** + * @deprecated Use {@link #TYPE} and its methods. + */ + @Deprecated + public static final String HOODIE_TABLE_TYPE_PROP_NAME = TYPE.key(); + /** + * @deprecated Use {@link #VERSION} and its methods. + */ + @Deprecated + public static final String HOODIE_TABLE_VERSION_PROP_NAME = VERSION.key(); + /** + * @deprecated Use {@link #PRECOMBINE_FIELD} and its methods. + */ + @Deprecated + public static final String HOODIE_TABLE_PRECOMBINE_FIELD = PRECOMBINE_FIELD.key(); + /** + * @deprecated Use {@link #BASE_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final String HOODIE_BASE_FILE_FORMAT_PROP_NAME = BASE_FILE_FORMAT.key(); + /** + * @deprecated Use {@link #LOG_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final String HOODIE_LOG_FILE_FORMAT_PROP_NAME = LOG_FILE_FORMAT.key(); + /** + * @deprecated Use {@link #TIMELINE_LAYOUT_VERSION} and its methods. + */ + @Deprecated + public static final String HOODIE_TIMELINE_LAYOUT_VERSION = TIMELINE_LAYOUT_VERSION.key(); + /** + * @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods. + */ + @Deprecated + public static final String HOODIE_PAYLOAD_CLASS_PROP_NAME = PAYLOAD_CLASS_NAME.key(); + /** + * @deprecated Use {@link #ARCHIVELOG_FOLDER} and its methods. + */ + @Deprecated + public static final String HOODIE_ARCHIVELOG_FOLDER_PROP_NAME = ARCHIVELOG_FOLDER.key(); + /** + * @deprecated Use {@link #BOOTSTRAP_INDEX_CLASS_NAME} and its methods. + */ + @Deprecated + public static final String HOODIE_BOOTSTRAP_INDEX_CLASS_PROP_NAME = BOOTSTRAP_INDEX_CLASS_NAME.key(); + /** + * @deprecated Use {@link #BOOTSTRAP_BASE_PATH} and its methods. + */ + @Deprecated + public static final String HOODIE_BOOTSTRAP_BASE_PATH = BOOTSTRAP_BASE_PATH.key(); + /** + * @deprecated Use {@link #TYPE} and its methods. + */ + @Deprecated + public static final HoodieTableType DEFAULT_TABLE_TYPE = TYPE.defaultValue(); + /** + * @deprecated Use {@link #VERSION} and its methods. + */ + @Deprecated + public static final HoodieTableVersion DEFAULT_TABLE_VERSION = VERSION.defaultValue(); + /** + * @deprecated Use {@link #BASE_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final HoodieFileFormat DEFAULT_BASE_FILE_FORMAT = BASE_FILE_FORMAT.defaultValue(); + /** + * @deprecated Use {@link #LOG_FILE_FORMAT} and its methods. + */ + @Deprecated + public static final HoodieFileFormat DEFAULT_LOG_FILE_FORMAT = LOG_FILE_FORMAT.defaultValue(); + /** + * @deprecated Use {@link #PAYLOAD_CLASS_NAME} and its methods. + */ + @Deprecated + public static final String DEFAULT_PAYLOAD_CLASS = PAYLOAD_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #BOOTSTRAP_INDEX_CLASS_NAME} and its methods. + */ + @Deprecated + public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue(); + /** + * @deprecated Use {@link #ARCHIVELOG_FOLDER} and its methods. + */ + @Deprecated + public static final String DEFAULT_ARCHIVELOG_FOLDER = ARCHIVELOG_FOLDER.defaultValue(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 6899b939c..e4494bd0c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -227,7 +227,7 @@ public class HoodieTableMetaClient implements Serializable { */ public String getArchivePath() { String archiveFolder = tableConfig.getArchivelogFolder(); - if (archiveFolder.equals(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue())) { + if (archiveFolder.equals(HoodieTableConfig.ARCHIVELOG_FOLDER.defaultValue())) { return getMetaPath(); } else { return getMetaPath() + "/" + archiveFolder; @@ -321,16 +321,16 @@ public class HoodieTableMetaClient implements Serializable { public void validateTableProperties(Properties properties, WriteOperationType operationType) { // once meta fields are disabled, it cant be re-enabled for a given table. if (!getTableConfig().populateMetaFields() - && Boolean.parseBoolean((String) properties.getOrDefault(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()))) { - throw new HoodieException(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key() + " already disabled for the table. Can't be re-enabled back"); + && Boolean.parseBoolean((String) properties.getOrDefault(HoodieTableConfig.POPULATE_META_FIELDS.key(), HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()))) { + throw new HoodieException(HoodieTableConfig.POPULATE_META_FIELDS.key() + " already disabled for the table. Can't be re-enabled back"); } // meta fields can be disabled only with SimpleKeyGenerator if (!getTableConfig().populateMetaFields() - && !properties.getProperty(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS.key(), "org.apache.hudi.keygen.SimpleKeyGenerator") + && !properties.getProperty(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key(), "org.apache.hudi.keygen.SimpleKeyGenerator") .equals("org.apache.hudi.keygen.SimpleKeyGenerator")) { throw new HoodieException("Only simple key generator is supported when meta fields are disabled. KeyGenerator used : " - + properties.getProperty(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS.key())); + + properties.getProperty(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key())); } } @@ -353,7 +353,7 @@ public class HoodieTableMetaClient implements Serializable { } // if anything other than default archive log folder is specified, create that too - String archiveLogPropVal = new HoodieConfig(props).getStringOrDefault(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP); + String archiveLogPropVal = new HoodieConfig(props).getStringOrDefault(HoodieTableConfig.ARCHIVELOG_FOLDER); if (!StringUtils.isNullOrEmpty(archiveLogPropVal)) { Path archiveLogDir = new Path(metaPathDir, archiveLogPropVal); if (!fs.exists(archiveLogDir)) { @@ -725,52 +725,52 @@ public class HoodieTableMetaClient implements Serializable { public PropertyBuilder fromProperties(Properties properties) { HoodieConfig hoodieConfig = new HoodieConfig(properties); - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_NAME_PROP)) { - setTableName(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_NAME_PROP)); + if (hoodieConfig.contains(HoodieTableConfig.NAME)) { + setTableName(hoodieConfig.getString(HoodieTableConfig.NAME)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP)) { - setTableType(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP)); + if (hoodieConfig.contains(HoodieTableConfig.TYPE)) { + setTableType(hoodieConfig.getString(HoodieTableConfig.TYPE)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP)) { + if (hoodieConfig.contains(HoodieTableConfig.ARCHIVELOG_FOLDER)) { setArchiveLogFolder( - hoodieConfig.getString(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP)); + hoodieConfig.getString(HoodieTableConfig.ARCHIVELOG_FOLDER)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP)) { + if (hoodieConfig.contains(HoodieTableConfig.PAYLOAD_CLASS_NAME)) { setPayloadClassName( - hoodieConfig.getString(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP)); + hoodieConfig.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TIMELINE_LAYOUT_VERSION_PROP)) { - setTimelineLayoutVersion(hoodieConfig.getInt(HoodieTableConfig.HOODIE_TIMELINE_LAYOUT_VERSION_PROP)); + if (hoodieConfig.contains(HoodieTableConfig.TIMELINE_LAYOUT_VERSION)) { + setTimelineLayoutVersion(hoodieConfig.getInt(HoodieTableConfig.TIMELINE_LAYOUT_VERSION)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP)) { + if (hoodieConfig.contains(HoodieTableConfig.BASE_FILE_FORMAT)) { setBaseFileFormat( - hoodieConfig.getString(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP)); + hoodieConfig.getString(HoodieTableConfig.BASE_FILE_FORMAT)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_CLASS_PROP)) { + if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME)) { setBootstrapIndexClass( - hoodieConfig.getString(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_CLASS_PROP)); + hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP)) { - setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP)); + if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) { + setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP)) { - setPreCombineField(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP)); + if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) { + setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP)) { + if (hoodieConfig.contains(HoodieTableConfig.PARTITION_FIELDS)) { setPartitionFields( - hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP)); + hoodieConfig.getString(HoodieTableConfig.PARTITION_FIELDS)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS)) { - setRecordKeyFields(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS)); + if (hoodieConfig.contains(HoodieTableConfig.RECORDKEY_FIELDS)) { + setRecordKeyFields(hoodieConfig.getString(HoodieTableConfig.RECORDKEY_FIELDS)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA)) { - setTableCreateSchema(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA)); + if (hoodieConfig.contains(HoodieTableConfig.CREATE_SCHEMA)) { + setTableCreateSchema(hoodieConfig.getString(HoodieTableConfig.CREATE_SCHEMA)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS)) { - setPopulateMetaFields(hoodieConfig.getBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS)); + if (hoodieConfig.contains(HoodieTableConfig.POPULATE_META_FIELDS)) { + setPopulateMetaFields(hoodieConfig.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS)); } - if (hoodieConfig.contains(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS)) { - setKeyGeneratorClassProp(hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS)); + if (hoodieConfig.contains(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)) { + setKeyGeneratorClassProp(hoodieConfig.getString(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)); } return this; } @@ -780,56 +780,56 @@ public class HoodieTableMetaClient implements Serializable { ValidationUtils.checkArgument(tableName != null, "tableName is null"); HoodieTableConfig tableConfig = new HoodieTableConfig(); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_NAME_PROP, tableName); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP, tableType.name()); - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_VERSION_PROP, + tableConfig.setValue(HoodieTableConfig.NAME, tableName); + tableConfig.setValue(HoodieTableConfig.TYPE, tableType.name()); + tableConfig.setValue(HoodieTableConfig.VERSION, String.valueOf(HoodieTableVersion.current().versionCode())); if (tableType == HoodieTableType.MERGE_ON_READ && payloadClassName != null) { - tableConfig.setValue(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP, payloadClassName); + tableConfig.setValue(HoodieTableConfig.PAYLOAD_CLASS_NAME, payloadClassName); } if (null != tableCreateSchema) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA, tableCreateSchema); + tableConfig.setValue(HoodieTableConfig.CREATE_SCHEMA, tableCreateSchema); } if (!StringUtils.isNullOrEmpty(archiveLogFolder)) { - tableConfig.setValue(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP, archiveLogFolder); + tableConfig.setValue(HoodieTableConfig.ARCHIVELOG_FOLDER, archiveLogFolder); } else { - tableConfig.setDefaultValue(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP); + tableConfig.setDefaultValue(HoodieTableConfig.ARCHIVELOG_FOLDER); } if (null != timelineLayoutVersion) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TIMELINE_LAYOUT_VERSION_PROP, + tableConfig.setValue(HoodieTableConfig.TIMELINE_LAYOUT_VERSION, String.valueOf(timelineLayoutVersion)); } if (null != baseFileFormat) { - tableConfig.setValue(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP, baseFileFormat.toUpperCase()); + tableConfig.setValue(HoodieTableConfig.BASE_FILE_FORMAT, baseFileFormat.toUpperCase()); } if (null != bootstrapIndexClass) { - tableConfig.setValue(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_CLASS_PROP, bootstrapIndexClass); + tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass); } if (null != bootstrapBasePath) { - tableConfig.setValue(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP, bootstrapBasePath); + tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath); } if (null != preCombineField) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP, preCombineField); + tableConfig.setValue(HoodieTableConfig.PRECOMBINE_FIELD, preCombineField); } if (null != partitionFields) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP, partitionFields); + tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, partitionFields); } if (null != recordKeyFields) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS, recordKeyFields); + tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, recordKeyFields); } if (null != populateMetaFields) { - tableConfig.setValue(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); + tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); } if (null != keyGeneratorClassProp) { - tableConfig.setValue(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS, keyGeneratorClassProp); + tableConfig.setValue(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME, keyGeneratorClassProp); } return tableConfig.getProps(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java index 603f6bc0c..b77b0d3a8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java @@ -40,7 +40,7 @@ import java.util.stream.Collectors; public class FileSystemViewStorageConfig extends HoodieConfig { // Property Names - public static final ConfigProperty FILESYSTEM_VIEW_STORAGE_TYPE = ConfigProperty + public static final ConfigProperty VIEW_TYPE = ConfigProperty .key("hoodie.filesystem.view.type") .defaultValue(FileSystemViewStorageType.MEMORY) .withDocumentation("File system view provides APIs for viewing the files on the underlying lake storage, " @@ -48,69 +48,69 @@ public class FileSystemViewStorageConfig extends HoodieConfig { + Arrays.stream(FileSystemViewStorageType.values()).map(Enum::name).collect(Collectors.joining(",")) + " which provide different trade offs for memory usage and API request performance."); - public static final ConfigProperty FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE = ConfigProperty + public static final ConfigProperty INCREMENTAL_TIMELINE_SYNC_ENABLE = ConfigProperty .key("hoodie.filesystem.view.incr.timeline.sync.enable") .defaultValue("false") .withDocumentation("Controls whether or not, the file system view is incrementally updated as " + "new actions are performed on the timeline."); - public static final ConfigProperty FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE = ConfigProperty + public static final ConfigProperty SECONDARY_VIEW_TYPE = ConfigProperty .key("hoodie.filesystem.view.secondary.type") .defaultValue(FileSystemViewStorageType.MEMORY) .withDocumentation("Specifies the secondary form of storage for file system view, if the primary (e.g timeline server) " + " is unavailable."); - public static final ConfigProperty FILESYSTEM_VIEW_REMOTE_HOST = ConfigProperty + public static final ConfigProperty REMOTE_HOST_NAME = ConfigProperty .key("hoodie.filesystem.view.remote.host") .defaultValue("localhost") .withDocumentation("We expect this to be rarely hand configured."); - public static final ConfigProperty FILESYSTEM_VIEW_REMOTE_PORT = ConfigProperty + public static final ConfigProperty REMOTE_PORT_NUM = ConfigProperty .key("hoodie.filesystem.view.remote.port") .defaultValue(26754) .withDocumentation("Port to serve file system view queries, when remote. We expect this to be rarely hand configured."); - public static final ConfigProperty FILESYSTEM_VIEW_SPILLABLE_DIR = ConfigProperty + public static final ConfigProperty SPILLABLE_DIR = ConfigProperty .key("hoodie.filesystem.view.spillable.dir") .defaultValue("/tmp/") .withDocumentation("Path on local storage to use, when file system view is held in a spillable map."); - public static final ConfigProperty FILESYSTEM_VIEW_SPILLABLE_MEM = ConfigProperty + public static final ConfigProperty SPILLABLE_MEMORY = ConfigProperty .key("hoodie.filesystem.view.spillable.mem") .defaultValue(100 * 1024 * 1024L) // 100 MB .withDocumentation("Amount of memory to be used for holding file system view, before spilling to disk."); - public static final ConfigProperty FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION = ConfigProperty + public static final ConfigProperty SPILLABLE_COMPACTION_MEM_FRACTION = ConfigProperty .key("hoodie.filesystem.view.spillable.compaction.mem.fraction") .defaultValue(0.8) .withDocumentation("Fraction of the file system view memory, to be used for holding compaction related metadata."); - public static final ConfigProperty FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION = ConfigProperty + public static final ConfigProperty BOOTSTRAP_BASE_FILE_MEM_FRACTION = ConfigProperty .key("hoodie.filesystem.view.spillable.bootstrap.base.file.mem.fraction") .defaultValue(0.05) .withDocumentation("Fraction of the file system view memory, to be used for holding mapping to bootstrap base files."); - public static final ConfigProperty FILESYSTEM_VIEW_REPLACED_MEM_FRACTION = ConfigProperty + public static final ConfigProperty SPILLABLE_REPLACED_MEM_FRACTION = ConfigProperty .key("hoodie.filesystem.view.spillable.replaced.mem.fraction") .defaultValue(0.01) .withDocumentation("Fraction of the file system view memory, to be used for holding replace commit related metadata."); - public static final ConfigProperty FILESYSTEM_VIEW_PENDING_CLUSTERING_MEM_FRACTION = ConfigProperty + public static final ConfigProperty SPILLABLE_CLUSTERING_MEM_FRACTION = ConfigProperty .key("hoodie.filesystem.view.spillable.clustering.mem.fraction") .defaultValue(0.01) .withDocumentation("Fraction of the file system view memory, to be used for holding clustering related metadata."); - public static final ConfigProperty ROCKSDB_BASE_PATH_PROP = ConfigProperty + public static final ConfigProperty ROCKSDB_BASE_PATH = ConfigProperty .key("hoodie.filesystem.view.rocksdb.base.path") .defaultValue("/tmp/hoodie_timeline_rocksdb") .withDocumentation("Path on local storage to use, when storing file system view in embedded kv store/rocksdb."); - public static final ConfigProperty FILESYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS = ConfigProperty + public static final ConfigProperty REMOTE_TIMEOUT_SECS = ConfigProperty .key("hoodie.filesystem.view.remote.timeout.secs") .defaultValue(5 * 60) // 5 min .withDocumentation("Timeout in seconds, to wait for API requests against a remote file system view. e.g timeline server."); - public static final ConfigProperty REMOTE_BACKUP_VIEW_HANDLER_ENABLE = ConfigProperty + public static final ConfigProperty REMOTE_BACKUP_VIEW_ENABLE = ConfigProperty .key("hoodie.filesystem.remote.backup.view.enable") .defaultValue("true") // Need to be disabled only for tests. .withDocumentation("Config to control whether backup needs to be configured if clients were not able to reach" @@ -125,70 +125,70 @@ public class FileSystemViewStorageConfig extends HoodieConfig { } public FileSystemViewStorageType getStorageType() { - return FileSystemViewStorageType.valueOf(getString(FILESYSTEM_VIEW_STORAGE_TYPE)); + return FileSystemViewStorageType.valueOf(getString(VIEW_TYPE)); } public boolean isIncrementalTimelineSyncEnabled() { - return getBoolean(FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE); + return getBoolean(INCREMENTAL_TIMELINE_SYNC_ENABLE); } public String getRemoteViewServerHost() { - return getString(FILESYSTEM_VIEW_REMOTE_HOST); + return getString(REMOTE_HOST_NAME); } public Integer getRemoteViewServerPort() { - return getInt(FILESYSTEM_VIEW_REMOTE_PORT); + return getInt(REMOTE_PORT_NUM); } public Integer getRemoteTimelineClientTimeoutSecs() { - return getInt(FILESYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS); + return getInt(REMOTE_TIMEOUT_SECS); } public long getMaxMemoryForFileGroupMap() { - long totalMemory = getLong(FILESYSTEM_VIEW_SPILLABLE_MEM); + long totalMemory = getLong(SPILLABLE_MEMORY); return totalMemory - getMaxMemoryForPendingCompaction() - getMaxMemoryForBootstrapBaseFile(); } public long getMaxMemoryForPendingCompaction() { - long totalMemory = getLong(FILESYSTEM_VIEW_SPILLABLE_MEM); - return new Double(totalMemory * getDouble(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION)) + long totalMemory = getLong(SPILLABLE_MEMORY); + return new Double(totalMemory * getDouble(SPILLABLE_COMPACTION_MEM_FRACTION)) .longValue(); } public long getMaxMemoryForBootstrapBaseFile() { - long totalMemory = getLong(FILESYSTEM_VIEW_SPILLABLE_MEM); + long totalMemory = getLong(SPILLABLE_MEMORY); long reservedForExternalDataFile = - new Double(totalMemory * getDouble(FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION)) + new Double(totalMemory * getDouble(BOOTSTRAP_BASE_FILE_MEM_FRACTION)) .longValue(); return reservedForExternalDataFile; } public long getMaxMemoryForReplacedFileGroups() { - long totalMemory = getLong(FILESYSTEM_VIEW_SPILLABLE_MEM); - return new Double(totalMemory * getDouble(FILESYSTEM_VIEW_REPLACED_MEM_FRACTION)) + long totalMemory = getLong(SPILLABLE_MEMORY); + return new Double(totalMemory * getDouble(SPILLABLE_REPLACED_MEM_FRACTION)) .longValue(); } public long getMaxMemoryForPendingClusteringFileGroups() { - long totalMemory = getLong(FILESYSTEM_VIEW_SPILLABLE_MEM); - return new Double(totalMemory * getDouble(FILESYSTEM_VIEW_PENDING_CLUSTERING_MEM_FRACTION)) + long totalMemory = getLong(SPILLABLE_MEMORY); + return new Double(totalMemory * getDouble(SPILLABLE_CLUSTERING_MEM_FRACTION)) .longValue(); } public String getSpillableDir() { - return getString(FILESYSTEM_VIEW_SPILLABLE_DIR); + return getString(SPILLABLE_DIR); } public FileSystemViewStorageType getSecondaryStorageType() { - return FileSystemViewStorageType.valueOf(getString(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE)); + return FileSystemViewStorageType.valueOf(getString(SECONDARY_VIEW_TYPE)); } public boolean shouldEnableBackupForRemoteFileSystemView() { - return getBoolean(REMOTE_BACKUP_VIEW_HANDLER_ENABLE); + return getBoolean(REMOTE_BACKUP_VIEW_ENABLE); } public String getRocksdbBasePath() { - return getString(ROCKSDB_BASE_PATH_PROP); + return getString(ROCKSDB_BASE_PATH); } /** @@ -211,73 +211,203 @@ public class FileSystemViewStorageConfig extends HoodieConfig { } public Builder withStorageType(FileSystemViewStorageType storageType) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_STORAGE_TYPE, storageType.name()); + fileSystemViewStorageConfig.setValue(VIEW_TYPE, storageType.name()); return this; } public Builder withSecondaryStorageType(FileSystemViewStorageType storageType) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE, storageType.name()); + fileSystemViewStorageConfig.setValue(SECONDARY_VIEW_TYPE, storageType.name()); return this; } public Builder withIncrementalTimelineSync(boolean enableIncrTimelineSync) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE, Boolean.toString(enableIncrTimelineSync)); + fileSystemViewStorageConfig.setValue(INCREMENTAL_TIMELINE_SYNC_ENABLE, Boolean.toString(enableIncrTimelineSync)); return this; } public Builder withRemoteServerHost(String remoteServerHost) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_REMOTE_HOST, remoteServerHost); + fileSystemViewStorageConfig.setValue(REMOTE_HOST_NAME, remoteServerHost); return this; } public Builder withRemoteServerPort(Integer remoteServerPort) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_REMOTE_PORT, remoteServerPort.toString()); + fileSystemViewStorageConfig.setValue(REMOTE_PORT_NUM, remoteServerPort.toString()); return this; } public Builder withMaxMemoryForView(Long maxMemoryForView) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_SPILLABLE_MEM, maxMemoryForView.toString()); + fileSystemViewStorageConfig.setValue(SPILLABLE_MEMORY, maxMemoryForView.toString()); return this; } public Builder withRemoteTimelineClientTimeoutSecs(Long timelineClientTimeoutSecs) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS, timelineClientTimeoutSecs.toString()); + fileSystemViewStorageConfig.setValue(REMOTE_TIMEOUT_SECS, timelineClientTimeoutSecs.toString()); return this; } public Builder withMemFractionForPendingCompaction(Double memFractionForPendingCompaction) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION, memFractionForPendingCompaction.toString()); + fileSystemViewStorageConfig.setValue(SPILLABLE_COMPACTION_MEM_FRACTION, memFractionForPendingCompaction.toString()); return this; } public Builder withMemFractionForExternalDataFile(Double memFractionForExternalDataFile) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION, memFractionForExternalDataFile.toString()); + fileSystemViewStorageConfig.setValue(BOOTSTRAP_BASE_FILE_MEM_FRACTION, memFractionForExternalDataFile.toString()); return this; } public Builder withBaseStoreDir(String baseStorePath) { - fileSystemViewStorageConfig.setValue(FILESYSTEM_VIEW_SPILLABLE_DIR, baseStorePath); + fileSystemViewStorageConfig.setValue(SPILLABLE_DIR, baseStorePath); return this; } public Builder withRocksDBPath(String basePath) { - fileSystemViewStorageConfig.setValue(ROCKSDB_BASE_PATH_PROP, basePath); + fileSystemViewStorageConfig.setValue(ROCKSDB_BASE_PATH, basePath); return this; } public Builder withEnableBackupForRemoteFileSystemView(boolean enable) { - fileSystemViewStorageConfig.setValue(REMOTE_BACKUP_VIEW_HANDLER_ENABLE, Boolean.toString(enable)); + fileSystemViewStorageConfig.setValue(REMOTE_BACKUP_VIEW_ENABLE, Boolean.toString(enable)); return this; } public FileSystemViewStorageConfig build() { fileSystemViewStorageConfig.setDefaults(FileSystemViewStorageConfig.class.getName()); // Validations - FileSystemViewStorageType.valueOf(fileSystemViewStorageConfig.getString(FILESYSTEM_VIEW_STORAGE_TYPE)); - FileSystemViewStorageType.valueOf(fileSystemViewStorageConfig.getString(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE)); - ValidationUtils.checkArgument(fileSystemViewStorageConfig.getInt(FILESYSTEM_VIEW_REMOTE_PORT) > 0); + FileSystemViewStorageType.valueOf(fileSystemViewStorageConfig.getString(VIEW_TYPE)); + FileSystemViewStorageType.valueOf(fileSystemViewStorageConfig.getString(SECONDARY_VIEW_TYPE)); + ValidationUtils.checkArgument(fileSystemViewStorageConfig.getInt(REMOTE_PORT_NUM) > 0); return fileSystemViewStorageConfig; } } + /** + * @deprecated Use {@link #VIEW_TYPE} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_STORAGE_TYPE = VIEW_TYPE.key(); + /** + * @deprecated Use {@link #VIEW_TYPE} and its methods. + */ + @Deprecated + public static final FileSystemViewStorageType DEFAULT_VIEW_STORAGE_TYPE = VIEW_TYPE.defaultValue(); + /** + * @deprecated Use {@link #INCREMENTAL_TIMELINE_SYNC_ENABLE} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE = INCREMENTAL_TIMELINE_SYNC_ENABLE.key(); + /** + * @deprecated Use {@link #INCREMENTAL_TIMELINE_SYNC_ENABLE} and its methods. + */ + @Deprecated + public static final String DEFAULT_FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE = INCREMENTAL_TIMELINE_SYNC_ENABLE.defaultValue(); + /** + * @deprecated Use {@link #SECONDARY_VIEW_TYPE} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE = SECONDARY_VIEW_TYPE.key(); + /** + * @deprecated Use {@link #SECONDARY_VIEW_TYPE} and its methods. + */ + @Deprecated + public static final FileSystemViewStorageType DEFAULT_SECONDARY_VIEW_STORAGE_TYPE = SECONDARY_VIEW_TYPE.defaultValue(); + /** + * @deprecated Use {@link #REMOTE_HOST_NAME} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_REMOTE_HOST = REMOTE_HOST_NAME.key(); + /** + * @deprecated Use {@link #REMOTE_HOST_NAME} and its methods. + */ + @Deprecated + public static final String DEFUALT_REMOTE_VIEW_SERVER_HOST = REMOTE_HOST_NAME.defaultValue(); + /** + * @deprecated Use {@link #REMOTE_PORT_NUM} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_REMOTE_PORT = REMOTE_PORT_NUM.key(); + /** + * @deprecated Use {@link #REMOTE_PORT_NUM} and its methods. + */ + @Deprecated + public static final Integer DEFAULT_REMOTE_VIEW_SERVER_PORT = REMOTE_PORT_NUM.defaultValue(); + /** + * @deprecated Use {@link #SPILLABLE_DIR} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_SPILLABLE_DIR = SPILLABLE_DIR.key(); + /** + * @deprecated Use {@link #SPILLABLE_DIR} and its methods. + */ + @Deprecated + public static final String DEFAULT_VIEW_SPILLABLE_DIR = SPILLABLE_DIR.defaultValue(); + /** + * @deprecated Use {@link #SPILLABLE_MEMORY} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_SPILLABLE_MEM = SPILLABLE_MEMORY.key(); + /** + * @deprecated Use {@link #SPILLABLE_MEMORY} and its methods. + */ + @Deprecated + private static final Long DEFAULT_MAX_MEMORY_FOR_VIEW = SPILLABLE_MEMORY.defaultValue(); + /** + * @deprecated Use {@link #SPILLABLE_COMPACTION_MEM_FRACTION} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION = SPILLABLE_COMPACTION_MEM_FRACTION.key(); + /** + * @deprecated Use {@link #SPILLABLE_COMPACTION_MEM_FRACTION} and its methods. + */ + @Deprecated + private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION = SPILLABLE_COMPACTION_MEM_FRACTION.defaultValue(); + /** + * @deprecated Use {@link #BOOTSTRAP_BASE_FILE_MEM_FRACTION} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION = BOOTSTRAP_BASE_FILE_MEM_FRACTION.key(); + /** + * @deprecated Use {@link #SPILLABLE_REPLACED_MEM_FRACTION} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_REPLACED_MEM_FRACTION = SPILLABLE_REPLACED_MEM_FRACTION.key(); + /** + * @deprecated Use {@link #SPILLABLE_REPLACED_MEM_FRACTION} and its methods. + */ + @Deprecated + private static final Double DEFAULT_MEM_FRACTION_FOR_REPLACED_FILEGROUPS = SPILLABLE_REPLACED_MEM_FRACTION.defaultValue(); + /** + * @deprecated Use {@link #SPILLABLE_CLUSTERING_MEM_FRACTION} and its methods. + */ + @Deprecated + public static final String FILESYSTEM_VIEW_PENDING_CLUSTERING_MEM_FRACTION = SPILLABLE_CLUSTERING_MEM_FRACTION.key(); + /** + * @deprecated Use {@link #SPILLABLE_CLUSTERING_MEM_FRACTION} and its methods. + */ + @Deprecated + private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_CLUSTERING_FILEGROUPS = SPILLABLE_CLUSTERING_MEM_FRACTION.defaultValue(); + /** + * @deprecated Use {@link #ROCKSDB_BASE_PATH} and its methods. + */ + @Deprecated + private static final String ROCKSDB_BASE_PATH_PROP = ROCKSDB_BASE_PATH.key(); + /** + * @deprecated Use {@link #ROCKSDB_BASE_PATH} and its methods. + */ + @Deprecated + public static final String DEFAULT_ROCKSDB_BASE_PATH = ROCKSDB_BASE_PATH.defaultValue(); + /** + * @deprecated Use {@link #REMOTE_TIMEOUT_SECS} and its methods. + */ + @Deprecated + public static final String FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS = REMOTE_TIMEOUT_SECS.key(); + /** + * @deprecated Use {@link #REMOTE_TIMEOUT_SECS} and its methods. + */ + @Deprecated + public static final Integer DEFAULT_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS = REMOTE_TIMEOUT_SECS.defaultValue(); + /** + * @deprecated Use {@link #BOOTSTRAP_BASE_FILE_MEM_FRACTION} and its methods. + */ + @Deprecated + private static final Double DEFAULT_MEM_FRACTION_FOR_EXTERNAL_DATA_FILE = BOOTSTRAP_BASE_FILE_MEM_FRACTION.defaultValue(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java index d781b3703..ab51b8e2a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java @@ -55,5 +55,36 @@ public class KeyGeneratorOptions extends HoodieConfig { .defaultValue("partitionpath") .withDocumentation("Partition path field. Value to be used at the partitionPath component of HoodieKey. " + "Actual value ontained by invoking .toString()"); + + /** + * @deprecated Use {@link #URL_ENCODE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String URL_ENCODE_PARTITIONING_OPT_KEY = URL_ENCODE_PARTITIONING.key(); + /** + * @deprecated Use {@link #URL_ENCODE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue(); + /** + * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING.key(); + /** + * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. + */ + @Deprecated + public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue(); + /** + * @deprecated Use {@link #RECORDKEY_FIELD} and its methods. + */ + @Deprecated + public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD.key(); + /** + * @deprecated Use {@link #PARTITIONPATH_FIELD} and its methods. + */ + @Deprecated + public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD.key(); } diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java index 999ea8e2a..453ec8f15 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java @@ -51,7 +51,7 @@ public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView { HoodieMetadataConfig metadataConfig) { super(metaClient, visibleActiveTimeline); this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(), - FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue()); + FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue()); } /** diff --git a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java index 391435e4d..aba11220f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java @@ -34,6 +34,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; @@ -52,9 +54,6 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -92,7 +91,7 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness { @Test public void testNoOpBootstrapIndex() throws IOException { Properties props = metaClient.getTableConfig().getProps(); - props.put(HoodieTableConfig.HOODIE_BOOTSTRAP_INDEX_ENABLE_PROP.key(), "false"); + props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), "false"); Properties properties = new Properties(); properties.putAll(props); HoodieTableConfig.createHoodieProperties(metaClient.getFs(), new Path(metaClient.getMetaPath()), properties); @@ -185,7 +184,7 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness { return Arrays.stream(partitions).map(partition -> { return Pair.of(partition, IntStream.range(0, numEntriesPerPartition).mapToObj(idx -> { String hudiFileId = UUID.randomUUID().toString(); - String sourceFileName = idx + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + String sourceFileName = idx + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); HoodieFileStatus sourceFileStatus = HoodieFileStatus.newBuilder() .setPath(HoodiePath.newBuilder().setUri(sourceBasePath + "/" + partition + "/" + sourceFileName).build()) .setLength(256 * 1024 * 1024L) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index 5a73667b3..abfeee9c6 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -18,14 +18,15 @@ package org.apache.hudi.common.fs; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.exception.HoodieException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.jupiter.api.BeforeEach; @@ -57,7 +58,7 @@ public class TestFSUtils extends HoodieCommonTestHarness { private final long minCleanToKeep = 10; private static String TEST_WRITE_TOKEN = "1-0-1"; - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); @Rule public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieDeltaWriteStat.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieDeltaWriteStat.java index 8393de591..b774e06ce 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieDeltaWriteStat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieDeltaWriteStat.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.model; import org.apache.hudi.common.table.HoodieTableConfig; + import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -34,7 +35,7 @@ public class TestHoodieDeltaWriteStat { @Test public void testBaseFileAndLogFiles() { HoodieDeltaWriteStat writeStat = new HoodieDeltaWriteStat(); - String baseFile = "file1" + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + String baseFile = "file1" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); String logFile1 = ".log1.log"; String logFile2 = ".log2.log"; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java index 5e73d1833..e0f1f42a8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java @@ -68,7 +68,7 @@ public class TestPriorityBasedFileSystemView { fsView = new PriorityBasedFileSystemView(primary, secondary); testBaseFileStream = Stream.of(new HoodieBaseFile("test")); testFileSliceStream = Stream.of(new FileSlice("2020-01-01", "20:20", - "file0001" + HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension())); + "file0001" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension())); } private void resetMocks() { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java index 960ed2813..bb6c0b491 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java @@ -51,16 +51,16 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; -import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeCompactionPlan; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeCleanMetadata; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeCleanerPlan; +import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeCompactionPlan; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeRequestedReplaceMetadata; import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeRollbackMetadata; public class FileCreateUtils { private static final String WRITE_TOKEN = "1-0-1"; - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); public static String baseFileName(String instantTime, String fileId) { return baseFileName(instantTime, fileId, BASE_FILE_EXTENSION); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java index 046fc9ed3..e6c488e49 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java @@ -397,7 +397,7 @@ public class HoodieTestTable { } public FileStatus[] listAllBaseFiles() throws IOException { - return listAllBaseFiles(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension()); + return listAllBaseFiles(HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()); } public FileStatus[] listAllBaseFiles(String fileExtension) throws IOException { diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java index cf83eaca7..bc1c18a79 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java @@ -65,7 +65,7 @@ public class HoodieTestUtils { public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException { Properties props = new Properties(); - props.setProperty(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP.key(), bootstrapBasePath); + props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath); return init(getDefaultHadoopConf(), basePath, tableType, props); } @@ -86,7 +86,7 @@ public class HoodieTestUtils { String tableName) throws IOException { Properties properties = new Properties(); - properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP.key(), tableName); + properties.setProperty(HoodieTableConfig.NAME.key(), tableName); return init(hadoopConf, basePath, tableType, properties); } @@ -94,7 +94,7 @@ public class HoodieTestUtils { HoodieFileFormat baseFileFormat) throws IOException { Properties properties = new Properties(); - properties.setProperty(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.key(), baseFileFormat.toString()); + properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), baseFileFormat.toString()); return init(hadoopConf, basePath, tableType, properties); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java index 49659d196..ab573f0d8 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java @@ -17,12 +17,13 @@ package org.apache.hudi.common.util; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -37,7 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; public final class TestTablePathUtils { - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); @TempDir static File tempDir; diff --git a/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala b/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala index ada5aeaa1..cb221fcef 100644 --- a/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala +++ b/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala @@ -18,11 +18,11 @@ package org.apache.hudi.examples.spark -import org.apache.hudi.DataSourceReadOptions.{BEGIN_INSTANTTIME, END_INSTANTTIME, QUERY_TYPE_INCREMENTAL_OPT_VAL, QUERY_TYPE} +import org.apache.hudi.DataSourceReadOptions.{BEGIN_INSTANTTIME, END_INSTANTTIME, QUERY_TYPE, QUERY_TYPE_INCREMENTAL_OPT_VAL} import org.apache.hudi.DataSourceWriteOptions.{PARTITIONPATH_FIELD, PRECOMBINE_FIELD, RECORDKEY_FIELD, PARTITIONS_TO_DELETE, OPERATION, DELETE_PARTITION_OPERATION_OPT_VAL, DELETE_OPERATION_OPT_VAL} import org.apache.hudi.QuickstartUtils.getQuickstartWriteConfigs import org.apache.hudi.common.model.HoodieAvroPayload -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.examples.common.{HoodieExampleDataGenerator, HoodieExampleSparkUtils} import org.apache.spark.sql.SaveMode.{Append, Overwrite} import org.apache.spark.sql.SparkSession @@ -76,12 +76,12 @@ object HoodieDataSourceExample { val inserts = dataGen.convertToStringList(dataGen.generateInserts(commitTime, 20)) val df = spark.read.json(spark.sparkContext.parallelize(inserts, 1)) df.write.format("org.apache.hudi"). - options(getQuickstartWriteConfigs). - option(PRECOMBINE_FIELD.key, "ts"). - option(RECORDKEY_FIELD.key, "uuid"). - option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). - mode(Overwrite). + options(getQuickstartWriteConfigs). + option(PRECOMBINE_FIELD.key, "ts"). + option(RECORDKEY_FIELD.key, "uuid"). + option(PARTITIONPATH_FIELD.key, "partitionpath"). + option(TBL_NAME.key, tableName). + mode(Overwrite). save(tablePath) } @@ -121,12 +121,12 @@ object HoodieDataSourceExample { val updates = dataGen.convertToStringList(dataGen.generateUpdates(commitTime, 10)) val df = spark.read.json(spark.sparkContext.parallelize(updates, 1)) df.write.format("org.apache.hudi"). - options(getQuickstartWriteConfigs). - option(PRECOMBINE_FIELD.key, "ts"). - option(RECORDKEY_FIELD.key, "uuid"). - option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). - mode(Append). + options(getQuickstartWriteConfigs). + option(PRECOMBINE_FIELD.key, "ts"). + option(RECORDKEY_FIELD.key, "uuid"). + option(PARTITIONPATH_FIELD.key, "partitionpath"). + option(TBL_NAME.key, tableName). + mode(Append). save(tablePath) } @@ -144,7 +144,7 @@ object HoodieDataSourceExample { option(PRECOMBINE_FIELD.key, "ts"). option(RECORDKEY_FIELD.key, "uuid"). option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). + option(TBL_NAME.key, tableName). option(OPERATION.key, DELETE_OPERATION_OPT_VAL). mode(Append). save(tablePath) @@ -160,7 +160,7 @@ object HoodieDataSourceExample { option(PRECOMBINE_FIELD.key, "ts"). option(RECORDKEY_FIELD.key, "uuid"). option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). + option(TBL_NAME.key, tableName). option(OPERATION.key, DELETE_PARTITION_OPERATION_OPT_VAL). option(PARTITIONS_TO_DELETE.key(), HoodieExampleDataGenerator.DEFAULT_PARTITION_PATHS.mkString(",")). mode(Append). diff --git a/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala b/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala index ace086703..2d96f9c6c 100644 --- a/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala +++ b/hudi-examples/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala @@ -25,7 +25,7 @@ import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.model.{HoodieAvroPayload, HoodieRecordPayload, HoodieTableType} import org.apache.hudi.common.util.Option -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.config.{HoodieCompactionConfig, HoodieWriteConfig} import org.apache.hudi.examples.common.{HoodieExampleDataGenerator, HoodieExampleSparkUtils} import org.apache.spark.sql.SaveMode.{Append, Overwrite} @@ -89,7 +89,7 @@ object HoodieMorCompactionJob { option(PRECOMBINE_FIELD.key, "ts"). option(RECORDKEY_FIELD.key, "uuid"). option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). + option(TBL_NAME.key, tableName). option(TABLE_TYPE.key, tableType). mode(Overwrite). save(tablePath) @@ -105,7 +105,7 @@ object HoodieMorCompactionJob { option(PRECOMBINE_FIELD.key, "ts"). option(RECORDKEY_FIELD.key, "uuid"). option(PARTITIONPATH_FIELD.key, "partitionpath"). - option(TABLE_NAME.key, tableName). + option(TBL_NAME.key, tableName). option(TABLE_TYPE.key, tableType). mode(Append). save(tablePath) diff --git a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java index 5a2751a4f..72e8fccb9 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java +++ b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java @@ -207,7 +207,7 @@ public class FlinkOptions extends HoodieConfig { // Write Options // ------------------------------------------------------------------------ public static final ConfigOption TABLE_NAME = ConfigOptions - .key(HoodieWriteConfig.TABLE_NAME.key()) + .key(HoodieWriteConfig.TBL_NAME.key()) .stringType() .noDefaultValue() .withDescription("Table name to register to Hive metastore"); @@ -240,7 +240,7 @@ public class FlinkOptions extends HoodieConfig { + "key value, we will pick the one with the largest value for the precombine field,\n" + "determined by Object.compareTo(..)"); - public static final ConfigOption PAYLOAD_CLASS = ConfigOptions + public static final ConfigOption PAYLOAD_CLASS_NAME = ConfigOptions .key("write.payload.class") .stringType() .defaultValue(OverwriteWithLatestAvroPayload.class.getName()) @@ -308,8 +308,8 @@ public class FlinkOptions extends HoodieConfig { + "If set true, the names of partition folders follow = format.\n" + "By default false (the names of partition folders are only partition values)"); - public static final ConfigOption KEYGEN_CLASS = ConfigOptions - .key(HoodieWriteConfig.KEYGENERATOR_CLASS.key()) + public static final ConfigOption KEYGEN_CLASS_NAME = ConfigOptions + .key(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key()) .stringType() .defaultValue("") .withDescription("Key generator class, that implements will extract the key out of incoming record"); @@ -549,7 +549,7 @@ public class FlinkOptions extends HoodieConfig { .defaultValue("") .withDescription("Partition fields for hive sync, default ''"); - public static final ConfigOption HIVE_SYNC_PARTITION_EXTRACTOR_CLASS = ConfigOptions + public static final ConfigOption HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME = ConfigOptions .key("hive_sync.partition_extractor_class") .stringType() .defaultValue(SlashEncodedDayPartitionValueExtractor.class.getCanonicalName()) diff --git a/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java b/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java index 3ccfb26d1..4d953c29f 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java +++ b/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java @@ -81,7 +81,7 @@ public class WriteProfile { /** * Total records to write for each bucket based on - * the config option {@link org.apache.hudi.config.HoodieStorageConfig#PARQUET_FILE_MAX_BYTES}. + * the config option {@link org.apache.hudi.config.HoodieStorageConfig#PARQUET_MAX_FILE_SIZE}. */ private long recordsPerBucket; diff --git a/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java b/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java index 192feb5ec..d374882e9 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java +++ b/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java @@ -76,7 +76,7 @@ public class HiveSyncContext { hiveSyncConfig.hivePass = conf.getString(FlinkOptions.HIVE_SYNC_PASSWORD); hiveSyncConfig.jdbcUrl = conf.getString(FlinkOptions.HIVE_SYNC_JDBC_URL); hiveSyncConfig.partitionFields = Arrays.asList(FilePathUtils.extractPartitionKeys(conf)); - hiveSyncConfig.partitionValueExtractorClass = conf.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS); + hiveSyncConfig.partitionValueExtractorClass = conf.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME); hiveSyncConfig.useJdbc = conf.getBoolean(FlinkOptions.HIVE_SYNC_USE_JDBC); hiveSyncConfig.useFileListingFromMetadata = conf.getBoolean(FlinkOptions.METADATA_ENABLED); hiveSyncConfig.verifyMetadataFileListing = false; diff --git a/hudi-flink/src/main/java/org/apache/hudi/sink/utils/PayloadCreation.java b/hudi-flink/src/main/java/org/apache/hudi/sink/utils/PayloadCreation.java index 20070b3eb..d10447f81 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/sink/utils/PayloadCreation.java +++ b/hudi-flink/src/main/java/org/apache/hudi/sink/utils/PayloadCreation.java @@ -66,7 +66,7 @@ public class PayloadCreation implements Serializable { } else { argTypes = new Class[] {Option.class}; } - final String clazz = conf.getString(FlinkOptions.PAYLOAD_CLASS); + final String clazz = conf.getString(FlinkOptions.PAYLOAD_CLASS_NAME); constructor = ReflectionUtils.getClass(clazz).getConstructor(argTypes); return new PayloadCreation(shouldCombine, constructor, preCombineField); } diff --git a/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java b/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java index a4591c835..75272abf4 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java +++ b/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java @@ -311,7 +311,7 @@ public class FlinkStreamerConfig extends Configuration { conf.setBoolean(FlinkOptions.INSERT_DEDUP, config.insertDedup); conf.setString(FlinkOptions.OPERATION, config.operation.value()); conf.setString(FlinkOptions.PRECOMBINE_FIELD, config.sourceOrderingField); - conf.setString(FlinkOptions.PAYLOAD_CLASS, config.payloadClassName); + conf.setString(FlinkOptions.PAYLOAD_CLASS_NAME, config.payloadClassName); conf.setBoolean(FlinkOptions.INSERT_DROP_DUPS, config.filterDupes); conf.setInteger(FlinkOptions.RETRY_TIMES, Integer.parseInt(config.instantRetryTimes)); conf.setLong(FlinkOptions.RETRY_INTERVAL_MS, Long.parseLong(config.instantRetryInterval)); @@ -319,7 +319,7 @@ public class FlinkStreamerConfig extends Configuration { conf.setString(FlinkOptions.RECORD_KEY_FIELD, config.recordKeyField); conf.setString(FlinkOptions.PARTITION_PATH_FIELD, config.partitionPathField); if (!StringUtils.isNullOrEmpty(config.keygenClass)) { - conf.setString(FlinkOptions.KEYGEN_CLASS, config.keygenClass); + conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, config.keygenClass); } else { conf.setString(FlinkOptions.KEYGEN_TYPE, config.keygenType); } @@ -362,7 +362,7 @@ public class FlinkStreamerConfig extends Configuration { conf.setString(FlinkOptions.HIVE_SYNC_JDBC_URL, config.hiveSyncJdbcUrl); conf.setString(FlinkOptions.HIVE_SYNC_METASTORE_URIS, config.hiveSyncMetastoreUri); conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_FIELDS, config.hiveSyncPartitionFields); - conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS, config.hiveSyncPartitionExtractorClass); + conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME, config.hiveSyncPartitionExtractorClass); conf.setBoolean(FlinkOptions.HIVE_SYNC_ASSUME_DATE_PARTITION, config.hiveSyncAssumeDatePartition); conf.setBoolean(FlinkOptions.HIVE_SYNC_USE_JDBC, config.hiveSyncUseJdbc); conf.setBoolean(FlinkOptions.HIVE_SYNC_AUTO_CREATE_DB, config.hiveSyncAutoCreateDb); diff --git a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java index c753dde54..89f7ed78c 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java +++ b/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java @@ -174,17 +174,17 @@ public class HoodieTableFactory implements DynamicTableSourceFactory, DynamicTab // tweak the key gen class if possible final String[] partitions = conf.getString(FlinkOptions.PARTITION_PATH_FIELD).split(","); if (partitions.length == 1 && partitions[0].equals("")) { - conf.setString(FlinkOptions.KEYGEN_CLASS, NonpartitionedAvroKeyGenerator.class.getName()); + conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, NonpartitionedAvroKeyGenerator.class.getName()); LOG.info("Table option [{}] is reset to {} because this is a non-partitioned table", - FlinkOptions.KEYGEN_CLASS.key(), NonpartitionedAvroKeyGenerator.class.getName()); + FlinkOptions.KEYGEN_CLASS_NAME.key(), NonpartitionedAvroKeyGenerator.class.getName()); return; } final String[] pks = conf.getString(FlinkOptions.RECORD_KEY_FIELD).split(","); boolean complexHoodieKey = pks.length > 1 || partitions.length > 1; - if (complexHoodieKey && FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.KEYGEN_CLASS)) { - conf.setString(FlinkOptions.KEYGEN_CLASS, ComplexAvroKeyGenerator.class.getName()); + if (complexHoodieKey && FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.KEYGEN_CLASS_NAME)) { + conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, ComplexAvroKeyGenerator.class.getName()); LOG.info("Table option [{}] is reset to {} because record key or partition path has two or more fields", - FlinkOptions.KEYGEN_CLASS.key(), ComplexAvroKeyGenerator.class.getName()); + FlinkOptions.KEYGEN_CLASS_NAME.key(), ComplexAvroKeyGenerator.class.getName()); } } @@ -220,8 +220,8 @@ public class HoodieTableFactory implements DynamicTableSourceFactory, DynamicTab * */ private static void setupHiveOptions(Configuration conf) { if (!conf.getBoolean(FlinkOptions.HIVE_STYLE_PARTITIONING) - && FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS)) { - conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS, MultiPartKeysValueExtractor.class.getName()); + && FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME)) { + conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME, MultiPartKeysValueExtractor.class.getName()); } } diff --git a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index f89d089ae..6dc6adde2 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -76,7 +76,7 @@ import java.util.Properties; import static org.apache.hudi.common.model.HoodieFileFormat.HOODIE_LOG; import static org.apache.hudi.common.model.HoodieFileFormat.ORC; import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER; /** * Utilities for Flink stream read and write. @@ -157,7 +157,7 @@ public class StreamerUtil { .withMergeAllowDuplicateOnInserts(allowDuplicateInserts(conf)) .withCompactionConfig( HoodieCompactionConfig.newBuilder() - .withPayloadClass(conf.getString(FlinkOptions.PAYLOAD_CLASS)) + .withPayloadClass(conf.getString(FlinkOptions.PAYLOAD_CLASS_NAME)) .withTargetIOPerCompactionInMB(conf.getLong(FlinkOptions.COMPACTION_TARGET_IO)) .withInlineCompactionTriggerStrategy( CompactionTriggerStrategy.valueOf(conf.getString(FlinkOptions.COMPACTION_TRIGGER_STRATEGY).toUpperCase(Locale.ROOT))) @@ -236,8 +236,8 @@ public class StreamerUtil { .setTableType(conf.getString(FlinkOptions.TABLE_TYPE)) .setTableName(conf.getString(FlinkOptions.TABLE_NAME)) .setRecordKeyFields(conf.getString(FlinkOptions.RECORD_KEY_FIELD, null)) - .setPayloadClassName(conf.getString(FlinkOptions.PAYLOAD_CLASS)) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setPayloadClassName(conf.getString(FlinkOptions.PAYLOAD_CLASS_NAME)) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .setPartitionFields(conf.getString(FlinkOptions.PARTITION_PATH_FIELD, null)) .setPreCombineField(conf.getString(FlinkOptions.PRECOMBINE_FIELD)) .setTimelineLayoutVersion(1) diff --git a/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java b/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java index bea86001e..d10421d66 100644 --- a/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java +++ b/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java @@ -152,7 +152,7 @@ public class TestBucketAssigner { @Test public void testInsertOverBucketAssigned() { - conf.setInteger(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE.key(), 2); + conf.setInteger(HoodieCompactionConfig.COPY_ON_WRITE_INSERT_SPLIT_SIZE.key(), 2); writeConfig = StreamerUtil.getHoodieClientConfig(conf); MockBucketAssigner mockBucketAssigner = new MockBucketAssigner(context, writeConfig); diff --git a/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java b/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java index 0439c4d08..6e4b21598 100644 --- a/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java +++ b/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java @@ -137,7 +137,7 @@ public class TestHoodieTableFactory { @Test void testSetupHoodieKeyOptionsForSource() { this.conf.setString(FlinkOptions.RECORD_KEY_FIELD, "dummyField"); - this.conf.setString(FlinkOptions.KEYGEN_CLASS, "dummyKeyGenClass"); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, "dummyKeyGenClass"); // definition with simple primary key and partition path ResolvedSchema schema1 = SchemaBuilder.instance() .field("f0", DataTypes.INT().notNull()) @@ -150,10 +150,10 @@ public class TestHoodieTableFactory { final HoodieTableSource tableSource1 = (HoodieTableSource) new HoodieTableFactory().createDynamicTableSource(sourceContext1); final Configuration conf1 = tableSource1.getConf(); assertThat(conf1.get(FlinkOptions.RECORD_KEY_FIELD), is("f0")); - assertThat(conf1.get(FlinkOptions.KEYGEN_CLASS), is("dummyKeyGenClass")); + assertThat(conf1.get(FlinkOptions.KEYGEN_CLASS_NAME), is("dummyKeyGenClass")); // definition with complex primary keys and partition paths - this.conf.setString(FlinkOptions.KEYGEN_CLASS, FlinkOptions.KEYGEN_CLASS.defaultValue()); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, FlinkOptions.KEYGEN_CLASS_NAME.defaultValue()); ResolvedSchema schema2 = SchemaBuilder.instance() .field("f0", DataTypes.INT().notNull()) .field("f1", DataTypes.VARCHAR(20).notNull()) @@ -165,15 +165,15 @@ public class TestHoodieTableFactory { final HoodieTableSource tableSource2 = (HoodieTableSource) new HoodieTableFactory().createDynamicTableSource(sourceContext2); final Configuration conf2 = tableSource2.getConf(); assertThat(conf2.get(FlinkOptions.RECORD_KEY_FIELD), is("f0,f1")); - assertThat(conf2.get(FlinkOptions.KEYGEN_CLASS), is(ComplexAvroKeyGenerator.class.getName())); + assertThat(conf2.get(FlinkOptions.KEYGEN_CLASS_NAME), is(ComplexAvroKeyGenerator.class.getName())); // definition with complex primary keys and empty partition paths - this.conf.setString(FlinkOptions.KEYGEN_CLASS, FlinkOptions.KEYGEN_CLASS.defaultValue()); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, FlinkOptions.KEYGEN_CLASS_NAME.defaultValue()); final MockContext sourceContext3 = MockContext.getInstance(this.conf, schema2, ""); final HoodieTableSource tableSource3 = (HoodieTableSource) new HoodieTableFactory().createDynamicTableSource(sourceContext3); final Configuration conf3 = tableSource3.getConf(); assertThat(conf3.get(FlinkOptions.RECORD_KEY_FIELD), is("f0,f1")); - assertThat(conf3.get(FlinkOptions.KEYGEN_CLASS), is(NonpartitionedAvroKeyGenerator.class.getName())); + assertThat(conf3.get(FlinkOptions.KEYGEN_CLASS_NAME), is(NonpartitionedAvroKeyGenerator.class.getName())); } @Test @@ -190,7 +190,7 @@ public class TestHoodieTableFactory { final MockContext sourceContext1 = MockContext.getInstance(this.conf, schema1, "f2"); final HoodieTableSource tableSource1 = (HoodieTableSource) new HoodieTableFactory().createDynamicTableSource(sourceContext1); final Configuration conf1 = tableSource1.getConf(); - assertThat(conf1.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS), is(MultiPartKeysValueExtractor.class.getName())); + assertThat(conf1.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME), is(MultiPartKeysValueExtractor.class.getName())); // set up hive style partitioning is true. this.conf.setBoolean(FlinkOptions.HIVE_STYLE_PARTITIONING, true); @@ -198,7 +198,7 @@ public class TestHoodieTableFactory { final MockContext sourceContext2 = MockContext.getInstance(this.conf, schema1, "f2"); final HoodieTableSource tableSource2 = (HoodieTableSource) new HoodieTableFactory().createDynamicTableSource(sourceContext2); final Configuration conf2 = tableSource2.getConf(); - assertThat(conf2.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS), is(SlashEncodedDayPartitionValueExtractor.class.getName())); + assertThat(conf2.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME), is(SlashEncodedDayPartitionValueExtractor.class.getName())); } @Test @@ -249,7 +249,7 @@ public class TestHoodieTableFactory { @Test void testSetupHoodieKeyOptionsForSink() { this.conf.setString(FlinkOptions.RECORD_KEY_FIELD, "dummyField"); - this.conf.setString(FlinkOptions.KEYGEN_CLASS, "dummyKeyGenClass"); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, "dummyKeyGenClass"); // definition with simple primary key and partition path ResolvedSchema schema1 = SchemaBuilder.instance() .field("f0", DataTypes.INT().notNull()) @@ -262,10 +262,10 @@ public class TestHoodieTableFactory { final HoodieTableSink tableSink1 = (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(sinkContext1); final Configuration conf1 = tableSink1.getConf(); assertThat(conf1.get(FlinkOptions.RECORD_KEY_FIELD), is("f0")); - assertThat(conf1.get(FlinkOptions.KEYGEN_CLASS), is("dummyKeyGenClass")); + assertThat(conf1.get(FlinkOptions.KEYGEN_CLASS_NAME), is("dummyKeyGenClass")); // definition with complex primary keys and partition paths - this.conf.setString(FlinkOptions.KEYGEN_CLASS, FlinkOptions.KEYGEN_CLASS.defaultValue()); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, FlinkOptions.KEYGEN_CLASS_NAME.defaultValue()); ResolvedSchema schema2 = SchemaBuilder.instance() .field("f0", DataTypes.INT().notNull()) .field("f1", DataTypes.VARCHAR(20).notNull()) @@ -277,15 +277,15 @@ public class TestHoodieTableFactory { final HoodieTableSink tableSink2 = (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(sinkContext2); final Configuration conf2 = tableSink2.getConf(); assertThat(conf2.get(FlinkOptions.RECORD_KEY_FIELD), is("f0,f1")); - assertThat(conf2.get(FlinkOptions.KEYGEN_CLASS), is(ComplexAvroKeyGenerator.class.getName())); + assertThat(conf2.get(FlinkOptions.KEYGEN_CLASS_NAME), is(ComplexAvroKeyGenerator.class.getName())); // definition with complex primary keys and empty partition paths - this.conf.setString(FlinkOptions.KEYGEN_CLASS, FlinkOptions.KEYGEN_CLASS.defaultValue()); + this.conf.setString(FlinkOptions.KEYGEN_CLASS_NAME, FlinkOptions.KEYGEN_CLASS_NAME.defaultValue()); final MockContext sinkContext3 = MockContext.getInstance(this.conf, schema2, ""); final HoodieTableSink tableSink3 = (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(sinkContext3); final Configuration conf3 = tableSink3.getConf(); assertThat(conf3.get(FlinkOptions.RECORD_KEY_FIELD), is("f0,f1")); - assertThat(conf3.get(FlinkOptions.KEYGEN_CLASS), is(NonpartitionedAvroKeyGenerator.class.getName())); + assertThat(conf3.get(FlinkOptions.KEYGEN_CLASS_NAME), is(NonpartitionedAvroKeyGenerator.class.getName())); } @Test @@ -302,7 +302,7 @@ public class TestHoodieTableFactory { final MockContext sinkContext1 = MockContext.getInstance(this.conf, schema1, "f2"); final HoodieTableSink tableSink1 = (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(sinkContext1); final Configuration conf1 = tableSink1.getConf(); - assertThat(conf1.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS), is(MultiPartKeysValueExtractor.class.getName())); + assertThat(conf1.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME), is(MultiPartKeysValueExtractor.class.getName())); // set up hive style partitioning is true. this.conf.setBoolean(FlinkOptions.HIVE_STYLE_PARTITIONING, true); @@ -310,7 +310,7 @@ public class TestHoodieTableFactory { final MockContext sinkContext2 = MockContext.getInstance(this.conf, schema1, "f2"); final HoodieTableSink tableSink2 = (HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(sinkContext2); final Configuration conf2 = tableSink2.getConf(); - assertThat(conf2.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS), is(SlashEncodedDayPartitionValueExtractor.class.getName())); + assertThat(conf2.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME), is(SlashEncodedDayPartitionValueExtractor.class.getName())); } @Test diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java index f4bddbd39..70c43011b 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java @@ -18,9 +18,6 @@ package org.apache.hudi.hadoop.utils; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.fs.FSUtils; @@ -51,6 +48,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.mapred.FileInputFormat; @@ -71,8 +71,8 @@ import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS; -import static org.apache.hudi.common.config.HoodieMetadataConfig.METADATA_ENABLE_PROP; -import static org.apache.hudi.common.config.HoodieMetadataConfig.METADATA_VALIDATE_PROP; +import static org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE; +import static org.apache.hudi.common.config.HoodieMetadataConfig.VALIDATE_ENABLE; public class HoodieInputFormatUtils { @@ -418,8 +418,8 @@ public class HoodieInputFormatUtils { public static HoodieMetadataConfig buildMetadataConfig(Configuration conf) { return HoodieMetadataConfig.newBuilder() - .enable(conf.getBoolean(METADATA_ENABLE_PROP.key(), DEFAULT_METADATA_ENABLE_FOR_READERS)) - .validate(conf.getBoolean(METADATA_VALIDATE_PROP.key(), METADATA_VALIDATE_PROP.defaultValue())) + .enable(conf.getBoolean(ENABLE.key(), DEFAULT_METADATA_ENABLE_FOR_READERS)) + .validate(conf.getBoolean(VALIDATE_ENABLE.key(), VALIDATE_ENABLE.defaultValue())) .build(); } diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java index ee6502cd9..3966aa339 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java @@ -28,9 +28,9 @@ import org.apache.hudi.hadoop.utils.HoodieHiveUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -138,9 +138,9 @@ public class TestInputPathHandler { static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, String tableName, HoodieTableType tableType) throws IOException { Properties properties = new Properties(); - properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP.key(), tableName); - properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP.key(), tableType.name()); - properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP.key(), HoodieAvroPayload.class.getName()); + properties.setProperty(HoodieTableConfig.NAME.key(), tableName); + properties.setProperty(HoodieTableConfig.TYPE.key(), tableType.name()); + properties.setProperty(HoodieTableConfig.PAYLOAD_CLASS_NAME.key(), HoodieAvroPayload.class.getName()); return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties); } diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java index 39112c14f..a647da9b9 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java @@ -31,12 +31,12 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.FileCreateUtils; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.SchemaTestUtil; -import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; import org.apache.hudi.hadoop.config.HoodieRealtimeConfig; +import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; @@ -124,11 +124,11 @@ public class TestHoodieRealtimeRecordReader { protected Properties getPropertiesForKeyGen() { Properties properties = new Properties(); - properties.put(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), "false"); - properties.put("hoodie.datasource.write.recordkey.field","_row_key"); - properties.put("hoodie.datasource.write.partitionpath.field","partition_path"); - properties.put(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS.key(), "_row_key"); - properties.put(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key(), "partition_path"); + properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + properties.put("hoodie.datasource.write.recordkey.field", "_row_key"); + properties.put("hoodie.datasource.write.partitionpath.field", "partition_path"); + properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key"); + properties.put(HoodieTableConfig.PARTITION_FIELDS.key(), "partition_path"); return properties; } diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java index 61e310a89..0bcbaf8f8 100644 --- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java +++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java @@ -61,7 +61,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER; /** * This is the entry point for running a Hudi Test Suite. Although this class has similarities with {@link HoodieDeltaStreamer} this class does not extend it since do not want to create a dependency @@ -111,7 +111,7 @@ public class HoodieTestSuiteJob { metaClient = HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .initTable(jsc.hadoopConfiguration(), cfg.targetBasePath); if (cfg.cleanInput) { diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java index 603fda4e3..1fd3d2f96 100644 --- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java +++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java @@ -34,13 +34,13 @@ import org.apache.hudi.common.table.view.TableFileSystemView; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.config.HoodieMemoryConfig; +import org.apache.hudi.io.storage.HoodieFileReader; +import org.apache.hudi.io.storage.HoodieFileReaderFactory; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; -import org.apache.hudi.io.storage.HoodieFileReader; -import org.apache.hudi.io.storage.HoodieFileReaderFactory; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -88,7 +88,7 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader { // calls in metrics as they are not part of normal HUDI operation. HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc); List partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getBasePath(), - HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false); + HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false); // Sort partition so we can pick last N partitions by default Collections.sort(partitionPaths); if (!partitionPaths.isEmpty()) { diff --git a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkBulkInsertNode.scala b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkBulkInsertNode.scala index d55bb9693..4d17570fc 100644 --- a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkBulkInsertNode.scala +++ b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkBulkInsertNode.scala @@ -59,7 +59,7 @@ class SparkBulkInsertNode(config1: Config) extends DagNode[RDD[WriteStatus]] { .option(DataSourceWriteOptions.ENABLE_ROW_WRITER.key(), String.valueOf(config.enableRowWriting())) .option(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX.key(), "deltastreamer.checkpoint.key") .option("deltastreamer.checkpoint.key", context.getWriterContext.getHoodieTestSuiteWriter.getLastCheckpoint.orElse("")) - .option(HoodieWriteConfig.TABLE_NAME.key(), context.getHoodieTestSuiteWriter.getCfg.targetTableName) + .option(HoodieWriteConfig.TBL_NAME.key(), context.getHoodieTestSuiteWriter.getCfg.targetTableName) .mode(saveMode) .save(context.getHoodieTestSuiteWriter.getWriteConfig.getBasePath) } diff --git a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkInsertNode.scala b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkInsertNode.scala index e9720fd47..f962e41bd 100644 --- a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkInsertNode.scala +++ b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkInsertNode.scala @@ -58,7 +58,7 @@ class SparkInsertNode(config1: Config) extends DagNode[RDD[WriteStatus]] { .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX.key, "deltastreamer.checkpoint.key") .option("deltastreamer.checkpoint.key", context.getWriterContext.getHoodieTestSuiteWriter.getLastCheckpoint.orElse("")) - .option(HoodieWriteConfig.TABLE_NAME.key, context.getHoodieTestSuiteWriter.getCfg.targetTableName) + .option(HoodieWriteConfig.TBL_NAME.key, context.getHoodieTestSuiteWriter.getCfg.targetTableName) .mode(SaveMode.Overwrite) .save(context.getHoodieTestSuiteWriter.getWriteConfig.getBasePath) } diff --git a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkUpsertNode.scala b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkUpsertNode.scala index 3666b3856..6486fede1 100644 --- a/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkUpsertNode.scala +++ b/hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkUpsertNode.scala @@ -58,7 +58,7 @@ class SparkUpsertNode(config1: Config) extends DagNode[RDD[WriteStatus]] { .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX.key, "deltastreamer.checkpoint.key") .option("deltastreamer.checkpoint.key", context.getWriterContext.getHoodieTestSuiteWriter.getLastCheckpoint.orElse("")) - .option(HoodieWriteConfig.TABLE_NAME.key, context.getHoodieTestSuiteWriter.getCfg.targetTableName) + .option(HoodieWriteConfig.TBL_NAME.key, context.getHoodieTestSuiteWriter.getCfg.targetTableName) .mode(SaveMode.Append) .save(context.getHoodieTestSuiteWriter.getWriteConfig.getBasePath) } diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java index 56aa7af7f..be6e55226 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java @@ -169,7 +169,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase { props.setProperty(DataSourceWriteOptions.HIVE_DATABASE().key(), "testdb1"); props.setProperty(DataSourceWriteOptions.HIVE_TABLE().key(), "table1"); props.setProperty(DataSourceWriteOptions.HIVE_PARTITION_FIELDS().key(), "datestr"); - props.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), TimestampBasedKeyGenerator.class.getName()); + props.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), TimestampBasedKeyGenerator.class.getName()); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider"); props.setProperty("hoodie.write.lock.hivemetastore.database", "testdb1"); diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java index 07ba8a935..7e043eb72 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java @@ -185,7 +185,7 @@ public class DataSourceUtils { return builder.forTable(tblName) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.BLOOM).build()) .withCompactionConfig(HoodieCompactionConfig.newBuilder() - .withPayloadClass(parameters.get(DataSourceWriteOptions.PAYLOAD_CLASS().key())) + .withPayloadClass(parameters.get(DataSourceWriteOptions.PAYLOAD_CLASS_NAME().key())) .withInlineCompaction(inlineCompact).build()) .withClusteringConfig(HoodieClusteringConfig.newBuilder() .withInlineClustering(inlineClusteringEnabled) @@ -300,10 +300,10 @@ public class DataSourceUtils { DataSourceWriteOptions.HIVE_AUTO_CREATE_DATABASE().defaultValue())); hiveSyncConfig.ignoreExceptions = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_IGNORE_EXCEPTIONS().key(), DataSourceWriteOptions.HIVE_IGNORE_EXCEPTIONS().defaultValue())); - hiveSyncConfig.skipROSuffix = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX().key(), - DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX().defaultValue())); - hiveSyncConfig.supportTimestamp = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP().key(), - DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP().defaultValue())); + hiveSyncConfig.skipROSuffix = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE().key(), + DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE().defaultValue())); + hiveSyncConfig.supportTimestamp = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP_TYPE().key(), + DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP_TYPE().defaultValue())); return hiveSyncConfig; } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal/BulkInsertDataInternalWriterHelper.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal/BulkInsertDataInternalWriterHelper.java index 0a845f701..823de99fc 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal/BulkInsertDataInternalWriterHelper.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal/BulkInsertDataInternalWriterHelper.java @@ -106,7 +106,7 @@ public class BulkInsertDataInternalWriterHelper { private Option getKeyGenerator(Properties properties) { TypedProperties typedProperties = new TypedProperties(); typedProperties.putAll(properties); - if (properties.get(DataSourceWriteOptions.KEYGENERATOR_CLASS().key()).equals(NonpartitionedKeyGenerator.class.getName())) { + if (properties.get(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()).equals(NonpartitionedKeyGenerator.class.getName())) { return Option.empty(); // Do not instantiate NonPartitionKeyGen } else { try { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala index 5044ab642..9881e431a 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -204,7 +204,7 @@ object DataSourceWriteOptions { val partitionColumns = optParams.get(SparkDataSourceUtils.PARTITIONING_COLUMNS_KEY) .map(SparkDataSourceUtils.decodePartitioningColumns) .getOrElse(Nil) - val keyGeneratorClass = optParams.getOrElse(DataSourceWriteOptions.KEYGENERATOR_CLASS.key(), + val keyGeneratorClass = optParams.getOrElse(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key(), DataSourceWriteOptions.DEFAULT_KEYGENERATOR_CLASS_OPT_VAL) val partitionPathField = @@ -244,7 +244,7 @@ object DataSourceWriteOptions { * Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. * This will render any value set for `PRECOMBINE_FIELD_OPT_VAL` in-effective */ - val PAYLOAD_CLASS = HoodieWriteConfig.WRITE_PAYLOAD_CLASS + val PAYLOAD_CLASS_NAME = HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME /** * Record key field. Value to be used as the `recordKey` component of `HoodieKey`. Actual value @@ -267,7 +267,7 @@ object DataSourceWriteOptions { */ val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING - val KEYGENERATOR_CLASS = ConfigProperty.key("hoodie.datasource.write.keygenerator.class") + val KEYGENERATOR_CLASS_NAME = ConfigProperty.key("hoodie.datasource.write.keygenerator.class") .defaultValue(classOf[SimpleKeyGenerator].getName) .withDocumentation("Key generator class, that implements `org.apache.hudi.keygen.KeyGenerator`") @@ -328,7 +328,7 @@ object DataSourceWriteOptions { .withDocumentation("Config to indicate whether to ignore any non exception error (e.g. writestatus error)" + " within a streaming microbatch") - val META_SYNC_CLIENT_TOOL_CLASS: ConfigProperty[String] = ConfigProperty + val META_SYNC_CLIENT_TOOL_CLASS_NAME: ConfigProperty[String] = ConfigProperty .key("hoodie.meta.sync.client.tool.class") .defaultValue(classOf[HiveSyncTool].getName) .withDocumentation("Sync tool class name used to sync to metastore. Defaults to Hive.") @@ -403,7 +403,7 @@ object DataSourceWriteOptions { .defaultValue("false") .withDocumentation("") - /* @deprecated We should use {@link HIVE_SYNC_MODE} instead of this config from 0.9.0 */ + /** @deprecated Use {@link HIVE_SYNC_MODE} instead of this config from 0.9.0 */ @Deprecated val HIVE_USE_JDBC: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.use_jdbc") @@ -421,12 +421,12 @@ object DataSourceWriteOptions { .defaultValue("false") .withDocumentation("") - val HIVE_SKIP_RO_SUFFIX: ConfigProperty[String] = ConfigProperty + val HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.skip_ro_suffix") .defaultValue("false") .withDocumentation("Skip the _ro suffix for Read optimized table, when registering") - val HIVE_SUPPORT_TIMESTAMP: ConfigProperty[String] = ConfigProperty + val HIVE_SUPPORT_TIMESTAMP_TYPE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.support_timestamp") .defaultValue("false") .withDocumentation("‘INT64’ with original type TIMESTAMP_MICROS is converted to hive ‘timestamp’ type. " + @@ -514,9 +514,12 @@ object DataSourceWriteOptions { /** @deprecated Use {@link STREAMING_IGNORE_FAILED_BATCH} and its methods instead */ @Deprecated val DEFAULT_STREAMING_IGNORE_FAILED_BATCH_OPT_VAL = STREAMING_IGNORE_FAILED_BATCH.defaultValue() - /** @deprecated Use {@link META_SYNC_CLIENT_TOOL_CLASS} and its methods instead */ + /** @deprecated Use {@link META_SYNC_CLIENT_TOOL_CLASS_NAME} and its methods instead */ @Deprecated - val DEFAULT_META_SYNC_CLIENT_TOOL_CLASS = META_SYNC_CLIENT_TOOL_CLASS.defaultValue() + val META_SYNC_CLIENT_TOOL_CLASS = META_SYNC_CLIENT_TOOL_CLASS_NAME.key() + /** @deprecated Use {@link META_SYNC_CLIENT_TOOL_CLASS_NAME} and its methods instead */ + @Deprecated + val DEFAULT_META_SYNC_CLIENT_TOOL_CLASS = META_SYNC_CLIENT_TOOL_CLASS_NAME.defaultValue() /** @deprecated Use {@link HIVE_SYNC_ENABLED} and its methods instead */ @Deprecated val HIVE_SYNC_ENABLED_OPT_KEY = HIVE_SYNC_ENABLED.key() @@ -550,10 +553,10 @@ object DataSourceWriteOptions { /** @deprecated Use {@link KEYGENERATOR_CLASS} and its methods instead */ @Deprecated - val DEFAULT_KEYGENERATOR_CLASS_OPT_VAL = KEYGENERATOR_CLASS.defaultValue() + val DEFAULT_KEYGENERATOR_CLASS_OPT_VAL = KEYGENERATOR_CLASS_NAME.defaultValue() /** @deprecated Use {@link KEYGENERATOR_CLASS} and its methods instead */ @Deprecated - val KEYGENERATOR_CLASS_OPT_KEY = HoodieWriteConfig.KEYGENERATOR_CLASS.key() + val KEYGENERATOR_CLASS_OPT_KEY = HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key() /** @deprecated Use {@link ENABLE_ROW_WRITER} and its methods instead */ @Deprecated val ENABLE_ROW_WRITER_OPT_KEY = ENABLE_ROW_WRITER.key() @@ -622,12 +625,12 @@ object DataSourceWriteOptions { @Deprecated val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue() - /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS} and its methods instead */ + /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME} and its methods instead */ @Deprecated - val PAYLOAD_CLASS_OPT_KEY = HoodieWriteConfig.WRITE_PAYLOAD_CLASS.key() - /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS} and its methods instead */ + val PAYLOAD_CLASS_OPT_KEY = HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key() + /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME} and its methods instead */ @Deprecated - val DEFAULT_PAYLOAD_OPT_VAL = PAYLOAD_CLASS.defaultValue() + val DEFAULT_PAYLOAD_OPT_VAL = PAYLOAD_CLASS_NAME.defaultValue() /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ @Deprecated @@ -638,7 +641,7 @@ object DataSourceWriteOptions { /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ @Deprecated - val STORAGE_TYPE_OPT = "hoodie.datasource.write.storage.type" + val STORAGE_TYPE_OPT_KEY = "hoodie.datasource.write.storage.type" @Deprecated val COW_STORAGE_TYPE_OPT_VAL = HoodieTableType.COPY_ON_WRITE.name @Deprecated @@ -698,12 +701,18 @@ object DataSourceWriteOptions { /** @deprecated Use {@link HIVE_IGNORE_EXCEPTIONS} and its methods instead */ @Deprecated val DEFAULT_HIVE_IGNORE_EXCEPTIONS_OPT_KEY = HIVE_IGNORE_EXCEPTIONS.defaultValue() - /** @deprecated Use {@link HIVE_SKIP_RO_SUFFIX} and its methods instead */ + /** @deprecated Use {@link HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE} and its methods instead */ @Deprecated - val DEFAULT_HIVE_SKIP_RO_SUFFIX_VAL = HIVE_SKIP_RO_SUFFIX.defaultValue() - /** @deprecated Use {@link HIVE_SUPPORT_TIMESTAMP} and its methods instead */ + val HIVE_SKIP_RO_SUFFIX = HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key() + /** @deprecated Use {@link HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE} and its methods instead */ @Deprecated - val DEFAULT_HIVE_SUPPORT_TIMESTAMP = HIVE_SUPPORT_TIMESTAMP.defaultValue() + val DEFAULT_HIVE_SKIP_RO_SUFFIX_VAL = HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.defaultValue() + /** @deprecated Use {@link HIVE_SUPPORT_TIMESTAMP_TYPE} and its methods instead */ + @Deprecated + val HIVE_SUPPORT_TIMESTAMP = HIVE_SUPPORT_TIMESTAMP_TYPE.key() + /** @deprecated Use {@link HIVE_SUPPORT_TIMESTAMP_TYPE} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_SUPPORT_TIMESTAMP = HIVE_SUPPORT_TIMESTAMP_TYPE.defaultValue() /** @deprecated Use {@link ASYNC_COMPACT_ENABLE} and its methods instead */ @Deprecated val ASYNC_COMPACT_ENABLE_OPT_KEY = ASYNC_COMPACT_ENABLE.key() @@ -713,6 +722,8 @@ object DataSourceWriteOptions { /** @deprecated Use {@link KAFKA_AVRO_VALUE_DESERIALIZER_CLASS} and its methods instead */ @Deprecated val KAFKA_AVRO_VALUE_DESERIALIZER = KAFKA_AVRO_VALUE_DESERIALIZER_CLASS.key() + @Deprecated + val SCHEMA_PROVIDER_CLASS_PROP = "hoodie.deltastreamer.schemaprovider.class" } object DataSourceOptionsHelper { @@ -723,13 +734,13 @@ object DataSourceOptionsHelper { val allConfigsWithAlternatives = List( DataSourceReadOptions.QUERY_TYPE, DataSourceWriteOptions.TABLE_TYPE, - HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP, - HoodieTableConfig.HOODIE_LOG_FILE_FORMAT_PROP + HoodieTableConfig.BASE_FILE_FORMAT, + HoodieTableConfig.LOG_FILE_FORMAT ) // put all the deprecated configs here val allDeprecatedConfigs: Set[String] = Set( - ConsistencyGuardConfig.CONSISTENCY_CHECK_ENABLED_PROP.key + ConsistencyGuardConfig.ENABLE.key ) // maps the deprecated config name to its latest name diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java index 690f613ae..b19b09844 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java +++ b/hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java @@ -72,10 +72,10 @@ public class HoodieBulkInsertInternalWriterTestBase extends HoodieClientTestHarn protected HoodieWriteConfig getWriteConfig(boolean populateMetaFields) { Properties properties = new Properties(); if (!populateMetaFields) { - properties.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), SimpleKeyGenerator.class.getName()); + properties.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), SimpleKeyGenerator.class.getName()); properties.setProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key(), SparkDatasetTestUtils.RECORD_KEY_FIELD_NAME); properties.setProperty(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), SparkDatasetTestUtils.PARTITION_PATH_FIELD_NAME); - properties.setProperty(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), "false"); + properties.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); } return getConfigBuilder(basePath).withProperties(properties).build(); } diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java index 47e97aaeb..c96d216e1 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java @@ -77,7 +77,7 @@ public class HoodieDatasetBulkInsertHelper { TypedProperties properties = new TypedProperties(); properties.putAll(config.getProps()); - String keyGeneratorClass = properties.getString(DataSourceWriteOptions.KEYGENERATOR_CLASS().key()); + String keyGeneratorClass = properties.getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()); BuiltinKeyGenerator keyGenerator = (BuiltinKeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass, properties); StructType structTypeForUDF = rows.schema(); diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieFileIndex.scala index f94d228c3..b87be995f 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieFileIndex.scala @@ -17,8 +17,6 @@ package org.apache.hudi -import java.util.Properties -import scala.collection.JavaConverters._ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hudi.DataSourceReadOptions.{QUERY_TYPE, QUERY_TYPE_SNAPSHOT_OPT_VAL} import org.apache.hudi.client.common.HoodieSparkEngineContext @@ -26,21 +24,23 @@ import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ -import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.table.view.{FileSystemViewStorageConfig, HoodieTableFileSystemView} +import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.{InternalRow, expressions} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.avro.SchemaConverters import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, Expression, InterpretedPredicate} import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} +import org.apache.spark.sql.catalyst.{InternalRow, expressions} import org.apache.spark.sql.execution.datasources.{FileIndex, FileStatusCache, NoopCache, PartitionDirectory} import org.apache.spark.sql.hudi.HoodieSqlUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.unsafe.types.UTF8String +import java.util.Properties +import scala.collection.JavaConverters._ import scala.collection.mutable /** @@ -122,12 +122,12 @@ case class HoodieFileIndex( // To support metadata listing via Spark SQL we allow users to pass the config via SQL Conf in spark session. Users // would be able to run SET hoodie.metadata.enable=true in the spark sql session to enable metadata listing. - properties.put(HoodieMetadataConfig.METADATA_ENABLE_PROP, - sqlConf.getConfString(HoodieMetadataConfig.METADATA_ENABLE_PROP.key(), + properties.put(HoodieMetadataConfig.ENABLE, + sqlConf.getConfString(HoodieMetadataConfig.ENABLE.key(), HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS.toString)) - properties.put(HoodieMetadataConfig.METADATA_VALIDATE_PROP, - sqlConf.getConfString(HoodieMetadataConfig.METADATA_VALIDATE_PROP.key(), - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue().toString)) + properties.put(HoodieMetadataConfig.VALIDATE_ENABLE, + sqlConf.getConfString(HoodieMetadataConfig.VALIDATE_ENABLE.key(), + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue().toString)) properties.putAll(options.asJava) properties } diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index 87e7b2202..83b37acd5 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -18,9 +18,6 @@ package org.apache.hudi -import java.util -import java.util.Properties - import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration @@ -35,7 +32,7 @@ import org.apache.hudi.common.model.{HoodieRecordPayload, HoodieTableType, Write import org.apache.hudi.common.table.timeline.HoodieActiveTimeline import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.util.{CommitUtils, ReflectionUtils} -import org.apache.hudi.config.HoodieBootstrapConfig.{BOOTSTRAP_BASE_PATH, BOOTSTRAP_INDEX_CLASS} +import org.apache.hudi.config.HoodieBootstrapConfig.{BASE_PATH, INDEX_CLASS_NAME} import org.apache.hudi.config.{HoodieInternalConfig, HoodieWriteConfig} import org.apache.hudi.exception.HoodieException import org.apache.hudi.execution.bulkinsert.{BulkInsertInternalPartitionerWithRowsFactory, NonSortPartitionerWithRows} @@ -53,6 +50,8 @@ import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{DataFrame, Dataset, Row, SQLContext, SaveMode, SparkSession} import org.apache.spark.{SPARK_VERSION, SparkContext} +import java.util +import java.util.Properties import scala.collection.JavaConversions._ import scala.collection.mutable.ListBuffer @@ -78,7 +77,7 @@ object HoodieSparkSqlWriter { val sparkContext = sqlContext.sparkContext val path = parameters.get("path") val hoodieConfig = HoodieWriterUtils.convertMapToHoodieConfig(parameters) - val tblNameOp = hoodieConfig.getStringOrThrow(HoodieWriteConfig.TABLE_NAME, s"'${HoodieWriteConfig.TABLE_NAME.key}' must be set.") + val tblNameOp = hoodieConfig.getStringOrThrow(HoodieWriteConfig.TBL_NAME, s"'${HoodieWriteConfig.TBL_NAME.key}' must be set.") asyncCompactionTriggerFnDefined = asyncCompactionTriggerFn.isDefined asyncClusteringTriggerFnDefined = asyncClusteringTriggerFn.isDefined if (path.isEmpty) { @@ -121,10 +120,10 @@ object HoodieSparkSqlWriter { val partitionColumns = HoodieSparkUtils.getPartitionColumns(keyGenerator, toProperties(parameters)) // Create the table if not present if (!tableExists) { - val baseFileFormat = hoodieConfig.getStringOrDefault(HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP) - val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP) + val baseFileFormat = hoodieConfig.getStringOrDefault(HoodieTableConfig.BASE_FILE_FORMAT) + val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.ARCHIVELOG_FOLDER) val recordKeyFields = hoodieConfig.getString(DataSourceWriteOptions.RECORDKEY_FIELD) - val populateMetaFields = parameters.getOrElse(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()).toBoolean + val populateMetaFields = parameters.getOrElse(HoodieTableConfig.POPULATE_META_FIELDS.key(), HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()).toBoolean val tableMetaClient = HoodieTableMetaClient.withPropertyBuilder() .setTableType(tableType) @@ -132,12 +131,12 @@ object HoodieSparkSqlWriter { .setRecordKeyFields(recordKeyFields) .setBaseFileFormat(baseFileFormat) .setArchiveLogFolder(archiveLogFolder) - .setPayloadClassName(hoodieConfig.getString(PAYLOAD_CLASS)) + .setPayloadClassName(hoodieConfig.getString(PAYLOAD_CLASS_NAME)) .setPreCombineField(hoodieConfig.getStringOrDefault(PRECOMBINE_FIELD, null)) .setPartitionFields(partitionColumns) .setPopulateMetaFields(populateMetaFields) .setRecordKeyFields(hoodieConfig.getString(RECORDKEY_FIELD)) - .setKeyGeneratorClassProp(hoodieConfig.getString(KEYGENERATOR_CLASS)) + .setKeyGeneratorClassProp(hoodieConfig.getString(KEYGENERATOR_CLASS_NAME)) .initTable(sparkContext.hadoopConfiguration, path.get) tableConfig = tableMetaClient.getTableConfig } @@ -169,8 +168,8 @@ object HoodieSparkSqlWriter { // Create a HoodieWriteClient & issue the delete. val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc, - null, path.get, tblName, - mapAsJavaMap(parameters - HoodieWriteConfig.HOODIE_AUTO_COMMIT.key))) + null, path.get, tblName, + mapAsJavaMap(parameters - HoodieWriteConfig.AUTO_COMMIT_ENABLE.key))) .asInstanceOf[SparkRDDWriteClient[HoodieRecordPayload[Nothing]]] if (isAsyncCompactionEnabled(client, tableConfig, parameters, jsc.hadoopConfiguration())) { @@ -201,7 +200,7 @@ object HoodieSparkSqlWriter { // Create a HoodieWriteClient & issue the delete. val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc, null, path.get, tblName, - mapAsJavaMap(parameters - HoodieWriteConfig.HOODIE_AUTO_COMMIT.key))) + mapAsJavaMap(parameters - HoodieWriteConfig.AUTO_COMMIT_ENABLE.key))) .asInstanceOf[SparkRDDWriteClient[HoodieRecordPayload[Nothing]]] // Issue delete partitions client.startCommitWithTime(instantTime, commitActionType) @@ -235,9 +234,9 @@ object HoodieSparkSqlWriter { .asInstanceOf[Comparable[_]] DataSourceUtils.createHoodieRecord(processedRecord, orderingVal, keyGenerator.getKey(gr), - hoodieConfig.getString(PAYLOAD_CLASS)) + hoodieConfig.getString(PAYLOAD_CLASS_NAME)) } else { - DataSourceUtils.createHoodieRecord(processedRecord, keyGenerator.getKey(gr), hoodieConfig.getString(PAYLOAD_CLASS)) + DataSourceUtils.createHoodieRecord(processedRecord, keyGenerator.getKey(gr), hoodieConfig.getString(PAYLOAD_CLASS_NAME)) } hoodieRecord }).toJavaRDD() @@ -245,7 +244,7 @@ object HoodieSparkSqlWriter { val writeSchema = if (dropPartitionColumns) generateSchemaWithoutPartitionColumns(partitionColumns, schema) else schema // Create a HoodieWriteClient & issue the write. val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc, writeSchema.toString, path.get, - tblName, mapAsJavaMap(parameters - HoodieWriteConfig.HOODIE_AUTO_COMMIT.key) + tblName, mapAsJavaMap(parameters - HoodieWriteConfig.AUTO_COMMIT_ENABLE.key) )).asInstanceOf[SparkRDDWriteClient[HoodieRecordPayload[Nothing]]] if (isAsyncCompactionEnabled(client, tableConfig, parameters, jsc.hadoopConfiguration())) { @@ -334,12 +333,12 @@ object HoodieSparkSqlWriter { val sparkContext = sqlContext.sparkContext val path = parameters.getOrElse("path", throw new HoodieException("'path' must be set.")) val hoodieConfig = HoodieWriterUtils.convertMapToHoodieConfig(parameters) - val tableName = hoodieConfig.getStringOrThrow(HoodieWriteConfig.TABLE_NAME, s"'${HoodieWriteConfig.TABLE_NAME.key}' must be set.") + val tableName = hoodieConfig.getStringOrThrow(HoodieWriteConfig.TBL_NAME, s"'${HoodieWriteConfig.TBL_NAME.key}' must be set.") val tableType = hoodieConfig.getStringOrDefault(TABLE_TYPE) - val bootstrapBasePath = hoodieConfig.getStringOrThrow(BOOTSTRAP_BASE_PATH, - s"'${BOOTSTRAP_BASE_PATH.key}' is required for '${BOOTSTRAP_OPERATION_OPT_VAL}'" + + val bootstrapBasePath = hoodieConfig.getStringOrThrow(BASE_PATH, + s"'${BASE_PATH.key}' is required for '${BOOTSTRAP_OPERATION_OPT_VAL}'" + " operation'") - val bootstrapIndexClass = hoodieConfig.getStringOrDefault(BOOTSTRAP_INDEX_CLASS) + val bootstrapIndexClass = hoodieConfig.getStringOrDefault(INDEX_CLASS_NAME) var schema: String = null if (df.schema.nonEmpty) { @@ -363,18 +362,18 @@ object HoodieSparkSqlWriter { } if (!tableExists) { - val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP) + val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.ARCHIVELOG_FOLDER) val partitionColumns = HoodieWriterUtils.getPartitionColumns(parameters) val recordKeyFields = hoodieConfig.getString(DataSourceWriteOptions.RECORDKEY_FIELD) - val keyGenProp = hoodieConfig.getString(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS) - val populateMetaFields = parameters.getOrElse(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()).toBoolean + val keyGenProp = hoodieConfig.getString(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME) + val populateMetaFields = parameters.getOrElse(HoodieTableConfig.POPULATE_META_FIELDS.key(), HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()).toBoolean HoodieTableMetaClient.withPropertyBuilder() .setTableType(HoodieTableType.valueOf(tableType)) .setTableName(tableName) .setRecordKeyFields(recordKeyFields) .setArchiveLogFolder(archiveLogFolder) - .setPayloadClassName(hoodieConfig.getStringOrDefault(PAYLOAD_CLASS)) + .setPayloadClassName(hoodieConfig.getStringOrDefault(PAYLOAD_CLASS_NAME)) .setPreCombineField(hoodieConfig.getStringOrDefault(PRECOMBINE_FIELD, null)) .setBootstrapIndexClass(bootstrapIndexClass) .setBootstrapBasePath(bootstrapBasePath) @@ -405,8 +404,8 @@ object HoodieSparkSqlWriter { instantTime: String, partitionColumns: String): (Boolean, common.util.Option[String]) = { val sparkContext = sqlContext.sparkContext - val populateMetaFields = parameters.getOrElse(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), - HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()).toBoolean + val populateMetaFields = parameters.getOrElse(HoodieTableConfig.POPULATE_META_FIELDS.key(), + HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()).toBoolean val dropPartitionColumns = parameters.getOrElse(DataSourceWriteOptions.DROP_PARTITION_COLUMNS.key(), DataSourceWriteOptions.DROP_PARTITION_COLUMNS.defaultValue()).toBoolean // register classes & schemas @@ -423,7 +422,7 @@ object HoodieSparkSqlWriter { if (parameters(INSERT_DROP_DUPS.key).toBoolean) { throw new HoodieException("Dropping duplicates with bulk_insert in row writer path is not supported yet") } - val params = parameters.updated(HoodieWriteConfig.AVRO_SCHEMA.key, schema.toString) + val params = parameters.updated(HoodieWriteConfig.AVRO_SCHEMA_STRING.key, schema.toString) val writeConfig = DataSourceUtils.createHoodieConfig(schema.toString, path.get, tblName, mapAsJavaMap(params)) val bulkInsertPartitionerRows : BulkInsertPartitioner[Dataset[Row]] = if (populateMetaFields) { val userDefinedBulkInsertPartitionerOpt = DataSourceUtils.createUserDefinedBulkInsertPartitionerWithRows(writeConfig) @@ -533,21 +532,21 @@ object HoodieSparkSqlWriter { hiveSyncConfig.hiveUser = hoodieConfig.getString(HIVE_USER) hiveSyncConfig.hivePass = hoodieConfig.getString(HIVE_PASS) hiveSyncConfig.jdbcUrl = hoodieConfig.getString(HIVE_URL) - hiveSyncConfig.skipROSuffix = hoodieConfig.getStringOrDefault(HIVE_SKIP_RO_SUFFIX, - DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX.defaultValue).toBoolean + hiveSyncConfig.skipROSuffix = hoodieConfig.getStringOrDefault(HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE, + DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.defaultValue).toBoolean hiveSyncConfig.partitionFields = ListBuffer(hoodieConfig.getString(HIVE_PARTITION_FIELDS).split(",").map(_.trim).filter(!_.isEmpty).toList: _*) hiveSyncConfig.partitionValueExtractorClass = hoodieConfig.getString(HIVE_PARTITION_EXTRACTOR_CLASS) hiveSyncConfig.useJdbc = hoodieConfig.getBoolean(HIVE_USE_JDBC) - hiveSyncConfig.useFileListingFromMetadata = hoodieConfig.getBoolean(HoodieMetadataConfig.METADATA_ENABLE_PROP) - hiveSyncConfig.verifyMetadataFileListing = hoodieConfig.getBoolean(HoodieMetadataConfig.METADATA_VALIDATE_PROP) + hiveSyncConfig.useFileListingFromMetadata = hoodieConfig.getBoolean(HoodieMetadataConfig.ENABLE) + hiveSyncConfig.verifyMetadataFileListing = hoodieConfig.getBoolean(HoodieMetadataConfig.VALIDATE_ENABLE) hiveSyncConfig.ignoreExceptions = hoodieConfig.getStringOrDefault(HIVE_IGNORE_EXCEPTIONS).toBoolean - hiveSyncConfig.supportTimestamp = hoodieConfig.getStringOrDefault(HIVE_SUPPORT_TIMESTAMP).toBoolean + hiveSyncConfig.supportTimestamp = hoodieConfig.getStringOrDefault(HIVE_SUPPORT_TIMESTAMP_TYPE).toBoolean hiveSyncConfig.autoCreateDatabase = hoodieConfig.getStringOrDefault(HIVE_AUTO_CREATE_DATABASE).toBoolean hiveSyncConfig.decodePartition = hoodieConfig.getStringOrDefault(URL_ENCODE_PARTITIONING).toBoolean hiveSyncConfig.batchSyncNum = hoodieConfig.getStringOrDefault(HIVE_BATCH_SYNC_PARTITION_NUM).toInt - hiveSyncConfig.syncAsSparkDataSourceTable = hoodieConfig.getStringOrDefault(HIVE_SYNC_AS_DATA_SOURCE_TABLE).toBoolean + hiveSyncConfig.syncAsSparkDataSourceTable = hoodieConfig.getStringOrDefault(HIVE_SYNC_AS_DATA_SOURCE_TABLE).toBoolean hiveSyncConfig.sparkSchemaLengthThreshold = sqlConf.getConf(StaticSQLConf.SCHEMA_STRING_LENGTH_THRESHOLD) hiveSyncConfig.createManagedTable = hoodieConfig.getBoolean(HIVE_CREATE_MANAGED_TABLE) hiveSyncConfig.syncMode = hoodieConfig.getString(HIVE_SYNC_MODE) @@ -561,7 +560,7 @@ object HoodieSparkSqlWriter { val hiveSyncEnabled = hoodieConfig.getStringOrDefault(HIVE_SYNC_ENABLED).toBoolean var metaSyncEnabled = hoodieConfig.getStringOrDefault(META_SYNC_ENABLED).toBoolean var syncClientToolClassSet = scala.collection.mutable.Set[String]() - hoodieConfig.getString(META_SYNC_CLIENT_TOOL_CLASS).split(",").foreach(syncClass => syncClientToolClassSet += syncClass) + hoodieConfig.getString(META_SYNC_CLIENT_TOOL_CLASS_NAME).split(",").foreach(syncClass => syncClientToolClassSet += syncClass) // for backward compatibility if (hiveSyncEnabled) { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala index da06cbc72..51b632b50 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala @@ -17,13 +17,12 @@ package org.apache.hudi -import java.util.Properties - import org.apache.hudi.DataSourceWriteOptions._ -import org.apache.hudi.common.config.HoodieMetadataConfig.{METADATA_ENABLE_PROP, METADATA_VALIDATE_PROP} +import org.apache.hudi.common.config.HoodieMetadataConfig.{ENABLE, VALIDATE_ENABLE} import org.apache.hudi.common.config.{HoodieConfig, TypedProperties} import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory +import java.util.Properties import scala.collection.JavaConversions.mapAsJavaMap import scala.collection.JavaConverters.{mapAsScalaMapConverter, _} @@ -46,18 +45,18 @@ object HoodieWriterUtils { Map(OPERATION.key -> OPERATION.defaultValue, TABLE_TYPE.key -> TABLE_TYPE.defaultValue, PRECOMBINE_FIELD.key -> PRECOMBINE_FIELD.defaultValue, - PAYLOAD_CLASS.key -> PAYLOAD_CLASS.defaultValue, + PAYLOAD_CLASS_NAME.key -> PAYLOAD_CLASS_NAME.defaultValue, RECORDKEY_FIELD.key -> RECORDKEY_FIELD.defaultValue, PARTITIONPATH_FIELD.key -> PARTITIONPATH_FIELD.defaultValue, - KEYGENERATOR_CLASS.key -> DEFAULT_KEYGENERATOR_CLASS_OPT_VAL, - METADATA_ENABLE_PROP.key -> METADATA_ENABLE_PROP.defaultValue.toString, - METADATA_VALIDATE_PROP.key -> METADATA_VALIDATE_PROP.defaultValue.toString, + KEYGENERATOR_CLASS_NAME.key -> DEFAULT_KEYGENERATOR_CLASS_OPT_VAL, + ENABLE.key -> ENABLE.defaultValue.toString, + VALIDATE_ENABLE.key -> VALIDATE_ENABLE.defaultValue.toString, COMMIT_METADATA_KEYPREFIX.key -> COMMIT_METADATA_KEYPREFIX.defaultValue, INSERT_DROP_DUPS.key -> INSERT_DROP_DUPS.defaultValue, STREAMING_RETRY_CNT.key -> STREAMING_RETRY_CNT.defaultValue, STREAMING_RETRY_INTERVAL_MS.key -> STREAMING_RETRY_INTERVAL_MS.defaultValue, STREAMING_IGNORE_FAILED_BATCH.key -> STREAMING_IGNORE_FAILED_BATCH.defaultValue, - META_SYNC_CLIENT_TOOL_CLASS.key -> META_SYNC_CLIENT_TOOL_CLASS.defaultValue, + META_SYNC_CLIENT_TOOL_CLASS_NAME.key -> META_SYNC_CLIENT_TOOL_CLASS_NAME.defaultValue, HIVE_SYNC_ENABLED.key -> HIVE_SYNC_ENABLED.defaultValue, META_SYNC_ENABLED.key -> META_SYNC_ENABLED.defaultValue, HIVE_DATABASE.key -> HIVE_DATABASE.defaultValue, diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala index 5120fe7c2..25d3026f1 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala @@ -42,26 +42,26 @@ object HoodieOptionConfig { val SQL_KEY_TABLE_PRIMARY_KEY: HoodieOption[String] = buildConf() .withSqlKey("primaryKey") .withHoodieKey(DataSourceWriteOptions.RECORDKEY_FIELD.key) - .withTableConfigKey(HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS.key) + .withTableConfigKey(HoodieTableConfig.RECORDKEY_FIELDS.key) .build() val SQL_KEY_TABLE_TYPE: HoodieOption[String] = buildConf() .withSqlKey("type") .withHoodieKey(DataSourceWriteOptions.TABLE_TYPE.key) - .withTableConfigKey(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP.key) + .withTableConfigKey(HoodieTableConfig.TYPE.key) .defaultValue(SQL_VALUE_TABLE_TYPE_COW) .build() val SQL_KEY_PRECOMBINE_FIELD: HoodieOption[String] = buildConf() .withSqlKey("preCombineField") .withHoodieKey(DataSourceWriteOptions.PRECOMBINE_FIELD.key) - .withTableConfigKey(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP.key) + .withTableConfigKey(HoodieTableConfig.PRECOMBINE_FIELD.key) .build() val SQL_PAYLOAD_CLASS: HoodieOption[String] = buildConf() .withSqlKey("payloadClass") - .withHoodieKey(DataSourceWriteOptions.PAYLOAD_CLASS.key) - .withTableConfigKey(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP.key) + .withHoodieKey(DataSourceWriteOptions.PAYLOAD_CLASS_NAME.key) + .withTableConfigKey(HoodieTableConfig.PAYLOAD_CLASS_NAME.key) .defaultValue(classOf[DefaultHoodieRecordPayload].getName) .build() diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala index 12a67bb80..00e8afb60 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala @@ -17,22 +17,19 @@ package org.apache.spark.sql.hudi.command -import scala.collection.JavaConverters._ -import java.util.{Locale, Properties} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} import org.apache.hudi.common.model.HoodieFileFormat import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.util.ValidationUtils import org.apache.hudi.hadoop.HoodieParquetInputFormat import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils +import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} import org.apache.spark.api.java.JavaSparkContext -import org.apache.spark.{SPARK_VERSION, SparkConf} import org.apache.spark.internal.Logging import org.apache.spark.sql.avro.SchemaConverters import org.apache.spark.sql.catalyst.TableIdentifier @@ -41,13 +38,16 @@ import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.hive.HiveClientUtils import org.apache.spark.sql.hive.HiveExternalCatalog._ -import org.apache.spark.sql.hudi.HoodieSqlUtils._ import org.apache.spark.sql.hudi.HoodieOptionConfig +import org.apache.spark.sql.hudi.HoodieSqlUtils._ import org.apache.spark.sql.hudi.command.CreateHoodieTableCommand.{initTableIfNeed, isEmptyPath} import org.apache.spark.sql.internal.StaticSQLConf.SCHEMA_STRING_LENGTH_THRESHOLD import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{AnalysisException, Row, SparkSession} +import org.apache.spark.{SPARK_VERSION, SparkConf} +import java.util.{Locale, Properties} +import scala.collection.JavaConverters._ import scala.collection.mutable /** @@ -106,28 +106,28 @@ case class CreateHoodieTableCommand(table: CatalogTable, ignoreIfExists: Boolean // (By default this config is enable for spark sql) upgrateConfig = if (isNotHiveStyledPartitionTable(allPartitionPaths, table)) { upgrateConfig + (DataSourceWriteOptions.HIVE_STYLE_PARTITIONING.key -> "false") - } else { - upgrateConfig - } - upgrateConfig = if (isUrlEncodeDisable(allPartitionPaths, table)) { - upgrateConfig + (DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key -> "false") } else { upgrateConfig } + upgrateConfig = if (isUrlEncodeDisable(allPartitionPaths, table)) { + upgrateConfig + (DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key -> "false") + } else { + upgrateConfig + } // Use the origin keygen to generate record key to keep the rowkey consistent with the old table for spark sql. // See SqlKeyGenerator#getRecordKey for detail. - upgrateConfig = if (originTableConfig.contains(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS.key)) { - upgrateConfig + (SqlKeyGenerator.ORIGIN_KEYGEN_CLASS -> originTableConfig(HoodieTableConfig.HOODIE_TABLE_KEY_GENERATOR_CLASS.key)) - } else { + upgrateConfig = if (originTableConfig.contains(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key)) { + upgrateConfig + (SqlKeyGenerator.ORIGIN_KEYGEN_CLASS_NAME -> originTableConfig(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key)) + } else { upgrateConfig - } - val options = originTableConfig ++ upgrateConfig ++ table.storage.properties + } + val options = originTableConfig ++ upgrateConfig ++ table.storage.properties - val userSpecifiedSchema = table.schema - if (userSpecifiedSchema.isEmpty && tableSchema.isDefined) { + val userSpecifiedSchema = table.schema + if (userSpecifiedSchema.isEmpty && tableSchema.isDefined) { (addMetaFields(tableSchema.get), options) - } else if (userSpecifiedSchema.nonEmpty) { + } else if (userSpecifiedSchema.nonEmpty) { (addMetaFields(userSpecifiedSchema), options) } else { throw new IllegalArgumentException(s"Missing schema for Create Table: $tableName") @@ -392,19 +392,19 @@ object CreateHoodieTableCommand extends Logging { val tableName = table.identifier.table logInfo(s"Init hoodie.properties for $tableName") val tableOptions = HoodieOptionConfig.mappingSqlOptionToTableConfig(table.storage.properties) - checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP.key) - checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key) - checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.HOODIE_TABLE_RECORDKEY_FIELDS.key) + checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.PRECOMBINE_FIELD.key) + checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.PARTITION_FIELDS.key) + checkTableConfigEqual(originTableConfig, tableOptions, HoodieTableConfig.RECORDKEY_FIELDS.key) // Save all the table config to the hoodie.properties. val parameters = originTableConfig ++ tableOptions val properties = new Properties() - properties.putAll(parameters.asJava) - HoodieTableMetaClient.withPropertyBuilder() - .fromProperties(properties) - .setTableName(tableName) - .setTableCreateSchema(SchemaConverters.toAvroType(table.schema).toString()) - .setPartitionFields(table.partitionColumnNames.mkString(",")) - .initTable(conf, location) + properties.putAll(parameters.asJava) + HoodieTableMetaClient.withPropertyBuilder() + .fromProperties(properties) + .setTableName(tableName) + .setTableCreateSchema(SchemaConverters.toAvroType(table.schema).toString()) + .setPartitionFields(table.partitionColumnNames.mkString(",")) + .initTable(conf, location) } def checkTableConfigEqual(originTableConfig: Map[String, String], diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala index cca9e1521..4d6d0a236 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala @@ -17,14 +17,13 @@ package org.apache.spark.sql.hudi.command -import org.apache.hudi.DataSourceWriteOptions.OPERATION -import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} -import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.DataSourceWriteOptions.{OPERATION, _} import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.hive.ddl.HiveSyncMode +import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, SubqueryAlias} +import org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.hudi.HoodieOptionConfig import org.apache.spark.sql.hudi.HoodieSqlUtils._ @@ -67,14 +66,14 @@ case class DeleteHoodieTableCommand(deleteTable: DeleteFromTable) extends Runnab withSparkConf(sparkSession, targetTable.storage.properties) { Map( "path" -> path, - KEYGENERATOR_CLASS.key -> classOf[SqlKeyGenerator].getCanonicalName, - TABLE_NAME.key -> tableId.table, + KEYGENERATOR_CLASS_NAME.key -> classOf[SqlKeyGenerator].getCanonicalName, + TBL_NAME.key -> tableId.table, OPERATION.key -> DataSourceWriteOptions.DELETE_OPERATION_OPT_VAL, PARTITIONPATH_FIELD.key -> targetTable.partitionColumnNames.mkString(","), HIVE_SYNC_MODE.key -> HiveSyncMode.HMS.name(), - HIVE_SUPPORT_TIMESTAMP.key -> "true", + HIVE_SUPPORT_TIMESTAMP_TYPE.key -> "true", HIVE_STYLE_PARTITIONING.key -> "true", - HoodieWriteConfig.DELETE_PARALLELISM.key -> "200", + HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key -> "200", SqlKeyGenerator.PARTITION_SCHEMA -> targetTable.partitionSchema.toDDL ) } diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala index ec3e41120..8ced2fede 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala @@ -17,15 +17,14 @@ package org.apache.spark.sql.hudi.command -import java.util.Properties import org.apache.avro.Schema import org.apache.avro.generic.{GenericRecord, IndexedRecord} +import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieRecord} import org.apache.hudi.common.util.{Option => HOption} -import org.apache.hudi.exception.HoodieDuplicateKeyException -import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME +import org.apache.hudi.exception.HoodieDuplicateKeyException import org.apache.hudi.hive.MultiPartKeysValueExtractor import org.apache.hudi.hive.ddl.HiveSyncMode import org.apache.hudi.sql.InsertMode @@ -33,22 +32,24 @@ import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkSqlWriter, HoodieWrit import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Alias, Literal} -import org.apache.spark.sql.{Dataset, Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.hudi.{HoodieOptionConfig, HoodieSqlUtils} import org.apache.spark.sql.hudi.HoodieSqlUtils._ +import org.apache.spark.sql.hudi.{HoodieOptionConfig, HoodieSqlUtils} import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.{Dataset, Row, SaveMode, SparkSession} + +import java.util.Properties /** * Command for insert into hoodie table. */ case class InsertIntoHoodieTableCommand( - logicalRelation: LogicalRelation, - query: LogicalPlan, - partition: Map[String, Option[String]], - overwrite: Boolean) + logicalRelation: LogicalRelation, + query: LogicalPlan, + partition: Map[String, Option[String]], + overwrite: Boolean) extends RunnableCommand { override def run(sparkSession: SparkSession): Seq[Row] = { @@ -262,13 +263,13 @@ object InsertIntoHoodieTableCommand extends Logging { Map( "path" -> path, TABLE_TYPE.key -> tableType, - TABLE_NAME.key -> table.identifier.table, + TBL_NAME.key -> table.identifier.table, PRECOMBINE_FIELD.key -> tableSchema.fields.last.name, OPERATION.key -> operation, - KEYGENERATOR_CLASS.key -> keyGenClass, + KEYGENERATOR_CLASS_NAME.key -> keyGenClass, RECORDKEY_FIELD.key -> primaryColumns.mkString(","), PARTITIONPATH_FIELD.key -> partitionFields, - PAYLOAD_CLASS.key -> payloadClassName, + PAYLOAD_CLASS_NAME.key -> payloadClassName, ENABLE_ROW_WRITER.key -> enableBulkInsert.toString, HoodieWriteConfig.COMBINE_BEFORE_INSERT.key -> isPrimaryKeyTable.toString, META_SYNC_ENABLED.key -> enableHive.toString, @@ -276,13 +277,13 @@ object InsertIntoHoodieTableCommand extends Logging { HIVE_USE_JDBC.key -> "false", HIVE_DATABASE.key -> table.identifier.database.getOrElse("default"), HIVE_TABLE.key -> table.identifier.table, - HIVE_SUPPORT_TIMESTAMP.key -> "true", + HIVE_SUPPORT_TIMESTAMP_TYPE.key -> "true", HIVE_STYLE_PARTITIONING.key -> "true", HIVE_PARTITION_FIELDS.key -> partitionFields, HIVE_PARTITION_EXTRACTOR_CLASS.key -> classOf[MultiPartKeysValueExtractor].getCanonicalName, URL_ENCODE_PARTITIONING.key -> "true", - HoodieWriteConfig.INSERT_PARALLELISM.key -> "200", - HoodieWriteConfig.UPSERT_PARALLELISM.key -> "200", + HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key -> "200", + HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key -> "200", SqlKeyGenerator.PARTITION_SCHEMA -> table.partitionSchema.toDDL ) } diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala index febd6f31b..c4b9aec75 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala @@ -17,24 +17,25 @@ package org.apache.spark.sql.hudi.command -import java.util.Base64 import org.apache.avro.Schema import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.hive.MultiPartKeysValueExtractor import org.apache.hudi.hive.ddl.HiveSyncMode import org.apache.hudi.{AvroConversionUtils, DataSourceWriteOptions, HoodieSparkSqlWriter, HoodieWriterUtils, SparkAdapterSupport} +import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, BoundReference, Cast, EqualTo, Expression, Literal} -import org.apache.spark.sql.catalyst.plans.logical.{Assignment, DeleteAction, InsertAction, MergeIntoTable, SubqueryAlias, UpdateAction} +import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.hudi.HoodieSqlUtils._ -import org.apache.spark.sql.types.{BooleanType, StructType} -import org.apache.spark.sql._ -import org.apache.spark.sql.hudi.{HoodieOptionConfig, SerDeUtils} import org.apache.spark.sql.hudi.command.payload.ExpressionPayload import org.apache.spark.sql.hudi.command.payload.ExpressionPayload._ +import org.apache.spark.sql.hudi.{HoodieOptionConfig, SerDeUtils} +import org.apache.spark.sql.types.{BooleanType, StructType} + +import java.util.Base64 /** * The Command for hoodie MergeIntoTable. @@ -431,24 +432,24 @@ case class MergeIntoHoodieTableCommand(mergeInto: MergeIntoTable) extends Runnab Map( "path" -> path, RECORDKEY_FIELD.key -> targetKey2SourceExpression.keySet.mkString(","), - KEYGENERATOR_CLASS.key -> classOf[SqlKeyGenerator].getCanonicalName, + KEYGENERATOR_CLASS_NAME.key -> classOf[SqlKeyGenerator].getCanonicalName, PRECOMBINE_FIELD.key -> targetKey2SourceExpression.keySet.head, // set a default preCombine field - TABLE_NAME.key -> targetTableName, + TBL_NAME.key -> targetTableName, PARTITIONPATH_FIELD.key -> targetTable.partitionColumnNames.mkString(","), - PAYLOAD_CLASS.key -> classOf[ExpressionPayload].getCanonicalName, + PAYLOAD_CLASS_NAME.key -> classOf[ExpressionPayload].getCanonicalName, META_SYNC_ENABLED.key -> enableHive.toString, HIVE_SYNC_MODE.key -> HiveSyncMode.HMS.name(), HIVE_USE_JDBC.key -> "false", HIVE_DATABASE.key -> targetTableDb, HIVE_TABLE.key -> targetTableName, - HIVE_SUPPORT_TIMESTAMP.key -> "true", + HIVE_SUPPORT_TIMESTAMP_TYPE.key -> "true", HIVE_STYLE_PARTITIONING.key -> "true", HIVE_PARTITION_FIELDS.key -> targetTable.partitionColumnNames.mkString(","), HIVE_PARTITION_EXTRACTOR_CLASS.key -> classOf[MultiPartKeysValueExtractor].getCanonicalName, URL_ENCODE_PARTITIONING.key -> "true", // enable the url decode for sql. - HoodieWriteConfig.INSERT_PARALLELISM.key -> "200", // set the default parallelism to 200 for sql - HoodieWriteConfig.UPSERT_PARALLELISM.key -> "200", - HoodieWriteConfig.DELETE_PARALLELISM.key -> "200", + HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key -> "200", // set the default parallelism to 200 for sql + HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key -> "200", + HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key -> "200", SqlKeyGenerator.PARTITION_SCHEMA -> targetTable.partitionSchema.toDDL ) }) diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala index bd2d69f98..b59984acf 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala @@ -43,12 +43,12 @@ class SqlKeyGenerator(props: TypedProperties) extends ComplexKeyGenerator(props) } // The origin key generator class for this table. private lazy val originKeyGen = { - val beforeKeyGenClassName = props.getString(SqlKeyGenerator.ORIGIN_KEYGEN_CLASS, null) + val beforeKeyGenClassName = props.getString(SqlKeyGenerator.ORIGIN_KEYGEN_CLASS_NAME, null) if (beforeKeyGenClassName != null) { val keyGenProps = new TypedProperties() keyGenProps.putAll(props) - keyGenProps.remove(SqlKeyGenerator.ORIGIN_KEYGEN_CLASS) - keyGenProps.put(HoodieWriteConfig.KEYGENERATOR_CLASS.key, beforeKeyGenClassName) + keyGenProps.remove(SqlKeyGenerator.ORIGIN_KEYGEN_CLASS_NAME) + keyGenProps.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key, beforeKeyGenClassName) Some(KeyGenUtils.createKeyGeneratorByClassName(keyGenProps)) } else { None @@ -118,7 +118,7 @@ class SqlKeyGenerator(props: TypedProperties) extends ComplexKeyGenerator(props) object SqlKeyGenerator { val PARTITION_SCHEMA = "hoodie.sql.partition.schema" - val ORIGIN_KEYGEN_CLASS = "hoodie.sql.origin.keygen.class" + val ORIGIN_KEYGEN_CLASS_NAME = "hoodie.sql.origin.keygen.class" private val timestampTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss") private val sqlTimestampFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.S") } diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala index c0b0d7847..20a827400 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala @@ -17,16 +17,16 @@ package org.apache.spark.sql.hudi.command -import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.hive.MultiPartKeysValueExtractor import org.apache.hudi.hive.ddl.HiveSyncMode +import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression} -import org.apache.spark.sql.catalyst.plans.logical.{Assignment, SubqueryAlias, UpdateTable} +import org.apache.spark.sql.catalyst.plans.logical.{Assignment, UpdateTable} import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.hudi.HoodieOptionConfig import org.apache.spark.sql.hudi.HoodieSqlUtils._ @@ -95,9 +95,9 @@ case class UpdateHoodieTableCommand(updateTable: UpdateTable) extends RunnableCo Map( "path" -> path, RECORDKEY_FIELD.key -> primaryColumns.mkString(","), - KEYGENERATOR_CLASS.key -> classOf[SqlKeyGenerator].getCanonicalName, + KEYGENERATOR_CLASS_NAME.key -> classOf[SqlKeyGenerator].getCanonicalName, PRECOMBINE_FIELD.key -> primaryColumns.head, //set the default preCombine field. - TABLE_NAME.key -> tableId.table, + TBL_NAME.key -> tableId.table, OPERATION.key -> DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL, PARTITIONPATH_FIELD.key -> targetTable.partitionColumnNames.mkString(","), META_SYNC_ENABLED.key -> enableHive.toString, @@ -108,9 +108,9 @@ case class UpdateHoodieTableCommand(updateTable: UpdateTable) extends RunnableCo HIVE_PARTITION_FIELDS.key -> targetTable.partitionColumnNames.mkString(","), HIVE_PARTITION_EXTRACTOR_CLASS.key -> classOf[MultiPartKeysValueExtractor].getCanonicalName, URL_ENCODE_PARTITIONING.key -> "true", - HIVE_SUPPORT_TIMESTAMP.key -> "true", + HIVE_SUPPORT_TIMESTAMP_TYPE.key -> "true", HIVE_STYLE_PARTITIONING.key -> "true", - HoodieWriteConfig.UPSERT_PARALLELISM.key -> "200", + HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key -> "200", SqlKeyGenerator.PARTITION_SCHEMA -> targetTable.partitionSchema.toDDL ) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaApp.java b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaApp.java index c254e2319..0557d7040 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaApp.java @@ -16,7 +16,6 @@ * limitations under the License. */ -import org.apache.hadoop.fs.Path; import org.apache.hudi.DataSourceReadOptions; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; @@ -33,6 +32,7 @@ import org.apache.hudi.keygen.SimpleKeyGenerator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; @@ -150,9 +150,9 @@ public class HoodieJavaApp { // use to combine duplicate records in input/with disk val .option(DataSourceWriteOptions.PRECOMBINE_FIELD().key(), "timestamp") // Used by hive sync and queries - .option(HoodieWriteConfig.TABLE_NAME.key(), tableName) + .option(HoodieWriteConfig.TBL_NAME.key(), tableName) // Add Key Extractor - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) .option(DataSourceWriteOptions.ASYNC_COMPACT_ENABLE().key(), "false") @@ -179,13 +179,13 @@ public class HoodieJavaApp { .option(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "_row_key") .option(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), "partition") .option(DataSourceWriteOptions.PRECOMBINE_FIELD().key(), "timestamp") - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) // Add Key Extractor .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1") .option(DataSourceWriteOptions.ASYNC_COMPACT_ENABLE().key(), "false") .option(DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE().key(), "true") - .option(HoodieWriteConfig.TABLE_NAME.key(), tableName).mode(SaveMode.Append); + .option(HoodieWriteConfig.TBL_NAME.key(), tableName).mode(SaveMode.Append); updateHiveSyncConfig(writer); writer.save(tablePath); @@ -207,13 +207,13 @@ public class HoodieJavaApp { .option(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "_row_key") .option(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), "partition") .option(DataSourceWriteOptions.PRECOMBINE_FIELD().key(), "_row_key") - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) // Add Key Extractor .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1") .option(DataSourceWriteOptions.ASYNC_COMPACT_ENABLE().key(), "false") .option(DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE().key(), "true") - .option(HoodieWriteConfig.TABLE_NAME.key(), tableName).mode(SaveMode.Append); + .option(HoodieWriteConfig.TBL_NAME.key(), tableName).mode(SaveMode.Append); updateHiveSyncConfig(writer); writer.save(tablePath); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaGenerateApp.java b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaGenerateApp.java index 43e0b20d3..cf86ba7ba 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaGenerateApp.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaGenerateApp.java @@ -175,9 +175,9 @@ public class HoodieJavaGenerateApp { // use to combine duplicate records in input/with disk val .option(DataSourceWriteOptions.PRECOMBINE_FIELD().key(), "timestamp") // Used by hive sync and queries - .option(HoodieWriteConfig.TABLE_NAME.key(), tableName) + .option(HoodieWriteConfig.TBL_NAME.key(), tableName) // Add Key Extractor - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) .mode(commitType); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java index ea71e55f0..c047ef196 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java @@ -16,7 +16,6 @@ * limitations under the License. */ -import java.util.stream.Collectors; import org.apache.hudi.DataSourceReadOptions; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; @@ -43,13 +42,14 @@ import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.streaming.DataStreamWriter; import org.apache.spark.sql.streaming.OutputMode; +import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.Trigger; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.apache.spark.sql.streaming.StreamingQuery; +import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings; @@ -363,7 +363,7 @@ public class HoodieJavaStreamingApp { .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "1") .option(DataSourceWriteOptions.ASYNC_COMPACT_ENABLE().key(), "true") .option(DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE().key(), "true") - .option(HoodieWriteConfig.TABLE_NAME.key(), tableName).option("checkpointLocation", checkpointLocation) + .option(HoodieWriteConfig.TBL_NAME.key(), tableName).option("checkpointLocation", checkpointLocation) .outputMode(OutputMode.Append()); updateHiveSyncConfig(writer); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java index 9ae6a05c7..5cbcf6cf8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java @@ -18,12 +18,6 @@ package org.apache.hudi.functional; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.mapred.JobConf; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.avro.model.HoodieFileStatus; import org.apache.hudi.client.SparkRDDWriteClient; @@ -49,6 +43,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetReaderIterator; +import org.apache.hudi.common.util.PartitionPathEncodeUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieBootstrapConfig; import org.apache.hudi.config.HoodieCompactionConfig; @@ -58,11 +53,17 @@ import org.apache.hudi.hadoop.HoodieParquetInputFormat; import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.keygen.NonpartitionedKeyGenerator; -import org.apache.hudi.common.util.PartitionPathEncodeUtils; import org.apache.hudi.keygen.SimpleKeyGenerator; import org.apache.hudi.table.action.bootstrap.BootstrapUtils; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapred.JobConf; import org.apache.parquet.avro.AvroParquetReader; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.avro.AvroSchemaConverter; @@ -374,7 +375,7 @@ public class TestBootstrap extends HoodieClientTestBase { List records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>()); assertEquals(totalRecords, records.size()); @@ -393,7 +394,7 @@ public class TestBootstrap extends HoodieClientTestBase { records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>()); assertEquals(totalRecords, records.size()); @@ -410,7 +411,7 @@ public class TestBootstrap extends HoodieClientTestBase { records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, true, HoodieRecord.HOODIE_META_COLUMNS); @@ -428,7 +429,7 @@ public class TestBootstrap extends HoodieClientTestBase { records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true, HoodieRecord.HOODIE_META_COLUMNS); @@ -444,7 +445,7 @@ public class TestBootstrap extends HoodieClientTestBase { records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, true, Arrays.asList("_row_key")); @@ -462,7 +463,7 @@ public class TestBootstrap extends HoodieClientTestBase { records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat( jsc.hadoopConfiguration(), FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(), false).stream() + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(), false).stream() .map(f -> basePath + "/" + f).collect(Collectors.toList()), basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true, Arrays.asList("_row_key")); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java index 071a12cbc..abcc24739 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java @@ -214,12 +214,12 @@ public class TestHoodieDatasetBulkInsertHelper extends HoodieClientTestBase { private Map getProps(boolean setAll, boolean setKeyGen, boolean setRecordKey, boolean setPartitionPath) { Map props = new HashMap<>(); if (setAll) { - props.put(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), "org.apache.hudi.keygen.SimpleKeyGenerator"); + props.put(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), "org.apache.hudi.keygen.SimpleKeyGenerator"); props.put(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "_row_key"); props.put(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), "partition"); } else { if (setKeyGen) { - props.put(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), "org.apache.hudi.keygen.SimpleKeyGenerator"); + props.put(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), "org.apache.hudi.keygen.SimpleKeyGenerator"); } if (setRecordKey) { props.put(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "_row_key"); diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala index 1ff30e2b6..1e2714b9d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala @@ -22,9 +22,8 @@ import org.apache.hadoop.fs.Path import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.common.config.HoodieConfig -import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} -import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord, HoodieRecordPayload, HoodieTableType, WriteOperationType} -import org.apache.hudi.common.table.HoodieTableConfig +import org.apache.hudi.common.model._ +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.testutils.HoodieTestDataGenerator import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieWriteConfig} import org.apache.hudi.exception.HoodieException @@ -73,7 +72,7 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { val session = SparkSession.builder().appName("hoodie_test").master("local").getOrCreate() try { val sqlContext = session.sqlContext - val options = Map("path" -> "hoodie/test/path", HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test_tbl") + val options = Map("path" -> "hoodie/test/path", HoodieWriteConfig.TBL_NAME.key -> "hoodie_test_tbl") val e = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.ErrorIfExists, options, session.emptyDataFrame)) assert(e.getMessage.contains("spark.serializer")) @@ -89,7 +88,7 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { val hoodieFooTableName = "hoodie_foo_tbl" //create a new table val fooTableModifier = Map("path" -> path.toAbsolutePath.toString, - HoodieWriteConfig.TABLE_NAME.key -> hoodieFooTableName, + HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName, "hoodie.insert.shuffle.parallelism" -> "4", "hoodie.upsert.shuffle.parallelism" -> "4") val fooTableParams = HoodieWriterUtils.parametersWithWriteDefaults(fooTableModifier) @@ -98,7 +97,7 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { //on same path try append with different("hoodie_bar_tbl") table name which should throw an exception val barTableModifier = Map("path" -> path.toAbsolutePath.toString, - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_bar_tbl", + HoodieWriteConfig.TBL_NAME.key -> "hoodie_bar_tbl", "hoodie.insert.shuffle.parallelism" -> "4", "hoodie.upsert.shuffle.parallelism" -> "4") val barTableParams = HoodieWriterUtils.parametersWithWriteDefaults(barTableModifier) @@ -151,8 +150,8 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { .updated("hoodie.bulkinsert.shuffle.parallelism", "4") .updated(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL) .updated(DataSourceWriteOptions.ENABLE_ROW_WRITER.key, "true") - .updated(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields)) - .updated(HoodieWriteConfig.BULKINSERT_SORT_MODE.key(), sortMode.name()) + .updated(HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields)) + .updated(HoodieWriteConfig.BULK_INSERT_SORT_MODE.key(), sortMode.name()) val fooTableParams = HoodieWriterUtils.parametersWithWriteDefaults(fooTableModifier) // generate the inserts @@ -200,7 +199,7 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { .updated("hoodie.bulkinsert.shuffle.parallelism", "4") .updated(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL) .updated(DataSourceWriteOptions.ENABLE_ROW_WRITER.key, "true") - .updated(HoodieWriteConfig.BULKINSERT_SORT_MODE.key(), BulkInsertSortMode.NONE.name()) + .updated(HoodieWriteConfig.BULK_INSERT_SORT_MODE.key(), BulkInsertSortMode.NONE.name()) val fooTableParams = HoodieWriterUtils.parametersWithWriteDefaults(fooTableModifier) // generate the inserts @@ -352,15 +351,15 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { try { val hoodieFooTableName = "hoodie_foo_tbl" val fooTableModifier = Map("path" -> path.toAbsolutePath.toString, - HoodieWriteConfig.TABLE_NAME.key -> hoodieFooTableName, + HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName, HoodieWriteConfig.BASE_FILE_FORMAT.key -> baseFileFormat, DataSourceWriteOptions.TABLE_TYPE.key -> tableType, - HoodieWriteConfig.INSERT_PARALLELISM.key -> "4", + HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key -> "4", DataSourceWriteOptions.OPERATION.key -> DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL, DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", - HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key() -> String.valueOf(populateMetaFields), - DataSourceWriteOptions.KEYGENERATOR_CLASS.key -> classOf[SimpleKeyGenerator].getCanonicalName) + HoodieTableConfig.POPULATE_META_FIELDS.key() -> String.valueOf(populateMetaFields), + DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key -> classOf[SimpleKeyGenerator].getCanonicalName) val fooTableParams = HoodieWriterUtils.parametersWithWriteDefaults(fooTableModifier) // generate the inserts @@ -428,14 +427,14 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { .save(srcPath.toAbsolutePath.toString) val fooTableModifier = Map("path" -> path.toAbsolutePath.toString, - HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH.key -> srcPath.toAbsolutePath.toString, - HoodieWriteConfig.TABLE_NAME.key -> hoodieFooTableName, + HoodieBootstrapConfig.BASE_PATH.key -> srcPath.toAbsolutePath.toString, + HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName, DataSourceWriteOptions.TABLE_TYPE.key -> tableType, - HoodieBootstrapConfig.BOOTSTRAP_PARALLELISM.key -> "4", + HoodieBootstrapConfig.PARALLELISM_VALUE.key -> "4", DataSourceWriteOptions.OPERATION.key -> DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL, DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", - HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS.key -> classOf[NonpartitionedKeyGenerator].getCanonicalName) + HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key -> classOf[NonpartitionedKeyGenerator].getCanonicalName) val fooTableParams = HoodieWriterUtils.parametersWithWriteDefaults(fooTableModifier) val client = spy(DataSourceUtils.createHoodieClient( @@ -552,7 +551,7 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { "path" -> basePath, DataSourceWriteOptions.TABLE_NAME.key -> "test_hoodie", DataSourceWriteOptions.HIVE_PARTITION_FIELDS.key -> "partition", - DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX.key -> "true", + DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key -> "true", DataSourceWriteOptions.HIVE_CREATE_MANAGED_TABLE.key -> "true" ) val parameters = HoodieWriterUtils.parametersWithWriteDefaults(params) @@ -621,8 +620,8 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "col3", DataSourceWriteOptions.RECORDKEY_FIELD.key -> "keyid", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "", - DataSourceWriteOptions.KEYGENERATOR_CLASS.key -> "org.apache.hudi.keygen.NonpartitionedKeyGenerator", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test") + DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key -> "org.apache.hudi.keygen.NonpartitionedKeyGenerator", + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test") try { val df = spark.range(0, 1000).toDF("keyid") .withColumn("col3", expr("keyid")) @@ -652,10 +651,10 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { df.write.mode(SaveMode.Overwrite).save(baseBootStrapPath) spark.emptyDataFrame.write.format("hudi") .options(options) - .option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH.key, baseBootStrapPath) - .option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS.key, classOf[NonpartitionedKeyGenerator].getCanonicalName) + .option(HoodieBootstrapConfig.BASE_PATH.key, baseBootStrapPath) + .option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getCanonicalName) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL) - .option(HoodieBootstrapConfig.BOOTSTRAP_PARALLELISM.key, "4") + .option(HoodieBootstrapConfig.PARALLELISM_VALUE.key, "4") .mode(SaveMode.Overwrite).save(basePath) df.write.format("hudi") @@ -750,13 +749,13 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { def getCommonParams(path: java.nio.file.Path, hoodieFooTableName: String, tableType: String) : Map[String, String] = { Map("path" -> path.toAbsolutePath.toString, - HoodieWriteConfig.TABLE_NAME.key -> hoodieFooTableName, + HoodieWriteConfig.TBL_NAME.key -> hoodieFooTableName, "hoodie.insert.shuffle.parallelism" -> "1", "hoodie.upsert.shuffle.parallelism" -> "1", DataSourceWriteOptions.TABLE_TYPE.key -> tableType, DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", - DataSourceWriteOptions.KEYGENERATOR_CLASS.key -> "org.apache.hudi.keygen.SimpleKeyGenerator") + DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key -> "org.apache.hudi.keygen.SimpleKeyGenerator") } test("test Non partition table with metatable support") { @@ -775,11 +774,11 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { .option(DataSourceWriteOptions.PRECOMBINE_FIELD.key, "col3") .option(DataSourceWriteOptions.RECORDKEY_FIELD.key, "keyid") .option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key, "") - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") .option(DataSourceWriteOptions.OPERATION.key, "insert") .option("hoodie.insert.shuffle.parallelism", "1") .option("hoodie.metadata.enable", "true") - .option(HoodieWriteConfig.TABLE_NAME.key, "hoodie_test") + .option(HoodieWriteConfig.TBL_NAME.key, "hoodie_test") .mode(SaveMode.Overwrite).save(basePath) // upsert same record again val df_update = spark.range(0, 10).toDF("keyid") @@ -790,11 +789,11 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { .option(DataSourceWriteOptions.PRECOMBINE_FIELD.key, "col3") .option(DataSourceWriteOptions.RECORDKEY_FIELD.key, "keyid") .option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key, "") - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") .option(DataSourceWriteOptions.OPERATION.key, "upsert") .option("hoodie.upsert.shuffle.parallelism", "1") .option("hoodie.metadata.enable", "true") - .option(HoodieWriteConfig.TABLE_NAME.key, "hoodie_test") + .option(HoodieWriteConfig.TBL_NAME.key, "hoodie_test") .mode(SaveMode.Append).save(basePath) assert(spark.read.format("hudi").load(basePath).count() == 10) assert(spark.read.format("hudi").load(basePath).where("age >= 2000").count() == 10) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala index 74da49137..94e9620d0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala @@ -29,10 +29,10 @@ import org.apache.hudi.keygen.ComplexKeyGenerator import org.apache.hudi.keygen.TimestampBasedAvroKeyGenerator.{Config, TimestampType} import org.apache.hudi.testutils.HoodieClientTestBase import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EqualTo, GreaterThanOrEqual, LessThan, Literal} import org.apache.spark.sql.execution.datasources.PartitionDirectory import org.apache.spark.sql.types.StringType +import org.apache.spark.sql.{SaveMode, SparkSession} import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.params.ParameterizedTest @@ -50,7 +50,7 @@ class TestHoodieFileIndex extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) var queryOpts = Map( @@ -95,7 +95,7 @@ class TestHoodieFileIndex extends HoodieClientTestBase { inputDF1.write.format("hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, keyGenerator) + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, keyGenerator) .option(Config.TIMESTAMP_TYPE_FIELD_PROP, TimestampType.DATE_STRING.name()) .option(Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP, "yyyy/MM/dd") .option(Config.TIMESTAMP_OUTPUT_DATE_FORMAT_PROP, "yyyy-MM-dd") @@ -116,7 +116,7 @@ class TestHoodieFileIndex extends HoodieClientTestBase { inputDF1.write.format("hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, keyGenerator) + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, keyGenerator) .option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key, "partition:simple") .mode(SaveMode.Overwrite) .save(basePath) @@ -176,14 +176,14 @@ class TestHoodieFileIndex extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "dt,hh") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, "false") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, useMetaFileList) + .option(HoodieMetadataConfig.ENABLE.key, useMetaFileList) .mode(SaveMode.Overwrite) .save(basePath) metaClient = HoodieTableMetaClient.reload(metaClient) val fileIndex = HoodieFileIndex(spark, metaClient, None, - queryOpts ++ Map(HoodieMetadataConfig.METADATA_ENABLE_PROP.key -> useMetaFileList.toString)) + queryOpts ++ Map(HoodieMetadataConfig.ENABLE.key -> useMetaFileList.toString)) val partitionFilter1 = And( EqualTo(attribute("dt"), literal("2021-03-01")), @@ -198,7 +198,7 @@ class TestHoodieFileIndex extends HoodieClientTestBase { assertEquals(getFileCountInPartitionPath("2021-03-01/10"), filesAfterPrune.size) val readDF1 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key(), useMetaFileList) + .option(HoodieMetadataConfig.ENABLE.key(), useMetaFileList) .load(basePath) assertEquals(10, readDF1.count()) assertEquals(5, readDF1.filter("dt = '2021-03-01' and hh = '10'").count()) @@ -213,9 +213,9 @@ class TestHoodieFileIndex extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "dt,hh") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, "false") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key(), useMetaFileList) + .option(HoodieMetadataConfig.ENABLE.key(), useMetaFileList) .mode(SaveMode.Overwrite) .save(basePath) @@ -235,7 +235,7 @@ class TestHoodieFileIndex extends HoodieClientTestBase { assertEquals(getFileCountInPartitionPaths("2021/03/01/10", "2021/03/02/10"), filesAfterPrune2.length) val readDF2 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, useMetaFileList) + .option(HoodieMetadataConfig.ENABLE.key, useMetaFileList) .load(basePath) assertEquals(10, readDF2.count()) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala index da448b85a..cbd05ebe9 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala @@ -56,7 +56,7 @@ class TestCOWDataSource extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) val verificationCol: String = "driver" @@ -117,7 +117,7 @@ class TestCOWDataSource extends HoodieClientTestBase { val tableMetaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath).build() val actualSchema = new TableSchemaResolver(tableMetaClient).getTableAvroSchemaWithoutMetadataFields - val (structName, nameSpace) = AvroConversionUtils.getAvroRecordNameAndNamespace(commonOpts(HoodieWriteConfig.TABLE_NAME.key)) + val (structName, nameSpace) = AvroConversionUtils.getAvroRecordNameAndNamespace(commonOpts(HoodieWriteConfig.TBL_NAME.key)) spark.sparkContext.getConf.registerKryoClasses( Array(classOf[org.apache.avro.generic.GenericData], classOf[org.apache.avro.Schema])) @@ -166,7 +166,7 @@ class TestCOWDataSource extends HoodieClientTestBase { inputDF1.write.format("org.apache.hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Overwrite) .save(basePath) @@ -175,7 +175,7 @@ class TestCOWDataSource extends HoodieClientTestBase { // Snapshot query val snapshotDF1 = spark.read.format("org.apache.hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") assertEquals(100, snapshotDF1.count()) @@ -185,13 +185,13 @@ class TestCOWDataSource extends HoodieClientTestBase { updateDf.write.format("org.apache.hudi") .options(commonOpts) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) val commitInstantTime2 = HoodieDataSourceHelpers.latestCommit(fs, basePath) val snapshotDF2 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") assertEquals(100, snapshotDF2.count()) assertEquals(updatedVerificationVal, snapshotDF2.filter(col("_row_key") === verificationRowKey).select(verificationCol).first.getString(0)) @@ -203,7 +203,7 @@ class TestCOWDataSource extends HoodieClientTestBase { inputDF2.write.format("org.apache.hudi") .options(commonOpts) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) @@ -212,7 +212,7 @@ class TestCOWDataSource extends HoodieClientTestBase { // Snapshot Query val snapshotDF3 = spark.read.format("org.apache.hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") assertEquals(100, snapshotDF3.count()) // still 100, since we only updated @@ -242,7 +242,7 @@ class TestCOWDataSource extends HoodieClientTestBase { val emptyDF = spark.read.json(spark.sparkContext.parallelize(emptyRecords, 1)) emptyDF.write.format("org.apache.hudi") .options(commonOpts) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) @@ -510,7 +510,7 @@ class TestCOWDataSource extends HoodieClientTestBase { inputDF1.write.format("org.apache.hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) - .option(HoodieWriteConfig.HOODIE_AUTO_COMMIT.key, "true") + .option(HoodieWriteConfig.AUTO_COMMIT_ENABLE.key, "true") .mode(SaveMode.Overwrite) .save(basePath) @@ -523,7 +523,7 @@ class TestCOWDataSource extends HoodieClientTestBase { inputDF.write.format("hudi") .options(commonOpts) - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, keyGenerator) + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, keyGenerator) .mode(SaveMode.Overwrite) } @@ -660,7 +660,7 @@ class TestCOWDataSource extends HoodieClientTestBase { .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, partitionEncode) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Overwrite) .save(basePath) val commitInstantTime1 = HoodieDataSourceHelpers.latestCommit(fs, basePath) @@ -668,7 +668,7 @@ class TestCOWDataSource extends HoodieClientTestBase { val countIn20160315 = records1.asScala.count(record => record.getPartitionPath == "2016/03/15") // query the partition by filter val count1 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath) .filter("partition = '2016/03/15'") .count() @@ -677,7 +677,7 @@ class TestCOWDataSource extends HoodieClientTestBase { // query the partition by path val partitionPath = if (partitionEncode) "2016%2F03%2F15" else "2016/03/15" val count2 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + s"/$partitionPath") .count() assertEquals(countIn20160315, count2) @@ -689,7 +689,7 @@ class TestCOWDataSource extends HoodieClientTestBase { .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, partitionEncode) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) // Incremental query without "*" in path diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala index 8c0a33763..3d35c03e4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala @@ -17,39 +17,39 @@ package org.apache.hudi.functional -import collection.JavaConverters._ import org.apache.hadoop.fs.FileSystem import org.apache.hudi.bootstrap.SparkParquetBootstrapDataProvider import org.apache.hudi.client.bootstrap.selector.FullRecordBootstrapModeSelector -import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieCompactionConfig, HoodieWriteConfig} import org.apache.hudi.keygen.SimpleKeyGenerator +import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.{SaveMode, SparkSession} import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import java.time.Instant import java.util.Collections +import scala.collection.JavaConverters._ class TestDataSourceForBootstrap { var spark: SparkSession = _ val commonOpts: Map[String, String] = Map( - HoodieWriteConfig.INSERT_PARALLELISM.key -> "4", - HoodieWriteConfig.UPSERT_PARALLELISM.key -> "4", - HoodieWriteConfig.DELETE_PARALLELISM.key -> "4", - HoodieWriteConfig.BULKINSERT_PARALLELISM.key -> "4", - HoodieWriteConfig.FINALIZE_WRITE_PARALLELISM.key -> "4", - HoodieBootstrapConfig.BOOTSTRAP_PARALLELISM.key -> "4", + HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key -> "4", + HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key -> "4", + HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key -> "4", + HoodieWriteConfig.BULKINSERT_PARALLELISM_VALUE.key -> "4", + HoodieWriteConfig.FINALIZE_WRITE_PARALLELISM_VALUE.key -> "4", + HoodieBootstrapConfig.PARALLELISM_VALUE.key -> "4", DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) var basePath: String = _ var srcPath: String = _ @@ -118,7 +118,7 @@ class TestDataSourceForBootstrap { .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.UPSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.COW_TABLE_TYPE_OPT_VAL) - .option(DataSourceWriteOptions.KEYGENERATOR_CLASS.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") + .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key, "org.apache.hudi.keygen.NonpartitionedKeyGenerator") .mode(SaveMode.Append) .save(basePath) @@ -429,10 +429,10 @@ class TestDataSourceForBootstrap { .options(commonOpts) .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key, "datestr") - .option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH.key, srcPath) - .option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS.key, classOf[SimpleKeyGenerator].getName) - .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR.key, classOf[FullRecordBootstrapModeSelector].getName) - .option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER.key, classOf[SparkParquetBootstrapDataProvider].getName) + .option(HoodieBootstrapConfig.BASE_PATH.key, srcPath) + .option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key, classOf[SimpleKeyGenerator].getName) + .option(HoodieBootstrapConfig.MODE_SELECTOR_CLASS_NAME.key, classOf[FullRecordBootstrapModeSelector].getName) + .option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.key, classOf[SparkParquetBootstrapDataProvider].getName) .mode(SaveMode.Overwrite) .save(basePath) @@ -480,8 +480,8 @@ class TestDataSourceForBootstrap { .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, tableType) .option(DataSourceWriteOptions.PARTITIONPATH_FIELD.key, partitionColumns.getOrElse("")) - .option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH.key, srcPath) - .option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS.key, classOf[SimpleKeyGenerator].getName) + .option(HoodieBootstrapConfig.BASE_PATH.key, srcPath) + .option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key, classOf[SimpleKeyGenerator].getName) .mode(SaveMode.Overwrite) .save(basePath) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala index b54e58d77..addd23ef8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala @@ -17,12 +17,12 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieClientTestBase +import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql.{SaveMode, SparkSession} -import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource @@ -34,7 +34,7 @@ class TestEmptyCommit extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) @BeforeEach override def setUp() { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala index b7daecdaa..1bd1c93cc 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala @@ -18,30 +18,28 @@ package org.apache.hudi.functional import org.apache.hadoop.fs.Path - -import scala.collection.JavaConverters._ -import org.apache.hudi.DataSourceWriteOptions.{KEYGENERATOR_CLASS, PARTITIONPATH_FIELD, PAYLOAD_CLASS, PRECOMBINE_FIELD, RECORDKEY_FIELD} +import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieTableType} import org.apache.hudi.common.table.HoodieTableMetaClient -import org.apache.hudi.common.table.view.HoodieTableFileSystemView import org.apache.hudi.common.testutils.HoodieTestDataGenerator -import org.apache.hudi.config.{HoodieIndexConfig, HoodieWriteConfig} -import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers, HoodieSparkUtils} import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings +import org.apache.hudi.config.{HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.index.HoodieIndex.IndexType import org.apache.hudi.keygen.NonpartitionedKeyGenerator import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieClientTestBase} +import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.log4j.LogManager import org.apache.spark.sql._ import org.apache.spark.sql.functions._ -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.{CsvSource, ValueSource} import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ /** * Tests on Spark DataSource for MOR table. @@ -56,7 +54,7 @@ class TestMORDataSource extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) val verificationCol: String = "driver" @@ -89,7 +87,7 @@ class TestMORDataSource extends HoodieClientTestBase { .option("hoodie.compact.inline", "false") // else fails due to compaction & deltacommit instant times being same .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Overwrite) .save(basePath) @@ -98,7 +96,7 @@ class TestMORDataSource extends HoodieClientTestBase { // Read RO View val hudiRODF1 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.QUERY_TYPE.key, DataSourceReadOptions.QUERY_TYPE_READ_OPTIMIZED_OPT_VAL) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") assertEquals(100, hudiRODF1.count()) // still 100, since we only updated @@ -111,7 +109,7 @@ class TestMORDataSource extends HoodieClientTestBase { val inputDF2: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records2, 2)) inputDF2.write.format("org.apache.hudi") .options(commonOpts) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) @@ -119,7 +117,7 @@ class TestMORDataSource extends HoodieClientTestBase { val updateCommitTime = HoodieDataSourceHelpers.latestCommit(fs, basePath) val hudiSnapshotDF2 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.QUERY_TYPE.key, DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") val updateCommitTimes = hudiSnapshotDF2.select("_hoodie_commit_time").distinct().collectAsList().map(r => r.getString(0)).toList @@ -131,12 +129,12 @@ class TestMORDataSource extends HoodieClientTestBase { inputDF3.write.format("org.apache.hudi") .options(commonOpts) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) val hudiSnapshotDF3 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + "/*/*/*") assertEquals(100, hudiSnapshotDF3.count()) assertEquals(updatedVerificationVal, hudiSnapshotDF3.filter(col("_row_key") === verificationRowKey).select(verificationCol).first.getString(0)) @@ -420,7 +418,7 @@ class TestMORDataSource extends HoodieClientTestBase { .option("hoodie.compact.inline", "false") // else fails due to compaction & deltacommit instant times being same .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) - .option(DataSourceWriteOptions.PAYLOAD_CLASS.key, classOf[DefaultHoodieRecordPayload].getName) + .option(DataSourceWriteOptions.PAYLOAD_CLASS_NAME.key, classOf[DefaultHoodieRecordPayload].getName) .mode(SaveMode.Overwrite) .save(basePath) val hudiSnapshotDF1 = spark.read.format("org.apache.hudi") @@ -582,12 +580,12 @@ class TestMORDataSource extends HoodieClientTestBase { df.write.format("org.apache.hudi") .options(commonOpts) // use DefaultHoodieRecordPayload here - .option(PAYLOAD_CLASS.key, classOf[DefaultHoodieRecordPayload].getCanonicalName) + .option(PAYLOAD_CLASS_NAME.key, classOf[DefaultHoodieRecordPayload].getCanonicalName) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "") - .option(KEYGENERATOR_CLASS.key, classOf[NonpartitionedKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getName) .mode(SaveMode.Append) .save(basePath) } @@ -639,7 +637,7 @@ class TestMORDataSource extends HoodieClientTestBase { .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, partitionEncode) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Overwrite) .save(basePath) val commitInstantTime1 = HoodieDataSourceHelpers.latestCommit(fs, basePath) @@ -647,7 +645,7 @@ class TestMORDataSource extends HoodieClientTestBase { val countIn20160315 = records1.asScala.count(record => record.getPartitionPath == "2016/03/15") // query the partition by filter val count1 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath) .filter("partition = '2016/03/15'") .count() @@ -656,7 +654,7 @@ class TestMORDataSource extends HoodieClientTestBase { // query the partition by path val partitionPath = if (partitionEncode) "2016%2F03%2F15" else "2016/03/15" val count2 = spark.read.format("hudi") - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .load(basePath + s"/$partitionPath") .count() assertEquals(countIn20160315, count2) @@ -669,7 +667,7 @@ class TestMORDataSource extends HoodieClientTestBase { .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .option(DataSourceWriteOptions.TABLE_TYPE.key, DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) .option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, partitionEncode) - .option(HoodieMetadataConfig.METADATA_ENABLE_PROP.key, isMetadataEnabled) + .option(HoodieMetadataConfig.ENABLE.key, isMetadataEnabled) .mode(SaveMode.Append) .save(basePath) // Incremental query without "*" in path diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala index 11f816b03..b628cc157 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala @@ -21,7 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions import org.apache.hudi.DataSourceWriteOptions.{PRECOMBINE_FIELD, RECORDKEY_FIELD} import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, MERGE_ON_READ} import org.apache.hudi.common.table.HoodieTableMetaClient -import org.apache.hudi.config.HoodieWriteConfig.{DELETE_PARALLELISM, INSERT_PARALLELISM, TABLE_NAME, UPSERT_PARALLELISM} +import org.apache.hudi.config.HoodieWriteConfig.{DELETE_PARALLELISM_VALUE, INSERT_PARALLELISM_VALUE, TBL_NAME, UPSERT_PARALLELISM_VALUE} import org.apache.log4j.Level import org.apache.spark.sql.streaming.StreamTest import org.apache.spark.sql.{Row, SaveMode} @@ -32,9 +32,9 @@ class TestStreamingSource extends StreamTest { private val commonOptions = Map( RECORDKEY_FIELD.key -> "id", PRECOMBINE_FIELD.key -> "ts", - INSERT_PARALLELISM.key -> "4", - UPSERT_PARALLELISM.key -> "4", - DELETE_PARALLELISM.key -> "4" + INSERT_PARALLELISM_VALUE.key -> "4", + UPSERT_PARALLELISM_VALUE.key -> "4", + DELETE_PARALLELISM_VALUE.key -> "4" ) private val columns = Seq("id", "name", "price", "ts") @@ -50,7 +50,7 @@ class TestStreamingSource extends StreamTest { HoodieTableMetaClient.withPropertyBuilder() .setTableType(COPY_ON_WRITE) .setTableName(getTableName(tablePath)) - .setPayloadClassName(DataSourceWriteOptions.PAYLOAD_CLASS.defaultValue) + .setPayloadClassName(DataSourceWriteOptions.PAYLOAD_CLASS_NAME.defaultValue) .initTable(spark.sessionState.newHadoopConf(), tablePath) addData(tablePath, Seq(("1", "a1", "10", "000"))) @@ -100,7 +100,7 @@ class TestStreamingSource extends StreamTest { HoodieTableMetaClient.withPropertyBuilder() .setTableType(MERGE_ON_READ) .setTableName(getTableName(tablePath)) - .setPayloadClassName(DataSourceWriteOptions.PAYLOAD_CLASS.defaultValue) + .setPayloadClassName(DataSourceWriteOptions.PAYLOAD_CLASS_NAME.defaultValue) .initTable(spark.sessionState.newHadoopConf(), tablePath) addData(tablePath, Seq(("1", "a1", "10", "000"))) @@ -143,7 +143,7 @@ class TestStreamingSource extends StreamTest { .write .format("org.apache.hudi") .options(commonOptions) - .option(TABLE_NAME.key, getTableName(inputPath)) + .option(TBL_NAME.key, getTableName(inputPath)) .mode(SaveMode.Append) .save(inputPath) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala index 9286ee09c..68b630be5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala @@ -50,7 +50,7 @@ class TestStructuredStreaming extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) @BeforeEach override def setUp() { @@ -194,11 +194,11 @@ class TestStructuredStreaming extends HoodieClientTestBase { def getClusteringOpts(isInlineClustering: String, isAsyncClustering: String, isAsyncCompaction: String, clusteringNumCommit: String, fileMaxRecordNum: Int):Map[String, String] = { commonOpts + (HoodieClusteringConfig.INLINE_CLUSTERING.key -> isInlineClustering, - HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT.key -> clusteringNumCommit, + HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit, DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE.key -> isAsyncClustering, DataSourceWriteOptions.ASYNC_COMPACT_ENABLE.key -> isAsyncCompaction, - HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMIT_PROP.key -> clusteringNumCommit, - HoodieStorageConfig.PARQUET_FILE_MAX_BYTES.key -> dataGen.getEstimatedFileSizeInBytes(fileMaxRecordNum).toString + HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit, + HoodieStorageConfig.PARQUET_MAX_FILE_SIZE.key -> dataGen.getEstimatedFileSizeInBytes(fileMaxRecordNum).toString ) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala index 96e9a17ac..bb102a4cd 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala @@ -17,13 +17,13 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions} import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.common.table.timeline.HoodieActiveTimeline import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator} import org.apache.hudi.testutils.HoodieClientTestBase +import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions} import org.apache.spark.sql.{Row, SaveMode, SparkSession} import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} import org.junit.jupiter.api.{AfterEach, BeforeEach} @@ -42,7 +42,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key", DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition", DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "timestamp", - HoodieWriteConfig.TABLE_NAME.key -> "hoodie_test" + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test" ) @BeforeEach override def setUp() { @@ -74,7 +74,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "") - .option(KEYGENERATOR_CLASS.key, classOf[NonpartitionedKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getName) .mode(SaveMode.Overwrite) .save(basePath) @@ -88,7 +88,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "") - .option(KEYGENERATOR_CLASS.key, classOf[NonpartitionedKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getName) .mode(SaveMode.Append) .save(basePath) metaClient.reloadActiveTimeline() @@ -102,7 +102,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "") - .option(KEYGENERATOR_CLASS.key, classOf[NonpartitionedKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getName) .mode(SaveMode.Append) .save(basePath) metaClient.reloadActiveTimeline() @@ -148,7 +148,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "dt") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) .mode(SaveMode.Overwrite) .save(basePath) @@ -162,7 +162,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "dt") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) .mode(SaveMode.Append) .save(basePath) metaClient.reloadActiveTimeline() @@ -176,7 +176,7 @@ class TestTimeTravelQuery extends HoodieClientTestBase { .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "version") .option(PARTITIONPATH_FIELD.key, "dt") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) .mode(SaveMode.Append) .save(basePath) metaClient.reloadActiveTimeline() diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala index f75af6109..f34d5f34c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala @@ -18,8 +18,6 @@ package org.apache.spark.sql.hudi import org.apache.hudi.DataSourceWriteOptions._ - -import scala.collection.JavaConverters._ import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.config.HoodieWriteConfig @@ -28,7 +26,9 @@ import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator, import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTableType -import org.apache.spark.sql.types.{DoubleType, IntegerType, LongType, StringType, StructField} +import org.apache.spark.sql.types._ + +import scala.collection.JavaConverters._ class TestCreateTable extends TestHoodieSqlBase { @@ -288,14 +288,14 @@ class TestCreateTable extends TestHoodieSqlBase { val df = Seq((1, "a1", 10, 1000, partitionValue)).toDF("id", "name", "value", "ts", "dt") // Write a table by spark dataframe. df.write.format("hudi") - .option(HoodieWriteConfig.TABLE_NAME.key, tableName) + .option(HoodieWriteConfig.TBL_NAME.key, tableName) .option(TABLE_TYPE.key, COW_TABLE_TYPE_OPT_VAL) .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "ts") .option(PARTITIONPATH_FIELD.key, "dt") - .option(KEYGENERATOR_CLASS.key, classOf[SimpleKeyGenerator].getName) - .option(HoodieWriteConfig.INSERT_PARALLELISM.key, "1") - .option(HoodieWriteConfig.UPSERT_PARALLELISM.key, "1") + .option(KEYGENERATOR_CLASS_NAME.key, classOf[SimpleKeyGenerator].getName) + .option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key, "1") + .option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key, "1") .mode(SaveMode.Overwrite) .save(tablePath) @@ -319,9 +319,9 @@ class TestCreateTable extends TestHoodieSqlBase { .setConf(spark.sessionState.newHadoopConf()) .build() val properties = metaClient.getTableConfig.getProps.asScala.toMap - assertResult(true)(properties.contains(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA.key)) - assertResult("dt")(properties(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key)) - assertResult("ts")(properties(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP.key)) + assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key)) + assertResult("dt")(properties(HoodieTableConfig.PARTITION_FIELDS.key)) + assertResult("ts")(properties(HoodieTableConfig.PRECOMBINE_FIELD.key)) // Test insert into spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000, '$partitionValue')") @@ -365,14 +365,14 @@ class TestCreateTable extends TestHoodieSqlBase { val df = Seq((1, "a1", 10, 1000, day, 12)).toDF("id", "name", "value", "ts", "day", "hh") // Write a table by spark dataframe. df.write.format("hudi") - .option(HoodieWriteConfig.TABLE_NAME.key, tableName) + .option(HoodieWriteConfig.TBL_NAME.key, tableName) .option(TABLE_TYPE.key, MOR_TABLE_TYPE_OPT_VAL) .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "ts") .option(PARTITIONPATH_FIELD.key, "day,hh") - .option(KEYGENERATOR_CLASS.key, classOf[ComplexKeyGenerator].getName) - .option(HoodieWriteConfig.INSERT_PARALLELISM.key, "1") - .option(HoodieWriteConfig.UPSERT_PARALLELISM.key, "1") + .option(KEYGENERATOR_CLASS_NAME.key, classOf[ComplexKeyGenerator].getName) + .option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key, "1") + .option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key, "1") .mode(SaveMode.Overwrite) .save(tablePath) @@ -396,9 +396,9 @@ class TestCreateTable extends TestHoodieSqlBase { .setConf(spark.sessionState.newHadoopConf()) .build() val properties = metaClient.getTableConfig.getProps.asScala.toMap - assertResult(true)(properties.contains(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA.key)) - assertResult("day,hh")(properties(HoodieTableConfig.HOODIE_TABLE_PARTITION_FIELDS_PROP.key)) - assertResult("ts")(properties(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP.key)) + assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key)) + assertResult("day,hh")(properties(HoodieTableConfig.PARTITION_FIELDS.key)) + assertResult("ts")(properties(HoodieTableConfig.PRECOMBINE_FIELD.key)) // Test insert into spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000, '$day', 12)") @@ -440,14 +440,14 @@ class TestCreateTable extends TestHoodieSqlBase { import spark.implicits._ val df = Seq((1, "a1", 10, 1000)).toDF("id", "name", "value", "ts") df.write.format("hudi") - .option(HoodieWriteConfig.TABLE_NAME.key, tableName) + .option(HoodieWriteConfig.TBL_NAME.key, tableName) .option(TABLE_TYPE.key, COW_TABLE_TYPE_OPT_VAL) .option(RECORDKEY_FIELD.key, "id") .option(PRECOMBINE_FIELD.key, "ts") .option(PARTITIONPATH_FIELD.key, "") - .option(KEYGENERATOR_CLASS.key, classOf[NonpartitionedKeyGenerator].getName) - .option(HoodieWriteConfig.INSERT_PARALLELISM.key, "1") - .option(HoodieWriteConfig.UPSERT_PARALLELISM.key, "1") + .option(KEYGENERATOR_CLASS_NAME.key, classOf[NonpartitionedKeyGenerator].getName) + .option(HoodieWriteConfig.INSERT_PARALLELISM_VALUE.key, "1") + .option(HoodieWriteConfig.UPSERT_PARALLELISM_VALUE.key, "1") .mode(SaveMode.Overwrite) .save(tmp.getCanonicalPath) @@ -470,8 +470,8 @@ class TestCreateTable extends TestHoodieSqlBase { .setConf(spark.sessionState.newHadoopConf()) .build() val properties = metaClient.getTableConfig.getProps.asScala.toMap - assertResult(true)(properties.contains(HoodieTableConfig.HOODIE_TABLE_CREATE_SCHEMA.key)) - assertResult("ts")(properties(HoodieTableConfig.HOODIE_TABLE_PRECOMBINE_FIELD_PROP.key)) + assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key)) + assertResult("ts")(properties(HoodieTableConfig.PRECOMBINE_FIELD.key)) // Test insert into spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000)") diff --git a/hudi-spark-datasource/hudi-spark2/src/main/java/org/apache/hudi/internal/DefaultSource.java b/hudi-spark-datasource/hudi-spark2/src/main/java/org/apache/hudi/internal/DefaultSource.java index e04b105c0..649440639 100644 --- a/hudi-spark-datasource/hudi-spark2/src/main/java/org/apache/hudi/internal/DefaultSource.java +++ b/hudi-spark-datasource/hudi-spark2/src/main/java/org/apache/hudi/internal/DefaultSource.java @@ -61,15 +61,15 @@ public class DefaultSource extends BaseDefaultSource implements DataSourceV2, DataSourceOptions options) { String instantTime = options.get(DataSourceInternalWriterHelper.INSTANT_TIME_OPT_KEY).get(); String path = options.get("path").get(); - String tblName = options.get(HoodieWriteConfig.TABLE_NAME.key()).get(); - boolean populateMetaFields = options.getBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), - Boolean.parseBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue())); + String tblName = options.get(HoodieWriteConfig.TBL_NAME.key()).get(); + boolean populateMetaFields = options.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS.key(), + Boolean.parseBoolean(HoodieTableConfig.POPULATE_META_FIELDS.defaultValue())); // 1st arg to createHooodieConfig is not really reuqired to be set. but passing it anyways. - HoodieWriteConfig config = DataSourceUtils.createHoodieConfig(options.get(HoodieWriteConfig.AVRO_SCHEMA.key()).get(), path, tblName, options.asMap()); + HoodieWriteConfig config = DataSourceUtils.createHoodieConfig(options.get(HoodieWriteConfig.AVRO_SCHEMA_STRING.key()).get(), path, tblName, options.asMap()); boolean arePartitionRecordsSorted = HoodieInternalConfig.getBulkInsertIsPartitionRecordsSorted( options.get(HoodieInternalConfig.BULKINSERT_ARE_PARTITIONER_RECORDS_SORTED).isPresent() ? options.get(HoodieInternalConfig.BULKINSERT_ARE_PARTITIONER_RECORDS_SORTED).get() : null); return Optional.of(new HoodieDataSourceInternalWriter(instantTime, config, schema, getSparkSession(), - getConfiguration(), options, populateMetaFields, arePartitionRecordsSorted)); + getConfiguration(), options, populateMetaFields, arePartitionRecordsSorted)); } } diff --git a/hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java b/hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java index 1c4397a47..116108809 100644 --- a/hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java +++ b/hudi-spark-datasource/hudi-spark3/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java @@ -48,13 +48,13 @@ public class DefaultSource extends BaseDefaultSource implements TableProvider { public Table getTable(StructType schema, Transform[] partitioning, Map properties) { String instantTime = properties.get(DataSourceInternalWriterHelper.INSTANT_TIME_OPT_KEY); String path = properties.get("path"); - String tblName = properties.get(HoodieWriteConfig.TABLE_NAME.key()); - boolean populateMetaFields = Boolean.parseBoolean(properties.getOrDefault(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), - HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue())); + String tblName = properties.get(HoodieWriteConfig.TBL_NAME.key()); + boolean populateMetaFields = Boolean.parseBoolean(properties.getOrDefault(HoodieTableConfig.POPULATE_META_FIELDS.key(), + HoodieTableConfig.POPULATE_META_FIELDS.defaultValue())); boolean arePartitionRecordsSorted = Boolean.parseBoolean(properties.getOrDefault(HoodieInternalConfig.BULKINSERT_ARE_PARTITIONER_RECORDS_SORTED, Boolean.toString(HoodieInternalConfig.DEFAULT_BULKINSERT_ARE_PARTITIONER_RECORDS_SORTED))); // 1st arg to createHooodieConfig is not really reuqired to be set. but passing it anyways. - HoodieWriteConfig config = DataSourceUtils.createHoodieConfig(properties.get(HoodieWriteConfig.AVRO_SCHEMA.key()), path, tblName, properties); + HoodieWriteConfig config = DataSourceUtils.createHoodieConfig(properties.get(HoodieWriteConfig.AVRO_SCHEMA_STRING.key()), path, tblName, properties); return new HoodieDataSourceInternalTable(instantTime, config, schema, getSparkSession(), getConfiguration(), properties, populateMetaFields, arePartitionRecordsSorted); } diff --git a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java index 352796683..06408bf61 100644 --- a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java +++ b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java @@ -18,11 +18,11 @@ package org.apache.hudi.dla; -import com.beust.jcommander.Parameter; - import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor; +import com.beust.jcommander.Parameter; + import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -74,7 +74,7 @@ public class DLASyncConfig implements Serializable { public Boolean useFileListingFromMetadata = HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS; @Parameter(names = {"--verify-metadata-file-listing"}, description = "Verify file listing from Hudi's metadata against file system") - public Boolean verifyMetadataFileListing = HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(); + public Boolean verifyMetadataFileListing = HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(); @Parameter(names = {"--help", "-h"}, help = true) public Boolean help = false; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java index d9d833d39..30d5a401b 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java @@ -90,7 +90,7 @@ public class HiveSyncConfig implements Serializable { public Boolean useFileListingFromMetadata = HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS; @Parameter(names = {"--verify-metadata-file-listing"}, description = "Verify file listing from Hudi's metadata against file system") - public Boolean verifyMetadataFileListing = HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(); + public Boolean verifyMetadataFileListing = HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(); @Parameter(names = {"--table-properties"}, description = "Table properties to hive table") public String tableProperties; diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index 4fa2f0ac3..77c7870f5 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -97,10 +97,10 @@ public class TimelineService { @Parameter(names = {"--base-store-path", "-sp"}, description = "Directory where spilled view entries will be stored. Used for SPILLABLE_DISK storage type") - public String baseStorePathForFileGroups = FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue(); + public String baseStorePathForFileGroups = FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(); @Parameter(names = {"--rocksdb-path", "-rp"}, description = "Root directory for RocksDB") - public String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH_PROP.defaultValue(); + public String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH.defaultValue(); @Parameter(names = {"--threads", "-t"}, description = "Number of threads to use for serving requests") public int numThreads = DEFAULT_NUM_THREADS; @@ -138,8 +138,8 @@ public class TimelineService { private FileSystemViewStorageType viewStorageType = FileSystemViewStorageType.SPILLABLE_DISK; private Integer maxViewMemPerTableInMB = 2048; private Double memFractionForCompactionPerTable = 0.001; - private String baseStorePathForFileGroups = FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue(); - private String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH_PROP.defaultValue(); + private String baseStorePathForFileGroups = FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(); + private String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH.defaultValue(); private int numThreads = DEFAULT_NUM_THREADS; private boolean async = false; private boolean compress = true; @@ -148,7 +148,8 @@ public class TimelineService { private long markerBatchIntervalMs = 50L; private int markerParallelism = 100; - public Builder() {} + public Builder() { + } public Builder serverPort(int serverPort) { this.serverPort = serverPort; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java index d3281ad88..20e19cf08 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java @@ -74,7 +74,7 @@ public class HoodieSnapshotCopier implements Serializable { public Boolean useFileListingFromMetadata = HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS; @Parameter(names = {"--verify-metadata-file-listing"}, description = "Verify file listing from Hudi's metadata against file system") - public Boolean verifyMetadataFileListing = HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(); + public Boolean verifyMetadataFileListing = HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(); } public void snapshot(JavaSparkContext jsc, String baseDir, final String outputDir, diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java index 594ed5685..9d164906e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallback.java @@ -35,12 +35,12 @@ import org.apache.log4j.Logger; import java.util.Properties; -import static org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME; -import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.CALLBACK_KAFKA_ACKS; -import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.CALLBACK_KAFKA_BOOTSTRAP_SERVERS; -import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.CALLBACK_KAFKA_PARTITION; -import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.CALLBACK_KAFKA_RETRIES; -import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.CALLBACK_KAFKA_TOPIC; +import static org.apache.hudi.config.HoodieWriteConfig.TBL_NAME; +import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.ACKS; +import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.BOOTSTRAP_SERVERS; +import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.PARTITION; +import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.RETRIES; +import static org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig.TOPIC; /** * Kafka implementation of {@link HoodieWriteCommitCallback}. @@ -55,8 +55,8 @@ public class HoodieWriteCommitKafkaCallback implements HoodieWriteCommitCallback public HoodieWriteCommitKafkaCallback(HoodieWriteConfig config) { this.hoodieConfig = config; - this.bootstrapServers = config.getString(CALLBACK_KAFKA_BOOTSTRAP_SERVERS); - this.topic = config.getString(CALLBACK_KAFKA_TOPIC); + this.bootstrapServers = config.getString(BOOTSTRAP_SERVERS); + this.topic = config.getString(TOPIC); validateKafkaConfig(); } @@ -85,10 +85,10 @@ public class HoodieWriteCommitKafkaCallback implements HoodieWriteCommitCallback kafkaProducerProps.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); // default "all" to ensure no message loss kafkaProducerProps.setProperty(ProducerConfig.ACKS_CONFIG, hoodieConfig - .getString(CALLBACK_KAFKA_ACKS)); + .getString(ACKS)); // retries 3 times by default kafkaProducerProps.setProperty(ProducerConfig.RETRIES_CONFIG, hoodieConfig - .getString(CALLBACK_KAFKA_RETRIES)); + .getString(RETRIES)); kafkaProducerProps.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); kafkaProducerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, @@ -109,13 +109,13 @@ public class HoodieWriteCommitKafkaCallback implements HoodieWriteCommitCallback * @return Callback {@link ProducerRecord} */ private ProducerRecord buildProducerRecord(HoodieConfig hoodieConfig, String callbackMsg) { - String partition = hoodieConfig.getString(CALLBACK_KAFKA_PARTITION); + String partition = hoodieConfig.getString(PARTITION); if (null != partition) { return new ProducerRecord(topic, Integer.valueOf(partition), hoodieConfig - .getString(TABLE_NAME), + .getString(TBL_NAME), callbackMsg); } else { - return new ProducerRecord(topic, hoodieConfig.getString(TABLE_NAME), callbackMsg); + return new ProducerRecord(topic, hoodieConfig.getString(TBL_NAME), callbackMsg); } } @@ -125,9 +125,9 @@ public class HoodieWriteCommitKafkaCallback implements HoodieWriteCommitCallback */ private void validateKafkaConfig() { ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(bootstrapServers), String.format("Config %s can not be " - + "null or empty", CALLBACK_KAFKA_BOOTSTRAP_SERVERS.key())); + + "null or empty", BOOTSTRAP_SERVERS.key())); ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(topic), String.format("Config %s can not be null or empty", - CALLBACK_KAFKA_TOPIC.key())); + TOPIC.key())); } /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java index 3b2f18bb0..0ec91e129 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java @@ -32,19 +32,19 @@ import static org.apache.hudi.config.HoodieWriteCommitCallbackConfig.CALLBACK_PR description = "Controls notifications sent to Kafka, on events happening to a hudi table.") public class HoodieWriteCommitKafkaCallbackConfig extends HoodieConfig { - public static final ConfigProperty CALLBACK_KAFKA_BOOTSTRAP_SERVERS = ConfigProperty + public static final ConfigProperty BOOTSTRAP_SERVERS = ConfigProperty .key(CALLBACK_PREFIX + "kafka.bootstrap.servers") .noDefaultValue() .sinceVersion("0.7.0") .withDocumentation("Bootstrap servers of kafka cluster, to be used for publishing commit metadata."); - public static final ConfigProperty CALLBACK_KAFKA_TOPIC = ConfigProperty + public static final ConfigProperty TOPIC = ConfigProperty .key(CALLBACK_PREFIX + "kafka.topic") .noDefaultValue() .sinceVersion("0.7.0") .withDocumentation("Kafka topic name to publish timeline activity into."); - public static final ConfigProperty CALLBACK_KAFKA_PARTITION = ConfigProperty + public static final ConfigProperty PARTITION = ConfigProperty .key(CALLBACK_PREFIX + "kafka.partition") .noDefaultValue() .sinceVersion("0.7.0") @@ -52,13 +52,13 @@ public class HoodieWriteCommitKafkaCallbackConfig extends HoodieConfig { + " for providing strict ordering. By default, Kafka messages are keyed by table name, which " + " guarantees ordering at the table level, but not globally (or when new partitions are added)"); - public static final ConfigProperty CALLBACK_KAFKA_ACKS = ConfigProperty + public static final ConfigProperty ACKS = ConfigProperty .key(CALLBACK_PREFIX + "kafka.acks") .defaultValue("all") .sinceVersion("0.7.0") .withDocumentation("kafka acks level, all by default to ensure strong durability."); - public static final ConfigProperty CALLBACK_KAFKA_RETRIES = ConfigProperty + public static final ConfigProperty RETRIES = ConfigProperty .key(CALLBACK_PREFIX + "kafka.retries") .defaultValue(3) .sinceVersion("0.7.0") @@ -68,8 +68,43 @@ public class HoodieWriteCommitKafkaCallbackConfig extends HoodieConfig { * Set default value for {@link HoodieWriteCommitKafkaCallbackConfig} if needed. */ public static void setCallbackKafkaConfigIfNeeded(HoodieConfig config) { - config.setDefaultValue(CALLBACK_KAFKA_ACKS); - config.setDefaultValue(CALLBACK_KAFKA_RETRIES); + config.setDefaultValue(ACKS); + config.setDefaultValue(RETRIES); } + /** + * @deprecated Use {@link #BOOTSTRAP_SERVERS} and its methods. + */ + @Deprecated + public static final String CALLBACK_KAFKA_BOOTSTRAP_SERVERS = BOOTSTRAP_SERVERS.key(); + /** + * @deprecated Use {@link #TOPIC} and its methods. + */ + @Deprecated + public static final String CALLBACK_KAFKA_TOPIC = TOPIC.key(); + /** + * @deprecated Use {@link #PARTITION} and its methods. + */ + @Deprecated + public static final String CALLBACK_KAFKA_PARTITION = PARTITION.key(); + /** + * @deprecated Use {@link #ACKS} and its methods. + */ + @Deprecated + public static final String CALLBACK_KAFKA_ACKS = ACKS.key(); + /** + * @deprecated Use {@link #ACKS} and its methods. + */ + @Deprecated + public static final String DEFAULT_CALLBACK_KAFKA_ACKS = ACKS.defaultValue(); + /** + * @deprecated Use {@link #RETRIES} and its methods. + */ + @Deprecated + public static final String CALLBACK_KAFKA_RETRIES = RETRIES.key(); + /** + * @deprecated Use {@link #RETRIES} and its methods. + */ + @Deprecated + public static final int DEFAULT_CALLBACK_KAFKA_RETRIES = RETRIES.defaultValue(); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java index 55f849c18..17fecdecc 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java @@ -49,7 +49,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER; /** * Performs bootstrap from a non-hudi source. @@ -111,14 +111,14 @@ public class BootstrapExecutor implements Serializable { this.configuration = conf; this.props = properties; - ValidationUtils.checkArgument(properties.containsKey(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP + ValidationUtils.checkArgument(properties.containsKey(HoodieTableConfig.BOOTSTRAP_BASE_PATH .key()), - HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP.key() + " must be specified."); - this.bootstrapBasePath = properties.getString(HoodieTableConfig.HOODIE_BOOTSTRAP_BASE_PATH_PROP.key()); + HoodieTableConfig.BOOTSTRAP_BASE_PATH.key() + " must be specified."); + this.bootstrapBasePath = properties.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key()); // Add more defaults if full bootstrap requested - this.props.putIfAbsent(DataSourceWriteOptions.PAYLOAD_CLASS().key(), - DataSourceWriteOptions.PAYLOAD_CLASS().defaultValue()); + this.props.putIfAbsent(DataSourceWriteOptions.PAYLOAD_CLASS_NAME().key(), + DataSourceWriteOptions.PAYLOAD_CLASS_NAME().defaultValue()); this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jssc); HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder().withPath(cfg.targetBasePath) @@ -175,7 +175,7 @@ public class BootstrapExecutor implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setBootstrapIndexClass(cfg.bootstrapIndexClass) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java index 2f9d148db..88d1d8c8a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java @@ -99,13 +99,13 @@ import java.util.stream.Collectors; import scala.collection.JavaConversions; -import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP; +import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER; import static org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE; import static org.apache.hudi.config.HoodieClusteringConfig.INLINE_CLUSTERING; import static org.apache.hudi.config.HoodieCompactionConfig.INLINE_COMPACT; +import static org.apache.hudi.config.HoodieWriteConfig.AUTO_COMMIT_ENABLE; import static org.apache.hudi.config.HoodieWriteConfig.COMBINE_BEFORE_INSERT; import static org.apache.hudi.config.HoodieWriteConfig.COMBINE_BEFORE_UPSERT; -import static org.apache.hudi.config.HoodieWriteConfig.HOODIE_AUTO_COMMIT; import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.CHECKPOINT_KEY; import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.CHECKPOINT_RESET_KEY; import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; @@ -252,14 +252,14 @@ public class DeltaSync implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setPartitionFields(partitionColumns) .setRecordKeyFields(props.getProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key())) - .setPopulateMetaFields(props.getBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), - Boolean.parseBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()))) - .setKeyGeneratorClassProp(props.getProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .setPopulateMetaFields(props.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS.key(), + Boolean.parseBoolean(HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()))) + .setKeyGeneratorClassProp(props.getProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), SimpleKeyGenerator.class.getName())) .setPreCombineField(cfg.sourceOrderingField) .initTable(new Configuration(jssc.hadoopConfiguration()), @@ -356,14 +356,14 @@ public class DeltaSync implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder(HOODIE_ARCHIVELOG_FOLDER_PROP.defaultValue()) + .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue()) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setPartitionFields(partitionColumns) .setRecordKeyFields(props.getProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key())) - .setPopulateMetaFields(props.getBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.key(), - Boolean.parseBoolean(HoodieTableConfig.HOODIE_POPULATE_META_FIELDS.defaultValue()))) - .setKeyGeneratorClassProp(props.getProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS().key(), + .setPopulateMetaFields(props.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS.key(), + Boolean.parseBoolean(HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()))) + .setKeyGeneratorClassProp(props.getProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key(), SimpleKeyGenerator.class.getName())) .initTable(new Configuration(jssc.hadoopConfiguration()), cfg.targetBasePath); } @@ -711,7 +711,7 @@ public class DeltaSync implements Serializable { ValidationUtils.checkArgument(config.isAsyncClusteringEnabled() == cfg.isAsyncClusteringEnabled(), String.format("%s should be set to %s", ASYNC_CLUSTERING_ENABLE.key(), cfg.isAsyncClusteringEnabled())); ValidationUtils.checkArgument(!config.shouldAutoCommit(), - String.format("%s should be set to %s", HOODIE_AUTO_COMMIT.key(), autoCommit)); + String.format("%s should be set to %s", AUTO_COMMIT_ENABLE.key(), autoCommit)); ValidationUtils.checkArgument(config.shouldCombineBeforeInsert() == cfg.filterDupes, String.format("%s should be set to %s", COMBINE_BEFORE_INSERT.key(), cfg.filterDupes)); ValidationUtils.checkArgument(config.shouldCombineBeforeUpsert(), diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java index eb5f2a803..53b2febcc 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java @@ -297,10 +297,10 @@ public class TimelineServerPerf implements Serializable { @Parameter(names = {"--base-store-path", "-sp"}, description = "Directory where spilled view entries will be stored. Used for SPILLABLE_DISK storage type") - public String baseStorePathForFileGroups = FileSystemViewStorageConfig.FILESYSTEM_VIEW_SPILLABLE_DIR.defaultValue(); + public String baseStorePathForFileGroups = FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(); @Parameter(names = {"--rocksdb-path", "-rp"}, description = "Root directory for RocksDB") - public String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH_PROP.defaultValue(); + public String rocksDBPath = FileSystemViewStorageConfig.ROCKSDB_BASE_PATH.defaultValue(); @Parameter(names = {"--wait-for-manual-queries", "-ww"}) public Boolean waitForManualQueries = false; @@ -309,7 +309,7 @@ public class TimelineServerPerf implements Serializable { public Boolean useFileListingFromMetadata = HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS; @Parameter(names = {"--verify-metadata-file-listing"}, description = "Verify file listing from Hudi's metadata against file system") - public Boolean verifyMetadataFileListing = HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue(); + public Boolean verifyMetadataFileListing = HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue(); @Parameter(names = {"--help", "-h"}) public Boolean help = false; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java index 9fa1527cd..fb6f5d649 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; public class TestKafkaConnectHdfsProvider extends HoodieCommonTestHarness { - private static final String BASE_FILE_EXTENSION = HoodieTableConfig.HOODIE_BASE_FILE_FORMAT_PROP.defaultValue().getFileExtension(); + private static final String BASE_FILE_EXTENSION = HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension(); @Test public void testValidKafkaConnectPath() throws Exception { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index 7848679fe..aab02da23 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -983,13 +983,13 @@ public class TestHoodieDeltaStreamer extends TestHoodieDeltaStreamerBase { configs.add(String.format("%s=%s", HoodieClusteringConfig.INLINE_CLUSTERING.key(), inlineCluster)); } if (!StringUtils.isNullOrEmpty(inlineClusterMaxCommit)) { - configs.add(String.format("%s=%s", HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT.key(), inlineClusterMaxCommit)); + configs.add(String.format("%s=%s", HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMITS.key(), inlineClusterMaxCommit)); } if (!StringUtils.isNullOrEmpty(asyncCluster)) { configs.add(String.format("%s=%s", HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE.key(), asyncCluster)); } if (!StringUtils.isNullOrEmpty(asyncClusterMaxCommit)) { - configs.add(String.format("%s=%s", HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMIT_PROP.key(), asyncClusterMaxCommit)); + configs.add(String.format("%s=%s", HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key(), asyncClusterMaxCommit)); } return configs; } @@ -1250,7 +1250,7 @@ public class TestHoodieDeltaStreamer extends TestHoodieDeltaStreamerBase { props.load(inputStream); } - assertEquals(new HoodieConfig(props).getString(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP), DummyAvroPayload.class.getName()); + assertEquals(new HoodieConfig(props).getString(HoodieTableConfig.PAYLOAD_CLASS_NAME), DummyAvroPayload.class.getName()); } @Test @@ -1276,7 +1276,7 @@ public class TestHoodieDeltaStreamer extends TestHoodieDeltaStreamerBase { props.load(inputStream); } - assertFalse(props.containsKey(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP.key())); + assertFalse(props.containsKey(HoodieTableConfig.PAYLOAD_CLASS_NAME.key())); } @Test diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java index 0e5cece6a..3f2e86e24 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java @@ -111,7 +111,7 @@ public class TestHoodieMultiTableDeltaStreamer extends TestHoodieDeltaStreamerBa assertEquals("uber_db.dummy_table_uber", executionContext.getConfig().targetTableName); assertEquals("topic1", executionContext.getProperties().getString(HoodieMultiTableDeltaStreamer.Constants.KAFKA_TOPIC_PROP)); assertEquals("_row_key", executionContext.getProperties().getString(DataSourceWriteOptions.RECORDKEY_FIELD().key())); - assertEquals(TestHoodieDeltaStreamer.TestGenerator.class.getName(), executionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS().key())); + assertEquals(TestHoodieDeltaStreamer.TestGenerator.class.getName(), executionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key())); assertEquals("uber_hive_dummy_table", executionContext.getProperties().getString(HoodieMultiTableDeltaStreamer.Constants.HIVE_SYNC_TABLE_PROP)); } @@ -224,11 +224,11 @@ public class TestHoodieMultiTableDeltaStreamer extends TestHoodieDeltaStreamerBa tableExecutionContexts.forEach(tableExecutionContext -> { switch (tableExecutionContext.getTableName()) { case "dummy_table_short_trip": - String tableLevelKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS().key()); + String tableLevelKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()); assertEquals(TestHoodieDeltaStreamer.TestTableLevelGenerator.class.getName(), tableLevelKeyGeneratorClass); break; default: - String defaultKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS().key()); + String defaultKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()); assertEquals(TestHoodieDeltaStreamer.TestGenerator.class.getName(), defaultKeyGeneratorClass); } }); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java index 9aa03e2e4..f68f8c6db 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java @@ -70,7 +70,7 @@ public class TestHoodieSnapshotCopier extends FunctionalTestHarness { HoodieSnapshotCopier copier = new HoodieSnapshotCopier(); copier.snapshot(jsc(), basePath, outputPath, true, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue()); + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue()); // Nothing changed; we just bail out assertEquals(fs.listStatus(new Path(basePath)).length, 1); @@ -124,7 +124,7 @@ public class TestHoodieSnapshotCopier extends FunctionalTestHarness { // Do a snapshot copy HoodieSnapshotCopier copier = new HoodieSnapshotCopier(); copier.snapshot(jsc(), basePath, outputPath, false, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, - HoodieMetadataConfig.METADATA_VALIDATE_PROP.defaultValue()); + HoodieMetadataConfig.VALIDATE_ENABLE.defaultValue()); // Check results assertTrue(fs.exists(new Path(outputPath + "/2016/05/01/" + file11.getName())));