From 18e6b79947b4aa0cbaeebd1918fcb14864d3f171 Mon Sep 17 00:00:00 2001 From: vinoth chandar Date: Sat, 14 Aug 2021 14:49:22 -0700 Subject: [PATCH] [MINOR] Adding back all old default val members to DataSourceOptions (#3474) - Added @Deprecated - Added @deprecated javadoc to keys and defaults suggested how to migrate - Moved all deprecated members to bottom to improve readability --- .../org/apache/hudi/DataSourceOptions.scala | 492 ++++++++++++------ 1 file changed, 327 insertions(+), 165 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala index 36c049393..168e79584 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -40,8 +40,6 @@ import org.apache.spark.sql.execution.datasources.{DataSourceUtils => SparkDataS */ object DataSourceReadOptions { - private val log = LogManager.getLogger(DataSourceReadOptions.getClass) - val QUERY_TYPE_SNAPSHOT_OPT_VAL = "snapshot" val QUERY_TYPE_READ_OPTIMIZED_OPT_VAL = "read_optimized" val QUERY_TYPE_INCREMENTAL_OPT_VAL = "incremental" @@ -52,8 +50,6 @@ object DataSourceReadOptions { .withDocumentation("Whether data needs to be read, in incremental mode (new data since an instantTime) " + "(or) Read Optimized mode (obtain latest view, based on base files) (or) Snapshot mode " + "(obtain latest view, by merging base and (if any) log files)") - @Deprecated - val QUERY_TYPE_OPT_KEY = QUERY_TYPE.key() val REALTIME_SKIP_MERGE_OPT_VAL = "skip_merge" val REALTIME_PAYLOAD_COMBINE_OPT_VAL = "payload_combine" @@ -63,15 +59,11 @@ object DataSourceReadOptions { .withDocumentation("For Snapshot query on merge on read table, control whether we invoke the record " + s"payload implementation to merge (${REALTIME_PAYLOAD_COMBINE_OPT_VAL}) or skip merging altogether" + s"${REALTIME_SKIP_MERGE_OPT_VAL}") - @Deprecated - val REALTIME_MERGE_OPT_KEY = REALTIME_MERGE.key() val READ_PATHS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.read.paths") .noDefaultValue() .withDocumentation("Comma separated list of file paths to read within a Hudi table.") - @Deprecated - val READ_PATHS_OPT_KEY = READ_PATHS.key() val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD @@ -81,6 +73,59 @@ object DataSourceReadOptions { .withDocumentation("Enables use of the spark file index implementation for Hudi, " + "that speeds up listing of large tables.") + val BEGIN_INSTANTTIME: ConfigProperty[String] = ConfigProperty + .key("hoodie.datasource.read.begin.instanttime") + .noDefaultValue() + .withDocumentation("Instant time to start incrementally pulling data from. The instanttime here need not necessarily " + + "correspond to an instant on the timeline. New data written with an instant_time > BEGIN_INSTANTTIME are fetched out. " + + "For e.g: ‘20170901080000’ will get all new data written after Sep 1, 2017 08:00AM.") + + val END_INSTANTTIME: ConfigProperty[String] = ConfigProperty + .key("hoodie.datasource.read.end.instanttime") + .noDefaultValue() + .withDocumentation("Instant time to limit incrementally fetched data to. " + + "New data written with an instant_time <= END_INSTANTTIME are fetched out.") + + val INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME: ConfigProperty[String] = ConfigProperty + .key("hoodie.datasource.read.schema.use.end.instanttime") + .defaultValue("false") + .withDocumentation("Uses end instant schema when incrementally fetched data to. Default: users latest instant schema.") + + val PUSH_DOWN_INCR_FILTERS: ConfigProperty[String] = ConfigProperty + .key("hoodie.datasource.read.incr.filters") + .defaultValue("") + .withDocumentation("For use-cases like DeltaStreamer which reads from Hoodie Incremental table and applies " + + "opaque map functions, filters appearing late in the sequence of transformations cannot be automatically " + + "pushed down. This option allows setting filters directly on Hoodie Source.") + + val INCR_PATH_GLOB: ConfigProperty[String] = ConfigProperty + .key("hoodie.datasource.read.incr.path.glob") + .defaultValue("") + .withDocumentation("For the use-cases like users only want to incremental pull from certain partitions " + + "instead of the full table. This option allows using glob pattern to directly filter on path.") + + val TIME_TRAVEL_AS_OF_INSTANT: ConfigProperty[String] = ConfigProperty + .key("as.of.instant") + .noDefaultValue() + .withDocumentation("The query instant for time travel. Without specified this option," + + " we query the latest snapshot.") + + /** @deprecated Use {@link QUERY_TYPE} and its methods instead */ + @Deprecated + val QUERY_TYPE_OPT_KEY = QUERY_TYPE.key() + /** @deprecated Use {@link QUERY_TYPE} and its methods instead */ + @Deprecated + val DEFAULT_QUERY_TYPE_OPT_VAL: String = QUERY_TYPE_SNAPSHOT_OPT_VAL + /** @deprecated Use {@link REALTIME_MERGE} and its methods instead */ + @Deprecated + val REALTIME_MERGE_OPT_KEY = REALTIME_MERGE.key() + /** @deprecated Use {@link REALTIME_MERGE} and its methods instead */ + @Deprecated + val DEFAULT_REALTIME_MERGE_OPT_VAL = REALTIME_PAYLOAD_COMBINE_OPT_VAL + /** @deprecated Use {@link READ_PATHS} and its methods instead */ + @Deprecated + val READ_PATHS_OPT_KEY = READ_PATHS.key() + /** @deprecated Use {@link QUERY_TYPE} and its methods instead */ @Deprecated val VIEW_TYPE_OPT_KEY = "hoodie.datasource.view.type" @Deprecated @@ -91,61 +136,36 @@ object DataSourceReadOptions { val VIEW_TYPE_REALTIME_OPT_VAL = "realtime" @Deprecated val DEFAULT_VIEW_TYPE_OPT_VAL = VIEW_TYPE_READ_OPTIMIZED_OPT_VAL - - val BEGIN_INSTANTTIME: ConfigProperty[String] = ConfigProperty - .key("hoodie.datasource.read.begin.instanttime") - .noDefaultValue() - .withDocumentation("Instant time to start incrementally pulling data from. The instanttime here need not necessarily " + - "correspond to an instant on the timeline. New data written with an instant_time > BEGIN_INSTANTTIME are fetched out. " + - "For e.g: ‘20170901080000’ will get all new data written after Sep 1, 2017 08:00AM.") + /** @deprecated Use {@link BEGIN_INSTANTTIME} and its methods instead */ @Deprecated val BEGIN_INSTANTTIME_OPT_KEY = BEGIN_INSTANTTIME.key() - - val END_INSTANTTIME: ConfigProperty[String] = ConfigProperty - .key("hoodie.datasource.read.end.instanttime") - .noDefaultValue() - .withDocumentation("Instant time to limit incrementally fetched data to. " + - "New data written with an instant_time <= END_INSTANTTIME are fetched out.") + /** @deprecated Use {@link END_INSTANTTIME} and its methods instead */ @Deprecated val END_INSTANTTIME_OPT_KEY = END_INSTANTTIME.key() - - val INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME: ConfigProperty[String] = ConfigProperty - .key("hoodie.datasource.read.schema.use.end.instanttime") - .defaultValue("false") - .withDocumentation("Uses end instant schema when incrementally fetched data to. Default: users latest instant schema.") + /** @deprecated Use {@link INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME} and its methods instead */ @Deprecated val INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME_OPT_KEY = INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME.key() - - val PUSH_DOWN_INCR_FILTERS: ConfigProperty[String] = ConfigProperty - .key("hoodie.datasource.read.incr.filters") - .defaultValue("") - .withDocumentation("For use-cases like DeltaStreamer which reads from Hoodie Incremental table and applies " - + "opaque map functions, filters appearing late in the sequence of transformations cannot be automatically " - + "pushed down. This option allows setting filters directly on Hoodie Source.") + /** @deprecated Use {@link INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME} and its methods instead */ + @Deprecated + val DEFAULT_INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME_OPT_VAL = INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME.defaultValue() + /** @deprecated Use {@link PUSH_DOWN_INCR_FILTERS} and its methods instead */ @Deprecated val PUSH_DOWN_INCR_FILTERS_OPT_KEY = PUSH_DOWN_INCR_FILTERS.key() - - val INCR_PATH_GLOB: ConfigProperty[String] = ConfigProperty - .key("hoodie.datasource.read.incr.path.glob") - .defaultValue("") - .withDocumentation("For the use-cases like users only want to incremental pull from certain partitions " - + "instead of the full table. This option allows using glob pattern to directly filter on path.") + /** @deprecated Use {@link PUSH_DOWN_INCR_FILTERS} and its methods instead */ + @Deprecated + val DEFAULT_PUSH_DOWN_FILTERS_OPT_VAL = PUSH_DOWN_INCR_FILTERS.defaultValue() + /** @deprecated Use {@link INCR_PATH_GLOB} and its methods instead */ @Deprecated val INCR_PATH_GLOB_OPT_KEY = INCR_PATH_GLOB.key() - - val TIME_TRAVEL_AS_OF_INSTANT: ConfigProperty[String] = ConfigProperty - .key("as.of.instant") - .noDefaultValue() - .withDocumentation("The query instant for time travel. Without specified this option," + - " we query the latest snapshot.") - + /** @deprecated Use {@link INCR_PATH_GLOB} and its methods instead */ + @Deprecated + val DEFAULT_INCR_PATH_GLOB_OPT_VAL = INCR_PATH_GLOB.defaultValue() } /** * Options supported for writing hoodie tables. */ object DataSourceWriteOptions { - private val log = LogManager.getLogger(DataSourceWriteOptions.getClass) val BULK_INSERT_OPERATION_OPT_VAL = WriteOperationType.BULK_INSERT.value val INSERT_OPERATION_OPT_VAL = WriteOperationType.INSERT.value @@ -160,8 +180,7 @@ object DataSourceWriteOptions { .withDocumentation("Whether to do upsert, insert or bulkinsert for the write operation. " + "Use bulkinsert to load new data into a table, and there on use upsert/insert. " + "bulk insert uses a disk based write path to scale to load large inputs without need to cache it.") - @Deprecated - val OPERATION_OPT_KEY = OPERATION.key() + val COW_TABLE_TYPE_OPT_VAL = HoodieTableType.COPY_ON_WRITE.name val MOR_TABLE_TYPE_OPT_VAL = HoodieTableType.MERGE_ON_READ.name @@ -170,17 +189,6 @@ object DataSourceWriteOptions { .defaultValue(COW_TABLE_TYPE_OPT_VAL) .withAlternatives("hoodie.datasource.write.storage.type") .withDocumentation("The table type for the underlying data, for this write. This can’t change between writes.") - @Deprecated - val TABLE_TYPE_OPT_KEY = TABLE_TYPE.key() - - @Deprecated - val STORAGE_TYPE_OPT = "hoodie.datasource.write.storage.type" - @Deprecated - val COW_STORAGE_TYPE_OPT_VAL = HoodieTableType.COPY_ON_WRITE.name - @Deprecated - val MOR_STORAGE_TYPE_OPT_VAL = HoodieTableType.MERGE_ON_READ.name - @Deprecated - val DEFAULT_STORAGE_TYPE_OPT_VAL = COW_STORAGE_TYPE_OPT_VAL /** * Translate spark parameters to hudi parameters @@ -223,81 +231,55 @@ object DataSourceWriteOptions { .key("hoodie.datasource.write.table.name") .noDefaultValue() .withDocumentation("Table name for the datasource write. Also used to register the table into meta stores.") - @Deprecated - val TABLE_NAME_OPT_KEY = TABLE_NAME.key() /** - * Field used in preCombining before actual write. When two records have the same - * key value, we will pick the one with the largest value for the precombine field, - * determined by Object.compareTo(..) - */ - val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD - @Deprecated - val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD.key() - - /** - * Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. - * This will render any value set for `PRECOMBINE_FIELD_OPT_VAL` in-effective - */ - val PAYLOAD_CLASS = HoodieWriteConfig.WRITE_PAYLOAD_CLASS - @Deprecated - val PAYLOAD_CLASS_OPT_KEY = HoodieWriteConfig.WRITE_PAYLOAD_CLASS.key() - - /** - * Record key field. Value to be used as the `recordKey` component of `HoodieKey`. Actual value - * will be obtained by invoking .toString() on the field value. Nested fields can be specified using - * the dot notation eg: `a.b.c` - * - */ - val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD - @Deprecated - val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD.key() - - /** - * Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual - * value obtained by invoking .toString() - */ - val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD - @Deprecated - val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD.key() - - /** - * Flag to indicate whether to use Hive style partitioning. - * If set true, the names of partition folders follow = format. - * By default false (the names of partition folders are only partition values) - */ - val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING - val URL_ENCODE_PARTITIONING = KeyGeneratorOptions.URL_ENCODE_PARTITIONING - @Deprecated - val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key() - @Deprecated - val URL_ENCODE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key() - - /** - * Key generator class, that implements will extract the key out of incoming record - * - */ - val KEYGENERATOR_CLASS = HoodieWriteConfig.KEYGENERATOR_CLASS - val DEFAULT_KEYGENERATOR_CLASS_OPT_VAL = classOf[SimpleKeyGenerator].getName - @Deprecated - val KEYGENERATOR_CLASS_OPT_KEY = HoodieWriteConfig.KEYGENERATOR_CLASS.key() - - /** - * - * By default, false (will be enabled as default in a future release) + * Field used in preCombining before actual write. When two records have the same + * key value, we will pick the one with the largest value for the precombine field, + * determined by Object.compareTo(..) */ + val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD + + /** + * Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. + * This will render any value set for `PRECOMBINE_FIELD_OPT_VAL` in-effective + */ + val PAYLOAD_CLASS = HoodieWriteConfig.WRITE_PAYLOAD_CLASS + + /** + * Record key field. Value to be used as the `recordKey` component of `HoodieKey`. Actual value + * will be obtained by invoking .toString() on the field value. Nested fields can be specified using + * the dot notation eg: `a.b.c` + * + */ + val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD + + /** + * Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual + * value obtained by invoking .toString() + */ + val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD + + /** + * Flag to indicate whether to use Hive style partitioning. + * If set true, the names of partition folders follow = format. + * By default false (the names of partition folders are only partition values) + */ + val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING + + val KEYGENERATOR_CLASS = ConfigProperty.key("hoodie.datasource.write.keygenerator.class") + .defaultValue(classOf[SimpleKeyGenerator].getName) + .withDocumentation("Key generator class, that implements `org.apache.hudi.keygen.KeyGenerator`") + val ENABLE_ROW_WRITER: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.write.row.writer.enable") .defaultValue("true") .withDocumentation("When set to true, will perform write operations directly using the spark native " + "`Row` representation, avoiding any additional conversion costs.") - @Deprecated - val ENABLE_ROW_WRITER_OPT_KEY = ENABLE_ROW_WRITER.key() /** * Enable the bulk insert for sql insert statement. */ - val SQL_ENABLE_BULK_INSERT:ConfigProperty[String] = ConfigProperty + val SQL_ENABLE_BULK_INSERT: ConfigProperty[String] = ConfigProperty .key("hoodie.sql.bulk.insert.enable") .defaultValue("false") .withDocumentation("When set to true, the sql insert statement will use bulk insert.") @@ -315,15 +297,11 @@ object DataSourceWriteOptions { .defaultValue("_") .withDocumentation("Option keys beginning with this prefix, are automatically added to the commit/deltacommit metadata. " + "This is useful to store checkpointing information, in a consistent way with the hudi timeline") - @Deprecated - val COMMIT_METADATA_KEYPREFIX_OPT_KEY = COMMIT_METADATA_KEYPREFIX.key() val INSERT_DROP_DUPS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.write.insert.drop.duplicates") .defaultValue("false") .withDocumentation("If set to true, filters out all duplicate records from incoming dataframe, during insert operations.") - @Deprecated - val INSERT_DROP_DUPS_OPT_KEY = INSERT_DROP_DUPS.key() val PARTITIONS_TO_DELETE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.write.partitions.to.delete") @@ -334,18 +312,13 @@ object DataSourceWriteOptions { .key("hoodie.datasource.write.streaming.retry.count") .defaultValue("3") .withDocumentation("Config to indicate how many times streaming job should retry for a failed micro batch.") - @Deprecated - val STREAMING_RETRY_CNT_OPT_KEY = STREAMING_RETRY_CNT.key() val STREAMING_RETRY_INTERVAL_MS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.write.streaming.retry.interval.ms") .defaultValue("2000") .withDocumentation(" Config to indicate how long (by millisecond) before a retry should issued for failed microbatch") - @Deprecated - val STREAMING_RETRY_INTERVAL_MS_OPT_KEY = STREAMING_RETRY_INTERVAL_MS.key() /** - * * By default true (in favor of streaming progressing over data integrity) */ val STREAMING_IGNORE_FAILED_BATCH: ConfigProperty[String] = ConfigProperty @@ -353,8 +326,6 @@ object DataSourceWriteOptions { .defaultValue("true") .withDocumentation("Config to indicate whether to ignore any non exception error (e.g. writestatus error)" + " within a streaming microbatch") - @Deprecated - val STREAMING_IGNORE_FAILED_BATCH_OPT_KEY = STREAMING_IGNORE_FAILED_BATCH.key() val META_SYNC_CLIENT_TOOL_CLASS: ConfigProperty[String] = ConfigProperty .key("hoodie.meta.sync.client.tool.class") @@ -375,109 +346,79 @@ object DataSourceWriteOptions { .key("hoodie.datasource.hive_sync.enable") .defaultValue("false") .withDocumentation("When set to true, register/sync the table to Apache Hive metastore") - @Deprecated - val HIVE_SYNC_ENABLED_OPT_KEY = HIVE_SYNC_ENABLED.key() val META_SYNC_ENABLED: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.meta.sync.enable") .defaultValue("false") .withDocumentation("") - @Deprecated - val META_SYNC_ENABLED_OPT_KEY = META_SYNC_ENABLED.key() val HIVE_DATABASE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.database") .defaultValue("default") .withDocumentation("database to sync to") - @Deprecated - val HIVE_DATABASE_OPT_KEY = HIVE_DATABASE.key() val HIVE_TABLE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.table") .defaultValue("unknown") .withDocumentation("table to sync to") - @Deprecated - val HIVE_TABLE_OPT_KEY = HIVE_TABLE.key() val HIVE_BASE_FILE_FORMAT: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.base_file_format") .defaultValue("PARQUET") .withDocumentation("Base file format for the sync.") - @Deprecated - val HIVE_BASE_FILE_FORMAT_OPT_KEY = HIVE_BASE_FILE_FORMAT.key() val HIVE_USER: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.username") .defaultValue("hive") .withDocumentation("hive user name to use") - @Deprecated - val HIVE_USER_OPT_KEY = HIVE_USER.key() val HIVE_PASS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.password") .defaultValue("hive") .withDocumentation("hive password to use") - @Deprecated - val HIVE_PASS_OPT_KEY = HIVE_PASS.key() val HIVE_URL: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.jdbcurl") .defaultValue("jdbc:hive2://localhost:10000") .withDocumentation("Hive metastore url") - @Deprecated - val HIVE_URL_OPT_KEY = HIVE_URL.key() val HIVE_PARTITION_FIELDS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.partition_fields") .defaultValue("") .withDocumentation("field in the table to use for determining hive partition columns.") - @Deprecated - val HIVE_PARTITION_FIELDS_OPT_KEY = HIVE_PARTITION_FIELDS.key() val HIVE_PARTITION_EXTRACTOR_CLASS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.partition_extractor_class") .defaultValue(classOf[SlashEncodedDayPartitionValueExtractor].getCanonicalName) .withDocumentation("") - @Deprecated - val HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY = HIVE_PARTITION_EXTRACTOR_CLASS.key() val HIVE_ASSUME_DATE_PARTITION: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.assume_date_partitioning") .defaultValue("false") .withDocumentation("Assume partitioning is yyyy/mm/dd") - @Deprecated - val HIVE_ASSUME_DATE_PARTITION_OPT_KEY = HIVE_ASSUME_DATE_PARTITION.key() val HIVE_USE_PRE_APACHE_INPUT_FORMAT: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.use_pre_apache_input_format") .defaultValue("false") .withDocumentation("") - @Deprecated - val HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY = HIVE_USE_PRE_APACHE_INPUT_FORMAT.key() - // We should use HIVE_SYNC_MODE instead of this config from 0.9.0 + /* @deprecated We should use {@link HIVE_SYNC_MODE} instead of this config from 0.9.0 */ @Deprecated val HIVE_USE_JDBC: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.use_jdbc") .defaultValue("true") .deprecatedAfter("0.9.0") .withDocumentation("Use JDBC when hive synchronization is enabled") - @Deprecated - val HIVE_USE_JDBC_OPT_KEY = HIVE_USE_JDBC.key() val HIVE_AUTO_CREATE_DATABASE: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.auto_create_database") .defaultValue("true") .withDocumentation("Auto create hive database if does not exists") - @Deprecated - val HIVE_AUTO_CREATE_DATABASE_OPT_KEY = HIVE_AUTO_CREATE_DATABASE.key() val HIVE_IGNORE_EXCEPTIONS: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.ignore_exceptions") .defaultValue("false") .withDocumentation("") - @Deprecated - val HIVE_IGNORE_EXCEPTIONS_OPT_KEY = HIVE_IGNORE_EXCEPTIONS.key() val HIVE_SKIP_RO_SUFFIX: ConfigProperty[String] = ConfigProperty .key("hoodie.datasource.hive_sync.skip_ro_suffix") @@ -550,6 +491,227 @@ object DataSourceWriteOptions { .defaultValue("false") .withDocumentation("When set to true, will not write the partition columns into hudi. " + "By default, false.") + + /** @deprecated Use {@link HIVE_ASSUME_DATE_PARTITION} and its methods instead */ + @Deprecated + val HIVE_ASSUME_DATE_PARTITION_OPT_KEY = HIVE_ASSUME_DATE_PARTITION.key() + /** @deprecated Use {@link HIVE_USE_PRE_APACHE_INPUT_FORMAT} and its methods instead */ + @Deprecated + val HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY = HIVE_USE_PRE_APACHE_INPUT_FORMAT.key() + /** @deprecated Use {@link HIVE_USE_JDBC} and its methods instead */ + @Deprecated + val HIVE_USE_JDBC_OPT_KEY = HIVE_USE_JDBC.key() + /** @deprecated Use {@link HIVE_AUTO_CREATE_DATABASE} and its methods instead */ + @Deprecated + val HIVE_AUTO_CREATE_DATABASE_OPT_KEY = HIVE_AUTO_CREATE_DATABASE.key() + /** @deprecated Use {@link HIVE_IGNORE_EXCEPTIONS} and its methods instead */ + @Deprecated + val HIVE_IGNORE_EXCEPTIONS_OPT_KEY = HIVE_IGNORE_EXCEPTIONS.key() + /** @deprecated Use {@link STREAMING_IGNORE_FAILED_BATCH} and its methods instead */ + @Deprecated + val STREAMING_IGNORE_FAILED_BATCH_OPT_KEY = STREAMING_IGNORE_FAILED_BATCH.key() + /** @deprecated Use {@link STREAMING_IGNORE_FAILED_BATCH} and its methods instead */ + @Deprecated + val DEFAULT_STREAMING_IGNORE_FAILED_BATCH_OPT_VAL = STREAMING_IGNORE_FAILED_BATCH.defaultValue() + /** @deprecated Use {@link META_SYNC_CLIENT_TOOL_CLASS} and its methods instead */ + @Deprecated + val DEFAULT_META_SYNC_CLIENT_TOOL_CLASS = META_SYNC_CLIENT_TOOL_CLASS.defaultValue() + /** @deprecated Use {@link HIVE_SYNC_ENABLED} and its methods instead */ + @Deprecated + val HIVE_SYNC_ENABLED_OPT_KEY = HIVE_SYNC_ENABLED.key() + /** @deprecated Use {@link META_SYNC_ENABLED} and its methods instead */ + @Deprecated + val META_SYNC_ENABLED_OPT_KEY = META_SYNC_ENABLED.key() + /** @deprecated Use {@link HIVE_DATABASE} and its methods instead */ + @Deprecated + val HIVE_DATABASE_OPT_KEY = HIVE_DATABASE.key() + /** @deprecated Use {@link HIVE_TABLE} and its methods instead */ + @Deprecated + val HIVE_TABLE_OPT_KEY = HIVE_TABLE.key() + /** @deprecated Use {@link HIVE_BASE_FILE_FORMAT} and its methods instead */ + @Deprecated + val HIVE_BASE_FILE_FORMAT_OPT_KEY = HIVE_BASE_FILE_FORMAT.key() + /** @deprecated Use {@link HIVE_USER} and its methods instead */ + @Deprecated + val HIVE_USER_OPT_KEY = HIVE_USER.key() + /** @deprecated Use {@link HIVE_PASS} and its methods instead */ + @Deprecated + val HIVE_PASS_OPT_KEY = HIVE_PASS.key() + /** @deprecated Use {@link HIVE_URL} and its methods instead */ + @Deprecated + val HIVE_URL_OPT_KEY = HIVE_URL.key() + /** @deprecated Use {@link HIVE_PARTITION_FIELDS} and its methods instead */ + @Deprecated + val HIVE_PARTITION_FIELDS_OPT_KEY = HIVE_PARTITION_FIELDS.key() + /** @deprecated Use {@link HIVE_PARTITION_EXTRACTOR_CLASS} and its methods instead */ + @Deprecated + val HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY = HIVE_PARTITION_EXTRACTOR_CLASS.key() + + /** @deprecated Use {@link KEYGENERATOR_CLASS} and its methods instead */ + @Deprecated + val DEFAULT_KEYGENERATOR_CLASS_OPT_VAL = KEYGENERATOR_CLASS.defaultValue() + /** @deprecated Use {@link KEYGENERATOR_CLASS} and its methods instead */ + @Deprecated + val KEYGENERATOR_CLASS_OPT_KEY = HoodieWriteConfig.KEYGENERATOR_CLASS.key() + /** @deprecated Use {@link ENABLE_ROW_WRITER} and its methods instead */ + @Deprecated + val ENABLE_ROW_WRITER_OPT_KEY = ENABLE_ROW_WRITER.key() + /** @deprecated Use {@link ENABLE_ROW_WRITER} and its methods instead */ + @Deprecated + val DEFAULT_ENABLE_ROW_WRITER_OPT_VAL = ENABLE_ROW_WRITER.defaultValue() + /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ + @Deprecated + val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key() + /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue() + + val URL_ENCODE_PARTITIONING = KeyGeneratorOptions.URL_ENCODE_PARTITIONING + /** @deprecated Use {@link URL_ENCODE_PARTITIONING} and its methods instead */ + @Deprecated + val URL_ENCODE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key() + /** @deprecated Use {@link URL_ENCODE_PARTITIONING} and its methods instead */ + @Deprecated + val DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue() + /** @deprecated Use {@link COMMIT_METADATA_KEYPREFIX} and its methods instead */ + @Deprecated + val COMMIT_METADATA_KEYPREFIX_OPT_KEY = COMMIT_METADATA_KEYPREFIX.key() + /** @deprecated Use {@link COMMIT_METADATA_KEYPREFIX} and its methods instead */ + @Deprecated + val DEFAULT_COMMIT_METADATA_KEYPREFIX_OPT_VAL = COMMIT_METADATA_KEYPREFIX.defaultValue() + /** @deprecated Use {@link INSERT_DROP_DUPS} and its methods instead */ + @Deprecated + val INSERT_DROP_DUPS_OPT_KEY = INSERT_DROP_DUPS.key() + /** @deprecated Use {@link INSERT_DROP_DUPS} and its methods instead */ + @Deprecated + val DEFAULT_INSERT_DROP_DUPS_OPT_VAL = INSERT_DROP_DUPS.defaultValue() + /** @deprecated Use {@link STREAMING_RETRY_CNT} and its methods instead */ + @Deprecated + val STREAMING_RETRY_CNT_OPT_KEY = STREAMING_RETRY_CNT.key() + /** @deprecated Use {@link STREAMING_RETRY_CNT} and its methods instead */ + @Deprecated + val DEFAULT_STREAMING_RETRY_CNT_OPT_VAL = STREAMING_RETRY_CNT.defaultValue() + /** @deprecated Use {@link STREAMING_RETRY_INTERVAL_MS} and its methods instead */ + @Deprecated + val STREAMING_RETRY_INTERVAL_MS_OPT_KEY = STREAMING_RETRY_INTERVAL_MS.key() + /** @deprecated Use {@link STREAMING_RETRY_INTERVAL_MS} and its methods instead */ + @Deprecated + val DEFAULT_STREAMING_RETRY_INTERVAL_MS_OPT_VAL = STREAMING_RETRY_INTERVAL_MS.defaultValue() + + /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ + @Deprecated + val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD.key() + /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ + @Deprecated + val DEFAULT_RECORDKEY_FIELD_OPT_VAL = RECORDKEY_FIELD.defaultValue() + /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ + @Deprecated + val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD.key() + /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ + @Deprecated + val DEFAULT_PARTITIONPATH_FIELD_OPT_VAL = PARTITIONPATH_FIELD.defaultValue() + + /** @deprecated Use {@link TABLE_NAME} and its methods instead */ + @Deprecated + val TABLE_NAME_OPT_KEY = TABLE_NAME.key() + /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ + @Deprecated + val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD.key() + /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ + @Deprecated + val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue() + + /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS} and its methods instead */ + @Deprecated + val PAYLOAD_CLASS_OPT_KEY = HoodieWriteConfig.WRITE_PAYLOAD_CLASS.key() + /** @deprecated Use {@link HoodieWriteConfig.WRITE_PAYLOAD_CLASS} and its methods instead */ + @Deprecated + val DEFAULT_PAYLOAD_OPT_VAL = PAYLOAD_CLASS.defaultValue() + + /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ + @Deprecated + val TABLE_TYPE_OPT_KEY = TABLE_TYPE.key() + /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ + @Deprecated + val DEFAULT_TABLE_TYPE_OPT_VAL = TABLE_TYPE.defaultValue() + + /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ + @Deprecated + val STORAGE_TYPE_OPT = "hoodie.datasource.write.storage.type" + @Deprecated + val COW_STORAGE_TYPE_OPT_VAL = HoodieTableType.COPY_ON_WRITE.name + @Deprecated + val MOR_STORAGE_TYPE_OPT_VAL = HoodieTableType.MERGE_ON_READ.name + /** @deprecated Use {@link TABLE_TYPE} and its methods instead */ + @Deprecated + val DEFAULT_STORAGE_TYPE_OPT_VAL = COW_STORAGE_TYPE_OPT_VAL + + /** @deprecated Use {@link OPERATION} and its methods instead */ + @Deprecated + val OPERATION_OPT_KEY = OPERATION.key() + /** @deprecated Use {@link OPERATION} and its methods instead */ + @Deprecated + val DEFAULT_OPERATION_OPT_VAL = OPERATION.defaultValue() + + /** @deprecated Use {@link HIVE_SYNC_ENABLED} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_SYNC_ENABLED_OPT_VAL = HIVE_SYNC_ENABLED.defaultValue() + /** @deprecated Use {@link META_SYNC_ENABLED} and its methods instead */ + @Deprecated + val DEFAULT_META_SYNC_ENABLED_OPT_VAL = META_SYNC_ENABLED.defaultValue() + /** @deprecated Use {@link HIVE_DATABASE} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_DATABASE_OPT_VAL = HIVE_DATABASE.defaultValue() + /** @deprecated Use {@link HIVE_TABLE} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_TABLE_OPT_VAL = HIVE_TABLE.defaultValue() + /** @deprecated Use {@link HIVE_BASE_FILE_FORMAT} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_BASE_FILE_FORMAT_OPT_VAL = HIVE_BASE_FILE_FORMAT.defaultValue() + /** @deprecated Use {@link HIVE_USER} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_USER_OPT_VAL = HIVE_USER.defaultValue() + /** @deprecated Use {@link HIVE_PASS} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_PASS_OPT_VAL = HIVE_PASS.defaultValue() + /** @deprecated Use {@link HIVE_URL} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_URL_OPT_VAL = HIVE_URL.defaultValue() + /** @deprecated Use {@link HIVE_PARTITION_FIELDS} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_PARTITION_FIELDS_OPT_VAL = HIVE_PARTITION_FIELDS.defaultValue() + /** @deprecated Use {@link HIVE_PARTITION_EXTRACTOR_CLASS} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL = HIVE_PARTITION_EXTRACTOR_CLASS.defaultValue() + /** @deprecated Use {@link HIVE_ASSUME_DATE_PARTITION} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_ASSUME_DATE_PARTITION_OPT_VAL = HIVE_ASSUME_DATE_PARTITION.defaultValue() + @Deprecated + val DEFAULT_USE_PRE_APACHE_INPUT_FORMAT_OPT_VAL = "false" + /** @deprecated Use {@link HIVE_USE_JDBC} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_USE_JDBC_OPT_VAL = HIVE_USE_JDBC.defaultValue() + /** @deprecated Use {@link HIVE_AUTO_CREATE_DATABASE} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_AUTO_CREATE_DATABASE_OPT_KEY = HIVE_AUTO_CREATE_DATABASE.defaultValue() + /** @deprecated Use {@link HIVE_IGNORE_EXCEPTIONS} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_IGNORE_EXCEPTIONS_OPT_KEY = HIVE_IGNORE_EXCEPTIONS.defaultValue() + /** @deprecated Use {@link HIVE_SKIP_RO_SUFFIX} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_SKIP_RO_SUFFIX_VAL = HIVE_SKIP_RO_SUFFIX.defaultValue() + /** @deprecated Use {@link HIVE_SUPPORT_TIMESTAMP} and its methods instead */ + @Deprecated + val DEFAULT_HIVE_SUPPORT_TIMESTAMP = HIVE_SUPPORT_TIMESTAMP.defaultValue() + /** @deprecated Use {@link ASYNC_COMPACT_ENABLE} and its methods instead */ + @Deprecated + val ASYNC_COMPACT_ENABLE_OPT_KEY = ASYNC_COMPACT_ENABLE.key() + /** @deprecated Use {@link ASYNC_COMPACT_ENABLE} and its methods instead */ + @Deprecated + val DEFAULT_ASYNC_COMPACT_ENABLE_OPT_VAL = ASYNC_COMPACT_ENABLE.defaultValue() + /** @deprecated Use {@link KAFKA_AVRO_VALUE_DESERIALIZER_CLASS} and its methods instead */ + @Deprecated + val KAFKA_AVRO_VALUE_DESERIALIZER = KAFKA_AVRO_VALUE_DESERIALIZER_CLASS.key() } object DataSourceOptionsHelper {