diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java index b9a7ac2e7..d008a1a3e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java @@ -22,6 +22,8 @@ import org.apache.hudi.client.bootstrap.BootstrapMode; import org.apache.hudi.client.bootstrap.selector.MetadataOnlyBootstrapModeSelector; import org.apache.hudi.client.bootstrap.translator.IdentityBootstrapPartitionPathTranslator; import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.table.HoodieTableConfig; @@ -35,6 +37,11 @@ import java.util.Properties; /** * Bootstrap specific configs. */ +@ConfigClassProperty(name = "Bootstrap Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control how you want to bootstrap your existing tables for the first time into hudi. " + + "The bootstrap operation can flexibly avoid copying data over before you can use Hudi and support running the existing " + + " writers and new hudi writers in parallel, to validate the migration.") public class HoodieBootstrapConfig extends HoodieConfig { public static final ConfigProperty BOOTSTRAP_BASE_PATH_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java index 3dbd5d2a4..521e53278 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -29,6 +31,10 @@ import java.util.Properties; /** * Clustering specific configs. */ +@ConfigClassProperty(name = "Clustering Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control the clustering table service in hudi, " + + "which optimizes the storage layout for better query performance by sorting and sizing data files.") public class HoodieClusteringConfig extends HoodieConfig { // Any strategy specific params can be saved with this prefix diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index e8d55934b..5bc3a7caa 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.model.HoodieCleaningPolicy; @@ -41,6 +43,11 @@ import java.util.stream.Collectors; * Compaction related config. */ @Immutable +@ConfigClassProperty(name = "Compaction Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control compaction " + + "(merging of log files onto a new base files) as well as " + + "cleaning (reclamation of older/unused file groups/slices).") public class HoodieCompactionConfig extends HoodieConfig { public static final ConfigProperty AUTO_CLEAN_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java index 95219d049..4e30046d7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; @@ -27,6 +29,11 @@ import java.io.FileReader; import java.io.IOException; import java.util.Properties; +@ConfigClassProperty(name = "HBase Index Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control indexing behavior " + + "(when HBase based indexing is enabled), which tags incoming " + + "records as either inserts or updates to older records.") public class HoodieHBaseIndexConfig extends HoodieConfig { public static final ConfigProperty HBASE_ZKQUORUM_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index 1c5f2b629..91524f175 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -19,6 +19,8 @@ package org.apache.hudi.config; import org.apache.hudi.common.bloom.BloomFilterTypeCode; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.engine.EngineType; @@ -36,6 +38,10 @@ import java.util.Properties; * Indexing related config. */ @Immutable +@ConfigClassProperty(name = "Index Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control indexing behavior, " + + "which tags incoming records as either inserts or updates to older records.") public class HoodieIndexConfig extends HoodieConfig { public static final ConfigProperty INDEX_TYPE_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java index 846437ee5..162dc6113 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java @@ -20,6 +20,8 @@ package org.apache.hudi.config; import org.apache.hudi.client.transaction.ConflictResolutionStrategy; import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy; import org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.lock.LockProvider; @@ -55,6 +57,11 @@ import static org.apache.hudi.common.config.LockConfiguration.ZK_SESSION_TIMEOUT /** * Hoodie Configs for Locks. */ +@ConfigClassProperty(name = "Locks Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configs that control locking mechanisms required for concurrency control " + + " between writers to a Hudi table. Concurrency between Hudi's own table services " + + " are auto managed internally.") public class HoodieLockConfig extends HoodieConfig { public static final ConfigProperty LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java index 2303f235b..319b7f609 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -32,6 +34,10 @@ import java.util.Properties; * Memory related config. */ @Immutable +@ConfigClassProperty(name = "Memory Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Controls memory usage for compaction " + + "and merges, performed internally by Hudi.") public class HoodieMemoryConfig extends HoodieConfig { // Default max memory fraction during hash-merge, excess spills to disk diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java index 599ae30af..b5434b0cc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.metrics.MetricsReporterType; @@ -33,6 +35,10 @@ import java.util.Properties; * Fetch the configurations used by the Metrics system. */ @Immutable +@ConfigClassProperty(name = "Metrics Configurations", + groupName = ConfigGroups.Names.METRICS, + description = "Enables reporting on Hudi metrics. Hudi publishes metrics on " + + "every commit, clean, rollback etc. The following sections list the supported reporters.") public class HoodieMetricsConfig extends HoodieConfig { public static final String METRIC_PREFIX = "hoodie.metrics"; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java index c86232b80..df70ce39b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -33,6 +35,10 @@ import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX; * {@link org.apache.hudi.metrics.MetricsReporterType#DATADOG} */ @Immutable +@ConfigClassProperty(name = "Metrics Configurations for Datadog reporter", + groupName = ConfigGroups.Names.METRICS, + description = "Enables reporting on Hudi metrics using the Datadog reporter type. " + + "Hudi publishes metrics on every commit, clean, rollback etc.") public class HoodieMetricsDatadogConfig extends HoodieConfig { public static final String DATADOG_PREFIX = METRIC_PREFIX + ".datadog"; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java index 504073720..a8eac6218 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -25,6 +27,10 @@ import java.util.Properties; import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX; +@ConfigClassProperty(name = "Metrics Configurations for Prometheus", + groupName = ConfigGroups.Names.METRICS, + description = "Enables reporting on Hudi metrics using Prometheus. " + + " Hudi publishes metrics on every commit, clean, rollback etc.") public class HoodieMetricsPrometheusConfig extends HoodieConfig { // Prometheus PushGateWay diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java index b91f1b68b..b243fbf0d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -32,6 +34,10 @@ import static org.apache.hudi.common.model.HoodiePayloadProps.PAYLOAD_ORDERING_F /** * Hoodie payload related configs. */ +@ConfigClassProperty(name = "Payload Configurations", + groupName = ConfigGroups.Names.RECORD_PAYLOAD, + description = "Payload related configs, that can be leveraged to " + + "control merges based on specific business fields in the data.") public class HoodiePayloadConfig extends HoodieConfig { public static final ConfigProperty PAYLOAD_ORDERING_FIELD_PROP = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java index ec1e15599..401537154 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -32,6 +34,9 @@ import java.util.Properties; * Storage related config. */ @Immutable +@ConfigClassProperty(name = "Storage Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control aspects around writing, sizing, reading base and log files.") public class HoodieStorageConfig extends HoodieConfig { public static final ConfigProperty PARQUET_FILE_MAX_BYTES = ConfigProperty diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java index 7e7d34eb4..df453596e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java @@ -17,6 +17,8 @@ package org.apache.hudi.config; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -28,6 +30,10 @@ import java.util.Properties; /** * Write callback related config. */ +@ConfigClassProperty(name = "Write commit callback configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Controls callback behavior into HTTP endpoints, to push " + + " notifications on commits on hudi tables.") public class HoodieWriteCommitCallbackConfig extends HoodieConfig { public static final String CALLBACK_PREFIX = "hoodie.write.commit.callback."; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index d84a68153..5536a7ea7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -21,6 +21,8 @@ package org.apache.hudi.config; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.client.bootstrap.BootstrapMode; import org.apache.hudi.client.transaction.ConflictResolutionStrategy; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.config.HoodieMetadataConfig; @@ -68,6 +70,10 @@ import java.util.stream.Collectors; * Class storing configs for the HoodieWriteClient. */ @Immutable +@ConfigClassProperty(name = "Write Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control write behavior on Hudi tables. These can be directly passed down from even " + + "higher level frameworks (e.g Spark datasources, Flink sink) and utilities (e.g DeltaStreamer).") public class HoodieWriteConfig extends HoodieConfig { private static final long serialVersionUID = 0L; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java index 0ddede128..7c5b6f385 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java @@ -18,9 +18,17 @@ package org.apache.hudi.keygen.constant; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; +@ConfigClassProperty(name = "Key Generator Options", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Hudi maintains keys (record key + partition path) " + + "for uniquely identifying a particular record. " + + "This config allows developers to setup the Key generator class that " + + "will extract these out of incoming records.") public class KeyGeneratorOptions extends HoodieConfig { public static final ConfigProperty URL_ENCODE_PARTITIONING_OPT_KEY = ConfigProperty diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java new file mode 100644 index 000000000..f5d72a6be --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.config; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) + +/** + * Annotation for superclasses of {@link HoodieConfig} that includes the + * human-readable name of the config class, the config group ({@link ConfigGroupName}) + * it belongs to (e.g., spark/ flink/ write) + * and the description of the config class. + */ +public @interface ConfigClassProperty { + String name(); + ConfigGroups.Names groupName(); + String description(); +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java new file mode 100644 index 000000000..4e53cca36 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.config; + +/** + * In Hudi, we have multiple superclasses, aka Config Classes of {@link HoodieConfig} that maintain + * several configs. This class group one or more of these superclasses into higher + * level groups, such as Spark Config, Flink Configs, Metrics .... + * This class maintains the human readable name and description of each config group. + */ +public class ConfigGroups { + public enum Names { + SPARK_DATASOURCE("Spark Datasource Configs"), + FLINK_SQL("Flink Sql Configs"), + WRITE_CLIENT("Write Client Configs"), + METRICS("Metrics Configs"), + RECORD_PAYLOAD("Record Payload Config"); + + public final String name; + + Names(String name) { + this.name = name; + } + } + + public static String getDescription(Names names) { + String description; + switch (names) { + case SPARK_DATASOURCE: + description = "These configs control the Hudi Spark Datasource, " + + "providing ability to define keys/partitioning, pick out the write operation, " + + "specify how to merge records or choosing query type to read."; + break; + case FLINK_SQL: + description = "These configs control the Hudi Flink SQL source/sink connectors, " + + "providing ability to define record keys, pick out the write operation, " + + "specify how to merge records, enable/disable asynchronous compaction " + + "or choosing query type to read."; + break; + case WRITE_CLIENT: + description = "Internally, the Hudi datasource uses a RDD based HoodieWriteClient API " + + "to actually perform writes to storage. These configs provide deep control over " + + "lower level aspects like file sizing, compression, parallelism, compaction, " + + "write schema, cleaning etc. Although Hudi provides sane defaults, from time-time " + + "these configs may need to be tweaked to optimize for specific workloads."; + break; + case RECORD_PAYLOAD: + description = "This is the lowest level of customization offered by Hudi. " + + "Record payloads define how to produce new values to upsert based on incoming " + + "new record and stored old record. Hudi provides default implementations such as " + + "OverwriteWithLatestAvroPayload which simply update table with the latest/last-written record. " + + "This can be overridden to a custom class extending HoodieRecordPayload class, " + + "on both datasource and WriteClient levels."; + break; + case METRICS: + description = "These set of configs are used to enable monitoring and reporting of key" + + "Hudi stats and metrics."; + break; + default: + description = "Please fill in the description for Config Group Name: " + names.name; + break; + } + return description; + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java index bacd3b9fc..7870d8551 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.config; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieException; import java.io.Serializable; @@ -76,6 +77,18 @@ public class ConfigProperty implements Serializable { return defaultValue != null; } + public String doc() { + return StringUtils.isNullOrEmpty(doc) ? StringUtils.EMPTY_STRING : doc; + } + + public Option getSinceVersion() { + return sinceVersion; + } + + public Option getDeprecatedVersion() { + return deprecatedVersion; + } + Option>> getInferFunc() { return inferFunction; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java index 205fbd7e3..11a75b6c4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java @@ -28,6 +28,11 @@ import java.util.Properties; * Configurations used by the HUDI Metadata Table. */ @Immutable +@ConfigClassProperty(name = "Metadata Configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations used by the Hudi Metadata Table. " + + "This table maintains the metadata about a given Hudi table (e.g file listings) " + + " to avoid overhead of accessing cloud storage, during queries.") public final class HoodieMetadataConfig extends HoodieConfig { public static final String METADATA_PREFIX = "hoodie.metadata"; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java index aa91af45e..9b12f89c7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.fs; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -29,6 +31,10 @@ import java.util.Properties; /** * The consistency guard relevant config options. */ +@ConfigClassProperty(name = "Consistency Guard Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "The consistency guard related config options, to help talk to eventually consistent object storage." + + "(Tip: S3 is NOT eventually consistent anymore!)") public class ConsistencyGuardConfig extends HoodieConfig { public static final ConfigProperty CONSISTENCY_CHECK_ENABLED_PROP = ConfigProperty diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 8dd40adf1..987d91033 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -20,6 +20,8 @@ package org.apache.hudi.common.table; import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex; import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.model.HoodieFileFormat; @@ -53,6 +55,14 @@ import java.util.stream.Collectors; * @see HoodieTableMetaClient * @since 0.3.0 */ +@ConfigClassProperty(name = "Table Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that persist across writes and read on a Hudi table " + + " like base, log file formats, table name, creation schema, table version layouts. " + + " Configurations are loaded from hoodie.properties, these properties are usually set during " + + "initializing a path as hoodie base path and rarely changes during " + + "the lifetime of the table. Writers/Queries' configurations are validated against these " + + " each time for compatibility.") public class HoodieTableConfig extends HoodieConfig implements Serializable { private static final Logger LOG = LogManager.getLogger(HoodieTableConfig.class); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java index c6ff0e956..80d0d5788 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.table.view; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.util.ValidationUtils; @@ -32,6 +34,9 @@ import java.util.stream.Collectors; /** * File System View Storage Configurations. */ +@ConfigClassProperty(name = "File System View Storage Configurations", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Configurations that control how file metadata is stored by Hudi, for transaction processing and queries.") public class FileSystemViewStorageConfig extends HoodieConfig { // Property Names diff --git a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java index fc2902091..e6f094138 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java +++ b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java @@ -18,6 +18,8 @@ package org.apache.hudi.configuration; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; @@ -44,6 +46,10 @@ import java.util.Set; * *

It has the options for Hoodie table read and write. It also defines some utilities. */ +@ConfigClassProperty(name = "Flink Options", + groupName = ConfigGroups.Names.FLINK_SQL, + description = "Flink jobs using the SQL can be configured through the options in WITH clause." + + " The actual datasource level configs are listed below.") public class FlinkOptions extends HoodieConfig { private FlinkOptions() { } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java index a198ddb1f..3b2f18bb0 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/callback/kafka/HoodieWriteCommitKafkaCallbackConfig.java @@ -17,6 +17,8 @@ package org.apache.hudi.utilities.callback.kafka; +import org.apache.hudi.common.config.ConfigClassProperty; +import org.apache.hudi.common.config.ConfigGroups; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; @@ -25,6 +27,9 @@ import static org.apache.hudi.config.HoodieWriteCommitCallbackConfig.CALLBACK_PR /** * Kafka write callback related config. */ +@ConfigClassProperty(name = "Write commit Kafka callback configs", + groupName = ConfigGroups.Names.WRITE_CLIENT, + description = "Controls notifications sent to Kafka, on events happening to a hudi table.") public class HoodieWriteCommitKafkaCallbackConfig extends HoodieConfig { public static final ConfigProperty CALLBACK_KAFKA_BOOTSTRAP_SERVERS = ConfigProperty