[HUDI-1241] Automate the generation of configs webpage as configs are added to Hudi repo (#3302)

hudi-common/src/main/java/org/apache/hudi/common/config/ConfigClassProperty.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.config;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+
+/**
+ * Annotation for superclasses of {@link HoodieConfig} that includes the
+ * human-readable name of the config class, the config group ({@link ConfigGroupName})
+ * it belongs to (e.g., spark/ flink/ write)
+ * and the description of the config class.
+ */
+public @interface ConfigClassProperty {
+  String name();
+  ConfigGroups.Names groupName();
+  String description();
+}

hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.config;
+
+/**
+ * In Hudi, we have multiple superclasses, aka Config Classes of {@link HoodieConfig} that maintain
+ * several configs. This class group one or more of these superclasses into higher
+ * level groups, such as Spark Config, Flink Configs, Metrics ....
+ * This class maintains the human readable name and description of each config group.
+ */
+public class ConfigGroups {
+  public enum Names {
+    SPARK_DATASOURCE("Spark Datasource Configs"),
+    FLINK_SQL("Flink Sql Configs"),
+    WRITE_CLIENT("Write Client Configs"),
+    METRICS("Metrics Configs"),
+    RECORD_PAYLOAD("Record Payload Config");
+
+    public final String name;
+
+    Names(String name) {
+      this.name = name;
+    }
+  }
+
+  public static String getDescription(Names names) {
+    String description;
+    switch (names) {
+      case SPARK_DATASOURCE:
+        description =  "These configs control the Hudi Spark Datasource, "
+            + "providing ability to define keys/partitioning, pick out the write operation, "
+            + "specify how to merge records or choosing query type to read.";
+        break;
+      case FLINK_SQL:
+        description = "These configs control the Hudi Flink SQL source/sink connectors, "
+            + "providing ability to define record keys, pick out the write operation, "
+            + "specify how to merge records, enable/disable asynchronous compaction "
+            + "or choosing query type to read.";
+        break;
+      case WRITE_CLIENT:
+        description = "Internally, the Hudi datasource uses a RDD based HoodieWriteClient API "
+            + "to actually perform writes to storage. These configs provide deep control over "
+            + "lower level aspects like file sizing, compression, parallelism, compaction, "
+            + "write schema, cleaning etc. Although Hudi provides sane defaults, from time-time "
+            + "these configs may need to be tweaked to optimize for specific workloads.";
+        break;
+      case RECORD_PAYLOAD:
+        description =  "This is the lowest level of customization offered by Hudi. "
+            + "Record payloads define how to produce new values to upsert based on incoming "
+            + "new record and stored old record. Hudi provides default implementations such as "
+            + "OverwriteWithLatestAvroPayload which simply update table with the latest/last-written record. "
+            + "This can be overridden to a custom class extending HoodieRecordPayload class, "
+            + "on both datasource and WriteClient levels.";
+        break;
+      case METRICS:
+        description = "These set of configs are used to enable monitoring and reporting of key"
+            + "Hudi stats and metrics.";
+        break;
+      default:
+        description = "Please fill in the description for Config Group Name: " + names.name;
+        break;
+    }
+    return description;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/config/ConfigProperty.java

@@ -19,6 +19,7 @@
 package org.apache.hudi.common.config;
 
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.exception.HoodieException;
 
 import java.io.Serializable;
@@ -76,6 +77,18 @@ public class ConfigProperty<T> implements Serializable {
     return defaultValue != null;
   }
 
+  public String doc() {
+    return StringUtils.isNullOrEmpty(doc) ? StringUtils.EMPTY_STRING : doc;
+  }
+
+  public Option<String> getSinceVersion() {
+    return sinceVersion;
+  }
+
+  public Option<String> getDeprecatedVersion() {
+    return deprecatedVersion;
+  }
+
   Option<Function<HoodieConfig, Option<T>>> getInferFunc() {
     return inferFunction;
   }

hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java

@@ -28,6 +28,11 @@ import java.util.Properties;
  * Configurations used by the HUDI Metadata Table.
  */
 @Immutable
+@ConfigClassProperty(name = "Metadata Configs",
+    groupName = ConfigGroups.Names.WRITE_CLIENT,
+    description = "Configurations used by the Hudi Metadata Table. "
+        + "This table maintains the metadata about a given Hudi table (e.g file listings) "
+        + " to avoid overhead of accessing cloud storage, during queries.")
 public final class HoodieMetadataConfig extends HoodieConfig {
 
   public static final String METADATA_PREFIX = "hoodie.metadata";

hudi-common/src/main/java/org/apache/hudi/common/fs/ConsistencyGuardConfig.java

@@ -18,6 +18,8 @@
 
 package org.apache.hudi.common.fs;
 
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
 import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.config.HoodieConfig;
 
@@ -29,6 +31,10 @@ import java.util.Properties;
 /**
  * The consistency guard relevant config options.
  */
+@ConfigClassProperty(name = "Consistency Guard Configurations",
+    groupName = ConfigGroups.Names.WRITE_CLIENT,
+    description = "The consistency guard related config options, to help talk to eventually consistent object storage."
+        + "(Tip: S3 is NOT eventually consistent anymore!)")
 public class ConsistencyGuardConfig extends HoodieConfig {
 
   public static final ConfigProperty<String> CONSISTENCY_CHECK_ENABLED_PROP = ConfigProperty

hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java

@@ -20,6 +20,8 @@ package org.apache.hudi.common.table;
 
 import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
 import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex;
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
 import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.config.HoodieConfig;
 import org.apache.hudi.common.model.HoodieFileFormat;
@@ -53,6 +55,14 @@ import java.util.stream.Collectors;
  * @see HoodieTableMetaClient
  * @since 0.3.0
  */
+@ConfigClassProperty(name = "Table Configurations",
+    groupName = ConfigGroups.Names.WRITE_CLIENT,
+    description = "Configurations that persist across writes and read on a Hudi table "
+        + " like  base, log file formats, table name, creation schema, table version layouts. "
+        + " Configurations are loaded from hoodie.properties, these properties are usually set during "
+        + "initializing a path as hoodie base path and rarely changes during "
+        + "the lifetime of the table. Writers/Queries' configurations are validated against these "
+        + " each time for compatibility.")
 public class HoodieTableConfig extends HoodieConfig implements Serializable {
 
   private static final Logger LOG = LogManager.getLogger(HoodieTableConfig.class);

hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java

@@ -18,6 +18,8 @@
 
 package org.apache.hudi.common.table.view;
 
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
 import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.config.HoodieConfig;
 import org.apache.hudi.common.util.ValidationUtils;
@@ -32,6 +34,9 @@ import java.util.stream.Collectors;
 /**
  * File System View Storage Configurations.
  */
+@ConfigClassProperty(name = "File System View Storage Configurations",
+    groupName = ConfigGroups.Names.WRITE_CLIENT,
+    description = "Configurations that control how file metadata is stored by Hudi, for transaction processing and queries.")
 public class FileSystemViewStorageConfig extends HoodieConfig {
 
   // Property Names