lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1241] Automate the generation of configs webpage as configs are added to Hudi repo (#3302)

Browse Source
This commit is contained in:
rmahindra123
2021-07-23 21:33:34 -07:00
committed by GitHub
parent b2f7fcb8c8
commit a14b19fdd5
24 changed files with 265 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieBootstrapConfig.java
View File

@@ -22,6 +22,8 @@ import org.apache.hudi.client.bootstrap.BootstrapMode;
import org.apache.hudi.client.bootstrap.selector.MetadataOnlyBootstrapModeSelector;
import org.apache.hudi.client.bootstrap.translator.IdentityBootstrapPartitionPathTranslator;
import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.table.HoodieTableConfig;
@@ -35,6 +37,11 @@ import java.util.Properties;
/**
* Bootstrap specific configs.
*/
@ConfigClassProperty(name = "Bootstrap Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control how you want to bootstrap your existing tables for the first time into hudi. "
+ "The bootstrap operation can flexibly avoid copying data over before you can use Hudi and support running the existing "
+ " writers and new hudi writers in parallel, to validate the migration.")
public class HoodieBootstrapConfig extends HoodieConfig {
public static final ConfigProperty<String> BOOTSTRAP_BASE_PATH_PROP = ConfigProperty

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -29,6 +31,10 @@ import java.util.Properties;
/**
* Clustering specific configs.
*/
@ConfigClassProperty(name = "Clustering Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control the clustering table service in hudi, "
+ "which optimizes the storage layout for better query performance by sorting and sizing data files.")
public class HoodieClusteringConfig extends HoodieConfig {
// Any strategy specific params can be saved with this prefix

7
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
@@ -41,6 +43,11 @@ import java.util.stream.Collectors;
* Compaction related config.
*/
@Immutable
@ConfigClassProperty(name = "Compaction Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control compaction "
+ "(merging of log files onto a new base files) as well as "
+ "cleaning (reclamation of older/unused file groups/slices).")
public class HoodieCompactionConfig extends HoodieConfig {
public static final ConfigProperty<String> AUTO_CLEAN_PROP = ConfigProperty

7
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator;
@@ -27,6 +29,11 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
@ConfigClassProperty(name = "HBase Index Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control indexing behavior "
+ "(when HBase based indexing is enabled), which tags incoming "
+ "records as either inserts or updates to older records.")
public class HoodieHBaseIndexConfig extends HoodieConfig {
public static final ConfigProperty<String> HBASE_ZKQUORUM_PROP = ConfigProperty

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
View File

@@ -19,6 +19,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.engine.EngineType;
@@ -36,6 +38,10 @@ import java.util.Properties;
* Indexing related config.
*/
@Immutable
@ConfigClassProperty(name = "Index Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control indexing behavior, "
+ "which tags incoming records as either inserts or updates to older records.")
public class HoodieIndexConfig extends HoodieConfig {
public static final ConfigProperty<String> INDEX_TYPE_PROP = ConfigProperty

7
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
View File

@@ -20,6 +20,8 @@ package org.apache.hudi.config;
import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy;
import org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.lock.LockProvider;
@@ -55,6 +57,11 @@ import static org.apache.hudi.common.config.LockConfiguration.ZK_SESSION_TIMEOUT
/**
* Hoodie Configs for Locks.
*/
@ConfigClassProperty(name = "Locks Configurations",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configs that control locking mechanisms required for concurrency control "
+ " between writers to a Hudi table. Concurrency between Hudi's own table services "
+ " are auto managed internally.")
public class HoodieLockConfig extends HoodieConfig {
public static final ConfigProperty<String> LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = ConfigProperty

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -32,6 +34,10 @@ import java.util.Properties;
* Memory related config.
*/
@Immutable
@ConfigClassProperty(name = "Memory Configurations",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Controls memory usage for compaction "
+ "and merges, performed internally by Hudi.")
public class HoodieMemoryConfig extends HoodieConfig {
// Default max memory fraction during hash-merge, excess spills to disk

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.metrics.MetricsReporterType;
@@ -33,6 +35,10 @@ import java.util.Properties;
* Fetch the configurations used by the Metrics system.
*/
@Immutable
@ConfigClassProperty(name = "Metrics Configurations",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics. Hudi publishes metrics on "
+ "every commit, clean, rollback etc. The following sections list the supported reporters.")
public class HoodieMetricsConfig extends HoodieConfig {
public static final String METRIC_PREFIX = "hoodie.metrics";

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -33,6 +35,10 @@ import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
* {@link org.apache.hudi.metrics.MetricsReporterType#DATADOG}
*/
@Immutable
@ConfigClassProperty(name = "Metrics Configurations for Datadog reporter",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using the Datadog reporter type. "
+ "Hudi publishes metrics on every commit, clean, rollback etc.")
public class HoodieMetricsDatadogConfig extends HoodieConfig {
public static final String DATADOG_PREFIX = METRIC_PREFIX + ".datadog";

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -25,6 +27,10 @@ import java.util.Properties;
import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
@ConfigClassProperty(name = "Metrics Configurations for Prometheus",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using Prometheus. "
+ " Hudi publishes metrics on every commit, clean, rollback etc.")
public class HoodieMetricsPrometheusConfig extends HoodieConfig {
// Prometheus PushGateWay

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -32,6 +34,10 @@ import static org.apache.hudi.common.model.HoodiePayloadProps.PAYLOAD_ORDERING_F
/**
* Hoodie payload related configs.
*/
@ConfigClassProperty(name = "Payload Configurations",
groupName = ConfigGroups.Names.RECORD_PAYLOAD,
description = "Payload related configs, that can be leveraged to "
+ "control merges based on specific business fields in the data.")
public class HoodiePayloadConfig extends HoodieConfig {
public static final ConfigProperty<String> PAYLOAD_ORDERING_FIELD_PROP = ConfigProperty

5
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -32,6 +34,9 @@ import java.util.Properties;
* Storage related config.
*/
@Immutable
@ConfigClassProperty(name = "Storage Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control aspects around writing, sizing, reading base and log files.")
public class HoodieStorageConfig extends HoodieConfig {
public static final ConfigProperty<String> PARQUET_FILE_MAX_BYTES = ConfigProperty

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java
View File

@@ -17,6 +17,8 @@
package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@@ -28,6 +30,10 @@ import java.util.Properties;
/**
* Write callback related config.
*/
@ConfigClassProperty(name = "Write commit callback configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Controls callback behavior into HTTP endpoints, to push "
+ " notifications on commits on hudi tables.")
public class HoodieWriteCommitCallbackConfig extends HoodieConfig {
public static final String CALLBACK_PREFIX = "hoodie.write.commit.callback.";

6
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -21,6 +21,8 @@ package org.apache.hudi.config;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.bootstrap.BootstrapMode;
import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.HoodieMetadataConfig;
@@ -68,6 +70,10 @@ import java.util.stream.Collectors;
* Class storing configs for the HoodieWriteClient.
*/
@Immutable
@ConfigClassProperty(name = "Write Configurations",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that control write behavior on Hudi tables. These can be directly passed down from even "
+ "higher level frameworks (e.g Spark datasources, Flink sink) and utilities (e.g DeltaStreamer).")
public class HoodieWriteConfig extends HoodieConfig {
private static final long serialVersionUID = 0L;

8
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java
View File

@@ -18,9 +18,17 @@
package org.apache.hudi.keygen.constant;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
@ConfigClassProperty(name = "Key Generator Options",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Hudi maintains keys (record key + partition path) "
+ "for uniquely identifying a particular record. "
+ "This config allows developers to setup the Key generator class that "
+ "will extract these out of incoming records.")
public class KeyGeneratorOptions extends HoodieConfig {
public static final ConfigProperty<String> URL_ENCODE_PARTITIONING_OPT_KEY = ConfigProperty