lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2423] Separate some config logic from HoodieMetricsConfig into HoodieMetricsGraphiteConfig HoodieMetricsJmxConfig (#3652)

Browse Source
This commit is contained in:
liujinhui
2021-09-16 15:08:10 +08:00
committed by GitHub
parent 2d5ac55195
commit 2791fb9a96
9 changed files with 344 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -40,6 +40,11 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig;
import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig;
import org.apache.hudi.config.metrics.HoodieMetricsPrometheusConfig;
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
@@ -1459,23 +1464,23 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public String getGraphiteServerHost() {
return getString(HoodieMetricsConfig.GRAPHITE_SERVER_HOST_NAME);
return getString(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_HOST_NAME);
}
public int getGraphiteServerPort() {
return getInt(HoodieMetricsConfig.GRAPHITE_SERVER_PORT_NUM);
return getInt(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_PORT_NUM);
}
public String getGraphiteMetricPrefix() {
return getString(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX_VALUE);
return getString(HoodieMetricsGraphiteConfig.GRAPHITE_METRIC_PREFIX_VALUE);
}
public String getJmxHost() {
return getString(HoodieMetricsConfig.JMX_HOST_NAME);
return getString(HoodieMetricsJmxConfig.JMX_HOST_NAME);
}
public String getJmxPort() {
return getString(HoodieMetricsConfig.JMX_PORT_NUM);
return getString(HoodieMetricsJmxConfig.JMX_PORT_NUM);
}
public int getDatadogReportPeriodSeconds() {
@@ -1777,6 +1782,8 @@ public class HoodieWriteConfig extends HoodieConfig {
private boolean isMetadataConfigSet = false;
private boolean isLockConfigSet = false;
private boolean isPreCommitValidationConfigSet = false;
private boolean isMetricsJmxConfigSet = false;
private boolean isMetricsGraphiteConfigSet = false;
public Builder withEngineType(EngineType engineType) {
this.engineType = engineType;
@@ -1931,6 +1938,18 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
public Builder withMetricsJmxConfig(HoodieMetricsJmxConfig metricsJmxConfig) {
writeConfig.getProps().putAll(metricsJmxConfig.getProps());
isMetricsJmxConfigSet = true;
return this;
}
public Builder withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig mericsGraphiteConfig) {
writeConfig.getProps().putAll(mericsGraphiteConfig.getProps());
isMetricsGraphiteConfigSet = true;
return this;
}
public Builder withPreCommitValidatorConfig(HoodiePreCommitValidatorConfig validatorConfig) {
writeConfig.getProps().putAll(validatorConfig.getProps());
isPreCommitValidationConfigSet = true;

112
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java → hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.config;
package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -55,42 +55,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
.sinceVersion("0.5.0")
.withDocumentation("Type of metrics reporter.");
// Graphite
public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite";
public static final ConfigProperty<String> GRAPHITE_SERVER_HOST_NAME = ConfigProperty
.key(GRAPHITE_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.0")
.withDocumentation("Graphite host to connect to");
public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT_NUM = ConfigProperty
.key(GRAPHITE_PREFIX + ".port")
.defaultValue(4756)
.sinceVersion("0.5.0")
.withDocumentation("Graphite port to connect to");
// Jmx
public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
public static final ConfigProperty<String> JMX_HOST_NAME = ConfigProperty
.key(JMX_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.1")
.withDocumentation("Jmx host to connect to");
public static final ConfigProperty<Integer> JMX_PORT_NUM = ConfigProperty
.key(JMX_PREFIX + ".port")
.defaultValue(9889)
.sinceVersion("0.5.1")
.withDocumentation("Jmx port to connect to");
public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty
.key(GRAPHITE_PREFIX + ".metric.prefix")
.noDefaultValue()
.sinceVersion("0.5.1")
.withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g");
// User defined
public static final ConfigProperty<String> METRICS_REPORTER_CLASS_NAME = ConfigProperty
.key(METRIC_PREFIX + ".reporter.class")
@@ -125,51 +89,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
*/
@Deprecated
public static final MetricsReporterType DEFAULT_METRICS_REPORTER_TYPE = METRICS_REPORTER_TYPE_VALUE.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String JMX_HOST = JMX_HOST_NAME.key();
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String JMX_PORT = JMX_PORT_NUM.key();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key();
/**
* @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods instead
*/
@@ -220,31 +139,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
return this;
}
public Builder toGraphiteHost(String host) {
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host);
return this;
}
public Builder onGraphitePort(int port) {
hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port));
return this;
}
public Builder toJmxHost(String host) {
hoodieMetricsConfig.setValue(JMX_HOST_NAME, host);
return this;
}
public Builder onJmxPort(String port) {
hoodieMetricsConfig.setValue(JMX_PORT_NUM, port);
return this;
}
public Builder usePrefix(String prefix) {
hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix);
return this;
}
public Builder withReporterClass(String className) {
hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS_NAME, className);
return this;
@@ -267,6 +161,10 @@ public class HoodieMetricsConfig extends HoodieConfig {
HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.PROMETHEUS,
HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.JMX,
HoodieMetricsJmxConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.GRAPHITE,
HoodieMetricsGraphiteConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
return hoodieMetricsConfig;
}
}

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java → hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.config;
package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -27,7 +27,7 @@ import javax.annotation.concurrent.Immutable;
import java.util.Properties;
import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
/**
* Configs for Datadog reporter type.

134
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java Normal file
View File

@@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
/**
* Configs for Graphite reporter type.
* <p>
* {@link org.apache.hudi.metrics.MetricsReporterType#GRAPHITE}
*/
@ConfigClassProperty(name = "Metrics Configurations for Graphite",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using Graphite. "
+ " Hudi publishes metrics on every commit, clean, rollback etc.")
public class HoodieMetricsGraphiteConfig extends HoodieConfig {
public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite";
public static final ConfigProperty<String> GRAPHITE_SERVER_HOST_NAME = ConfigProperty
.key(GRAPHITE_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.0")
.withDocumentation("Graphite host to connect to.");
public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT_NUM = ConfigProperty
.key(GRAPHITE_PREFIX + ".port")
.defaultValue(4756)
.sinceVersion("0.5.0")
.withDocumentation("Graphite port to connect to.");
public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX_VALUE = ConfigProperty
.key(GRAPHITE_PREFIX + ".metric.prefix")
.noDefaultValue()
.sinceVersion("0.5.1")
.withDocumentation("Standard prefix applied to all metrics. This helps to add datacenter, environment information for e.g");
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_GRAPHITE_SERVER_HOST = GRAPHITE_SERVER_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.key();
/**
* @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_GRAPHITE_SERVER_PORT = GRAPHITE_SERVER_PORT_NUM.defaultValue();
/**
* @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods instead
*/
@Deprecated
public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_METRIC_PREFIX_VALUE.key();
private HoodieMetricsGraphiteConfig() {
super();
}
public static HoodieMetricsGraphiteConfig.Builder newBuilder() {
return new HoodieMetricsGraphiteConfig.Builder();
}
public static class Builder {
private final HoodieMetricsGraphiteConfig hoodieMetricsGraphiteConfig = new HoodieMetricsGraphiteConfig();
public HoodieMetricsGraphiteConfig.Builder fromFile(File propertiesFile) throws IOException {
try (FileReader reader = new FileReader(propertiesFile)) {
this.hoodieMetricsGraphiteConfig.getProps().load(reader);
return this;
}
}
public HoodieMetricsGraphiteConfig.Builder fromProperties(Properties props) {
this.hoodieMetricsGraphiteConfig.getProps().putAll(props);
return this;
}
public HoodieMetricsGraphiteConfig.Builder toGraphiteHost(String host) {
hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host);
return this;
}
public HoodieMetricsGraphiteConfig.Builder onGraphitePort(int port) {
hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_PORT_NUM, String.valueOf(port));
return this;
}
public HoodieMetricsGraphiteConfig.Builder usePrefix(String prefix) {
hoodieMetricsGraphiteConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix);
return this;
}
public HoodieMetricsGraphiteConfig build() {
hoodieMetricsGraphiteConfig.setDefaults(HoodieMetricsGraphiteConfig.class.getName());
return hoodieMetricsGraphiteConfig;
}
}
}

118
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java Normal file
View File

@@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
/**
* Configs for Jmx reporter type.
* <p>
* {@link org.apache.hudi.metrics.MetricsReporterType#JMX}
*/
@ConfigClassProperty(name = "Metrics Configurations for Jmx",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using Jmx. "
+ " Hudi publishes metrics on every commit, clean, rollback etc.")
public class HoodieMetricsJmxConfig extends HoodieConfig {
public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
public static final ConfigProperty<String> JMX_HOST_NAME = ConfigProperty
.key(JMX_PREFIX + ".host")
.defaultValue("localhost")
.sinceVersion("0.5.1")
.withDocumentation("Jmx host to connect to");
public static final ConfigProperty<Integer> JMX_PORT_NUM = ConfigProperty
.key(JMX_PREFIX + ".port")
.defaultValue(9889)
.sinceVersion("0.5.1")
.withDocumentation("Jmx port to connect to");
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String JMX_HOST = JMX_HOST_NAME.key();
/**
* @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
*/
@Deprecated
public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final String JMX_PORT = JMX_PORT_NUM.key();
/**
* @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
*/
@Deprecated
public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue();
private HoodieMetricsJmxConfig() {
super();
}
public static HoodieMetricsJmxConfig.Builder newBuilder() {
return new HoodieMetricsJmxConfig.Builder();
}
public static class Builder {
private final HoodieMetricsJmxConfig hoodieMetricsJmxConfig = new HoodieMetricsJmxConfig();
public HoodieMetricsJmxConfig.Builder fromFile(File propertiesFile) throws IOException {
try (FileReader reader = new FileReader(propertiesFile)) {
this.hoodieMetricsJmxConfig.getProps().load(reader);
return this;
}
}
public HoodieMetricsJmxConfig.Builder fromProperties(Properties props) {
this.hoodieMetricsJmxConfig.getProps().putAll(props);
return this;
}
public HoodieMetricsJmxConfig.Builder toJmxHost(String host) {
hoodieMetricsJmxConfig.setValue(JMX_HOST_NAME, host);
return this;
}
public HoodieMetricsJmxConfig.Builder onJmxPort(String port) {
hoodieMetricsJmxConfig.setValue(JMX_PORT_NUM, port);
return this;
}
public HoodieMetricsJmxConfig build() {
hoodieMetricsJmxConfig.setDefaults(HoodieMetricsJmxConfig.class.getName());
return hoodieMetricsJmxConfig;
}
}
}

45
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java → hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.config;
package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -25,8 +25,14 @@ import org.apache.hudi.common.config.HoodieConfig;
import java.util.Properties;
import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
/**
* Configs for Prometheus/Pushgaeway reporter type.
* <p>
* {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS}
* {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS_PUSHGATEWAY}
*/
@ConfigClassProperty(name = "Metrics Configurations for Prometheus",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using Prometheus. "
@@ -174,6 +180,41 @@ public class HoodieMetricsPrometheusConfig extends HoodieConfig {
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayHostName(String hostName) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_HOST_NAME, String.valueOf(hostName));
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayPortNum(Integer pushgatewayPortNum) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_PORT_NUM, String.valueOf(pushgatewayPortNum));
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayReportPeriodInSeconds(String periodTime) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS, periodTime);
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayDeleteOnShutdownEnable(boolean deleteOnShutdownEnable) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE, String.valueOf(deleteOnShutdownEnable));
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayJobname(String jobname) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_JOBNAME, jobname);
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPushgatewayRandomJobnameSuffix(boolean randomJobnameSuffix) {
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX, String.valueOf(randomJobnameSuffix));
return this;
}
public HoodieMetricsPrometheusConfig.Builder withPrometheusPortNum(int prometheusPortNum) {
hoodieMetricsPrometheusConfig.setValue(PROMETHEUS_PORT_NUM, String.valueOf(prometheusPortNum));
return this;
}
public HoodieMetricsPrometheusConfig build() {
hoodieMetricsPrometheusConfig.setDefaults(HoodieMetricsPrometheusConfig.class.getName());
return hoodieMetricsPrometheusConfig;

22
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
View File

@@ -46,8 +46,10 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieMetricsConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieMetadataException;
@@ -175,19 +177,22 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
.withFinalizeWriteParallelism(parallelism);
if (writeConfig.isMetricsOn()) {
HoodieMetricsConfig.Builder metricsConfig = HoodieMetricsConfig.newBuilder()
builder.withMetricsConfig(HoodieMetricsConfig.newBuilder()
.withReporterType(writeConfig.getMetricsReporterType().toString())
.withExecutorMetrics(writeConfig.isExecutorMetricsEnabled())
.on(true);
.on(true).build());
switch (writeConfig.getMetricsReporterType()) {
case GRAPHITE:
metricsConfig.onGraphitePort(writeConfig.getGraphiteServerPort())
builder.withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder()
.onGraphitePort(writeConfig.getGraphiteServerPort())
.toGraphiteHost(writeConfig.getGraphiteServerHost())
.usePrefix(writeConfig.getGraphiteMetricPrefix());
.usePrefix(writeConfig.getGraphiteMetricPrefix()).build());
break;
case JMX:
metricsConfig.onJmxPort(writeConfig.getJmxPort())
.toJmxHost(writeConfig.getJmxHost());
builder.withMetricsJmxConfig(HoodieMetricsJmxConfig.newBuilder()
.onJmxPort(writeConfig.getJmxPort())
.toJmxHost(writeConfig.getJmxHost())
.build());
break;
case DATADOG:
case PROMETHEUS:
@@ -198,10 +203,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
default:
throw new HoodieMetadataException("Unsupported Metrics Reporter type " + writeConfig.getMetricsReporterType());
}
builder.withMetricsConfig(metricsConfig.build());
}
return builder.build();
}

2
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java
View File

@@ -19,7 +19,7 @@
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.config.HoodieMetricsDatadogConfig;
import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.Test;

7
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
View File

@@ -47,9 +47,10 @@ import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieMetricsConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.index.HoodieIndex;
@@ -1305,7 +1306,9 @@ public class TestHoodieBackedMetadata extends HoodieClientTestHarness {
.enable(useFileListingMetadata)
.enableMetrics(enableMetrics).build())
.withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics)
.withExecutorMetrics(true).usePrefix("unit-test").build());
.withExecutorMetrics(true).build())
.withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder()
.usePrefix("unit-test").build());
}
@Override