lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2801] Add Amazon CloudWatch metrics reporter (#4081)

Browse Source
This commit is contained in:
Udit Mehrotra
2021-11-25 13:33:16 -08:00
committed by GitHub
parent 8e1379384a
commit e0125a7911
15 changed files with 799 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -1568,6 +1568,22 @@ public class HoodieWriteConfig extends HoodieConfig {
HoodieMetricsDatadogConfig.METRIC_TAG_VALUES, ",").split("\\s*,\\s*")).collect(Collectors.toList());
}
public int getCloudWatchReportPeriodSeconds() {
return getInt(HoodieMetricsCloudWatchConfig.REPORT_PERIOD_SECONDS);
}
public String getCloudWatchMetricPrefix() {
return getString(HoodieMetricsCloudWatchConfig.METRIC_PREFIX);
}
public String getCloudWatchMetricNamespace() {
return getString(HoodieMetricsCloudWatchConfig.METRIC_NAMESPACE);
}
public int getCloudWatchMaxDatumsPerRequest() {
return getInt(HoodieMetricsCloudWatchConfig.MAX_DATUMS_PER_REQUEST);
}
public String getMetricReporterClassName() {
return getString(HoodieMetricsConfig.METRICS_REPORTER_CLASS_NAME);
}

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
View File

@@ -22,6 +22,7 @@ import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.config.HoodieMetricsCloudWatchConfig;
import org.apache.hudi.metrics.MetricsReporterType;
import javax.annotation.concurrent.Immutable;
@@ -165,6 +166,8 @@ public class HoodieMetricsConfig extends HoodieConfig {
HoodieMetricsJmxConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.GRAPHITE,
HoodieMetricsGraphiteConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType == MetricsReporterType.CLOUDWATCH,
HoodieMetricsCloudWatchConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
return hoodieMetricsConfig;
}
}

1
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
View File

@@ -241,6 +241,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
case PROMETHEUS_PUSHGATEWAY:
case CONSOLE:
case INMEMORY:
case CLOUDWATCH:
break;
default:
throw new HoodieMetadataException("Unsupported Metrics Reporter type " + writeConfig.getMetricsReporterType());

4
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
View File

@@ -22,6 +22,7 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metrics.cloudwatch.CloudWatchMetricsReporter;
import org.apache.hudi.metrics.datadog.DatadogMetricsReporter;
import com.codahale.metrics.MetricRegistry;
@@ -77,6 +78,9 @@ public class MetricsReporterFactory {
case CONSOLE:
reporter = new ConsoleMetricsReporter(registry);
break;
case CLOUDWATCH:
reporter = new CloudWatchMetricsReporter(config, registry);
break;
default:
LOG.error("Reporter type[" + type + "] is not supported.");
break;

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
View File

@@ -22,5 +22,5 @@ package org.apache.hudi.metrics;
* Types of the reporter supported, hudi also supports user defined reporter.
*/
public enum MetricsReporterType {
GRAPHITE, INMEMORY, JMX, DATADOG, CONSOLE, PROMETHEUS_PUSHGATEWAY, PROMETHEUS
GRAPHITE, INMEMORY, JMX, DATADOG, CONSOLE, PROMETHEUS_PUSHGATEWAY, PROMETHEUS, CLOUDWATCH
}

85
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.metrics.cloudwatch;
import org.apache.hudi.aws.cloudwatch.CloudWatchReporter;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.MetricsReporter;
import com.codahale.metrics.MetricRegistry;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.Closeable;
import java.util.concurrent.TimeUnit;
/**
* Hudi Amazon CloudWatch metrics reporter. Responsible for reading Hoodie metrics configurations and hooking up with
* {@link org.apache.hudi.metrics.Metrics}. Internally delegates reporting tasks to {@link CloudWatchReporter}.
*/
public class CloudWatchMetricsReporter extends MetricsReporter {
private static final Logger LOG = LogManager.getLogger(CloudWatchMetricsReporter.class);
private final MetricRegistry registry;
private final HoodieWriteConfig config;
private final CloudWatchReporter reporter;
public CloudWatchMetricsReporter(HoodieWriteConfig config, MetricRegistry registry) {
this.config = config;
this.registry = registry;
this.reporter = createCloudWatchReporter();
}
CloudWatchMetricsReporter(HoodieWriteConfig config, MetricRegistry registry, CloudWatchReporter reporter) {
this.config = config;
this.registry = registry;
this.reporter = reporter;
}
private CloudWatchReporter createCloudWatchReporter() {
return CloudWatchReporter.forRegistry(registry)
.prefixedWith(config.getCloudWatchMetricPrefix())
.namespace(config.getCloudWatchMetricNamespace())
.maxDatumsPerRequest(config.getCloudWatchMaxDatumsPerRequest())
.build(config.getProps());
}
@Override
public void start() {
LOG.info("Starting CloudWatch Metrics Reporter.");
reporter.start(config.getCloudWatchReportPeriodSeconds(), TimeUnit.SECONDS);
}
@Override
public void report() {
reporter.report();
}
@Override
public Closeable getReporter() {
return reporter;
}
@Override
public void stop() {
LOG.info("Stopping CloudWatch Metrics Reporter.");
reporter.stop();
}
}

62
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java Normal file
View File

@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.metrics.cloudwatch;
import org.apache.hudi.aws.cloudwatch.CloudWatchReporter;
import org.apache.hudi.config.HoodieWriteConfig;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.concurrent.TimeUnit;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestCloudWatchMetricsReporter {
@Mock
private HoodieWriteConfig config;
@Mock
private MetricRegistry registry;
@Mock
private CloudWatchReporter reporter;
@Test
public void testReporter() {
when(config.getCloudWatchReportPeriodSeconds()).thenReturn(30);
CloudWatchMetricsReporter metricsReporter = new CloudWatchMetricsReporter(config, registry, reporter);
metricsReporter.start();
verify(reporter, times(1)).start(30, TimeUnit.SECONDS);
metricsReporter.report();
verify(reporter, times(1)).report();
metricsReporter.stop();
verify(reporter, times(1)).stop();
}
}