From c21209cb58a5cff3267aca9f17fe7d2c4edd3190 Mon Sep 17 00:00:00 2001 From: Prashant Wason Date: Tue, 4 Aug 2020 17:14:36 -0700 Subject: [PATCH] [HUDI-1149] Added a console metrics reporter and associated unit tests. --- .../hudi/metrics/ConsoleMetricsReporter.java | 72 +++++++++++++++++++ .../hudi/metrics/MetricsReporterFactory.java | 3 + .../hudi/metrics/MetricsReporterType.java | 2 +- .../metrics/TestHoodieConsoleMetrics.java | 46 ++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 hudi-client/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java create mode 100644 hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java new file mode 100644 index 000000000..ca961096c --- /dev/null +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import java.io.Closeable; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; + +/** + * Hudi Console metrics reporter. Reports the metrics by printing them to the stdout on the console. + */ +public class ConsoleMetricsReporter extends MetricsReporter { + + private static final Logger LOG = LogManager.getLogger(ConsoleMetricsReporter.class); + private final ConsoleReporter consoleReporter; + + public ConsoleMetricsReporter(MetricRegistry registry) { + this.consoleReporter = ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .filter(MetricFilter.ALL).build(); + } + + @Override + public void start() { + if (consoleReporter != null) { + consoleReporter.start(30, TimeUnit.SECONDS); + } else { + LOG.error("Cannot start as the consoleReporter is null."); + } + } + + @Override + public void report() { + if (consoleReporter != null) { + consoleReporter.report(); + } else { + LOG.error("Cannot report metrics as the consoleReporter is null."); + } + } + + @Override + public Closeable getReporter() { + return consoleReporter; + } + + @Override + public void stop() { + } +} diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index ae9d65cea..f2466d3c0 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -66,6 +66,9 @@ public class MetricsReporterFactory { case DATADOG: reporter = new DatadogMetricsReporter(config, registry); break; + case CONSOLE: + reporter = new ConsoleMetricsReporter(registry); + break; default: LOG.error("Reporter type[" + type + "] is not supported."); break; diff --git a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java index cfe6a2b3a..a595b9a29 100644 --- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java @@ -22,5 +22,5 @@ package org.apache.hudi.metrics; * Types of the reporter. Right now we only support Graphite. We can include JMX and CSV in the future. */ public enum MetricsReporterType { - GRAPHITE, INMEMORY, JMX, DATADOG + GRAPHITE, INMEMORY, JMX, DATADOG, CONSOLE } diff --git a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java new file mode 100644 index 000000000..e64473070 --- /dev/null +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.config.HoodieWriteConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.apache.hudi.metrics.Metrics.registerGauge; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestHoodieConsoleMetrics { + + HoodieWriteConfig config = mock(HoodieWriteConfig.class); + + @BeforeEach + public void start() { + when(config.isMetricsOn()).thenReturn(true); + when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.CONSOLE); + new HoodieMetrics(config, "raw_table"); + } + + @Test + public void testRegisterGauge() { + registerGauge("metric1", 123L); + assertEquals("123", Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString()); + } +}