lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-836] Implement datadog metrics reporter (#1572)

Browse Source 
- Adds support for emitting metrics to datadog
- Tests, configs..
This commit is contained in:
Raymond Xu
2020-05-22 09:14:21 -07:00
committed by GitHub
parent 802d16c8c9
commit f34de3fb27
13 changed files with 906 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
hudi-client/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java Normal file
View File

@@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestDatadogHttpClient {
@Mock
AppenderSkeleton appender;
@Captor
ArgumentCaptor<LoggingEvent> logCaptor;
@Mock
CloseableHttpClient httpClient;
@Mock
CloseableHttpResponse httpResponse;
@Mock
StatusLine statusLine;
private void mockResponse(int statusCode) {
when(statusLine.getStatusCode()).thenReturn(statusCode);
when(httpResponse.getStatusLine()).thenReturn(statusLine);
try {
when(httpClient.execute(any())).thenReturn(httpResponse);
} catch (IOException e) {
fail(e.getMessage(), e);
}
}
@Test
public void validateApiKeyShouldThrowExceptionWhenRequestFailed() throws IOException {
when(httpClient.execute(any())).thenThrow(IOException.class);
Throwable t = assertThrows(IllegalStateException.class, () -> {
new DatadogHttpClient(ApiSite.EU, "foo", false, httpClient);
});
assertEquals("Failed to connect to Datadog to validate API key.", t.getMessage());
}
@Test
public void validateApiKeyShouldThrowExceptionWhenResponseNotSuccessful() {
mockResponse(500);
Throwable t = assertThrows(IllegalStateException.class, () -> {
new DatadogHttpClient(ApiSite.EU, "foo", false, httpClient);
});
assertEquals("API key is invalid.", t.getMessage());
}
@Test
public void sendPayloadShouldLogWhenRequestFailed() throws IOException {
Logger.getRootLogger().addAppender(appender);
when(httpClient.execute(any())).thenThrow(IOException.class);
DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
ddClient.send("{}");
verify(appender).doAppend(logCaptor.capture());
assertEquals("Failed to send to Datadog.", logCaptor.getValue().getRenderedMessage());
assertEquals(Level.WARN, logCaptor.getValue().getLevel());
}
@Test
public void sendPayloadShouldLogUnsuccessfulSending() {
Logger.getRootLogger().addAppender(appender);
mockResponse(401);
when(httpResponse.toString()).thenReturn("unauthorized");
DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
ddClient.send("{}");
verify(appender).doAppend(logCaptor.capture());
assertEquals("Failed to send to Datadog. Response was unauthorized", logCaptor.getValue().getRenderedMessage());
assertEquals(Level.WARN, logCaptor.getValue().getLevel());
}
@Test
public void sendPayloadShouldLogSuccessfulSending() {
Logger.getRootLogger().addAppender(appender);
mockResponse(202);
DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
ddClient.send("{}");
verify(appender).doAppend(logCaptor.capture());
assertTrue(logCaptor.getValue().getRenderedMessage().startsWith("Sent metrics data"));
assertEquals(Level.DEBUG, logCaptor.getValue().getLevel());
}
public static List<Arguments> getApiSiteAndDomain() {
return Arrays.asList(
Arguments.of("US", "com"),
Arguments.of("EU", "eu")
);
}
@ParameterizedTest
@MethodSource("getApiSiteAndDomain")
public void testApiSiteReturnCorrectDomain(String apiSite, String domain) {
assertEquals(domain, ApiSite.valueOf(apiSite).getDomain());
}
}

77
hudi-client/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java Normal file
View File

@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestDatadogMetricsReporter {
@Mock
HoodieWriteConfig config;
@Mock
MetricRegistry registry;
@Test
public void instantiationShouldFailWhenNoApiKey() {
when(config.getDatadogApiKey()).thenReturn("");
Throwable t = assertThrows(IllegalStateException.class, () -> {
new DatadogMetricsReporter(config, registry);
});
assertEquals("Datadog cannot be initialized: API key is null or empty.", t.getMessage());
}
@Test
public void instantiationShouldFailWhenNoMetricPrefix() {
when(config.getDatadogApiKey()).thenReturn("foo");
when(config.getDatadogMetricPrefix()).thenReturn("");
Throwable t = assertThrows(IllegalStateException.class, () -> {
new DatadogMetricsReporter(config, registry);
});
assertEquals("Datadog cannot be initialized: Metric prefix is null or empty.", t.getMessage());
}
@Test
public void instantiationShouldSucceed() {
when(config.getDatadogApiSite()).thenReturn(ApiSite.EU);
when(config.getDatadogApiKey()).thenReturn("foo");
when(config.getDatadogApiKeySkipValidation()).thenReturn(true);
when(config.getDatadogMetricPrefix()).thenReturn("bar");
when(config.getDatadogMetricHost()).thenReturn("foo");
when(config.getDatadogMetricTags()).thenReturn(Arrays.asList("baz", "foo"));
assertDoesNotThrow(() -> {
new DatadogMetricsReporter(config, registry);
});
}
}

105
hudi-client/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java Normal file
View File

@@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.metrics.datadog.DatadogReporter.MetricType;
import org.apache.hudi.metrics.datadog.DatadogReporter.PayloadBuilder;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
@ExtendWith(MockitoExtension.class)
public class TestDatadogReporter {
@Mock
AppenderSkeleton appender;
@Captor
ArgumentCaptor<LoggingEvent> logCaptor;
@Mock
MetricRegistry registry;
@Mock
DatadogHttpClient client;
@Test
public void stopShouldCloseEnclosedClient() throws IOException {
new DatadogReporter(registry, client, "foo", Option.empty(), Option.empty(),
MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS).stop();
verify(client).close();
}
@Test
public void stopShouldLogWhenEnclosedClientFailToClose() throws IOException {
Logger.getRootLogger().addAppender(appender);
doThrow(IOException.class).when(client).close();
new DatadogReporter(registry, client, "foo", Option.empty(), Option.empty(),
MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS).stop();
verify(appender).doAppend(logCaptor.capture());
assertEquals("Error disconnecting from Datadog.", logCaptor.getValue().getRenderedMessage());
assertEquals(Level.WARN, logCaptor.getValue().getLevel());
}
@Test
public void prefixShouldPrepend() {
DatadogReporter reporter = new DatadogReporter(
registry, client, "foo", Option.empty(), Option.empty(),
MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS);
assertEquals("foo.bar", reporter.prefix("bar"));
}
@Test
public void payloadBuilderShouldBuildExpectedPayloadString() {
String payload = new PayloadBuilder()
.withMetricType(MetricType.gauge)
.addGauge("foo", 0, 0)
.addGauge("bar", 1, 999)
.withHost("xhost")
.withTags(Arrays.asList("tag1", "tag2"))
.build();
assertEquals(
"{\"series\":["
+ "{\"metric\":\"foo\",\"points\":[[0,0]],\"host\":\"xhost\",\"tags\":[\"tag1\",\"tag2\"]},"
+ "{\"metric\":\"bar\",\"points\":[[1,999]],\"host\":\"xhost\",\"tags\":[\"tag1\",\"tag2\"]}]}",
payload);
}
}