lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1620] Fix Metrics UT (#2894)

Browse Source 
Make sure shutdown Metrics between unit test cases to ensure isolation
This commit is contained in:
Raymond Xu
2021-04-30 11:20:41 -07:00
committed by GitHub
parent 46de9e0f3f
commit 3418a92de8
8 changed files with 83 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
azure-pipelines.yml
View File

@@ -56,7 +56,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pazp-unit-tests -pl hudi-client/hudi-spark-client
options: -Punit-tests -pl hudi-client/hudi-spark-client
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests spark client'
@@ -88,7 +88,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pazp-unit-tests -pl hudi-utilities
options: -Punit-tests -pl hudi-utilities
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests utilities'
@@ -120,7 +120,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pazp-unit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client
options: -Punit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests other modules'

8
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
View File

@@ -22,15 +22,19 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestHoodieConsoleMetrics {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
@Mock
HoodieWriteConfig config;
@BeforeEach
public void start() {

16
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
View File

@@ -21,23 +21,34 @@ package org.apache.hudi.metrics;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Test for the Jmx metrics report.
*/
@ExtendWith(MockitoExtension.class)
public class TestHoodieJmxMetrics {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
@Mock
HoodieWriteConfig config;
@AfterEach
void shutdownMetrics() {
Metrics.shutdown();
}
@Test
public void testRegisterGauge() {
when(config.isMetricsOn()).thenReturn(true);
when(config.getTableName()).thenReturn("foo");
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
when(config.getJmxHost()).thenReturn("localhost");
when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
@@ -50,6 +61,7 @@ public class TestHoodieJmxMetrics {
@Test
public void testRegisterGaugeByRangerPort() {
when(config.isMetricsOn()).thenReturn(true);
when(config.getTableName()).thenReturn("foo");
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
when(config.getJmxHost()).thenReturn("localhost");
when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));

17
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
View File

@@ -24,8 +24,12 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import com.codahale.metrics.Timer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Random;
import java.util.stream.Stream;
@@ -36,18 +40,25 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestHoodieMetrics {
private HoodieMetrics metrics;
@Mock
HoodieWriteConfig config;
HoodieMetrics metrics;
@BeforeEach
public void start() {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
void setUp() {
when(config.isMetricsOn()).thenReturn(true);
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
metrics = new HoodieMetrics(config, "raw_table");
}
@AfterEach
void shutdownMetrics() {
Metrics.shutdown();
}
@Test
public void testRegisterGauge() {
registerGauge("metric1", 123L);

7
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
View File

@@ -19,9 +19,11 @@
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
@@ -43,6 +45,11 @@ public class TestDatadogMetricsReporter {
@Mock
MetricRegistry registry;
@AfterEach
void shutdownMetrics() {
Metrics.shutdown();
}
@Test
public void instantiationShouldFailWhenNoApiKey() {
when(config.getDatadogApiKey()).thenReturn("");

18
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
View File

@@ -20,16 +20,28 @@ package org.apache.hudi.metrics.prometheus;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestPrometheusReporter {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
@Mock
HoodieWriteConfig config;
@AfterEach
void shutdownMetrics() {
Metrics.shutdown();
}
@Test
public void testRegisterGauge() {
@@ -40,4 +52,4 @@ public class TestPrometheusReporter {
new HoodieMetrics(config, "raw_table");
});
}
}
}

28
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
View File

@@ -22,16 +22,28 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestPushGateWayReporter {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
@Mock
HoodieWriteConfig config;
@AfterEach
void shutdownMetrics() {
Metrics.shutdown();
}
@Test
public void testRegisterGauge() {
@@ -39,9 +51,17 @@ public class TestPushGateWayReporter {
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.PROMETHEUS_PUSHGATEWAY);
when(config.getPushGatewayHost()).thenReturn("localhost");
when(config.getPushGatewayPort()).thenReturn(9091);
new HoodieMetrics(config, "raw_table");
when(config.getPushGatewayReportPeriodSeconds()).thenReturn(30);
when(config.getPushGatewayDeleteOnShutdown()).thenReturn(true);
when(config.getPushGatewayJobName()).thenReturn("foo");
when(config.getPushGatewayRandomJobNameSuffix()).thenReturn(false);
assertDoesNotThrow(() -> {
new HoodieMetrics(config, "raw_table");
});
registerGauge("pushGateWayReporter_metric", 123L);
assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
.get("pushGateWayReporter_metric").getValue().toString());
}
}
}

34
pom.xml
View File

@@ -1273,40 +1273,6 @@
</plugins>
</build>
</profile>
<profile>
<id>azp-unit-tests</id>
<properties>
<skipUTs>false</skipUTs>
<skipFTs>true</skipFTs>
<skipITs>true</skipITs>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.jupiter.version}</version>
</dependency>
</dependencies>
<configuration combine.self="append">
<skip>${skipUTs}</skip>
<forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
<excludedGroups>functional</excludedGroups>
<excludes>
<exclude>**/*FunctionalTestSuite.java</exclude>
<exclude>**/IT*.java</exclude>
<exclude>**/testsuite/**/Test*.java</exclude>
<exclude>**/TestPushGateWayReporter.java</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>javadocs</id>
<build>