lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-896] Report test coverage by modules & parallelize CI (#1753)

Browse Source 
- use codecov flags for each module to report coverage
- parallelize CI jobs for shorter time
- add a testcase for MetricsReporterFactory (to trigger codecov comment)
This commit is contained in:
Raymond Xu
2020-06-27 23:16:12 -07:00
committed by GitHub
parent 2603cfb33e
commit 31247e9b34
11 changed files with 347 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
.codecov.yml
View File

@@ -19,6 +19,11 @@
# Check if this file is valid by running in bash:
# curl -X POST --data-binary @.codecov.yml https://codecov.io/validate
coverage:
precision: 2
round: down
range: "50...100"
# Ignoring Paths
# --------------
# which folders/files to ignore
@@ -42,3 +47,43 @@ ignore:
- "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java"
- "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java"
comment:
layout: "reach, diff, flags, files"
behavior: default
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: no # [yes :: must have a head report to post]
branches: # https://docs.codecov.io/docs/pull-request-comments#branches
- "master"
flags:
hudicli:
paths:
- hudi-cli/src/main/
hudiclient:
paths:
- hudi-client/src/main/
hudicommon:
paths:
- hudi-common/src/main/
hudiexamples:
paths:
- hudi-examples/src/main/
hudihadoopmr:
paths:
- hudi-hadoop-mr/src/main/
hudihivesync:
paths:
- hudi-hive-sync/src/main/
hudiintegtest:
paths:
- hudi-integ-test/src/main/
hudispark:
paths:
- hudi-spark/src/main/
huditimelineservice:
paths:
- hudi-timeline-service/src/main/
hudiutilities:
paths:
- hudi-utilities/src/main/

21
.travis.yml
View File

@@ -13,14 +13,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
language: java
os: linux
dist: trusty
language: java
jdk:
- openjdk8
sudo: required
env:
- HUDI_QUIETER_LOGGING=1 TEST_SUITE=unit
- TEST_SUITE=integration
jobs:
include:
- name: "Unit tests except hudi-client"
env: MODE=unit MODULES='!hudi-client' HUDI_QUIETER_LOGGING=1
- name: "Unit tests for hudi-client"
env: MODE=unit MODULES=hudi-client HUDI_QUIETER_LOGGING=1
- name: "Functional tests"
env: MODE=functional HUDI_QUIETER_LOGGING=1
- name: "Integration tests"
env: MODE=integration
install: true
services:
- docker
@@ -36,6 +43,6 @@ script:
# ping stdout every 9 minutes or Travis kills build
# https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received
- while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
- scripts/run_travis_tests.sh $TEST_SUITE
- scripts/run_travis_tests.sh $MODE $MODULES
after_success:
- bash <(curl -s https://codecov.io/bash)
- scripts/report_coverage.sh

25
hudi-cli/pom.xml
View File

@@ -123,29 +123,8 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.0</version>
<configuration>
<includes>
<include>**/ITT*.java</include>
</includes>
</configuration>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
<execution>
<id>verify</id>
<phase>verify</phase>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
</plugins>
</build>

48
hudi-client/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hudi.metrics;
import org.apache.hudi.config.HoodieWriteConfig;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class TestMetricsReporterFactory {
@Mock
HoodieWriteConfig config;
@Mock
MetricRegistry registry;
@Test
public void metricsReporterFactoryShouldReturnReporter() {
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry);
assertTrue(reporter instanceof InMemoryMetricsReporter);
}
}

28
hudi-integ-test/pom.xml
View File

@@ -143,7 +143,6 @@
<properties>
<dockerCompose.envFile>${project.basedir}/compose_env</dockerCompose.envFile>
<dockerCompose.file>${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark244.yml</dockerCompose.file>
<skipITs>false</skipITs>
<docker.compose.skip>${skipITs}</docker.compose.skip>
<checkstyle.skip>true</checkstyle.skip>
<main.basedir>${project.parent.basedir}</main.basedir>
@@ -165,7 +164,7 @@
<configuration>
<executable>/bin/bash</executable>
<arguments>
<argument> -c </argument>
<argument>-c</argument>
<argument>echo HUDI_WS=`dirname ${project.basedir}`</argument>
</arguments>
<outputFile>${dockerCompose.envFile}</outputFile>
@@ -174,29 +173,8 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.0</version>
<configuration>
<includes>
<include>**/ITT*.java</include>
</includes>
</configuration>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
<execution>
<id>verify</id>
<phase>verify</phase>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.dkanejs.maven.plugins</groupId>

1
hudi-spark/pom.xml
View File

@@ -132,6 +132,7 @@
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<skipTests>${skipTests}</skipTests>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>TestSuite.txt</filereports>

4
hudi-timeline-service/pom.xml
View File

@@ -55,10 +55,6 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
</plugins>
<resources>

18
packaging/hudi-spark-bundle/pom.xml
View File

@@ -160,24 +160,6 @@
</execution>
</executions>
</plugin>
<!-- Adding test coverage report aggregation to this module as this seems to cover all of the required dependencies -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report-aggregate</goal>
</goals>
<configuration>
<!-- Sets the output directory for the code coverage report. -->
<outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>

233
pom.xml
View File

@@ -70,6 +70,7 @@
<properties>
<maven-jar-plugin.version>2.6</maven-jar-plugin.version>
<maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
<maven-failsafe-plugin.version>3.0.0-M4</maven-failsafe-plugin.version>
<maven-shade-plugin.version>3.1.1</maven-shade-plugin.version>
<maven-javadoc-plugin.version>3.1.1</maven-javadoc-plugin.version>
<maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
@@ -108,8 +109,9 @@
<codehaus-jackson.version>1.9.13</codehaus-jackson.version>
<h2.version>1.4.199</h2.version>
<skipTests>false</skipTests>
<skipITs>${skipTests}</skipITs>
<skipUTs>${skipTests}</skipUTs>
<skipFTs>${skipTests}</skipFTs>
<skipITs>${skipTests}</skipITs>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<main.basedir>${project.basedir}</main.basedir>
<spark.bundle.hive.scope>provided</spark.bundle.hive.scope>
@@ -118,7 +120,7 @@
<spark.bundle.spark.shade.prefix>org.apache.hudi.spark.</spark.bundle.spark.shade.prefix>
<utilities.bundle.hive.scope>provided</utilities.bundle.hive.scope>
<utilities.bundle.hive.shade.prefix></utilities.bundle.hive.shade.prefix>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
<argLine>-Xmx2g</argLine>
<jacoco.version>0.8.5</jacoco.version>
</properties>
@@ -245,20 +247,28 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skip>${skipUTs}</skip>
<argLine>-Xmx2g</argLine>
<forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
<argLine>@{argLine}</argLine>
<systemPropertyVariables>
<log4j.configuration>
${surefire-log4j.file}
</log4j.configuration>
</systemPropertyVariables>
<!-- Excludes integration tests when unit tests are run. -->
<excludes>
<exclude>**/IT*.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven-failsafe-plugin.version}</version>
<configuration>
<skip>${skipITs}</skip>
<argLine>@{argLine}</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.version}</version>
</plugin>
</plugins>
<pluginManagement>
@@ -277,38 +287,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.version}</version>
<executions>
<!--
Prepares the property pointing to the JaCoCo runtime agent which
is passed as VM argument when Maven the Surefire plugin is executed.
-->
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<!--
Ensures that the code coverage report for unit tests is created after
unit tests have been run.
-->
<execution>
<id>post-unit-test</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<!-- Sets the output directory for the code coverage report. -->
<outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!-- excludes are inherited -->
<groupId>org.apache.rat</groupId>
@@ -965,9 +943,180 @@
</property>
</activation>
<properties>
<surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties</surefire-log4j.file>
<surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties
</surefire-log4j.file>
</properties>
</profile>
<profile>
<id>unit-tests</id>
<properties>
<skipUTs>false</skipUTs>
<skipFTs>true</skipFTs>
<skipITs>true</skipITs>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.jupiter.version}</version>
</dependency>
</dependencies>
<configuration combine.self="append">
<skip>${skipUTs}</skip>
<forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
<excludedGroups>functional</excludedGroups>
<excludes>
<exclude>**/*FunctionalTestSuite.java</exclude>
<exclude>**/IT*.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-unit-tests</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>functional-tests</id>
<properties>
<skipUTs>true</skipUTs>
<skipFTs>false</skipFTs>
<skipITs>true</skipITs>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>${maven-surefire-plugin.version}</version>
</dependency>
</dependencies>
<configuration combine.self="append">
<skip>${skipFTs}</skip>
<forkCount>1</forkCount>
<reuseForks>true</reuseForks>
<includes>
<include>**/*FunctionalTestSuite.java</include>
</includes>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-functional-tests</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<outputDirectory>${project.reporting.outputDirectory}/jacoco-ft</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>integration-tests</id>
<properties>
<skipUTs>true</skipUTs>
<skipFTs>true</skipFTs>
<skipITs>false</skipITs>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration combine.self="override">
<skip>${skipUTs}</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration combine.self="override">
<skip>${skipITs}</skip>
<includes>
<include>**/IT*.java</include>
</includes>
</configuration>
<executions>
<execution>
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
<execution>
<id>verify-integration-test</id>
<phase>verify</phase>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>post-integration-tests</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<outputDirectory>${project.reporting.outputDirectory}/jacoco-it</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>javadocs</id>
<build>

33
scripts/report_coverage.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
modules=(
hudi-cli
hudi-client
hudi-common
hudi-examples
hudi-hadoop-mr
hudi-hive-sync
hudi-integ-test
hudi-spark
hudi-timeline-service
hudi-utilities
)
for module in "${modules[@]}"; do
bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}"
done

18
scripts/run_travis_tests.sh
View File

@@ -17,24 +17,26 @@
# limitations under the License.
mode=$1
modules=$2
sparkVersion=2.4.4
hadoopVersion=2.7
if [ "$mode" = "unit" ];
then
if [ "$mode" = "unit" ]; then
mvn clean install -Punit-tests -DskipTests -q
echo "Running Unit Tests"
mvn test -DskipITs=true -B
elif [ "$mode" = "integration" ];
then
mvn test -Punit-tests -pl "$modules" -B
elif [ "$mode" = "functional" ]; then
echo "Running Functional Tests"
mvn test -Pfunctional-tests -B
elif [ "$mode" = "integration" ]; then
echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}"
wget http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz -O /tmp/spark-${sparkVersion}.tgz
tar -xvf /tmp/spark-${sparkVersion}.tgz
export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion}
mkdir /tmp/spark-events/
echo "Running Integration Tests"
mvn verify -DskipUTs=true -B
mvn verify -Pintegration-tests -B
else
echo "Unknown mode $mode"
exit 1;
exit 1
fi