[HUDI-896] Report test coverage by modules & parallelize CI (#1753)

- use codecov flags for each module to report coverage
- parallelize CI jobs for shorter time
- add a testcase for MetricsReporterFactory (to trigger codecov comment)

.codecov.yml

@@ -19,6 +19,11 @@
 # Check if this file is valid by running in bash:
 # curl -X POST --data-binary @.codecov.yml https://codecov.io/validate
 
+coverage:
+  precision: 2
+  round: down
+  range: "50...100"
+
 # Ignoring Paths
 # --------------
 # which folders/files to ignore
@@ -42,3 +47,43 @@ ignore:
   - "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java"
   - "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java"
 
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false  # if true: only post the comment if coverage changes
+  require_base: no        # [yes :: must have a base report to post]
+  require_head: no        # [yes :: must have a head report to post]
+  branches:               # https://docs.codecov.io/docs/pull-request-comments#branches
+    - "master"
+
+flags:
+  hudicli:
+    paths:
+      - hudi-cli/src/main/
+  hudiclient:
+    paths:
+      - hudi-client/src/main/
+  hudicommon:
+    paths:
+      - hudi-common/src/main/
+  hudiexamples:
+    paths:
+      - hudi-examples/src/main/
+  hudihadoopmr:
+    paths:
+      - hudi-hadoop-mr/src/main/
+  hudihivesync:
+    paths:
+      - hudi-hive-sync/src/main/
+  hudiintegtest:
+    paths:
+      - hudi-integ-test/src/main/
+  hudispark:
+    paths:
+      - hudi-spark/src/main/
+  huditimelineservice:
+    paths:
+      - hudi-timeline-service/src/main/
+  hudiutilities:
+    paths:
+      - hudi-utilities/src/main/

.travis.yml

@@ -13,14 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-language: java
+os: linux
 dist: trusty
+language: java
 jdk:
   - openjdk8
-sudo: required
+jobs:
-env:
+  include:
-  - HUDI_QUIETER_LOGGING=1 TEST_SUITE=unit
+    - name: "Unit tests except hudi-client"
-  - TEST_SUITE=integration
+      env: MODE=unit MODULES='!hudi-client' HUDI_QUIETER_LOGGING=1
+    - name: "Unit tests for hudi-client"
+      env: MODE=unit MODULES=hudi-client HUDI_QUIETER_LOGGING=1
+    - name: "Functional tests"
+      env: MODE=functional HUDI_QUIETER_LOGGING=1
+    - name: "Integration tests"
+      env: MODE=integration
 install: true
 services:
   - docker
@@ -36,6 +43,6 @@ script:
   # ping stdout every 9 minutes or Travis kills build
   # https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received
   - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
-  - scripts/run_travis_tests.sh $TEST_SUITE
+  - scripts/run_travis_tests.sh $MODE $MODULES
 after_success:
-  - bash <(curl -s https://codecov.io/bash)
+  - scripts/report_coverage.sh

hudi-cli/pom.xml

@@ -123,29 +123,8 @@
         </configuration>
       </plugin>
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
+        <groupId>org.jacoco</groupId>
-        <artifactId>maven-failsafe-plugin</artifactId>
+        <artifactId>jacoco-maven-plugin</artifactId>
-        <version>2.22.0</version>
-        <configuration>
-          <includes>
-            <include>**/ITT*.java</include>
-          </includes>
-        </configuration>
-        <executions>
-          <execution>
-            <phase>integration-test</phase>
-            <goals>
-              <goal>integration-test</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>verify</id>
-            <phase>verify</phase>
-            <goals>
-              <goal>verify</goal>
-            </goals>
-          </execution>
-        </executions>
       </plugin>
     </plugins>
   </build>

hudi-client/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import com.codahale.metrics.MetricRegistry;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestMetricsReporterFactory {
+
+  @Mock
+  HoodieWriteConfig config;
+
+  @Mock
+  MetricRegistry registry;
+
+  @Test
+  public void metricsReporterFactoryShouldReturnReporter() {
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
+    MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry);
+    assertTrue(reporter instanceof InMemoryMetricsReporter);
+  }
+}

hudi-integ-test/pom.xml

@@ -143,7 +143,6 @@
   <properties>
     <dockerCompose.envFile>${project.basedir}/compose_env</dockerCompose.envFile>
     <dockerCompose.file>${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark244.yml</dockerCompose.file>
-    <skipITs>false</skipITs>
     <docker.compose.skip>${skipITs}</docker.compose.skip>
     <checkstyle.skip>true</checkstyle.skip>
     <main.basedir>${project.parent.basedir}</main.basedir>
@@ -165,7 +164,7 @@
             <configuration>
               <executable>/bin/bash</executable>
               <arguments>
-                <argument> -c </argument>
+                <argument>-c</argument>
                 <argument>echo HUDI_WS=`dirname ${project.basedir}`</argument>
               </arguments>
               <outputFile>${dockerCompose.envFile}</outputFile>
@@ -174,29 +173,8 @@
         </executions>
       </plugin>
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
+        <groupId>org.jacoco</groupId>
-        <artifactId>maven-failsafe-plugin</artifactId>
+        <artifactId>jacoco-maven-plugin</artifactId>
-        <version>2.22.0</version>
-        <configuration>
-          <includes>
-            <include>**/ITT*.java</include>
-          </includes>
-        </configuration>
-        <executions>
-          <execution>
-            <phase>integration-test</phase>
-            <goals>
-              <goal>integration-test</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>verify</id>
-            <phase>verify</phase>
-            <goals>
-              <goal>verify</goal>
-            </goals>
-          </execution>
-        </executions>
       </plugin>
       <plugin>
         <groupId>com.dkanejs.maven.plugins</groupId>

hudi-spark/pom.xml

@@ -132,6 +132,7 @@
         <artifactId>scalatest-maven-plugin</artifactId>
         <version>1.0</version>
         <configuration>
+          <skipTests>${skipTests}</skipTests>
           <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
           <junitxml>.</junitxml>
           <filereports>TestSuite.txt</filereports>

hudi-timeline-service/pom.xml

@@ -55,10 +55,6 @@
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>
       </plugin>
-      <plugin>
-        <groupId>org.jacoco</groupId>
-        <artifactId>jacoco-maven-plugin</artifactId>
-      </plugin>
     </plugins>
 
     <resources>

packaging/hudi-spark-bundle/pom.xml

@@ -160,24 +160,6 @@
           </execution>
         </executions>
       </plugin>
-      <!-- Adding test coverage report aggregation to this module as this seems to cover all of the required dependencies -->
-      <plugin>
-        <groupId>org.jacoco</groupId>
-        <artifactId>jacoco-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>post-unit-test</id>
-            <phase>test</phase>
-            <goals>
-              <goal>report-aggregate</goal>
-            </goals>
-            <configuration>
-              <!-- Sets the output directory for the code coverage report. -->
-              <outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
      <resources>
        <resource>

pom.xml

@@ -70,6 +70,7 @@
   <properties>
     <maven-jar-plugin.version>2.6</maven-jar-plugin.version>
     <maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
+    <maven-failsafe-plugin.version>3.0.0-M4</maven-failsafe-plugin.version>
     <maven-shade-plugin.version>3.1.1</maven-shade-plugin.version>
     <maven-javadoc-plugin.version>3.1.1</maven-javadoc-plugin.version>
     <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
@@ -108,8 +109,9 @@
     <codehaus-jackson.version>1.9.13</codehaus-jackson.version>
     <h2.version>1.4.199</h2.version>
     <skipTests>false</skipTests>
-    <skipITs>${skipTests}</skipITs>
     <skipUTs>${skipTests}</skipUTs>
+    <skipFTs>${skipTests}</skipFTs>
+    <skipITs>${skipTests}</skipITs>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <main.basedir>${project.basedir}</main.basedir>
     <spark.bundle.hive.scope>provided</spark.bundle.hive.scope>
@@ -118,7 +120,7 @@
     <spark.bundle.spark.shade.prefix>org.apache.hudi.spark.</spark.bundle.spark.shade.prefix>
     <utilities.bundle.hive.scope>provided</utilities.bundle.hive.scope>
     <utilities.bundle.hive.shade.prefix></utilities.bundle.hive.shade.prefix>
-    <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+    <argLine>-Xmx2g</argLine>
     <jacoco.version>0.8.5</jacoco.version>
   </properties>
 
@@ -245,20 +247,28 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${maven-surefire-plugin.version}</version>
         <configuration>
-          <skip>${skipUTs}</skip>
+          <argLine>@{argLine}</argLine>
-          <argLine>-Xmx2g</argLine>
-          <forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
           <systemPropertyVariables>
             <log4j.configuration>
               ${surefire-log4j.file}
             </log4j.configuration>
           </systemPropertyVariables>
-          <!-- Excludes integration tests when unit tests are run. -->
-          <excludes>
-            <exclude>**/IT*.java</exclude>
-          </excludes>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>${maven-failsafe-plugin.version}</version>
+        <configuration>
+          <skip>${skipITs}</skip>
+          <argLine>@{argLine}</argLine>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <version>${jacoco.version}</version>
+      </plugin>
     </plugins>
 
     <pluginManagement>
@@ -277,38 +287,6 @@
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
         </plugin>
-        <plugin>
-          <groupId>org.jacoco</groupId>
-          <artifactId>jacoco-maven-plugin</artifactId>
-          <version>${jacoco.version}</version>
-          <executions>
-            <!--
-                Prepares the property pointing to the JaCoCo runtime agent which
-                is passed as VM argument when Maven the Surefire plugin is executed.
-            -->
-            <execution>
-              <id>pre-unit-test</id>
-              <goals>
-                <goal>prepare-agent</goal>
-              </goals>
-            </execution>
-            <!--
-                Ensures that the code coverage report for unit tests is created after
-                unit tests have been run.
-            -->
-            <execution>
-              <id>post-unit-test</id>
-              <phase>test</phase>
-              <goals>
-                <goal>report</goal>
-              </goals>
-              <configuration>
-                <!-- Sets the output directory for the code coverage report. -->
-                <outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
-              </configuration>
-            </execution>
-          </executions>
-        </plugin>
         <plugin>
           <!-- excludes are inherited -->
           <groupId>org.apache.rat</groupId>
@@ -965,9 +943,180 @@
         </property>
       </activation>
       <properties>
-        <surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties</surefire-log4j.file>
+        <surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties
+        </surefire-log4j.file>
       </properties>
     </profile>
+    <profile>
+      <id>unit-tests</id>
+      <properties>
+        <skipUTs>false</skipUTs>
+        <skipFTs>true</skipFTs>
+        <skipITs>true</skipITs>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.junit.jupiter</groupId>
+                <artifactId>junit-jupiter-engine</artifactId>
+                <version>${junit.jupiter.version}</version>
+              </dependency>
+            </dependencies>
+            <configuration combine.self="append">
+              <skip>${skipUTs}</skip>
+              <forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
+              <excludedGroups>functional</excludedGroups>
+              <excludes>
+                <exclude>**/*FunctionalTestSuite.java</exclude>
+                <exclude>**/IT*.java</exclude>
+              </excludes>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>post-unit-tests</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <configuration>
+                  <outputDirectory>${project.reporting.outputDirectory}/jacoco-ut</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>functional-tests</id>
+      <properties>
+        <skipUTs>true</skipUTs>
+        <skipFTs>false</skipFTs>
+        <skipITs>true</skipITs>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.maven.surefire</groupId>
+                <artifactId>surefire-junit47</artifactId>
+                <version>${maven-surefire-plugin.version}</version>
+              </dependency>
+            </dependencies>
+            <configuration combine.self="append">
+              <skip>${skipFTs}</skip>
+              <forkCount>1</forkCount>
+              <reuseForks>true</reuseForks>
+              <includes>
+                <include>**/*FunctionalTestSuite.java</include>
+              </includes>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>post-functional-tests</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <configuration>
+                  <outputDirectory>${project.reporting.outputDirectory}/jacoco-ft</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>integration-tests</id>
+      <properties>
+        <skipUTs>true</skipUTs>
+        <skipFTs>true</skipFTs>
+        <skipITs>false</skipITs>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration combine.self="override">
+              <skip>${skipUTs}</skip>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration combine.self="override">
+              <skip>${skipITs}</skip>
+              <includes>
+                <include>**/IT*.java</include>
+              </includes>
+            </configuration>
+            <executions>
+              <execution>
+                <phase>integration-test</phase>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>verify-integration-test</id>
+                <phase>verify</phase>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>post-integration-tests</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <configuration>
+                  <outputDirectory>${project.reporting.outputDirectory}/jacoco-it</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>javadocs</id>
       <build>

scripts/report_coverage.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+
+modules=(
+  hudi-cli
+  hudi-client
+  hudi-common
+  hudi-examples
+  hudi-hadoop-mr
+  hudi-hive-sync
+  hudi-integ-test
+  hudi-spark
+  hudi-timeline-service
+  hudi-utilities
+)
+for module in "${modules[@]}"; do
+  bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}"
+done

scripts/run_travis_tests.sh

@@ -17,24 +17,26 @@
 # limitations under the License.
 
 mode=$1
+modules=$2
 sparkVersion=2.4.4
 hadoopVersion=2.7
 
-if [ "$mode" = "unit" ];
+if [ "$mode" = "unit" ]; then
-then
+  mvn clean install -Punit-tests -DskipTests -q
   echo "Running Unit Tests"
-  mvn test -DskipITs=true -B
+  mvn test -Punit-tests -pl "$modules" -B
-elif [ "$mode" = "integration" ];
+elif [ "$mode" = "functional" ]; then
-then
+  echo "Running Functional Tests"
+  mvn test -Pfunctional-tests -B
+elif [ "$mode" = "integration" ]; then
   echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}"
   wget http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz -O /tmp/spark-${sparkVersion}.tgz
   tar -xvf /tmp/spark-${sparkVersion}.tgz
   export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion}
   mkdir /tmp/spark-events/
   echo "Running Integration Tests"
-  mvn verify -DskipUTs=true -B
+  mvn verify -Pintegration-tests -B
 else
   echo "Unknown mode $mode"
-  exit 1;
+  exit 1
 fi
-