diff --git a/.codecov.yml b/.codecov.yml index ddd5c70f9..c63c6853b 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -19,6 +19,11 @@ # Check if this file is valid by running in bash: # curl -X POST --data-binary @.codecov.yml https://codecov.io/validate +coverage: + precision: 2 + round: down + range: "50...100" + # Ignoring Paths # -------------- # which folders/files to ignore @@ -42,3 +47,43 @@ ignore: - "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java" - "hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java" +comment: + layout: "reach, diff, flags, files" + behavior: default + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + require_head: no # [yes :: must have a head report to post] + branches: # https://docs.codecov.io/docs/pull-request-comments#branches + - "master" + +flags: + hudicli: + paths: + - hudi-cli/src/main/ + hudiclient: + paths: + - hudi-client/src/main/ + hudicommon: + paths: + - hudi-common/src/main/ + hudiexamples: + paths: + - hudi-examples/src/main/ + hudihadoopmr: + paths: + - hudi-hadoop-mr/src/main/ + hudihivesync: + paths: + - hudi-hive-sync/src/main/ + hudiintegtest: + paths: + - hudi-integ-test/src/main/ + hudispark: + paths: + - hudi-spark/src/main/ + huditimelineservice: + paths: + - hudi-timeline-service/src/main/ + hudiutilities: + paths: + - hudi-utilities/src/main/ diff --git a/.travis.yml b/.travis.yml index 76aeacf7c..1454c6dc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,14 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -language: java +os: linux dist: trusty +language: java jdk: - openjdk8 -sudo: required -env: - - HUDI_QUIETER_LOGGING=1 TEST_SUITE=unit - - TEST_SUITE=integration +jobs: + include: + - name: "Unit tests except hudi-client" + env: MODE=unit MODULES='!hudi-client' HUDI_QUIETER_LOGGING=1 + - name: "Unit tests for hudi-client" + env: MODE=unit MODULES=hudi-client HUDI_QUIETER_LOGGING=1 + - name: "Functional tests" + env: MODE=functional HUDI_QUIETER_LOGGING=1 + - name: "Integration tests" + env: MODE=integration install: true services: - docker @@ -36,6 +43,6 @@ script: # ping stdout every 9 minutes or Travis kills build # https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done & - - scripts/run_travis_tests.sh $TEST_SUITE + - scripts/run_travis_tests.sh $MODE $MODULES after_success: - - bash <(curl -s https://codecov.io/bash) + - scripts/report_coverage.sh diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml index dbb44639f..13bc84862 100644 --- a/hudi-cli/pom.xml +++ b/hudi-cli/pom.xml @@ -123,29 +123,8 @@ - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - **/ITT*.java - - - - - integration-test - - integration-test - - - - verify - verify - - verify - - - + org.jacoco + jacoco-maven-plugin diff --git a/hudi-client/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java new file mode 100644 index 000000000..c4143a42f --- /dev/null +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.config.HoodieWriteConfig; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TestMetricsReporterFactory { + + @Mock + HoodieWriteConfig config; + + @Mock + MetricRegistry registry; + + @Test + public void metricsReporterFactoryShouldReturnReporter() { + when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY); + MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry); + assertTrue(reporter instanceof InMemoryMetricsReporter); + } +} diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml index a4381ac1d..b9bcc6d96 100644 --- a/hudi-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -143,7 +143,6 @@ ${project.basedir}/compose_env ${project.basedir}/../docker/compose/docker-compose_hadoop284_hive233_spark244.yml - false ${skipITs} true ${project.parent.basedir} @@ -165,7 +164,7 @@ /bin/bash - -c + -c echo HUDI_WS=`dirname ${project.basedir}` ${dockerCompose.envFile} @@ -174,29 +173,8 @@ - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - **/ITT*.java - - - - - integration-test - - integration-test - - - - verify - verify - - verify - - - + org.jacoco + jacoco-maven-plugin com.dkanejs.maven.plugins diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml index 6be663afa..e670ff729 100644 --- a/hudi-spark/pom.xml +++ b/hudi-spark/pom.xml @@ -132,6 +132,7 @@ scalatest-maven-plugin 1.0 + ${skipTests} ${project.build.directory}/surefire-reports . TestSuite.txt diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml index 7ab3bcddb..2751ab5cb 100644 --- a/hudi-timeline-service/pom.xml +++ b/hudi-timeline-service/pom.xml @@ -55,10 +55,6 @@ org.apache.rat apache-rat-plugin - - org.jacoco - jacoco-maven-plugin - diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml index 4e9cd3cba..2cc31cb2b 100644 --- a/packaging/hudi-spark-bundle/pom.xml +++ b/packaging/hudi-spark-bundle/pom.xml @@ -160,24 +160,6 @@ - - - org.jacoco - jacoco-maven-plugin - - - post-unit-test - test - - report-aggregate - - - - ${project.reporting.outputDirectory}/jacoco-ut - - - - diff --git a/pom.xml b/pom.xml index cc39f4136..d51e04abd 100644 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,7 @@ 2.6 3.0.0-M4 + 3.0.0-M4 3.1.1 3.1.1 3.8.0 @@ -108,8 +109,9 @@ 1.9.13 1.4.199 false - ${skipTests} ${skipTests} + ${skipTests} + ${skipTests} UTF-8 ${project.basedir} provided @@ -118,7 +120,7 @@ org.apache.hudi.spark. provided - -Xmx1024m -XX:MaxPermSize=256m + -Xmx2g 0.8.5 @@ -245,20 +247,28 @@ maven-surefire-plugin ${maven-surefire-plugin.version} - ${skipUTs} - -Xmx2g - 120 + @{argLine} ${surefire-log4j.file} - - - **/IT*.java - + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + ${skipITs} + @{argLine} + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + @@ -277,38 +287,6 @@ org.apache.maven.plugins maven-compiler-plugin - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - - pre-unit-test - - prepare-agent - - - - - post-unit-test - test - - report - - - - ${project.reporting.outputDirectory}/jacoco-ut - - - - org.apache.rat @@ -965,9 +943,180 @@ - file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties + file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties + + + unit-tests + + false + true + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.junit.jupiter + junit-jupiter-engine + ${junit.jupiter.version} + + + + ${skipUTs} + 120 + functional + + **/*FunctionalTestSuite.java + **/IT*.java + + + + + org.jacoco + jacoco-maven-plugin + + + + prepare-agent + + + + post-unit-tests + test + + report + + + ${project.reporting.outputDirectory}/jacoco-ut + + + + + + + + + functional-tests + + true + false + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit47 + ${maven-surefire-plugin.version} + + + + ${skipFTs} + 1 + true + + **/*FunctionalTestSuite.java + + + + + org.jacoco + jacoco-maven-plugin + + + + prepare-agent + + + + post-functional-tests + test + + report + + + ${project.reporting.outputDirectory}/jacoco-ft + + + + + + + + + integration-tests + + true + true + false + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipUTs} + + + + org.apache.maven.plugins + maven-failsafe-plugin + + ${skipITs} + + **/IT*.java + + + + + integration-test + + integration-test + + + + verify-integration-test + verify + + verify + + + + + + org.jacoco + jacoco-maven-plugin + + + + prepare-agent + + + + post-integration-tests + test + + report + + + ${project.reporting.outputDirectory}/jacoco-it + + + + + + + javadocs diff --git a/scripts/report_coverage.sh b/scripts/report_coverage.sh new file mode 100755 index 000000000..e6f933920 --- /dev/null +++ b/scripts/report_coverage.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +modules=( + hudi-cli + hudi-client + hudi-common + hudi-examples + hudi-hadoop-mr + hudi-hive-sync + hudi-integ-test + hudi-spark + hudi-timeline-service + hudi-utilities +) +for module in "${modules[@]}"; do + bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}" +done diff --git a/scripts/run_travis_tests.sh b/scripts/run_travis_tests.sh index f3cb39cb4..3921cc8ea 100755 --- a/scripts/run_travis_tests.sh +++ b/scripts/run_travis_tests.sh @@ -17,24 +17,26 @@ # limitations under the License. mode=$1 +modules=$2 sparkVersion=2.4.4 hadoopVersion=2.7 -if [ "$mode" = "unit" ]; -then +if [ "$mode" = "unit" ]; then + mvn clean install -Punit-tests -DskipTests -q echo "Running Unit Tests" - mvn test -DskipITs=true -B -elif [ "$mode" = "integration" ]; -then + mvn test -Punit-tests -pl "$modules" -B +elif [ "$mode" = "functional" ]; then + echo "Running Functional Tests" + mvn test -Pfunctional-tests -B +elif [ "$mode" = "integration" ]; then echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}" wget http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz -O /tmp/spark-${sparkVersion}.tgz tar -xvf /tmp/spark-${sparkVersion}.tgz export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion} mkdir /tmp/spark-events/ echo "Running Integration Tests" - mvn verify -DskipUTs=true -B + mvn verify -Pintegration-tests -B else echo "Unknown mode $mode" - exit 1; + exit 1 fi -