From 38c9b85aa8aa8f72876b112e376f300076fafacd Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Wed, 1 Sep 2021 08:51:00 -0700 Subject: [PATCH] [HUDI-2280] Use GitHub Actions to build different scala spark versions (#3556) --- .github/actions/bot/package.json | 36 ----------- .github/actions/bot/src/action.js | 100 ------------------------------ .github/workflows/bot.yml | 64 +++++++++---------- .travis.yml | 50 --------------- pom.xml | 5 +- scripts/report_coverage.sh | 34 ---------- scripts/run_travis_tests.sh | 42 ------------- 7 files changed, 32 insertions(+), 299 deletions(-) delete mode 100644 .github/actions/bot/package.json delete mode 100644 .github/actions/bot/src/action.js delete mode 100644 .travis.yml delete mode 100755 scripts/report_coverage.sh delete mode 100755 scripts/run_travis_tests.sh diff --git a/.github/actions/bot/package.json b/.github/actions/bot/package.json deleted file mode 100644 index 0d6e9d1e9..000000000 --- a/.github/actions/bot/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "github_action_ci_bot", - "version": "1.1.1", - "description": "CI Bot for GitHub Actions", - "main": "dist/index.js", - "scripts": { - "lint": "eslint 'src/**.js' 'tests/**.js' --fix", - "test": "eslint 'src/**.js' 'tests/**.js' && jest --coverage", - "build": "ncc build src/action.js" - }, - "author": "lamber-ken", - "license": "Apache LICENSE 2.0", - "homepage": "https://github.com/apache/hudi", - "bugs": { - "url": "https://github.com/apache/hudi/issues" - }, - "dependencies": { - "@actions/core": "^1.2.4", - "@actions/github": "^2.2.0", - "@actions/io": "^1.0.2" - }, - "devDependencies": { - "@types/jest": "^25.1.4", - "@typescript-eslint/eslint-plugin": "^2.33.0", - "@typescript-eslint/parser": "^2.33.0", - "@zeit/ncc": "^0.22.0", - "eslint": "^7.0.0", - "eslint-config-prettier": "^6.11.0", - "husky": "^4.2.5", - "jest": "^25.1.0", - "npm-run-all": "^4.1.5", - "prettier": "^2.0.5", - "ts-jest": "^25.2.1", - "typescript": "^3.8.3" - } -} diff --git a/.github/actions/bot/src/action.js b/.github/actions/bot/src/action.js deleted file mode 100644 index cb4f6916d..000000000 --- a/.github/actions/bot/src/action.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -async function check(core, context, github) { - - try { - const provider = process.env.PROVIDER; - const repository = process.env.REPOSITORY; - const command = context.payload.comment.body; - - if (command !== 'rerun tests') { - console.log("Invalid command:" + command); - return; - } - - const { - data: { - head: { - sha: ref, - } - } - } = await github.pulls.get({ - owner: provider, - repo: repository, - pull_number: context.issue.number, - }); - - const checks = await github.checks.listForRef({ - owner: provider, - repo: repository, - ref: ref - }); - - checks.data.check_runs.forEach(run => { - - if (run.app.owner.login === 'travis-ci') { - console.log("rerun travis ci check: " + run.external_id); - rebuild(run.external_id) - } else { - console.log("ignore github action check: " + run.id); - } - - }); - - } catch (e) { - console.log("check bot run failed: " + e); - } - -} - -function rebuild(buildId) { - const https = require('https'); - const token = process.env.HUDI_TRAVIS_ORG_TOKEN - - const options = { - hostname: 'api.travis-ci.org', - port: 443, - path: `/build/${buildId}/restart`, - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Travis-API-Version': 3, - 'Authorization': `token ${token}`, - } - }; - - const req = https.request(options, function (res) { - res.setEncoding('utf8'); - res.on('data', function (data) { - console.log('data: ' + data); - }); - res.on('error', function (error) { - console.log('error: ' + error); - }); - }); - req.on('error', function (e) { - console.log('problem with request: ' + e.message); - }); - - req.end(); -} - -module.exports = ({core}, {context}, {github}) => { - return check(core, context, github); -} diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index ca34cc37e..3cc43fe75 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1,43 +1,35 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI BOT +name: Java CI on: - issue_comment: - types: [created] + push: + branches: + - master + - 'release-*' + pull_request: + branches: + - master + - 'release-*' jobs: - bot: + build: runs-on: ubuntu-latest + strategy: + matrix: + include: + - scala: "scala-2.11" + spark: "spark2" + - scala: "scala-2.12" + spark: "spark3" steps: - - name: clone repository - uses: actions/checkout@v2 - - - name: bot actions - uses: actions/github-script@v1 - env: - PROVIDER : 'apache' - REPOSITORY: 'hudi' - HUDI_TRAVIS_ORG_TOKEN: ${{ secrets.HUDI_TRAVIS_ORG_TOKEN }} + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 with: - script: | - const path = require('path') - const scriptPath = path.resolve('.github/actions/bot/src/action.js') - require(scriptPath)({core}, {context}, {github}) \ No newline at end of file + java-version: '8' + distribution: 'adopt' + architecture: x64 + - name: Build Project + env: + SCALA_PROFILE: ${{ matrix.scala }} + SPAKR_PROFILE: ${{ matrix.spark }} + run: mvn install -P "$SCALA_PROFILE,$SPAKR_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 532099c55..000000000 --- a/.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -os: linux -dist: trusty -language: java -jdk: - - openjdk8 -jobs: - include: - - name: "Unit tests for hudi-spark-client" - env: MODE=unit MODULES=hudi-client/hudi-spark-client HUDI_QUIETER_LOGGING=1 - - name: "Unit tests for hudi-utilities" - env: MODE=unit MODULES=hudi-utilities HUDI_QUIETER_LOGGING=1 - - name: "All other unit tests" - env: MODE=unit MODULES='!hudi-utilities,!hudi-client/hudi-spark-client' HUDI_QUIETER_LOGGING=1 - - name: "Functional tests" - env: MODE=functional HUDI_QUIETER_LOGGING=1 - - name: "Integration tests" - env: MODE=integration HUDI_QUIETER_LOGGING=1 -install: true -services: - - docker -cache: - directories: - - "$HOME/.m2" -notifications: - slack: - rooms: - - secure: WNIZPBY//xf/xTJL1YUPzvPUDwjawaMM4IJ6IqxjRGcZCmuhNVu2XTJ3aL1g6X7ZcJKxJuwoU/TbSO8Dl6rgWSo/2OfyzBd4ks+hgeCsdycccTcvO8giQO1DOUGUSRdvUzOvKjWVK7iARYzQhoZawAYwI09UJLlwhYRCJ1IKc1ZksrEt964GeEmPyJbwMoZOJVUU84jJIAZPIpOFGTKM652FMermg9yaY2W5oSjDXaV98z0/mJV4Ry++J2v0fvoDs5HxkXYhZJP+dpWR82KDr6Q6LGL5/IlJ+b+IH3pF8LyKR4nCH6l1EZ8KpoFZapyYWYQpXMfQoF2K/JEQkpz1EqBCeEDSJ2+j1PPLhOWXd7ok4DsS26S8BP2ImvyXwua51THN1/r1fCGSIdxiQ5C8aeYmPCSr+oLChCVivEG2eeU34Z1nQJ5aDymNGeFE9qUUpjS0ETfFcjI/WQaA+FiYiPkDfeAoT1+6ySdY7l9gJhMygupILjq57IHbqx4nEr/8AB3Rqb8iIDTWDXgUBI9xKmty36zjIGcVOsCT/SGPccxvEJBXQk8uQqs/rDhaA/ErJPMLX/2b7ElSSObKFdjpMaxVvZIE6wvMLJpIYfChDoXwgfhN6zlAFZrEib7PFI4dGkS8u4wkkHkBS7C+uz2e92EhsAB+BIhUR1M3NQ33+Is= - on_pull_requests: false -script: - # ping stdout every 9 minutes or Travis kills build - # https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received - - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done & - - scripts/run_travis_tests.sh $MODE $MODULES -after_success: - - scripts/report_coverage.sh diff --git a/pom.xml b/pom.xml index 0fd756ee4..65e391ab1 100644 --- a/pom.xml +++ b/pom.xml @@ -1418,7 +1418,6 @@ scala-2.11 - scala-2.12 @@ -1460,6 +1459,10 @@ + + + spark2 + spark3 diff --git a/scripts/report_coverage.sh b/scripts/report_coverage.sh deleted file mode 100755 index 79bbce00f..000000000 --- a/scripts/report_coverage.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -modules=( - hudi-cli - hudi-client - hudi-common - hudi-examples - hudi-flink - hudi-hadoop-mr - hudi-integ-test - hudi-spark-datasource - hudi-sync - hudi-timeline-service - hudi-utilities -) -for module in "${modules[@]}"; do - bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}" -done diff --git a/scripts/run_travis_tests.sh b/scripts/run_travis_tests.sh deleted file mode 100755 index 63fb959c9..000000000 --- a/scripts/run_travis_tests.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -mode=$1 -modules=$2 -sparkVersion=2.4.4 -hadoopVersion=2.7 - -if [ "$mode" = "unit" ]; then - mvn clean install -DskipTests -q - echo "Running Unit Tests" - mvn test -Punit-tests -pl "$modules" -B -elif [ "$mode" = "functional" ]; then - echo "Running Functional Tests" - mvn test -Pfunctional-tests -B -elif [ "$mode" = "integration" ]; then - echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}" - wget http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz -O /tmp/spark-${sparkVersion}.tgz - tar -xvf /tmp/spark-${sparkVersion}.tgz - export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion} - mkdir /tmp/spark-events/ - echo "Running Integration Tests" - mvn verify -Pintegration-tests -B -else - echo "Unknown mode $mode" - exit 1 -fi