[HUDI-2280] Use GitHub Actions to build different scala spark versions (#3556)

.github/actions/bot/package.json

@@ -1,36 +0,0 @@
-{
-  "name": "github_action_ci_bot",
-  "version": "1.1.1",
-  "description": "CI Bot for GitHub Actions",
-  "main": "dist/index.js",
-  "scripts": {
-    "lint": "eslint 'src/**.js' 'tests/**.js' --fix",
-    "test": "eslint 'src/**.js' 'tests/**.js' && jest --coverage",
-    "build": "ncc build src/action.js"
-  },
-  "author": "lamber-ken",
-  "license": "Apache LICENSE 2.0",
-  "homepage": "https://github.com/apache/hudi",
-  "bugs": {
-    "url": "https://github.com/apache/hudi/issues"
-  },
-  "dependencies": {
-    "@actions/core": "^1.2.4",
-    "@actions/github": "^2.2.0",
-    "@actions/io": "^1.0.2"
-  },
-  "devDependencies": {
-    "@types/jest": "^25.1.4",
-    "@typescript-eslint/eslint-plugin": "^2.33.0",
-    "@typescript-eslint/parser": "^2.33.0",
-    "@zeit/ncc": "^0.22.0",
-    "eslint": "^7.0.0",
-    "eslint-config-prettier": "^6.11.0",
-    "husky": "^4.2.5",
-    "jest": "^25.1.0",
-    "npm-run-all": "^4.1.5",
-    "prettier": "^2.0.5",
-    "ts-jest": "^25.2.1",
-    "typescript": "^3.8.3"
-  }
-}

.github/actions/bot/src/action.js

@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-async function check(core, context, github) {
-
-    try {
-        const provider = process.env.PROVIDER;
-        const repository = process.env.REPOSITORY;
-        const command = context.payload.comment.body;
-
-        if (command !== 'rerun tests') {
-            console.log("Invalid command:" + command);
-            return;
-        }
-
-        const {
-            data: {
-                head: {
-                    sha: ref,
-                }
-            }
-        } = await github.pulls.get({
-            owner: provider,
-            repo: repository,
-            pull_number: context.issue.number,
-        });
-
-        const checks = await github.checks.listForRef({
-            owner: provider,
-            repo: repository,
-            ref: ref
-        });
-
-        checks.data.check_runs.forEach(run => {
-
-            if (run.app.owner.login === 'travis-ci') {
-                console.log("rerun travis ci check: " + run.external_id);
-                rebuild(run.external_id)
-            } else {
-                console.log("ignore github action check: " + run.id);
-            }
-
-        });
-
-    } catch (e) {
-        console.log("check bot run failed: " + e);
-    }
-
-}
-
-function rebuild(buildId) {
-    const https = require('https');
-    const token = process.env.HUDI_TRAVIS_ORG_TOKEN
-
-    const options = {
-        hostname: 'api.travis-ci.org',
-        port: 443,
-        path: `/build/${buildId}/restart`,
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-            'Travis-API-Version': 3,
-            'Authorization': `token ${token}`,
-        }
-    };
-
-    const req = https.request(options, function (res) {
-        res.setEncoding('utf8');
-        res.on('data', function (data) {
-            console.log('data: ' + data);
-        });
-        res.on('error', function (error) {
-            console.log('error: ' + error);
-        });
-    });
-    req.on('error', function (e) {
-        console.log('problem with request: ' + e.message);
-    });
-
-    req.end();
-}
-
-module.exports = ({core}, {context}, {github}) => {
-    return check(core, context, github);
-}

.github/workflows/bot.yml

@@ -1,43 +1,35 @@
-#
+name: Java CI
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI BOT
 
 on:
-  issue_comment:
+  push:
-    types: [created]
+    branches:
+      - master
+      - 'release-*'
+  pull_request:
+    branches:
+      - master
+      - 'release-*'
 
 jobs:
-  bot:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - scala: "scala-2.11"
+            spark: "spark2"
+          - scala: "scala-2.12"
+            spark: "spark3"
     steps:
-      - name: clone repository
+      - uses: actions/checkout@v2
-        uses: actions/checkout@v2
+      - name: Set up JDK 8
-
+        uses: actions/setup-java@v2
-      - name: bot actions
-        uses: actions/github-script@v1
-        env:
-          PROVIDER : 'apache'
-          REPOSITORY: 'hudi'
-          HUDI_TRAVIS_ORG_TOKEN: ${{ secrets.HUDI_TRAVIS_ORG_TOKEN }}
         with:
-          script: |
+          java-version: '8'
-            const path = require('path')
+          distribution: 'adopt'
-            const scriptPath = path.resolve('.github/actions/bot/src/action.js')
+          architecture: x64
-            require(scriptPath)({core}, {context}, {github})
+      - name: Build Project
+        env:
+          SCALA_PROFILE: ${{ matrix.scala }}
+          SPAKR_PROFILE: ${{ matrix.spark }}
+        run: mvn install -P "$SCALA_PROFILE,$SPAKR_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V

.travis.yml

@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-os: linux
-dist: trusty
-language: java
-jdk:
-  - openjdk8
-jobs:
-  include:
-    - name: "Unit tests for hudi-spark-client"
-      env: MODE=unit MODULES=hudi-client/hudi-spark-client HUDI_QUIETER_LOGGING=1
-    - name: "Unit tests for hudi-utilities"
-      env: MODE=unit MODULES=hudi-utilities HUDI_QUIETER_LOGGING=1
-    - name: "All other unit tests"
-      env: MODE=unit MODULES='!hudi-utilities,!hudi-client/hudi-spark-client' HUDI_QUIETER_LOGGING=1
-    - name: "Functional tests"
-      env: MODE=functional HUDI_QUIETER_LOGGING=1
-    - name: "Integration tests"
-      env: MODE=integration HUDI_QUIETER_LOGGING=1
-install: true
-services:
-  - docker
-cache:
-  directories:
-    - "$HOME/.m2"
-notifications:
-  slack:
-    rooms:
-      - secure: WNIZPBY//xf/xTJL1YUPzvPUDwjawaMM4IJ6IqxjRGcZCmuhNVu2XTJ3aL1g6X7ZcJKxJuwoU/TbSO8Dl6rgWSo/2OfyzBd4ks+hgeCsdycccTcvO8giQO1DOUGUSRdvUzOvKjWVK7iARYzQhoZawAYwI09UJLlwhYRCJ1IKc1ZksrEt964GeEmPyJbwMoZOJVUU84jJIAZPIpOFGTKM652FMermg9yaY2W5oSjDXaV98z0/mJV4Ry++J2v0fvoDs5HxkXYhZJP+dpWR82KDr6Q6LGL5/IlJ+b+IH3pF8LyKR4nCH6l1EZ8KpoFZapyYWYQpXMfQoF2K/JEQkpz1EqBCeEDSJ2+j1PPLhOWXd7ok4DsS26S8BP2ImvyXwua51THN1/r1fCGSIdxiQ5C8aeYmPCSr+oLChCVivEG2eeU34Z1nQJ5aDymNGeFE9qUUpjS0ETfFcjI/WQaA+FiYiPkDfeAoT1+6ySdY7l9gJhMygupILjq57IHbqx4nEr/8AB3Rqb8iIDTWDXgUBI9xKmty36zjIGcVOsCT/SGPccxvEJBXQk8uQqs/rDhaA/ErJPMLX/2b7ElSSObKFdjpMaxVvZIE6wvMLJpIYfChDoXwgfhN6zlAFZrEib7PFI4dGkS8u4wkkHkBS7C+uz2e92EhsAB+BIhUR1M3NQ33+Is=
-    on_pull_requests: false
-script:
-  # ping stdout every 9 minutes or Travis kills build
-  # https://docs.travis-ci.com/user/common-build-problems/#Build-times-out-because-no-output-was-received
-  - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
-  - scripts/run_travis_tests.sh $MODE $MODULES
-after_success:
-  - scripts/report_coverage.sh

pom.xml

@@ -1418,7 +1418,6 @@
     <profile>
       <id>scala-2.11</id>
     </profile>
-
     <profile>
       <id>scala-2.12</id>
       <properties>
@@ -1460,6 +1459,10 @@
       </build>
     </profile>
 
+    <!-- Exists for backwards compatibility; profile doesn't do anything -->
+    <profile>
+      <id>spark2</id>
+    </profile>
     <profile>
       <id>spark3</id>
       <properties>

scripts/report_coverage.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-
-modules=(
-  hudi-cli
-  hudi-client
-  hudi-common
-  hudi-examples
-  hudi-flink
-  hudi-hadoop-mr
-  hudi-integ-test
-  hudi-spark-datasource
-  hudi-sync
-  hudi-timeline-service
-  hudi-utilities
-)
-for module in "${modules[@]}"; do
-  bash <(curl -s https://codecov.io/bash) -s "$module" -F "${module//-/}"
-done

scripts/run_travis_tests.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-
-mode=$1
-modules=$2
-sparkVersion=2.4.4
-hadoopVersion=2.7
-
-if [ "$mode" = "unit" ]; then
-  mvn clean install -DskipTests -q
-  echo "Running Unit Tests"
-  mvn test -Punit-tests -pl "$modules" -B
-elif [ "$mode" = "functional" ]; then
-  echo "Running Functional Tests"
-  mvn test -Pfunctional-tests -B
-elif [ "$mode" = "integration" ]; then
-  echo "Downloading Apache Spark-${sparkVersion}-bin-hadoop${hadoopVersion}"
-  wget http://archive.apache.org/dist/spark/spark-${sparkVersion}/spark-${sparkVersion}-bin-hadoop${hadoopVersion}.tgz -O /tmp/spark-${sparkVersion}.tgz
-  tar -xvf /tmp/spark-${sparkVersion}.tgz
-  export SPARK_HOME=$PWD/spark-${sparkVersion}-bin-hadoop${hadoopVersion}
-  mkdir /tmp/spark-events/
-  echo "Running Integration Tests"
-  mvn verify -Pintegration-tests -B
-else
-  echo "Unknown mode $mode"
-  exit 1
-fi