hudi/azure-pipelines.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

trigger:
  branches:
    include:
      - '*'  # must quote since "*" is a YAML reserved character; we want a string

pool:
  vmImage: 'ubuntu-18.04'

variables:
  MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
  SPARK_VERSION: '2.4.4'
  HADOOP_VERSION: '2.7'
  SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)

stages:
  - stage: test
    jobs:
      - job: UT_FT_1
        displayName: UT FT common & flink & UT client/spark-client
        steps:
          - task: Cache@2
            displayName: set cache
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
          - task: Maven@3
            displayName: maven install
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'install'
              options: -DskipTests
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: UT common flink client/spark-client
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl hudi-common,hudi-flink,hudi-client/hudi-spark-client
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: FT common flink
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Pfunctional-tests -pl hudi-common,hudi-flink
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: UT_FT_2
        displayName: FT client/spark-client
        steps:
          - task: Cache@2
            displayName: set cache
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
          - task: Maven@3
            displayName: maven install
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'install'
              options: -DskipTests
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: FT client/spark-client
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Pfunctional-tests -pl hudi-client/hudi-spark-client
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: UT_FT_3
        displayName: UT FT cli & utilities & sync/hive-sync
        steps:
          - task: Cache@2
            displayName: set cache
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
          - task: Maven@3
            displayName: maven install
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'install'
              options: -DskipTests
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: UT cli & utilities & sync/hive-sync
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: FT cli & utilities & sync/hive-sync
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Pfunctional-tests -pl hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: UT_FT_4
        displayName: UT FT other modules
        steps:
          - task: Cache@2
            displayName: set cache
            inputs:
              key: 'maven | "$(Agent.OS)" | **/pom.xml'
              restoreKeys: |
                maven | "$(Agent.OS)"
                maven
              path: $(MAVEN_CACHE_FOLDER)
          - task: Maven@3
            displayName: maven install
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'install'
              options: -DskipTests
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: UT other modules
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Punit-tests -pl !hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
          - task: Maven@3
            displayName: FT other modules
            inputs:
              mavenPomFile: 'pom.xml'
              goals: 'test'
              options: -Pfunctional-tests -pl !hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
              publishJUnitResults: false
              jdkVersionOption: '1.8'
              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
      - job: IT
        steps:
          - script: |
              echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
              wget https://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz
              tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/
              mkdir /tmp/spark-events/
              mvn $(MAVEN_OPTS) -Pintegration-tests verify
            displayName: IT