[HUDI-1620] Add azure pipelines configs (#2582)
This commit is contained in:
Raymond Xu
165
azure-pipelines.yml
Normal file
165
azure-pipelines.yml
Normal file
@@ -0,0 +1,165 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- '*' # must quote since "*" is a YAML reserved character; we want a string
|
||||
|
||||
pool:
|
||||
vmImage: 'ubuntu-16.04'
|
||||
|
||||
variables:
|
||||
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
|
||||
MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
|
||||
SPARK_VERSION: '2.4.4'
|
||||
HADOOP_VERSION: '2.7'
|
||||
SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
|
||||
SKIP_IT: true
|
||||
|
||||
stages:
|
||||
- stage: test
|
||||
displayName: "Run tests"
|
||||
jobs:
|
||||
- job: unit_tests_spark_client
|
||||
steps:
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'clean install'
|
||||
options: -DskipTests
|
||||
publishJUnitResults: false
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
||||
restoreKeys: |
|
||||
maven | "$(Agent.OS)"
|
||||
maven
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
displayName: Cache Maven local repo
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'test'
|
||||
options: -Pazp-unit-tests -pl hudi-client/hudi-spark-client
|
||||
publishJUnitResults: false
|
||||
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
||||
testRunTitle: 'unit tests spark client'
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- job: unit_tests_utilities
|
||||
steps:
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'clean install'
|
||||
options: -DskipTests
|
||||
publishJUnitResults: false
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
||||
restoreKeys: |
|
||||
maven | "$(Agent.OS)"
|
||||
maven
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
displayName: Cache Maven local repo
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'test'
|
||||
options: -Pazp-unit-tests -pl hudi-utilities
|
||||
publishJUnitResults: false
|
||||
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
||||
testRunTitle: 'unit tests utilities'
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- job: unit_tests_other_modules
|
||||
steps:
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'clean install'
|
||||
options: -DskipTests
|
||||
publishJUnitResults: false
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
||||
restoreKeys: |
|
||||
maven | "$(Agent.OS)"
|
||||
maven
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
displayName: Cache Maven local repo
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'test'
|
||||
options: -Pazp-unit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client
|
||||
publishJUnitResults: false
|
||||
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
||||
testRunTitle: 'unit tests other modules'
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- job: functional_tests
|
||||
steps:
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'test'
|
||||
options: -Pfunctional-tests
|
||||
publishJUnitResults: false
|
||||
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
||||
testRunTitle: 'functional tests'
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
- job: integration_tests
|
||||
condition: eq(variables.SKIP_IT, false)
|
||||
steps:
|
||||
- script: |
|
||||
echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
|
||||
wget http://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O /tmp/spark-$(SPARK_VERSION).tgz
|
||||
tar -xvf /tmp/spark-$(SPARK_VERSION).tgz
|
||||
mkdir /tmp/spark-events/
|
||||
- task: Maven@3
|
||||
inputs:
|
||||
mavenPomFile: 'pom.xml'
|
||||
goals: 'verify'
|
||||
options: -Pintegration-tests
|
||||
publishJUnitResults: false
|
||||
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
||||
testRunTitle: 'unit tests utilities'
|
||||
javaHomeOption: 'JDKVersion'
|
||||
jdkVersionOption: '1.8'
|
||||
jdkArchitectureOption: 'x64'
|
||||
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
||||
Reference in New Issue
Block a user