119 lines
4.6 KiB
YAML
119 lines
4.6 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- '*' # must quote since "*" is a YAML reserved character; we want a string
|
|
|
|
pool:
|
|
vmImage: 'ubuntu-16.04'
|
|
|
|
variables:
|
|
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
|
|
MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
|
|
SPARK_VERSION: '2.4.4'
|
|
HADOOP_VERSION: '2.7'
|
|
SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
|
|
|
|
stages:
|
|
- stage: test
|
|
displayName: "Run tests"
|
|
jobs:
|
|
- job: unit_tests_spark_client
|
|
steps:
|
|
- task: Cache@2
|
|
inputs:
|
|
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
|
restoreKeys: |
|
|
maven | "$(Agent.OS)"
|
|
maven
|
|
path: $(MAVEN_CACHE_FOLDER)
|
|
displayName: Cache Maven local repo
|
|
- script: |
|
|
mvn $(MAVEN_OPTS) clean install -DskipTests
|
|
- task: Maven@3
|
|
inputs:
|
|
mavenPomFile: 'pom.xml'
|
|
goals: 'test'
|
|
options: -Punit-tests -pl hudi-client/hudi-spark-client
|
|
publishJUnitResults: false
|
|
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
|
testRunTitle: 'unit tests spark client'
|
|
javaHomeOption: 'JDKVersion'
|
|
jdkVersionOption: '1.8'
|
|
jdkArchitectureOption: 'x64'
|
|
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
|
- job: unit_tests_utilities
|
|
steps:
|
|
- task: Cache@2
|
|
inputs:
|
|
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
|
restoreKeys: |
|
|
maven | "$(Agent.OS)"
|
|
maven
|
|
path: $(MAVEN_CACHE_FOLDER)
|
|
displayName: Cache Maven local repo
|
|
- script: |
|
|
mvn $(MAVEN_OPTS) clean install -DskipTests
|
|
- task: Maven@3
|
|
inputs:
|
|
mavenPomFile: 'pom.xml'
|
|
goals: 'test'
|
|
options: -Punit-tests -pl hudi-utilities
|
|
publishJUnitResults: false
|
|
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
|
testRunTitle: 'unit tests utilities'
|
|
javaHomeOption: 'JDKVersion'
|
|
jdkVersionOption: '1.8'
|
|
jdkArchitectureOption: 'x64'
|
|
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
|
- job: unit_tests_other_modules
|
|
steps:
|
|
- task: Cache@2
|
|
inputs:
|
|
key: 'maven | "$(Agent.OS)" | **/pom.xml'
|
|
restoreKeys: |
|
|
maven | "$(Agent.OS)"
|
|
maven
|
|
path: $(MAVEN_CACHE_FOLDER)
|
|
displayName: Cache Maven local repo
|
|
- script: |
|
|
mvn $(MAVEN_OPTS) clean install -DskipTests
|
|
- task: Maven@3
|
|
inputs:
|
|
mavenPomFile: 'pom.xml'
|
|
goals: 'test'
|
|
options: -Punit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client
|
|
publishJUnitResults: false
|
|
testResultsFiles: '**/surefire-reports/TEST-*.xml'
|
|
testRunTitle: 'unit tests other modules'
|
|
javaHomeOption: 'JDKVersion'
|
|
jdkVersionOption: '1.8'
|
|
jdkArchitectureOption: 'x64'
|
|
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
|
|
- job: functional_tests
|
|
steps:
|
|
- script: |
|
|
mvn $(MAVEN_OPTS) -Pfunctional-tests test
|
|
- job: integration_tests
|
|
steps:
|
|
- script: |
|
|
echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
|
|
wget https://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz
|
|
tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/
|
|
mkdir /tmp/spark-events/
|
|
mvn -Pintegration-tests verify
|