lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2399] Rebalance CI jobs for shorter wait time (#3604)

Browse Source
This commit is contained in:
Raymond Xu
2021-09-05 09:25:57 -07:00
committed by GitHub
parent e9bf1c1186
commit 7592ddd776
1 changed files with 100 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
azure-pipelines.yml
View File

@@ -23,92 +23,158 @@ pool:
variables: variables:
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true' MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
SPARK_VERSION: '2.4.4' SPARK_VERSION: '2.4.4'
HADOOP_VERSION: '2.7' HADOOP_VERSION: '2.7'
SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION) SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
stages: stages:
- stage: test - stage: test
displayName: "Run tests"
jobs: jobs:
- job: unit_tests_spark_client - job: UT_FT_1
displayName: UT FT common & flink & UT client/spark-client
steps: steps:
- task: Cache@2 - task: Cache@2
displayName: set cache
inputs: inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml' key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: | restoreKeys: |
maven | "$(Agent.OS)" maven | "$(Agent.OS)"
maven maven
path: $(MAVEN_CACHE_FOLDER) path: $(MAVEN_CACHE_FOLDER)
displayName: Cache Maven local repo
- script: |
mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3 - task: Maven@3
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
goals: 'install'
options: -DskipTests
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT common flink client/spark-client
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'test' goals: 'test'
options: -Punit-tests -pl hudi-client/hudi-spark-client options: -Punit-tests -pl hudi-common,hudi-flink,hudi-client/hudi-spark-client
publishJUnitResults: false publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests spark client'
javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: unit_tests_utilities - task: Maven@3
displayName: FT common flink
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl hudi-common,hudi-flink
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: UT_FT_2
displayName: FT client/spark-client
steps: steps:
- task: Cache@2 - task: Cache@2
displayName: set cache
inputs: inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml' key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: | restoreKeys: |
maven | "$(Agent.OS)" maven | "$(Agent.OS)"
maven maven
path: $(MAVEN_CACHE_FOLDER) path: $(MAVEN_CACHE_FOLDER)
displayName: Cache Maven local repo
- script: |
mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3 - task: Maven@3
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
goals: 'install'
options: -DskipTests
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- task: Maven@3
displayName: FT client/spark-client
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'test' goals: 'test'
options: -Punit-tests -pl hudi-utilities options: -Pfunctional-tests -pl hudi-client/hudi-spark-client
publishJUnitResults: false publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests utilities'
javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: unit_tests_other_modules - job: UT_FT_3
displayName: UT FT cli & utilities & sync/hive-sync
steps: steps:
- task: Cache@2 - task: Cache@2
displayName: set cache
inputs: inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml' key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: | restoreKeys: |
maven | "$(Agent.OS)" maven | "$(Agent.OS)"
maven maven
path: $(MAVEN_CACHE_FOLDER) path: $(MAVEN_CACHE_FOLDER)
displayName: Cache Maven local repo
- script: |
mvn $(MAVEN_OPTS) clean install -DskipTests
- task: Maven@3 - task: Maven@3
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
goals: 'install'
options: -DskipTests
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT cli & utilities & sync/hive-sync
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'test' goals: 'test'
options: -Punit-tests -pl !hudi-utilities,!hudi-client/hudi-spark-client options: -Punit-tests -pl hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
publishJUnitResults: false publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests other modules'
javaHomeOption: 'JDKVersion'
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
jdkArchitectureOption: 'x64'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: functional_tests - task: Maven@3
displayName: FT cli & utilities & sync/hive-sync
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: UT_FT_4
displayName: UT FT other modules
steps: steps:
- script: | - task: Cache@2
mvn $(MAVEN_OPTS) -Pfunctional-tests test displayName: set cache
- job: integration_tests inputs:
key: 'maven | "$(Agent.OS)" | **/pom.xml'
restoreKeys: |
maven | "$(Agent.OS)"
maven
path: $(MAVEN_CACHE_FOLDER)
- task: Maven@3
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
goals: 'install'
options: -DskipTests
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT other modules
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Punit-tests -pl !hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- task: Maven@3
displayName: FT other modules
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl !hudi-common,!hudi-flink,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: IT
steps: steps:
- script: | - script: |
echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)' echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
@@ -116,3 +182,4 @@ stages:
tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/ tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/
mkdir /tmp/spark-events/ mkdir /tmp/spark-events/
mvn $(MAVEN_OPTS) -Pintegration-tests verify mvn $(MAVEN_OPTS) -Pintegration-tests verify
displayName: IT