lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Azure CI IT tasks clean up (#4337)

Browse Source
This commit is contained in:
Raymond Xu
2021-12-18 17:00:56 -08:00
committed by GitHub
parent 77abb5ccb9
commit f57e28fe39
1 changed files with 10 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
azure-pipelines.yml
View File

@@ -26,7 +26,7 @@ variables:
MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true' MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
SPARK_VERSION: '2.4.4' SPARK_VERSION: '2.4.4'
HADOOP_VERSION: '2.7' HADOOP_VERSION: '2.7'
SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION) SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
stages: stages:
- stage: test - stage: test
@@ -47,7 +47,7 @@ stages:
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'install' goals: 'install'
options: -DskipTests options: -T 2.5C -DskipTests
publishJUnitResults: false publishJUnitResults: false
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -85,7 +85,7 @@ stages:
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'install' goals: 'install'
options: -DskipTests options: -T 2.5C -DskipTests
publishJUnitResults: false publishJUnitResults: false
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -114,7 +114,7 @@ stages:
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'install' goals: 'install'
options: -DskipTests options: -T 2.5C -DskipTests
publishJUnitResults: false publishJUnitResults: false
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -152,7 +152,7 @@ stages:
inputs: inputs:
mavenPomFile: 'pom.xml' mavenPomFile: 'pom.xml'
goals: 'install' goals: 'install'
options: -DskipTests options: -T 2.5C -DskipTests
publishJUnitResults: false publishJUnitResults: false
jdkVersionOption: '1.8' jdkVersionOption: '1.8'
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -177,18 +177,16 @@ stages:
- job: IT - job: IT
steps: steps:
- task: AzureCLI@2 - task: AzureCLI@2
displayName: Azure CLI displayName: Prepare for IT
inputs: inputs:
azureSubscription: apachehudici-service-connection azureSubscription: apachehudici-service-connection
scriptType: bash scriptType: bash
scriptLocation: inlineScript scriptLocation: inlineScript
inlineScript: | inlineScript: |
az --version echo 'Downloading $(SPARK_ARCHIVE)'
az account show az storage blob download -c ci-caches -n $(SPARK_ARCHIVE).tgz -f $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz --account-name apachehudici
echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)' tar -xvf $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz -C $(Pipeline.Workspace)/
az storage blob download -c ci-caches -n spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -f $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz --account-name apachehudici mkdir /tmp/spark-events/
tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -C $(Pipeline.Workspace)/
- script: | - script: |
mkdir /tmp/spark-events/
mvn $(MAVEN_OPTS) -Pintegration-tests verify mvn $(MAVEN_OPTS) -Pintegration-tests verify
displayName: IT displayName: IT