lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3028] Use blob storage to speed up CI downloads (#4331)

Browse Source 
Co-authored-by: Sivabalan Narayanan <n.siva.b@gmail.com>
This commit is contained in:
Raymond Xu
2021-12-15 15:33:33 -08:00
committed by GitHub
parent 3b89457e2b
commit 27907dea4c
1 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
azure-pipelines.yml
View File

@@ -176,10 +176,19 @@ stages:
mavenOptions: '-Xmx2g $(MAVEN_OPTS)' mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
- job: IT - job: IT
steps: steps:
- script: | - task: AzureCLI@2
displayName: Azure CLI
inputs:
azureSubscription: apachehudici-service-connection
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
az --version
az account show
echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)' echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
wget https://archive.apache.org/dist/spark/spark-$(SPARK_VERSION)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -O $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz az storage blob download -c ci-caches -n spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -f $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz --account-name apachehudici
tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION).tgz -C $(Pipeline.Workspace)/ tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -C $(Pipeline.Workspace)/
- script: |
mkdir /tmp/spark-events/ mkdir /tmp/spark-events/
mvn $(MAVEN_OPTS) -Pintegration-tests verify mvn $(MAVEN_OPTS) -Pintegration-tests verify
displayName: IT displayName: IT