From f57e28fe396951ce6e4690373b6e75035362409f Mon Sep 17 00:00:00 2001
From: Raymond Xu <2701446+xushiyan@users.noreply.github.com>
Date: Sat, 18 Dec 2021 17:00:56 -0800
Subject: [PATCH] [MINOR] Azure CI IT tasks clean up (#4337)

---
 azure-pipelines.yml | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6d96ed51f..88eddcc3d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -26,7 +26,7 @@ variables:
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
   SPARK_VERSION: '2.4.4'
   HADOOP_VERSION: '2.7'
-  SPARK_HOME: $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
+  SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
 
 stages:
   - stage: test
@@ -47,7 +47,7 @@ stages:
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'install'
-              options: -DskipTests
+              options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -85,7 +85,7 @@ stages:
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'install'
-              options: -DskipTests
+              options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -114,7 +114,7 @@ stages:
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'install'
-              options: -DskipTests
+              options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -152,7 +152,7 @@ stages:
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'install'
-              options: -DskipTests
+              options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -177,18 +177,16 @@ stages:
       - job: IT
         steps:
           - task: AzureCLI@2
-            displayName: Azure CLI
+            displayName: Prepare for IT
             inputs:
               azureSubscription: apachehudici-service-connection
               scriptType: bash
               scriptLocation: inlineScript
               inlineScript: |
-                az --version
-                az account show
-                echo 'Downloading spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)'
-                az storage blob download -c ci-caches -n spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -f $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz --account-name apachehudici
-                tar -xvf $(Pipeline.Workspace)/spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION).tgz -C $(Pipeline.Workspace)/
+                echo 'Downloading $(SPARK_ARCHIVE)'
+                az storage blob download -c ci-caches -n $(SPARK_ARCHIVE).tgz -f $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz --account-name apachehudici
+                tar -xvf $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz -C $(Pipeline.Workspace)/
+                mkdir /tmp/spark-events/
           - script: |
-              mkdir /tmp/spark-events/
               mvn $(MAVEN_OPTS) -Pintegration-tests verify
             displayName: IT