[HOTFIX] close spark session in functional test suite and disable spark3 test for spark2 (#2727)

hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java

@@ -152,6 +152,8 @@ public class FunctionalTestHarness implements SparkProvider, DFSProvider, Hoodie
         hdfsTestService.stop();
         hdfsTestService = null;
 
+        jsc.close();
+        jsc = null;
         spark.stop();
         spark = null;
       }));
@@ -166,5 +168,19 @@ public class FunctionalTestHarness implements SparkProvider, DFSProvider, Hoodie
     for (FileStatus f : fileStatuses) {
       fs.delete(f.getPath(), true);
     }
+    if (hdfsTestService != null) {
+      hdfsTestService.stop();
+      hdfsTestService = null;
+    }
+    if (spark != null) {
+      spark.stop();
+      spark = null;
+    }
+    if (jsc != null) {
+      jsc.close();
+      jsc = null;
+    }
+    sqlContext = null;
+    context = null;
   }
 }

hudi-spark-datasource/hudi-spark2/pom.xml

@@ -151,7 +151,7 @@
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
-      <version>${scala11.version}</version>
+      <version>${scala.version}</version>
     </dependency>
 
     <!-- Hoodie -->

hudi-spark-datasource/hudi-spark3/pom.xml

@@ -124,6 +124,13 @@
           <skip>false</skip>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <skipTests>${skip.hudi-spark3.unit.tests}</skipTests>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.rat</groupId>
         <artifactId>apache-rat-plugin</artifactId>

hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java

@@ -124,15 +124,19 @@ public class UtilitiesTestBase {
   public static void cleanupClass() {
     if (hdfsTestService != null) {
       hdfsTestService.stop();
+      hdfsTestService = null;
     }
     if (hiveServer != null) {
       hiveServer.stop();
+      hiveServer = null;
     }
     if (hiveTestService != null) {
       hiveTestService.stop();
+      hiveTestService = null;
     }
     if (zookeeperTestService != null) {
       zookeeperTestService.stop();
+      zookeeperTestService = null;
     }
   }
 
@@ -150,6 +154,11 @@ public class UtilitiesTestBase {
     TestDataSource.resetDataGen();
     if (jsc != null) {
       jsc.stop();
+      jsc = null;
+    }
+    if (sparkSession != null) {
+      sparkSession.close();
+      sparkSession = null;
     }
     if (context != null) {
       context = null;

pom.xml

@@ -133,6 +133,7 @@
     <skipFTs>${skipTests}</skipFTs>
     <skipITs>${skipTests}</skipITs>
     <skip.hudi-spark2.unit.tests>${skipTests}</skip.hudi-spark2.unit.tests>
+    <skip.hudi-spark3.unit.tests>${skipTests}</skip.hudi-spark3.unit.tests>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <main.basedir>${project.basedir}</main.basedir>
     <spark.bundle.hive.scope>provided</spark.bundle.hive.scope>
@@ -1424,6 +1425,7 @@
       <properties>
         <scala.version>${scala12.version}</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
+        <skip.hudi-spark3.unit.tests>true</skip.hudi-spark3.unit.tests>
       </properties>
       <activation>
         <property>