[HUDI-2195] Sync Hive Failed When Execute CTAS In Spark2 And Spark3 (#3299)

hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala

@@ -181,8 +181,7 @@ object HoodieSqlUtils extends SparkAdapterSupport {
   }
 
   /**
-   * Append the SparkSession config and table options to the baseConfig.
-   * We add the "spark" prefix to hoodie's config key.
+   * Append the spark config and table options to the baseConfig.
    */
   def withSparkConf(spark: SparkSession, options: Map[String, String])
                    (baseConfig: Map[String, String]): Map[String, String] = {

packaging/hudi-spark-bundle/pom.xml

@@ -115,6 +115,8 @@
                   <include>org.apache.curator:curator-client</include>
                   <include>org.apache.curator:curator-recipes</include>
                   <include>commons-codec:commons-codec</include>
+                  <include>org.json:json</include>
+                  <include>org.apache.calcite:calcite-core</include>
                 </includes>
               </artifactSet>
               <relocations>
@@ -367,7 +369,18 @@
       <artifactId>curator-recipes</artifactId>
       <version>${zk-curator.version}</version>
     </dependency>
-
+<!--    import json to fix the crash when sync meta in spark-->
+    <dependency>
+      <groupId>org.json</groupId>
+      <artifactId>json</artifactId>
+      <version>${json.version}</version>
+    </dependency>
+<!--    import calcite to support hive3 & spark 3-->
+    <dependency>
+      <groupId>org.apache.calcite</groupId>
+      <artifactId>calcite-core</artifactId>
+      <version>${calcite.version}</version>
+    </dependency>
     <!-- TODO: Reinvestigate PR 633 -->
   </dependencies>
 

pom.xml

@@ -152,6 +152,8 @@
     <presto.bundle.bootstrap.shade.prefix>org.apache.hudi.</presto.bundle.bootstrap.shade.prefix>
     <shadeSources>true</shadeSources>
     <zk-curator.version>2.7.1</zk-curator.version>
+    <json.version>20200518</json.version>
+    <calcite.version>1.16.0</calcite.version>
   </properties>
 
   <scm>