lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3934] Fix Spark32HoodieParquetFileFormat not being compatible w/ Spark 3.2.0 (#5378)

Browse Source 
- Due to the fact that Spark 3.2.1 is non-BWC w/ 3.2.0, we have to handle all these incompatibilities in Spark32HoodieParquetFileFormat. This PR is addressing that.

Co-authored-by: Raymond Xu <2701446+xushiyan@users.noreply.github.com>
This commit is contained in:
Alexey Kudinkin
2022-04-21 18:00:38 -07:00
committed by GitHub
parent c4bc2deea0
commit c05a4e7b6f
5 changed files with 229 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
View File

@@ -53,13 +53,15 @@ object HoodieSparkUtils extends SparkAdapterSupport {
def isSpark3_1: Boolean = SPARK_VERSION.startsWith("3.1")
def gteqSpark3_1: Boolean = SPARK_VERSION > "3.1"
def gteqSpark3_1_3: Boolean = SPARK_VERSION >= "3.1.3"
def isSpark3_2: Boolean = SPARK_VERSION.startsWith("3.2")
def gteqSpark3_2: Boolean = SPARK_VERSION > "3.2"
def gteqSpark3_1: Boolean = SPARK_VERSION > "3.1"
def gteqSpark3_1_3: Boolean = SPARK_VERSION >= "3.1.3"
def gteqSpark3_2_1: Boolean = SPARK_VERSION >= "3.2.1"
def getMetaSchema: StructType = {
StructType(HoodieRecord.HOODIE_META_COLUMNS.asScala.map(col => {