From 56866a11fe8b7a0ef8340f221da30c83c72b85da Mon Sep 17 00:00:00 2001 From: steven zhang Date: Wed, 25 Nov 2020 11:55:33 +0800 Subject: [PATCH] [HUDI-1392] lose partition info when using spark parameter basePath (#2243) Co-authored-by: zhang wen --- .../main/scala/org/apache/hudi/DataSourceOptions.scala | 10 +++++++--- .../src/main/scala/org/apache/hudi/DefaultSource.scala | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala index fc52b3851..73f70e72d 100644 --- a/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -81,9 +81,13 @@ object DataSourceReadOptions { val translation = Map(VIEW_TYPE_READ_OPTIMIZED_OPT_VAL -> QUERY_TYPE_SNAPSHOT_OPT_VAL, VIEW_TYPE_INCREMENTAL_OPT_VAL -> QUERY_TYPE_INCREMENTAL_OPT_VAL, VIEW_TYPE_REALTIME_OPT_VAL -> QUERY_TYPE_SNAPSHOT_OPT_VAL) - if (optParams.contains(VIEW_TYPE_OPT_KEY) && !optParams.contains(QUERY_TYPE_OPT_KEY)) { - log.warn(VIEW_TYPE_OPT_KEY + " is deprecated and will be removed in a later release. Please use " + QUERY_TYPE_OPT_KEY) - optParams ++ Map(QUERY_TYPE_OPT_KEY -> translation(optParams(VIEW_TYPE_OPT_KEY))) + if (!optParams.contains(QUERY_TYPE_OPT_KEY)) { + if (optParams.contains(VIEW_TYPE_OPT_KEY)) { + log.warn(VIEW_TYPE_OPT_KEY + " is deprecated and will be removed in a later release. Please use " + QUERY_TYPE_OPT_KEY) + optParams ++ Map(QUERY_TYPE_OPT_KEY -> translation(optParams(VIEW_TYPE_OPT_KEY))) + } else { + optParams ++ Map(QUERY_TYPE_OPT_KEY -> DEFAULT_QUERY_TYPE_OPT_VAL) + } } else { optParams } diff --git a/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala index 1cf9bdb49..4a7837816 100644 --- a/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -55,7 +55,7 @@ class DefaultSource extends RelationProvider optParams: Map[String, String], schema: StructType): BaseRelation = { // Add default options for unspecified read options keys. - val parameters = Map(QUERY_TYPE_OPT_KEY -> DEFAULT_QUERY_TYPE_OPT_VAL) ++ translateViewTypesToQueryTypes(optParams) + val parameters = translateViewTypesToQueryTypes(optParams) val path = parameters.get("path") val readPathsStr = parameters.get(DataSourceReadOptions.READ_PATHS_OPT_KEY)