lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1806] Honoring skipROSuffix in spark ds (#2882)

Browse Source 
* Honoring skipROSuffix in spark ds

* Adding tests

* fixing scala checkstype issue
This commit is contained in:
Sivabalan Narayanan
2021-05-18 19:11:39 -04:00
committed by GitHub
parent 7d2971d4e2
commit 5d1f592395
2 changed files with 37 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
View File

@@ -412,6 +412,8 @@ private[hudi] object HoodieSparkSqlWriter {
hiveSyncConfig.hiveUser = parameters(HIVE_USER_OPT_KEY)
hiveSyncConfig.hivePass = parameters(HIVE_PASS_OPT_KEY)
hiveSyncConfig.jdbcUrl = parameters(HIVE_URL_OPT_KEY)
hiveSyncConfig.skipROSuffix = parameters.getOrElse(HIVE_SKIP_RO_SUFFIX,
DataSourceWriteOptions.DEFAULT_HIVE_SKIP_RO_SUFFIX_VAL).toBoolean
hiveSyncConfig.partitionFields =
ListBuffer(parameters(HIVE_PARTITION_FIELDS_OPT_KEY).split(",").map(_.trim).filter(!_.isEmpty).toList: _*)
hiveSyncConfig.partitionValueExtractorClass = parameters(HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY)