lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-936] Fix fetch ordering val in HoodieSparkSqlWriter to remove unnecessary conversion to String (#1659)

Browse Source
This commit is contained in:
Bhavani Sudha Saktheeswaran
2020-05-26 21:09:02 -07:00
committed by GitHub
parent 6c450957ce
commit 9697fbf71e
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
View File

@@ -106,8 +106,8 @@ private[hudi] object HoodieSparkSqlWriter {
val keyGenerator = DataSourceUtils.createKeyGenerator(toProperties(parameters)) val keyGenerator = DataSourceUtils.createKeyGenerator(toProperties(parameters))
val genericRecords: RDD[GenericRecord] = AvroConversionUtils.createRdd(df, structName, nameSpace) val genericRecords: RDD[GenericRecord] = AvroConversionUtils.createRdd(df, structName, nameSpace)
val hoodieAllIncomingRecords = genericRecords.map(gr => { val hoodieAllIncomingRecords = genericRecords.map(gr => {
val orderingVal = DataSourceUtils.getNestedFieldValAsString( val orderingVal = DataSourceUtils.getNestedFieldVal(gr, parameters(PRECOMBINE_FIELD_OPT_KEY), false)
gr, parameters(PRECOMBINE_FIELD_OPT_KEY), false).asInstanceOf[Comparable[_]] .asInstanceOf[Comparable[_]]
DataSourceUtils.createHoodieRecord(gr, DataSourceUtils.createHoodieRecord(gr,
orderingVal, keyGenerator.getKey(gr), parameters(PAYLOAD_CLASS_OPT_KEY)) orderingVal, keyGenerator.getKey(gr), parameters(PAYLOAD_CLASS_OPT_KEY))
}).toJavaRDD() }).toJavaRDD()