lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2495] Resolve inconsistent key generation for timestamp types by GenericRecord and Row (#3944)

Browse Source
This commit is contained in:
Yann Byron
2021-11-11 11:54:34 +08:00
committed by GitHub
parent 77b0440eb4
commit 90529aa552
3 changed files with 121 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala
View File

@@ -17,6 +17,7 @@
package org.apache.spark.sql.hudi.command
import java.sql.Timestamp
import java.util.concurrent.TimeUnit.{MICROSECONDS, MILLISECONDS}
import org.apache.avro.generic.GenericRecord
@@ -96,7 +97,7 @@ class SqlKeyGenerator(props: TypedProperties) extends ComplexKeyGenerator(props)
val timeMs = if (rowType) { // In RowType, the partitionPathValue is the time format string, convert to millis
SqlKeyGenerator.sqlTimestampFormat.parseMillis(_partitionValue)
} else {
MILLISECONDS.convert(_partitionValue.toLong, MICROSECONDS)
Timestamp.valueOf(_partitionValue).getTime
}
val timestampFormat = PartitionPathEncodeUtils.escapePathName(
SqlKeyGenerator.timestampTimeFormat.print(timeMs))