lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1794] Moved static COMMIT_FORMATTER to thread local variable as SimpleDateFormat is not thread safe. (#2819)

Browse Source
This commit is contained in:
Prashant Wason
2021-11-05 06:31:42 -07:00
committed by GitHub
parent 3af6568d31
commit b7ee341e14
19 changed files with 196 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala
View File

@@ -293,12 +293,12 @@ object HoodieSqlUtils extends SparkAdapterSupport {
*/
def formatQueryInstant(queryInstant: String): String = {
if (queryInstant.length == 19) { // for yyyy-MM-dd HH:mm:ss
HoodieActiveTimeline.COMMIT_FORMATTER.format(defaultDateTimeFormat.parse(queryInstant))
HoodieActiveTimeline.formatInstantTime(defaultDateTimeFormat.parse(queryInstant))
} else if (queryInstant.length == 14) { // for yyyyMMddHHmmss
HoodieActiveTimeline.COMMIT_FORMATTER.parse(queryInstant) // validate the format
HoodieActiveTimeline.parseInstantTime(queryInstant) // validate the format
queryInstant
} else if (queryInstant.length == 10) { // for yyyy-MM-dd
HoodieActiveTimeline.COMMIT_FORMATTER.format(defaultDateFormat.parse(queryInstant))
HoodieActiveTimeline.formatInstantTime(defaultDateFormat.parse(queryInstant))
} else {
throw new IllegalArgumentException(s"Unsupported query instant time format: $queryInstant,"
+ s"Supported time format are: 'yyyy-MM-dd: HH:mm:ss' or 'yyyy-MM-dd' or 'yyyyMMddHHmmss'")

4
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
View File

@@ -179,10 +179,10 @@ class HoodieStreamSource(
startOffset match {
case INIT_OFFSET => startOffset.commitTime
case HoodieSourceOffset(commitTime) =>
val time = HoodieActiveTimeline.COMMIT_FORMATTER.parse(commitTime).getTime
val time = HoodieActiveTimeline.parseInstantTime(commitTime).getTime
// As we consume the data between (start, end], start is not included,
// so we +1s to the start commit time here.
HoodieActiveTimeline.COMMIT_FORMATTER.format(new Date(time + 1000))
HoodieActiveTimeline.formatInstantTime(new Date(time + 1000))
case _=> throw new IllegalStateException("UnKnow offset type.")
}
}