lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3772] Fixing auto adjustment of lock configs for deltastreamer (#5207)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-04-02 23:44:10 -07:00
committed by GitHub
parent cc3737be50
commit 84064a9b08
5 changed files with 102 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
View File

@@ -81,7 +81,9 @@ class HoodieStreamingSink(sqlContext: SQLContext,
// Override to use direct markers. In Structured streaming, timeline server is closed after
// first micro-batch and subsequent micro-batches do not have timeline server running.
// Thus, we can't use timeline-server-based markers.
val updatedOptions = options.updated(HoodieWriteConfig.MARKERS_TYPE.key(), MarkerType.DIRECT.name())
var updatedOptions = options.updated(HoodieWriteConfig.MARKERS_TYPE.key(), MarkerType.DIRECT.name())
// we need auto adjustment enabled for streaming sink since async table services are feasible within the same JVM.
updatedOptions = updatedOptions.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
retry(retryCnt, retryIntervalMs)(
Try(