lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1918] Fix incorrect keyBy field cause serious data skew, to avoid multiple subtasks write to a partition at the same time (#2972)

Browse Source
This commit is contained in:
zhangminglei
2021-05-21 21:59:47 +08:00
committed by GitHub
parent a96034d38d
commit 99b14a78e3
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java
View File

@@ -88,8 +88,8 @@ public class HoodieFlinkStreamer {
.name("kafka_source") .name("kafka_source")
.uid("uid_kafka_source") .uid("uid_kafka_source")
.map(new RowDataToHoodieFunction<>(rowType, conf), TypeInformation.of(HoodieRecord.class)) .map(new RowDataToHoodieFunction<>(rowType, conf), TypeInformation.of(HoodieRecord.class))
// Key-by partition path, to avoid multiple subtasks write to a partition at the same time // Key-by record key, to avoid multiple subtasks write to a partition at the same time
.keyBy(HoodieRecord::getPartitionPath) .keyBy(HoodieRecord::getRecordKey)
.transform( .transform(
"bucket_assigner", "bucket_assigner",
TypeInformation.of(HoodieRecord.class), TypeInformation.of(HoodieRecord.class),