From 1ff99ca7d7bbdcb24ebae96c434a35000eded1fc Mon Sep 17 00:00:00 2001 From: Danny Chan Date: Mon, 12 Apr 2021 17:03:58 +0800 Subject: [PATCH] [HUDI-1786] Add option for merge max memory (#2805) --- .../java/org/apache/hudi/configuration/FlinkOptions.java | 6 ++++++ .../src/main/java/org/apache/hudi/util/StreamerUtil.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java index b120714b3..c47ea9584 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java +++ b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java @@ -273,6 +273,12 @@ public class FlinkOptions { .defaultValue(128) .withDescription("Max log block size in MB for log file, default 128MB"); + public static final ConfigOption WRITE_MERGE_MAX_MEMORY = ConfigOptions + .key("write.merge.max_memory") + .intType() + .defaultValue(100) // default 100 MB + .withDescription("Max memory in MB for merge, default 100MB"); + // ------------------------------------------------------------------------ // Compaction Options // ------------------------------------------------------------------------ diff --git a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index 3cc5d561d..a7295fb0f 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -207,7 +207,7 @@ public class StreamerUtil { .withMemoryConfig( HoodieMemoryConfig.newBuilder() .withMaxMemoryMaxSize( - conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L, + conf.getInteger(FlinkOptions.WRITE_MERGE_MAX_MEMORY) * 1024 * 1024L, conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L ).build()) .forTable(conf.getString(FlinkOptions.TABLE_NAME))