lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1786] Add option for merge max memory (#2805)

Browse Source
This commit is contained in:
Danny Chan
2021-04-12 17:03:58 +08:00
committed by GitHub
parent 040756d8c0
commit 1ff99ca7d7
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
View File

@@ -273,6 +273,12 @@ public class FlinkOptions {
.defaultValue(128) .defaultValue(128)
.withDescription("Max log block size in MB for log file, default 128MB"); .withDescription("Max log block size in MB for log file, default 128MB");
public static final ConfigOption<Integer> WRITE_MERGE_MAX_MEMORY = ConfigOptions
.key("write.merge.max_memory")
.intType()
.defaultValue(100) // default 100 MB
.withDescription("Max memory in MB for merge, default 100MB");
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// Compaction Options // Compaction Options
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------

2
hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java
View File

@@ -207,7 +207,7 @@ public class StreamerUtil {
.withMemoryConfig( .withMemoryConfig(
HoodieMemoryConfig.newBuilder() HoodieMemoryConfig.newBuilder()
.withMaxMemoryMaxSize( .withMaxMemoryMaxSize(
conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L, conf.getInteger(FlinkOptions.WRITE_MERGE_MAX_MEMORY) * 1024 * 1024L,
conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L conf.getInteger(FlinkOptions.COMPACTION_MAX_MEMORY) * 1024 * 1024L
).build()) ).build())
.forTable(conf.getString(FlinkOptions.TABLE_NAME)) .forTable(conf.getString(FlinkOptions.TABLE_NAME))