From 6be03ca56a9cb5453b61f1203e64910818262ebd Mon Sep 17 00:00:00 2001 From: v-zhangjc9 Date: Thu, 30 Jun 2022 19:13:51 +0800 Subject: [PATCH] Down the reader mem check --- .../main/java/org/apache/hudi/sink/StreamWriteFunction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java index 2748af529..687a6c178 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java @@ -325,10 +325,10 @@ public class StreamWriteFunction extends AbstractStreamWriteFunction { private final double maxBufferSize; TotalSizeTracer(Configuration conf) { - long mergeReaderMem = 100; // constant 100MB + long mergeReaderMem = 10; // constant 100MB long mergeMapMaxMem = conf.getInteger(FlinkOptions.WRITE_MERGE_MAX_MEMORY); this.maxBufferSize = (conf.getDouble(FlinkOptions.WRITE_TASK_MAX_SIZE) - mergeReaderMem - mergeMapMaxMem) * 1024 * 1024; - final String errMsg = String.format("'%s' should be at least greater than '%s' plus merge reader memory(constant 100MB now)", + final String errMsg = String.format("'%s' should be at least greater than '%s' plus merge reader memory(constant 50MB now)", FlinkOptions.WRITE_TASK_MAX_SIZE.key(), FlinkOptions.WRITE_MERGE_MAX_MEMORY.key()); ValidationUtils.checkState(this.maxBufferSize > 0, errMsg); }