lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add option to control use hsync or not

Browse Source
This commit is contained in:
v-zhangjc9
2024-04-24 15:33:03 +08:00
parent 181df2240a
commit d9581682a2
7 changed files with 40 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
View File

@@ -144,6 +144,8 @@ public interface HoodieLogFormat {
// Rollover Log file write token
private String rolloverLogWriteToken;
private Boolean useHSync;
public WriterBuilder withBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
return this;
@@ -204,6 +206,11 @@ public interface HoodieLogFormat {
return this;
}
public WriterBuilder withUseHSync(boolean useHSync) {
this.useHSync = useHSync;
return this;
}
public Writer build() throws IOException {
LOG.info("Building HoodieLogFormat Writer");
if (fs == null) {
@@ -264,7 +271,10 @@ public interface HoodieLogFormat {
if (sizeThreshold == null) {
sizeThreshold = DEFAULT_SIZE_THRESHOLD;
}
return new HoodieLogFormatWriter(fs, logFile, bufferSize, replication, sizeThreshold, rolloverLogWriteToken);
if (useHSync == null) {
useHSync = true;
}
return new HoodieLogFormatWriter(fs, logFile, bufferSize, replication, sizeThreshold, rolloverLogWriteToken, useHSync);
}
}

9
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
View File

@@ -58,15 +58,18 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
private boolean closed = false;
private transient Thread shutdownThread = null;
private final boolean useHSync;
private static final String APPEND_UNAVAILABLE_EXCEPTION_MESSAGE = "not sufficiently replicated yet";
HoodieLogFormatWriter(FileSystem fs, HoodieLogFile logFile, Integer bufferSize, Short replication, Long sizeThreshold, String rolloverLogWriteToken) {
HoodieLogFormatWriter(FileSystem fs, HoodieLogFile logFile, Integer bufferSize, Short replication, Long sizeThreshold, String rolloverLogWriteToken, Boolean useHSync) {
this.fs = fs;
this.logFile = logFile;
this.sizeThreshold = sizeThreshold;
this.bufferSize = bufferSize;
this.replication = replication;
this.rolloverLogWriteToken = rolloverLogWriteToken;
this.useHSync = useHSync;
addShutDownHook();
}
@@ -258,7 +261,9 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
output.flush();
// NOTE : the following API call makes sure that the data is flushed to disk on DataNodes (akin to POSIX fsync())
// See more details here : https://issues.apache.org/jira/browse/HDFS-744
output.hsync();
if (useHSync) {
output.hsync();
}
}
@Override