lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add option to control use hsync or not

Browse Source
This commit is contained in:
v-zhangjc9
2024-04-24 15:33:03 +08:00
parent 181df2240a
commit d9581682a2
7 changed files with 40 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
View File

@@ -117,7 +117,7 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
if (this.writer == null) {
return HoodieLogFormat.newWriterBuilder().onParentPath(archiveFilePath.getParent())
.withFileId(archiveFilePath.getName()).withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION)
.withFs(metaClient.getFs()).overBaseCommit("").build();
.withFs(metaClient.getFs()).overBaseCommit("").withUseHSync(config.getUseHSync()).build();
} else {
return this.writer;
}

15
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -481,6 +481,12 @@ public class HoodieWriteConfig extends HoodieConfig {
.sinceVersion("0.11.0")
.withDocumentation("Auto adjust lock configurations when metadata table is enabled and for async table services.");
public static final ConfigProperty<Boolean> USE_HSYNC = ConfigProperty
.key("hoodie.write.use.hsync")
.defaultValue(true)
.sinceVersion("0.12.0")
.withDocumentation("Use hsync or not");
private ConsistencyGuardConfig consistencyGuardConfig;
private FileSystemRetryConfig fileSystemRetryConfig;
@@ -2063,6 +2069,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MODE));
}
public Boolean getUseHSync() {
return getBooleanOrDefault(USE_HSYNC);
}
/**
* Are any table services configured to run inline for both scheduling and execution?
*
@@ -2542,6 +2552,11 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
public Builder withUseHSync(boolean useHSync) {
writeConfig.setValue(USE_HSYNC, String.valueOf(useHSync));
return this;
}
protected void setDefaults() {
writeConfig.setDefaultValue(MARKERS_TYPE, getDefaultMarkersType(engineType));
// Check for mandatory properties

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
View File

@@ -477,7 +477,8 @@ public class HoodieAppendHandle<T extends HoodieRecordPayload, I, K, O> extends
.withSizeThreshold(config.getLogFileMaxSize()).withFs(fs)
.withRolloverLogWriteToken(writeToken)
.withLogWriteToken(latestLogFile.map(x -> FSUtils.getWriteTokenFromLogPath(x.getPath())).orElse(writeToken))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
.withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withUseHSync(config.getUseHSync()).build();
}
/**

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
View File

@@ -716,7 +716,8 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
.withFs(dataMetaClient.getFs())
.withRolloverLogWriteToken(HoodieLogFormat.DEFAULT_WRITE_TOKEN)
.withLogWriteToken(HoodieLogFormat.DEFAULT_WRITE_TOKEN)
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
.withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withUseHSync(dataWriteConfig.getUseHSync()).build();
writer.appendBlock(block);
writer.close();
} catch (InterruptedException e) {

3
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
View File

@@ -129,7 +129,8 @@ public class BaseRollbackHelper implements Serializable {
.withFileId(fileId)
.overBaseCommit(latestBaseInstant)
.withFs(metaClient.getFs())
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
.withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withUseHSync(config.getUseHSync()).build();
// generate metadata
if (doDelete) {