lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1895] Close the file handles gracefully for flink write function to avoid corrupted files (#2938)

Browse Source
This commit is contained in:
Danny Chan
2021-05-12 18:44:10 +08:00
committed by GitHub
parent 5a8b2a4f86
commit b98c9ab439
10 changed files with 103 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java
View File

@@ -200,7 +200,7 @@ public class StreamWriteFunction<K, I, O>
@Override
public void close() {
if (this.writeClient != null) {
this.writeClient.cleanHandles();
this.writeClient.cleanHandlesGracefully();
this.writeClient.close();
}
}

2
hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
View File

@@ -249,7 +249,7 @@ public class StreamWriteOperatorCoordinator
"The coordinator can only handle BatchWriteSuccessEvent");
BatchWriteSuccessEvent event = (BatchWriteSuccessEvent) operatorEvent;
// the write task does not block after checkpointing(and before it receives a checkpoint success event),
// if it it checkpoints succeed then flushes the data buffer again before this coordinator receives a checkpoint
// if it checkpoints succeed then flushes the data buffer again before this coordinator receives a checkpoint
// success event, the data buffer would flush with an older instant time.
ValidationUtils.checkState(
HoodieTimeline.compareTimestamps(instant, HoodieTimeline.GREATER_THAN_OR_EQUALS, event.getInstantTime()),

14
hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java
View File

@@ -102,7 +102,7 @@ public class HoodieTableFactory implements DynamicTableSourceFactory, DynamicTab
// Utilities
// -------------------------------------------------------------------------
/** Validate required options. e.g record key and pre combine key.
/** Validate required options. For e.g, record key and pre_combine key.
*
* @param conf The table options
* @param schema The table schema
@@ -115,17 +115,17 @@ public class HoodieTableFactory implements DynamicTableSourceFactory, DynamicTab
Arrays.stream(conf.get(FlinkOptions.RECORD_KEY_FIELD).split(","))
.filter(field -> !fields.contains(field))
.findAny()
.ifPresent(e -> {
throw new ValidationException("The " + e + " field not exists in table schema."
+ "Please define primary key or modify hoodie.datasource.write.recordkey.field option.");
.ifPresent(f -> {
throw new ValidationException("Field '" + f + "' does not exist in the table schema."
+ "Please define primary key or modify 'hoodie.datasource.write.recordkey.field' option.");
});
}
// validate pre combine key
// validate pre_combine key
String preCombineField = conf.get(FlinkOptions.PRECOMBINE_FIELD);
if (!fields.contains(preCombineField)) {
throw new ValidationException("The " + preCombineField + " field not exists in table schema."
+ "Please check write.precombine.field option.");
throw new ValidationException("Field " + preCombineField + " does not exist in the table schema."
+ "Please check 'write.precombine.field' option.");
}
}