lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2735] Allow empty commits in Kafka Connect Sink for Hudi (#4544)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-01-10 12:31:25 -08:00
committed by GitHub
parent 251d4eb3b6
commit bc95571caa
2 changed files with 32 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-kafka-connect/src/main/java/org/apache/hudi/connect/transaction/ConnectTransactionCoordinator.java
View File

@@ -294,7 +294,7 @@ public class ConnectTransactionCoordinator implements TransactionCoordinator, Ru
long totalRecords = (long) allWriteStatuses.stream().mapToDouble(WriteStatus::getTotalRecords).sum();
boolean hasErrors = totalErrorRecords > 0;
if ((!hasErrors || configs.allowCommitOnErrors()) && !allWriteStatuses.isEmpty()) {
if (!hasErrors || configs.allowCommitOnErrors()) {
boolean success = transactionServices.endCommit(currentCommitTime,
allWriteStatuses,
transformKafkaOffsets(currentConsumedKafkaOffsets));
@@ -319,8 +319,6 @@ public class ConnectTransactionCoordinator implements TransactionCoordinator, Ru
ws.getErrors().forEach((key, value) -> LOG.trace("Error for key:" + key + " is " + value));
}
});
} else {
LOG.warn("Empty write statuses were received from all Participants");
}
// Submit the next start commit, that will rollback the current commit.