lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3072] Fixing conflict resolution in transaction management code path for auto commit code path (#4588)

Browse Source 
* Fixing conflict resolution in transaction management code path for auto commit code path

* Addressing comments

* Fixing test failures
This commit is contained in:
Sivabalan Narayanan
2022-01-24 05:43:28 -05:00
committed by GitHub
parent cfde45b548
commit e00a9042e9
8 changed files with 205 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java
View File

@@ -181,6 +181,11 @@ public class SparkBootstrapCommitActionExecutor<T extends HoodieRecordPayload<T>
return null;
}
@Override
protected void setCommitMetadata(HoodieWriteMetadata<JavaRDD<WriteStatus>> result) {
result.setCommitMetadata(Option.of(new HoodieCommitMetadata()));
}
@Override
protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<JavaRDD<WriteStatus>> result) {
// Perform bootstrap index write and then commit. Make sure both record-key and bootstrap-index

10
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
View File

@@ -273,6 +273,13 @@ public abstract class BaseSparkCommitActionExecutor<T extends HoodieRecordPayloa
return table.getMetaClient().getCommitActionType();
}
@Override
protected void setCommitMetadata(HoodieWriteMetadata<JavaRDD<WriteStatus>> result) {
result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(result.getWriteStatuses().map(WriteStatus::getStat).collect(),
result.getPartitionToReplaceFileIds(),
extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType())));
}
@Override
protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<JavaRDD<WriteStatus>> result) {
context.setJobStatus(this.getClass().getSimpleName(), "Commit write status collect");
@@ -288,8 +295,7 @@ public abstract class BaseSparkCommitActionExecutor<T extends HoodieRecordPayloa
finalizeWrite(instantTime, writeStats, result);
try {
HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
HoodieCommitMetadata metadata = CommitUtils.buildMetadata(writeStats, result.getPartitionToReplaceFileIds(),
extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType());
HoodieCommitMetadata metadata = result.getCommitMetadata().get();
writeTableMetadata(metadata, actionType);
activeTimeline.saveAsComplete(new HoodieInstant(true, getCommitActionType(), instantTime),
Option.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)));