lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2198] Clean and reset the bootstrap events for coordinator when task failover (#3304)

Browse Source
This commit is contained in:
Danny Chan
2021-07-21 10:13:05 +08:00
committed by GitHub
parent 634163a990
commit 858e84b5b2
6 changed files with 57 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java
View File

@@ -214,6 +214,7 @@ public class StreamWriteFunction<K, I, O>
}
// blocks flushing until the coordinator starts a new instant
this.confirming = true;
this.currentInstant = this.writeClient.getLastPendingInstant(this.actionType);
}
@Override
@@ -534,6 +535,7 @@ public class StreamWriteFunction<K, I, O>
DataBucket bucket = this.buckets.computeIfAbsent(bucketID,
k -> new DataBucket(this.config.getDouble(FlinkOptions.WRITE_BATCH_SIZE), value));
final DataItem item = DataItem.fromHoodieRecord(value);
boolean flushBucket = bucket.detector.detect(item);
boolean flushBuffer = this.tracer.trace(bucket.detector.lastRecordSize);
if (flushBucket) {

8
hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
View File

@@ -243,7 +243,9 @@ public class StreamWriteOperatorCoordinator
@Override
public void subtaskFailed(int i, @Nullable Throwable throwable) {
// no operation
// reset the event
this.eventBuffer[i] = null;
LOG.warn("Reset the event for task [" + i + "]", throwable);
}
@Override
@@ -328,8 +330,8 @@ public class StreamWriteOperatorCoordinator
}
private void handleBootstrapEvent(WriteMetadataEvent event) {
addEventToBuffer(event);
if (Arrays.stream(eventBuffer).allMatch(Objects::nonNull)) {
this.eventBuffer[event.getTaskID()] = event;
if (Arrays.stream(eventBuffer).allMatch(evt -> evt != null && evt.isBootstrap())) {
// start to initialize the instant.
initInstant(event.getInstantTime());
}

7
hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java
View File

@@ -194,9 +194,10 @@ public class BucketAssignFunction<K, I, O extends HoodieRecord<?>>
} else {
location = getNewRecordLocation(partitionPath);
this.context.setCurrentKey(recordKey);
if (isChangingRecords) {
updateIndexState(partitionPath, location);
}
}
// always refresh the index
if (isChangingRecords) {
updateIndexState(partitionPath, location);
}
record.setCurrentLocation(location);
out.collect((O) record);