lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2198] Clean and reset the bootstrap events for coordinator when task failover (#3304)

Browse Source
This commit is contained in:
Danny Chan
2021-07-21 10:13:05 +08:00
committed by GitHub
parent 634163a990
commit 858e84b5b2
6 changed files with 57 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
View File

@@ -218,6 +218,29 @@ public class TestWriteCopyOnWrite {
funcWrapper.checkpointFails(2);
}
@Test
public void testSubtaskFails() throws Exception {
// open the function and ingest data
funcWrapper.openFunction();
// no data written and triggers checkpoint fails,
// then we should revert the start instant
// this triggers the data write and event send
funcWrapper.checkpointFunction(1);
funcWrapper.getNextEvent();
String instant1 = funcWrapper.getWriteClient().getLastPendingInstant(getTableType());
assertNotNull(instant1);
// fails the subtask
funcWrapper.subTaskFails(0);
String instant2 = funcWrapper.getWriteClient().getLastPendingInstant(getTableType());
assertNotEquals(instant2, instant1, "The previous instant should be rolled back when starting new instant");
checkInstantState(funcWrapper.getWriteClient(), HoodieInstant.State.COMPLETED, null);
}
@Test
public void testInsert() throws Exception {
// open the function and ingest data

31
hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java
View File

@@ -110,7 +110,7 @@ public class StreamWriteFunctionWrapper<I> {
this.gateway = new MockOperatorEventGateway();
this.conf = conf;
// one function
this.coordinatorContext = new MockOperatorCoordinatorContext(new OperatorID(), 1);
this.coordinatorContext = new MockOperatorCoordinatorContext(new OperatorID(), 1, false);
this.coordinator = new StreamWriteOperatorCoordinator(conf, this.coordinatorContext);
this.compactFunctionWrapper = new CompactFunctionWrapper(this.conf);
this.bucketAssignOperatorContext = new MockBucketAssignOperatorContext();
@@ -139,14 +139,7 @@ public class StreamWriteFunctionWrapper<I> {
bucketAssignerFunction.setContext(bucketAssignOperatorContext);
bucketAssignerFunction.initializeState(this.functionInitializationContext);
writeFunction = new StreamWriteFunction<>(conf);
writeFunction.setRuntimeContext(runtimeContext);
writeFunction.setOperatorEventGateway(gateway);
writeFunction.initializeState(this.functionInitializationContext);
writeFunction.open(conf);
// handle the bootstrap event
coordinator.handleEventFromOperator(0, getNextEvent());
setupWriteFunction();
if (asyncCompaction) {
compactFunctionWrapper.openFunction();
@@ -240,6 +233,11 @@ public class StreamWriteFunctionWrapper<I> {
coordinator.notifyCheckpointAborted(checkpointId);
}
public void subTaskFails(int taskID) throws Exception {
coordinator.subtaskFailed(taskID, new RuntimeException("Dummy exception"));
setupWriteFunction();
}
public void close() throws Exception {
coordinator.close();
ioManager.close();
@@ -270,6 +268,21 @@ public class StreamWriteFunctionWrapper<I> {
return this.bootstrapFunction.isAlreadyBootstrap();
}
// -------------------------------------------------------------------------
// Utilities
// -------------------------------------------------------------------------
private void setupWriteFunction() throws Exception {
writeFunction = new StreamWriteFunction<>(conf);
writeFunction.setRuntimeContext(runtimeContext);
writeFunction.setOperatorEventGateway(gateway);
writeFunction.initializeState(this.functionInitializationContext);
writeFunction.open(conf);
// handle the bootstrap event
coordinator.handleEventFromOperator(0, getNextEvent());
}
// -------------------------------------------------------------------------
// Inner Class
// -------------------------------------------------------------------------