lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2040] Make flink writer as exactly-once by default (#3106)

Browse Source
This commit is contained in:
Danny Chan
2021-06-18 13:55:23 +08:00
committed by GitHub
parent aa6342c3c9
commit cdb9b48170
6 changed files with 23 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
View File

@@ -651,7 +651,6 @@ public class TestWriteCopyOnWrite {
@Test
public void testWriteExactlyOnce() throws Exception {
// reset the config option
conf.setBoolean(FlinkOptions.WRITE_EXACTLY_ONCE_ENABLED, true);
conf.setLong(FlinkOptions.WRITE_COMMIT_ACK_TIMEOUT, 3);
conf.setDouble(FlinkOptions.WRITE_TASK_MAX_SIZE, 200.0006); // 630 bytes buffer size
funcWrapper = new StreamWriteFunctionWrapper<>(tempFile.getAbsolutePath(), conf);

17
hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java
View File

@@ -23,10 +23,10 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.sink.bootstrap.BootstrapFunction;
import org.apache.hudi.sink.bootstrap.IndexRecord;
import org.apache.hudi.sink.StreamWriteFunction;
import org.apache.hudi.sink.StreamWriteOperatorCoordinator;
import org.apache.hudi.sink.bootstrap.BootstrapFunction;
import org.apache.hudi.sink.bootstrap.IndexRecord;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.partitioner.BucketAssignOperator;
@@ -48,13 +48,12 @@ import org.apache.flink.streaming.api.operators.collect.utils.MockOperatorEventG
import org.apache.flink.table.data.RowData;
import org.apache.flink.util.Collector;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
/**
* A wrapper class to manipulate the {@link StreamWriteFunction} instance for testing.
@@ -213,18 +212,8 @@ public class StreamWriteFunctionWrapper<I> {
public void checkpointComplete(long checkpointId) {
functionInitializationContext.getOperatorStateStore().checkpointSuccess(checkpointId);
if (asyncCompaction) {
// sleep for a while to give a change for scheduling compaction,
// see HoodieActiveTimeline#createNewInstantTime for details.
try {
TimeUnit.SECONDS.sleep(2);
} catch (InterruptedException e) {
throw new HoodieException("Waiting for checkpoint success exception", e);
}
}
coordinator.notifyCheckpointComplete(checkpointId);
this.bucketAssignerFunction.notifyCheckpointComplete(checkpointId);
this.writeFunction.notifyCheckpointComplete(checkpointId);
if (asyncCompaction) {
try {
compactFunctionWrapper.compact(checkpointId);