lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2274] Allows INSERT duplicates for Flink MOR table (#3403)

Browse Source
This commit is contained in:
Danny Chan
2021-08-06 10:30:52 +08:00
committed by GitHub
parent 0dcd6a8fca
commit b7586a5632
9 changed files with 20 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
View File

@@ -538,6 +538,7 @@ public class TestWriteCopyOnWrite {
// reset the config option
conf.setDouble(FlinkOptions.WRITE_BATCH_SIZE, 0.0006); // 630 bytes batch size
conf.setString(FlinkOptions.OPERATION, WriteOperationType.INSERT.value());
conf.setBoolean(FlinkOptions.INSERT_DEDUP, false);
funcWrapper = new StreamWriteFunctionWrapper<>(tempFile.getAbsolutePath(), conf);
// open the function and ingest data

6
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java
View File

@@ -37,7 +37,6 @@ import org.apache.avro.Schema;
import org.apache.flink.configuration.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.Comparator;
@@ -68,11 +67,6 @@ public class TestWriteMergeOnRead extends TestWriteCopyOnWrite {
conf.setBoolean(FlinkOptions.COMPACTION_ASYNC_ENABLED, false);
}
@Test
public void testInsertAllowsDuplication() {
// ignore the test because only COW table supports INSERT duplication
}
@Override
protected void checkWrittenData(File baseFile, Map<String, String> expected, int partitions) throws Exception {
HoodieTableMetaClient metaClient = HoodieFlinkTable.create(writeConfig, context).getMetaClient();

6
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
View File

@@ -22,7 +22,6 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.flink.configuration.Configuration;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.Map;
@@ -38,11 +37,6 @@ public class TestWriteMergeOnReadWithCompact extends TestWriteCopyOnWrite {
conf.setInteger(FlinkOptions.COMPACTION_DELTA_COMMITS, 1);
}
@Test
public void testInsertAllowsDuplication() {
// ignore the test because only COW table supports INSERT duplication
}
@Override
protected Map<String, String> getExpectedBeforeCheckpointComplete() {
return EXPECTED1;

18
hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java
View File

@@ -340,24 +340,6 @@ public class TestHoodieTableFactory {
assertThat(conf2.getInteger(FlinkOptions.ARCHIVE_MAX_COMMITS), is(45));
}
@Test
void testMorTableInsertAllowDuplication() {
TableSchema schema = TableSchema.builder()
.field("f0", DataTypes.INT().notNull())
.field("f1", DataTypes.VARCHAR(20))
.field("f2", DataTypes.TIMESTAMP(3))
.field("ts", DataTypes.TIMESTAMP(3))
.primaryKey("f0")
.build();
// overwrite the operation
this.conf.setString(FlinkOptions.OPERATION.key(), "insert");
this.conf.setString(FlinkOptions.TABLE_TYPE.key(), FlinkOptions.TABLE_TYPE_MERGE_ON_READ);
final MockContext sinkContext = MockContext.getInstance(this.conf, schema, "f2");
assertThrows(ValidationException.class, () -> new HoodieTableFactory().createDynamicTableSink(sinkContext),
"Option 'write.insert.allow_dup' is only allowed for COPY_ON_WRITE table.");
}
// -------------------------------------------------------------------------
// Inner Class
// -------------------------------------------------------------------------