lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2556] Tweak some default config options for flink (#3800)

Browse Source 
* rename write.insert.drop.duplicates to write.precombine and set it as true for COW table
* set index.global.enabled default as true
* set compaction.target_io default as 500GB
This commit is contained in:
Danny Chan
2021-10-14 19:42:56 +08:00
committed by GitHub
parent f897e6d73e
commit 2c370cbae0
10 changed files with 57 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
View File

@@ -275,7 +275,7 @@ public class TestWriteCopyOnWrite {
@Test
public void testInsertDuplicates() throws Exception {
// reset the config option
conf.setBoolean(FlinkOptions.INSERT_DROP_DUPS, true);
conf.setBoolean(FlinkOptions.PRE_COMBINE, true);
funcWrapper = new StreamWriteFunctionWrapper<>(tempFile.getAbsolutePath(), conf);
// open the function and ingest data
@@ -470,7 +470,7 @@ public class TestWriteCopyOnWrite {
public void testInsertWithDeduplication() throws Exception {
// reset the config option
conf.setDouble(FlinkOptions.WRITE_BATCH_SIZE, 0.0008); // 839 bytes batch size
conf.setBoolean(FlinkOptions.INSERT_DROP_DUPS, true);
conf.setBoolean(FlinkOptions.PRE_COMBINE, true);
funcWrapper = new StreamWriteFunctionWrapper<>(tempFile.getAbsolutePath(), conf);
// open the function and ingest data

4
hudi-flink/src/test/java/org/apache/hudi/table/HoodieDataSourceITCase.java
View File

@@ -652,7 +652,7 @@ public class HoodieDataSourceITCase extends AbstractTestBase {
String hoodieTableDDL = sql("t1")
.option(FlinkOptions.PATH, tempFile.getAbsolutePath())
.option(FlinkOptions.INDEX_GLOBAL_ENABLED, true)
.option(FlinkOptions.INSERT_DROP_DUPS, true)
.option(FlinkOptions.PRE_COMBINE, true)
.end();
streamTableEnv.executeSql(hoodieTableDDL);
@@ -674,7 +674,7 @@ public class HoodieDataSourceITCase extends AbstractTestBase {
String hoodieTableDDL = sql("t1")
.option(FlinkOptions.PATH, tempFile.getAbsolutePath())
.option(FlinkOptions.INDEX_GLOBAL_ENABLED, false)
.option(FlinkOptions.INSERT_DROP_DUPS, true)
.option(FlinkOptions.PRE_COMBINE, true)
.end();
streamTableEnv.executeSql(hoodieTableDDL);

15
hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java
View File

@@ -394,6 +394,21 @@ public class TestHoodieTableFactory {
is("UTC"));
}
@Test
void testSetupWriteOptionsForSink() {
final HoodieTableSink tableSink1 =
(HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(MockContext.getInstance(this.conf));
final Configuration conf1 = tableSink1.getConf();
assertThat(conf1.get(FlinkOptions.PRE_COMBINE), is(true));
// set up operation as 'insert'
this.conf.setString(FlinkOptions.OPERATION, "insert");
HoodieTableSink tableSink2 =
(HoodieTableSink) new HoodieTableFactory().createDynamicTableSink(MockContext.getInstance(this.conf));
Configuration conf2 = tableSink2.getConf();
assertThat(conf2.get(FlinkOptions.PRE_COMBINE), is(false));
}
// -------------------------------------------------------------------------
// Inner Class
// -------------------------------------------------------------------------