lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3083] Support component data types for flink bulk_insert (#4470)

Browse Source 
* [HUDI-3083] Support component data types for flink bulk_insert

* add nested row type test
This commit is contained in:
Ron
2021-12-30 11:15:54 +08:00
committed by GitHub
parent 5c0e4ce005
commit 674c149234
24 changed files with 3031 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
hudi-flink/src/test/java/org/apache/hudi/table/HoodieDataSourceITCase.java
View File

@@ -1081,6 +1081,62 @@ public class HoodieDataSourceITCase extends AbstractTestBase {
assertRowsEquals(result, expected);
}
@ParameterizedTest
@ValueSource(strings = {"insert", "upsert", "bulk_insert"})
void testParquetComplexTypes(String operation) {
TableEnvironment tableEnv = batchTableEnv;
String hoodieTableDDL = sql("t1")
.field("f_int int")
.field("f_array array<varchar(10)>")
.field("f_map map<varchar(20), int>")
.field("f_row row(f_row_f0 int, f_row_f1 varchar(10))")
.pkField("f_int")
.noPartition()
.option(FlinkOptions.PATH, tempFile.getAbsolutePath())
.option(FlinkOptions.OPERATION, operation)
.end();
tableEnv.executeSql(hoodieTableDDL);
execInsertSql(tableEnv, TestSQL.COMPLEX_TYPE_INSERT_T1);
List<Row> result = CollectionUtil.iterableToList(
() -> tableEnv.sqlQuery("select * from t1").execute().collect());
final String expected = "["
+ "+I[1, [abc1, def1], {abc1=1, def1=3}, +I[1, abc1]], "
+ "+I[2, [abc2, def2], {def2=3, abc2=1}, +I[2, abc2]], "
+ "+I[3, [abc3, def3], {def3=3, abc3=1}, +I[3, abc3]]]";
assertRowsEquals(result, expected);
}
@ParameterizedTest
@ValueSource(strings = {"insert", "upsert", "bulk_insert"})
void testParquetComplexNestedRowTypes(String operation) {
TableEnvironment tableEnv = batchTableEnv;
String hoodieTableDDL = sql("t1")
.field("f_int int")
.field("f_array array<varchar(10)>")
.field("f_map map<varchar(20), int>")
.field("f_row row(f_nested_array array<varchar(10)>, f_nested_row row(f_row_f0 int, f_row_f1 varchar(10)))")
.pkField("f_int")
.noPartition()
.option(FlinkOptions.PATH, tempFile.getAbsolutePath())
.option(FlinkOptions.OPERATION, operation)
.end();
tableEnv.executeSql(hoodieTableDDL);
execInsertSql(tableEnv, TestSQL.COMPLEX_NESTED_ROW_TYPE_INSERT_T1);
List<Row> result = CollectionUtil.iterableToList(
() -> tableEnv.sqlQuery("select * from t1").execute().collect());
final String expected = "["
+ "+I[1, [abc1, def1], {abc1=1, def1=3}, +I[[abc1, def1], +I[1, abc1]]], "
+ "+I[2, [abc2, def2], {def2=3, abc2=1}, +I[[abc2, def2], +I[2, abc2]]], "
+ "+I[3, [abc3, def3], {def3=3, abc3=1}, +I[[abc3, def3], +I[3, abc3]]]]";
assertRowsEquals(result, expected);
}
// -------------------------------------------------------------------------
// Utilities
// -------------------------------------------------------------------------

10
hudi-flink/src/test/java/org/apache/hudi/utils/TestSQL.java
View File

@@ -51,4 +51,14 @@ public class TestSQL {
+ "('id9','Jane',19,TIMESTAMP '1970-01-01 00:00:06','par3'),\n"
+ "('id10','Ella',38,TIMESTAMP '1970-01-01 00:00:07','par4'),\n"
+ "('id11','Phoebe',52,TIMESTAMP '1970-01-01 00:00:08','par4')";
public static final String COMPLEX_TYPE_INSERT_T1 = "insert into t1 values\n"
+ "(1, array['abc1', 'def1'], map['abc1', 1, 'def1', 3], row(1, 'abc1')),\n"
+ "(2, array['abc2', 'def2'], map['abc2', 1, 'def2', 3], row(2, 'abc2')),\n"
+ "(3, array['abc3', 'def3'], map['abc3', 1, 'def3', 3], row(3, 'abc3'))";
public static final String COMPLEX_NESTED_ROW_TYPE_INSERT_T1 = "insert into t1 values\n"
+ "(1, array['abc1', 'def1'], map['abc1', 1, 'def1', 3], row(array['abc1', 'def1'], row(1, 'abc1'))),\n"
+ "(2, array['abc2', 'def2'], map['abc2', 1, 'def2', 3], row(array['abc2', 'def2'], row(2, 'abc2'))),\n"
+ "(3, array['abc3', 'def3'], map['abc3', 1, 'def3', 3], row(array['abc3', 'def3'], row(3, 'abc3')))";
}