lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4160] Make database regex of MaxwellJsonKafkaSourcePostProcessor optional (#5697)

Browse Source
This commit is contained in:
wangxianghu
2022-05-28 11:13:24 +04:00
committed by GitHub
parent 93fe5a497e
commit 58014c147a
2 changed files with 25 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSourcePostProcessor.java
View File

@@ -178,6 +178,12 @@ public class TestJsonKafkaSourcePostProcessor extends TestJsonKafkaSource {
+ "\"name\":\"andy\",\"age\":17,\"insert_time\":\"2022-03-12 08:31:56\","
+ "\"update_time\":\"2022-03-12 08:31:56\"}}";
// database hudi_02, table hudi_maxwell_01, insert
String hudi02Maxwell01Insert = "{\"database\":\"hudi_02\",\"table\":\"hudi_maxwell_01\",\"type\":\"insert\","
+ "\"ts\":1647073916,\"xid\":4990,\"commit\":true,\"data\":{\"id\":\"9bb17f316ee8488cb107621ddf0f3cb0\","
+ "\"name\":\"andy\",\"age\":17,\"insert_time\":\"2022-03-12 08:31:56\","
+ "\"update_time\":\"2022-03-12 08:31:56\"}}";
// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
@@ -248,6 +254,14 @@ public class TestJsonKafkaSourcePostProcessor extends TestJsonKafkaSource {
// ddl data will be ignored, ths count should be 0
long ddlDataNum = processor.process(ddlData).count();
assertEquals(0, ddlDataNum);
// test table regex without database regex
props.remove(MaxwellJsonKafkaSourcePostProcessor.Config.DATABASE_NAME_REGEX_PROP.key());
props.setProperty(MaxwellJsonKafkaSourcePostProcessor.Config.TABLE_NAME_REGEX_PROP.key(), "hudi_maxwell(_)?[0-9]{0,2}");
JavaRDD<String> dataWithoutDatabaseRegex = jsc().parallelize(Arrays.asList(hudiMaxwell01Insert, hudi02Maxwell01Insert));
long countWithoutDatabaseRegex = processor.process(dataWithoutDatabaseRegex).count();
assertEquals(2, countWithoutDatabaseRegex);
}
/**