lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3575] Use HoodieTestDataGenerator#TRIP_SCHEMA as example schema in TestSchemaPostProcessor (#5019)

Browse Source
This commit is contained in:
wangxianghu
2022-03-11 15:03:42 +04:00
committed by GitHub
parent faed6996ee
commit b00180342e
1 changed files with 14 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestSchemaPostProcessor.java
View File

@@ -44,13 +44,16 @@ import static org.junit.jupiter.api.Assertions.assertNull;
public class TestSchemaPostProcessor extends UtilitiesTestBase { public class TestSchemaPostProcessor extends UtilitiesTestBase {
private TypedProperties properties = new TypedProperties(); private final TypedProperties properties = new TypedProperties();
private static String ORIGINAL_SCHEMA = "{\"name\":\"t3_biz_operation_t_driver\",\"type\":\"record\",\"fields\":[{\"name\":\"ums_id_\",\"type\":[\"null\",\"string\"],\"default\":null}," private static final String ORIGINAL_SCHEMA = "{\"type\":\"record\",\"name\":\"tripUberRec\",\"fields\":"
+ "{\"name\":\"ums_ts_\",\"type\":[\"null\",\"string\"],\"default\":null}]}"; + "[{\"name\":\"timestamp\",\"type\":\"long\"},{\"name\":\"_row_key\",\"type\":\"string\"},{\"name\":\"rider\","
+ "\"type\":\"string\"},{\"name\":\"driver\",\"type\":\"string\"},{\"name\":\"fare\",\"type\":\"double\"}]}";
private static String RESULT_SCHEMA = "{\"type\":\"record\",\"name\":\"hoodie_source\",\"namespace\":\"hoodie.source\",\"fields\":[{\"name\":\"ums_id_\",\"type\":[\"null\",\"string\"]," private static final String RESULT_SCHEMA = "{\"type\":\"record\",\"name\":\"hoodie_source\","
+ "\"default\":null},{\"name\":\"ums_ts_\",\"type\":[\"null\",\"string\"],\"default\":null}]}"; + "\"namespace\":\"hoodie.source\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"long\"},"
+ "{\"name\":\"_row_key\",\"type\":\"string\"},{\"name\":\"rider\",\"type\":\"string\"},{\"name\":\"driver\","
+ "\"type\":\"string\"},{\"name\":\"fare\",\"type\":\"double\"}]}";
@Test @Test
public void testPostProcessor() throws IOException { public void testPostProcessor() throws IOException {
@@ -102,7 +105,7 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA); Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
Schema targetSchema = processor.processSchema(schema); Schema targetSchema = processor.processSchema(schema);
assertNull(targetSchema.getField("ums_id_")); assertNull(targetSchema.getField("_row_key"));
assertNull(targetSchema.getField("_hoodie_is_deleted")); assertNull(targetSchema.getField("_hoodie_is_deleted"));
assertNotNull(targetSchema.getField("testString")); assertNotNull(targetSchema.getField("testString"));
@@ -114,7 +117,7 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
schema = new Schema.Parser().parse(ORIGINAL_SCHEMA); schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
targetSchema = processor.processSchema(schema); targetSchema = processor.processSchema(schema);
assertNull(targetSchema.getField("ums_id_")); assertNull(targetSchema.getField("_row_key"));
assertNotNull(targetSchema.getField("_hoodie_is_deleted")); assertNotNull(targetSchema.getField("_hoodie_is_deleted"));
assertNotNull(targetSchema.getField("testString")); assertNotNull(targetSchema.getField("testString"));
} }
@@ -122,19 +125,19 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
@Test @Test
public void testDeleteColumn() { public void testDeleteColumn() {
// remove column ums_id_ from source schema // remove column ums_id_ from source schema
properties.put(DropColumnSchemaPostProcessor.Config.DELETE_COLUMN_POST_PROCESSOR_COLUMN_PROP, "ums_id_"); properties.put(DropColumnSchemaPostProcessor.Config.DELETE_COLUMN_POST_PROCESSOR_COLUMN_PROP, "rider");
DropColumnSchemaPostProcessor processor = new DropColumnSchemaPostProcessor(properties, null); DropColumnSchemaPostProcessor processor = new DropColumnSchemaPostProcessor(properties, null);
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA); Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
Schema targetSchema = processor.processSchema(schema); Schema targetSchema = processor.processSchema(schema);
assertNull(targetSchema.getField("ums_id_")); assertNull(targetSchema.getField("rider"));
assertNotNull(targetSchema.getField("ums_ts_")); assertNotNull(targetSchema.getField("_row_key"));
} }
@Test @Test
public void testDeleteColumnThrows() { public void testDeleteColumnThrows() {
// remove all columns from source schema // remove all columns from source schema
properties.put(DropColumnSchemaPostProcessor.Config.DELETE_COLUMN_POST_PROCESSOR_COLUMN_PROP, "ums_id_,ums_ts_"); properties.put(DropColumnSchemaPostProcessor.Config.DELETE_COLUMN_POST_PROCESSOR_COLUMN_PROP, "timestamp,_row_key,rider,driver,fare");
DropColumnSchemaPostProcessor processor = new DropColumnSchemaPostProcessor(properties, null); DropColumnSchemaPostProcessor processor = new DropColumnSchemaPostProcessor(properties, null);
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA); Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);