lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1716]: Resolving default values for schema from dataframe (#2765)

Browse Source 
- Adding default values and setting null as first entry in UNION data types in avro schema. 

Co-authored-by: Aditya Tiwari <aditya.tiwari@flipkart.com>
This commit is contained in:
Aditya Tiwari
2021-04-19 19:35:20 +05:30
committed by GitHub
parent dab5114f16
commit ec2334ceac
9 changed files with 405 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestSchemaPostProcessor.java
View File

@@ -47,8 +47,8 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
private static String ORIGINAL_SCHEMA = "{\"name\":\"t3_biz_operation_t_driver\",\"type\":\"record\",\"fields\":[{\"name\":\"ums_id_\",\"type\":[\"null\",\"string\"],\"default\":null},"
+ "{\"name\":\"ums_ts_\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
private static String RESULT_SCHEMA = "{\"type\":\"record\",\"name\":\"hoodie_source\",\"namespace\":\"hoodie.source\",\"fields\":[{\"name\":\"ums_id_\",\"type\":[\"string\",\"null\"]},"
+ "{\"name\":\"ums_ts_\",\"type\":[\"string\",\"null\"]}]}";
private static String RESULT_SCHEMA = "{\"type\":\"record\",\"name\":\"hoodie_source\",\"namespace\":\"hoodie.source\",\"fields\":[{\"name\":\"ums_id_\",\"type\":[\"null\",\"string\"],"
+ "\"default\":null},{\"name\":\"ums_ts_\",\"type\":[\"null\",\"string\"],\"default\":null}]}";
@Test
public void testPostProcessor() throws IOException {

24
hudi-utilities/src/test/resources/delta-streamer-config/source-jdbc.avsc
View File

@@ -26,34 +26,42 @@
},
{
"name": "TIMESTAMP",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
},
{
"name": "RIDER",
"type": ["string", "null"]
"type": ["null", "string"],
"default": null
},
{
"name": "DRIVER",
"type": ["string", "null"]
"type": ["null" ,"string"],
"default": null
},
{
"name": "BEGIN_LAT",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
},
{
"name": "BEGIN_LON",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
},
{
"name": "END_LAT",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
},
{
"name": "END_LON",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
},
{
"name": "FARE",
"type": ["double", "null"]
"type": ["null", "double"],
"default": null
} ]
}