lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-713] Fix conversion of Spark array of struct type to Avro schema (#1406)

Browse Source 
Co-authored-by: Wenning Ding <wenningd@amazon.com>
This commit is contained in:
wenningd
2020-03-30 15:52:15 -07:00
committed by GitHub
parent dbc9acd23a
commit ce0a4c64d0
6 changed files with 75 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
hudi-utilities/src/test/resources/delta-streamer-config/source.avsc
View File

@@ -44,6 +44,13 @@
"name" : "end_lon",
"type" : "double"
},
{
"name" :"city_to_state",
"type" : {
"type" : "map",
"values": "string"
}
},
{
"name" : "fare",
"type" : {
@@ -61,6 +68,26 @@
]
}
},
{
"name" : "tip_history",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "tip_history",
"fields" : [
{
"name" : "amount",
"type" : "double"
},
{
"name" : "currency",
"type" : "string"
}
]
}
}
},
{
"name" : "_hoodie_is_deleted",
"type" : "boolean",

2
hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties
View File

@@ -16,4 +16,4 @@
# limitations under the License.
###
include=base.properties
hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.fare, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a
hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.city_to_state, a.fare, a.tip_history, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a

26
hudi-utilities/src/test/resources/delta-streamer-config/target.avsc
View File

@@ -43,6 +43,12 @@
}, {
"name" : "end_lon",
"type" : "double"
}, {
"name" :"city_to_state",
"type" : {
"type" : "map",
"values": "string"
}
}, {
"name" : "fare",
"type" : {
@@ -60,6 +66,26 @@
]
}
},
{
"name" : "tip_history",
"type" : {
"type" : "array",
"items" : {
"type" : "record",
"name" : "tip_history",
"fields" : [
{
"name" : "amount",
"type" : "double"
},
{
"name" : "currency",
"type" : "string"
}
]
}
}
},
{
"name" : "_hoodie_is_deleted",
"type" : "boolean",