[HUDI-703] Add test for HoodieSyncCommand (#1774)
This commit is contained in:
hongdd
31
docker/demo/config/hoodie-incr.properties
Normal file
31
docker/demo/config/hoodie-incr.properties
Normal file
@@ -0,0 +1,31 @@
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
hoodie.upsert.shuffle.parallelism=2
|
||||
hoodie.insert.shuffle.parallelism=2
|
||||
hoodie.bulkinsert.shuffle.parallelism=2
|
||||
hoodie.datasource.write.recordkey.field=_row_key
|
||||
hoodie.datasource.write.partitionpath.field=partition
|
||||
hoodie.deltastreamer.schemaprovider.source.schema.file=file:///var/hoodie/ws/docker/demo/config/hoodie-schema.avsc
|
||||
hoodie.deltastreamer.schemaprovider.target.schema.file=file:///var/hoodie/ws/docker/demo/config/hoodie-schema.avsc
|
||||
hoodie.deltastreamer.source.hoodieincr.partition.fields=partition
|
||||
hoodie.deltastreamer.source.hoodieincr.path=/docker_hoodie_sync_valid_test
|
||||
hoodie.deltastreamer.source.hoodieincr.read_latest_on_missing_ckpt=true
|
||||
# hive sync
|
||||
hoodie.datasource.hive_sync.table=docker_hoodie_sync_valid_test_2
|
||||
hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000
|
||||
hoodie.datasource.hive_sync.partition_fields=partition
|
||||
145
docker/demo/config/hoodie-schema.avsc
Normal file
145
docker/demo/config/hoodie-schema.avsc
Normal file
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
{
|
||||
"type": "record",
|
||||
"name": "triprec",
|
||||
"fields": [
|
||||
{
|
||||
"name": "timestamp",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "_row_key",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "rider",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "driver",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "begin_lat",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "begin_lon",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "end_lat",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "end_lon",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "distance_in_meters",
|
||||
"type": "int"
|
||||
},
|
||||
{
|
||||
"name": "seconds_since_epoch",
|
||||
"type": "long"
|
||||
},
|
||||
{
|
||||
"name": "weight",
|
||||
"type": "float"
|
||||
},
|
||||
{
|
||||
"name": "nation",
|
||||
"type": "bytes"
|
||||
},
|
||||
{
|
||||
"name": "current_date",
|
||||
"type": {
|
||||
"type": "int",
|
||||
"logicalType": "date"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "current_ts",
|
||||
"type": {
|
||||
"type": "long",
|
||||
"logicalType": "timestamp-micros"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": {
|
||||
"type": "fixed",
|
||||
"name": "abc",
|
||||
"size": 5,
|
||||
"logicalType": "decimal",
|
||||
"precision": 10,
|
||||
"scale": 6
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "city_to_state",
|
||||
"type": {
|
||||
"type": "map",
|
||||
"values": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "fare",
|
||||
"type": {
|
||||
"type": "record",
|
||||
"name": "fare",
|
||||
"fields": [
|
||||
{
|
||||
"name": "amount",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "currency",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tip_history",
|
||||
"type": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "record",
|
||||
"name": "tip_history",
|
||||
"fields": [
|
||||
{
|
||||
"name": "amount",
|
||||
"type": "double"
|
||||
},
|
||||
{
|
||||
"name": "currency",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "_hoodie_is_deleted",
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
}
|
||||
]
|
||||
}
|
||||
19
docker/demo/sync-validate.commands
Normal file
19
docker/demo/sync-validate.commands
Normal file
@@ -0,0 +1,19 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
connect --path /docker_hoodie_sync_valid_test
|
||||
commits sync --path /docker_hoodie_sync_valid_test_2
|
||||
sync validate --sourceDb default --targetDb default --hiveServerUrl jdbc:hive2://hiveserver:10000 --hiveUser hive --hivePass hive
|
||||
Reference in New Issue
Block a user