lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-394] Provide a basic implementation of test suite

Browse Source
This commit is contained in:
Nishith Agarwal
2019-11-01 12:40:09 -07:00
committed by n3nash
parent d5b593b7d9
commit 2fc2b01d86
102 changed files with 8633 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
docker/demo/config/test-suite/base.properties Normal file
View File

@@ -0,0 +1,21 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
hoodie.upsert.shuffle.parallelism=2
hoodie.insert.shuffle.parallelism=2
hoodie.bulkinsert.shuffle.parallelism=2
hoodie.datasource.write.partitionpath.field=timestamp

97
docker/demo/config/test-suite/complex-dag-cow.yaml Normal file
View File

@@ -0,0 +1,97 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
first_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 5
num_records_insert: 1000
type: InsertNode
deps: none
second_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 5
num_records_insert: 10000
deps: first_insert
type: InsertNode
third_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 2
num_records_insert: 300
deps: second_insert
type: InsertNode
first_rollback:
config:
deps: third_insert
type: RollbackNode
first_upsert:
config:
record_size: 70000
num_insert_partitions: 1
num_records_insert: 300
repeat_count: 5
num_records_upsert: 100
num_upsert_partitions: 10
type: UpsertNode
deps: first_rollback
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_upsert
first_hive_query:
config:
hive_props:
prop1: "set hive.execution.engine=spark"
prop2: "set spark.yarn.queue="
prop3: "set hive.strict.checks.large.query=false"
prop4: "set hive.stats.autogather=false"
hive_queries:
query1: "select count(*) from testdb1.table1 group by `_row_key` having count(*) > 1"
result1: 0
query2: "select count(*) from testdb1.table1"
result2: 22100000
type: HiveQueryNode
deps: first_hive_sync
second_upsert:
config:
record_size: 70000
num_insert_partitions: 1
num_records_insert: 300
repeat_count: 5
num_records_upsert: 100
num_upsert_partitions: 10
type: UpsertNode
deps: first_hive_query
second_hive_query:
config:
hive_props:
prop1: "set hive.execution.engine=mr"
prop2: "set mapred.job.queue.name="
prop3: "set hive.strict.checks.large.query=false"
prop4: "set hive.stats.autogather=false"
hive_queries:
query1: "select count(*) from testdb1.table1 group by `_row_key` having count(*) > 1"
result1: 0
query2: "select count(*) from testdb1.table1"
result2: 22100
type: HiveQueryNode
deps: second_upsert

119
docker/demo/config/test-suite/complex-dag-mor.yaml Normal file
View File

@@ -0,0 +1,119 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
first_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 5
num_records_insert: 100
type: InsertNode
deps: none
second_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 5
num_records_insert: 100
deps: first_insert
type: InsertNode
third_insert:
config:
record_size: 70000
num_insert_partitions: 1
repeat_count: 2
num_records_insert: 300
deps: second_insert
type: InsertNode
first_rollback:
config:
deps: third_insert
type: RollbackNode
first_upsert:
config:
record_size: 70000
num_insert_partitions: 1
num_records_insert: 300
repeat_count: 5
num_records_upsert: 100
num_upsert_partitions: 10
type: UpsertNode
deps: first_rollback
first_hive_sync:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveSyncNode
deps: first_upsert
first_hive_query:
config:
queue_name: "adhoc"
engine: "mr"
type: HiveQueryNode
deps: first_hive_sync
second_upsert:
config:
record_size: 70000
num_insert_partitions: 1
num_records_insert: 300
repeat_count: 5
num_records_upsert: 100
num_upsert_partitions: 10
type: UpsertNode
deps: first_hive_query
second_hive_query:
config:
queue_name: "adhoc"
engine: "mr"
hive_queries:
query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
result1: 0
query2: "select count(*) from testdb.table1"
result2: 3100
query3: "select count(*) from testdb.table1_rt group by `_row_key` having count(*) > 1"
result3: 0
query4: "select count(*) from testdb.table1_rt"
result4: 3100
type: HiveQueryNode
deps: second_upsert
first_schedule_compact:
config:
type: ScheduleCompactNode
deps: second_hive_query
third_upsert:
config:
record_size: 70000
num_insert_partitions: 1
num_records_insert: 300
repeat_count: 5
num_records_upsert: 100
num_upsert_partitions: 10
type: UpsertNode
deps: first_schedule_compact
first_compact:
config:
type: CompactNode
deps: first_schedule_compact
third_hive_query:
config:
queue_name: "adhoc"
engine: "mr"
hive_queries:
query1: "select count(*) from testdb.table1 group by `_row_key` having count(*) > 1"
result1: 0
query2: "select count(*) from testdb.table1"
result2: 2210
type: HiveQueryNode
deps: second_upsert

466
docker/demo/config/test-suite/complex-source.avsc Normal file
View File

@@ -0,0 +1,466 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
{
"name": "COMPLEX",
"fields": [
{
"default": null,
"type": [
"null",
{
"items": "string",
"type": "array"
}
],
"name": "array_of_string_fields1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field2"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field3"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field4"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field5"
},
{
"default": null,
"type": [
"null",
"boolean"
],
"name": "boolean_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field7"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field8"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field9"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field10"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field11"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field12"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field13"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field14"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field2"
},
{
"default": null,
"type": [
"null",
{
"items": {
"fields": [
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field15"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field16"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field17"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field3"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field4"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field2"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field3"
}
],
"type": "record",
"name": "record_field1"
},
"type": "array"
}
],
"name": "record_name1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field18"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field5"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field4"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field5"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field19"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field20"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field7"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field21"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field22"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field23"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field8"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field7"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field24"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field10"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field25"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field26"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field11"
},
{
"default": null,
"type": [
"null",
"boolean"
],
"name": "boolean_field3"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field12"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field8"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field13"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field27"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field28"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field29"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field30"
}
],
"type": "record"
}

51
docker/demo/config/test-suite/source.avsc Normal file
View File

@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
{
"type" : "record",
"name" : "triprec",
"fields" : [
{
"name" : "timestamp",
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"
}, {
"name" : "rider",
"type" : "string"
}, {
"name" : "driver",
"type" : "string"
}, {
"name" : "begin_lat",
"type" : "double"
}, {
"name" : "begin_lon",
"type" : "double"
}, {
"name" : "end_lat",
"type" : "double"
}, {
"name" : "end_lon",
"type" : "double"
}, {
"name" : "fare",
"type" : "double"
} ]
}

54
docker/demo/config/test-suite/target.avsc Normal file
View File

@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
{
"type" : "record",
"name" : "triprec",
"fields" : [
{
"name" : "timestamp",
"type" : "double"
}, {
"name" : "_row_key",
"type" : "string"
}, {
"name" : "rider",
"type" : "string"
}, {
"name" : "driver",
"type" : "string"
}, {
"name" : "begin_lat",
"type" : "double"
}, {
"name" : "begin_lon",
"type" : "double"
}, {
"name" : "end_lat",
"type" : "double"
}, {
"name" : "end_lon",
"type" : "double"
}, {
"name" : "fare",
"type" : "double"
}, {
"name" : "haversine_distance",
"type" : "double"
}]
}

27
docker/demo/config/test-suite/test-source.properties Normal file
View File

@@ -0,0 +1,27 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
hoodie.datasource.write.recordkey.field=_row_key
hoodie.deltastreamer.source.dfs.root=/user/hive/warehouse/hudi-bench/input
hoodie.datasource.write.keygenerator.class=org.apache.hudi.ComplexKeyGenerator
hoodie.datasource.write.partitionpath.field=timestamp
hoodie.deltastreamer.schemaprovider.source.schema.file=/var/hoodie/ws/docker/demo/config/bench/source.avsc
hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000/
hoodie.datasource.hive_sync.database=testdb
hoodie.datasource.hive_sync.table=test_table
hoodie.datasource.hive_sync.partition_extractor_class=org.apache.hudi.hive.NonPartitionedExtractor
hoodie.datasource.hive_sync.assume_date_partitioning=true
hoodie.datasource.write.keytranslator.class=org.apache.hudi.DayBasedPartitionPathKeyTranslator
hoodie.deltastreamer.schemaprovider.target.schema.file=/var/hoodie/ws/docker/demo/config/bench/source.avsc

1
docker/hoodie/hadoop/hive_base/Dockerfile
View File

@@ -55,6 +55,7 @@ ENV HUDI_HADOOP_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hood
ENV HUDI_HIVE_SYNC_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-sync-bundle.jar
ENV HUDI_SPARK_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar
ENV HUDI_UTILITIES_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-utilities.jar
ENV HUDI_INTEG_TEST_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-integ-test-bundle.jar
COPY startup.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/startup.sh

4
docker/hoodie/hadoop/hive_base/pom.xml
View File

@@ -59,9 +59,7 @@
<copy file="${project.basedir}/../../../../packaging/hudi-hadoop-mr-bundle/target/hudi-hadoop-mr-bundle-${project.version}.jar" tofile="target/hoodie-hadoop-mr-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-hive-sync-bundle/target/hudi-hive-sync-bundle-${project.version}.jar" tofile="target/hoodie-hive-sync-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hudi-spark-bundle/target/hudi-spark-bundle_${scala.binary.version}-${project.version}.jar" tofile="target/hoodie-spark-bundle.jar" />
<copy
file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_${scala.binary.version}-${project.version}.jar"
tofile="target/hoodie-utilities.jar"/>
<copy file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_${scala.binary.version}-${project.version}.jar" tofile="target/hoodie-utilities.jar"/>
</tasks>
</configuration>
<goals>