Docker Container Build and Run setup with foundations for adding docker integration tests. Docker images built with Hadoop 2.8.4 Hive 2.3.3 and Spark 2.3.1 and published to docker-hub

Look at quickstart document for how to setup docker and run demo

docker/demo/config/base.properties

@@ -0,0 +1,21 @@
+#
+#  Copyright (c) 2017 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#           http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Common hoodie client configs
+hoodie.upsert.shuffle.parallelism=2
+hoodie.insert.shuffle.parallelism=2
+hoodie.bulkinsert.shuffle.parallelism=2

docker/demo/config/kafka-source.properties

@@ -0,0 +1,29 @@
+#
+#  Copyright (c) 2017 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#           http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+include=base.properties
+# Key fields, for kafka example
+hoodie.datasource.write.recordkey.field=key
+hoodie.datasource.write.partitionpath.field=date
+# Schema provider props (change to absolute path based on your installation)
+hoodie.deltastreamer.schemaprovider.source.schema.file=/var/demo/config/schema.avsc
+hoodie.deltastreamer.schemaprovider.target.schema.file=/var/demo/config/schema.avsc
+# Kafka Source
+hoodie.deltastreamer.source.kafka.topic=stock_ticks
+#Kafka props
+metadata.broker.list=kafkabroker:9092
+auto.offset.reset=smallest

docker/demo/config/schema.avsc

@@ -0,0 +1,41 @@
+{
+  "type":"record",
+  "name":"stock_ticks",
+  "fields":[{
+     "name": "volume",
+     "type": "long"
+  }, {
+     "name": "ts", 
+     "type": "string"
+  }, {
+     "name": "symbol", 
+     "type": "string"
+  },{
+     "name": "year", 
+     "type": "int"
+  },{
+     "name": "month", 
+     "type": "string"
+  },{
+     "name": "high", 
+     "type": "double"
+  },{
+     "name": "low", 
+     "type": "double"
+  },{
+     "name": "key", 
+     "type": "string"
+  },{
+     "name": "date", 
+     "type":"string"
+  }, {
+     "name": "close", 
+     "type": "double"
+  }, {
+     "name": "open", 
+     "type": "double"
+  }, {
+     "name": "day", 
+     "type":"string"
+  }
+]}

docker/demo/config/spark-defaults.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default system properties included when running spark-submit.
+# This is useful for setting default environmental settings.
+
+# Example:
+spark.master                     local[3]
+spark.eventLog.dir               hdfs://namenode:8020/tmp/spark-events
+spark.serializer                 org.apache.spark.serializer.KryoSerializer
+#spark.executor.memory            4g
+# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"