bde7a7043e
add hoodie delta streamer mock source example and dfs source and kafka source examples Signed-off-by: dengziming <dengziming1993@gmail.com> add defaultSparkConf utils method change version of hudi-examples to 0.5.2-SNAPSHOT change the artifcatId of hudi-spark and hudi-utilities alter some code to adapt kafka2.0 Update scritps Add license
40 lines
1.6 KiB
Bash
Executable File
40 lines
1.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
EXAMPLES_DIR="$(dirname $(dirname "${BASH_SOURCE[0]}"))"
|
|
PROJECT_DIR="$(dirname ${EXAMPLES_DIR})"
|
|
|
|
JAR_FILE=`ls ${PROJECT_DIR}/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_*.jar | grep -v sources | grep -v tests.jar`
|
|
EXAMPLES_JARS=`ls ${PROJECT_DIR}/hudi-examples/target/hudi-examples-*.jar | grep -v sources | grep -v tests.jar`
|
|
|
|
if [ -z "${SPARK_MASTER}" ]; then
|
|
SPARK_MASTER="yarn-cluster"
|
|
fi
|
|
|
|
exec "${SPARK_HOME}"/bin/spark-submit \
|
|
--master ${SPARK_MASTER} \
|
|
--conf spark.serializer="org.apache.spark.serializer.KryoSerializer" \
|
|
--conf spark.kryoserializer.buffer.max=128m \
|
|
--conf spark.yarn.queue=root.default \
|
|
--conf spark.yarn.submit.waitAppCompletion=false \
|
|
--packages org.apache.spark:spark-avro_2.11:2.4.4 \
|
|
--jars ${EXAMPLES_JARS} \
|
|
--class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer \
|
|
"${JAR_FILE}" \
|
|
$@
|