788d236c44
1. Use the DAG Node's label from the yaml as its name instead of UUID names which are not descriptive when debugging issues from logs. 2. Fix CleanNode constructor which is not correctly implemented 3. When generating upsets, allows more granualar control over the number of inserts and upserts - zero or more inserts and upserts can be specified instead of always requiring both inserts and upserts. 4. Fixed generation of records of specific size - The current code was using a class variable "shouldAddMore" which was reset to false after the first record generation causing subsequent records to be of minimum size. - In this change, we pre-calculate the extra size of the complex fields. When generating records, for complex fields we read the field size from this map. 5. Refresh the timeline of the DeltaSync service before calling readFromSource. This ensures that only the newest generated data is read and data generated in the older Dag Nodes is ignored (as their AVRO files will have an older timestamp). 6. Making --workload-generator-classname an optional parameter as most probably the default will be used
52 lines
2.6 KiB
Bash
52 lines
2.6 KiB
Bash
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore
|
|
HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
|
|
HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
|
|
HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
|
|
HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
|
|
HIVE_SITE_CONF_hive_metastore_uris=thrift://hivemetastore:9083
|
|
|
|
HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
|
|
HDFS_CONF_dfs_webhdfs_enabled=true
|
|
HDFS_CONF_dfs_permissions_enabled=false
|
|
#HDFS_CONF_dfs_client_use_datanode_hostname=true
|
|
#HDFS_CONF_dfs_namenode_use_datanode_hostname=true
|
|
HDFS_CONF_dfs_replication=1
|
|
|
|
CORE_CONF_fs_defaultFS=hdfs://namenode:8020
|
|
CORE_CONF_hadoop_http_staticuser_user=root
|
|
CORE_CONF_hadoop_proxyuser_hue_hosts=*
|
|
CORE_CONF_hadoop_proxyuser_hue_groups=*
|
|
|
|
YARN_CONF_yarn_log___aggregation___enable=true
|
|
YARN_CONF_yarn_resourcemanager_recovery_enabled=true
|
|
YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
|
|
YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
|
|
YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
|
|
YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
|
|
YARN_CONF_yarn_timeline___service_enabled=true
|
|
YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
|
|
YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
|
|
YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
|
|
YARN_CONF_yarn_timeline___service_hostname=historyserver
|
|
YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
|
|
YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
|
|
YARN_CONF_yarn_resourcemanager_resource___tracker_address=resourcemanager:8031
|
|
YARN_CONF_yarn_nodemanager_vmem___check___enabled=false
|