diff --git a/bin/upload-config.sh b/bin/upload-config.sh index f1bd3e3..257d838 100755 --- a/bin/upload-config.sh +++ b/bin/upload-config.sh @@ -3,6 +3,9 @@ root_path=$(dirname $(cd $(dirname $0); pwd)) clusters=( + b2a4 + b2b1 + b2b5 b2b12 b9b9 ) diff --git a/config/b1e1/core-site.xml b/config/b1e1/core-site.xml deleted file mode 100644 index 9d72427..0000000 --- a/config/b1e1/core-site.xml +++ /dev/null @@ -1,262 +0,0 @@ - - - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - - - fs.defaultFS - hdfs://b1 - true - - - - fs.s3a.fast.upload - true - - - - fs.s3a.fast.upload.buffer - disk - - - - fs.s3a.multipart.size - 67108864 - - - - fs.trash.interval - 360 - - - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 - - - - ha.zookeeper.acl - sasl:nn:rwcda - - - - ha.zookeeper.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - hadoop.http.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - hadoop.http.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - hadoop.http.authentication.signature.secret.file - /etc/security/http_secret - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - - - hadoop.http.authentication.type - simple - - - - hadoop.http.cross-origin.allowed-headers - X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding - - - - hadoop.http.cross-origin.allowed-methods - GET,PUT,POST,OPTIONS,HEAD,DELETE - - - - hadoop.http.cross-origin.allowed-origins - * - - - - hadoop.http.cross-origin.max-age - 1800 - - - - hadoop.http.filter.initializers - org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer - - - - hadoop.proxyuser.hdfs.groups - * - - - - hadoop.proxyuser.hdfs.hosts - * - - - - hadoop.proxyuser.hive.groups - * - - - - hadoop.proxyuser.hive.hosts - * - - - - hadoop.proxyuser.HTTP.groups - * - - - - hadoop.proxyuser.HTTP.hosts - * - - - - hadoop.proxyuser.iap.groups - * - - - - hadoop.proxyuser.iap.hosts - * - - - - hadoop.proxyuser.livy.groups - * - - - - hadoop.proxyuser.livy.hosts - * - - - - hadoop.proxyuser.yarn.groups - * - - - - hadoop.proxyuser.yarn.hosts - * - - - - hadoop.rpc.protection - authentication,privacy - - - - hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/ -RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// -RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ -RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/ -RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/ -RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/ -RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/ -RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/ -RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/ -RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/ -DEFAULT - - - - hadoop.security.authentication - kerberos - - - - hadoop.security.authorization - true - - - - hadoop.security.instrumentation.requires.admin - false - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec - - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.file.buffer.size - 131072 - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - - - - ipc.client.connect.max.retries - 50 - - - - ipc.client.connection.maxidletime - 30000 - - - - ipc.client.idlethreshold - 8000 - - - - ipc.server.tcpnodelay - true - - - - mapreduce.jobtracker.webinterface.trusted - false - - - - ipc.client.fallback-to-simple-auth-allowed - true - - - - fs.hdfs.impl.disable.cache - true - - - diff --git a/config/b1e1/hdfs-site.xml b/config/b1e1/hdfs-site.xml deleted file mode 100644 index 9b4eda4..0000000 --- a/config/b1e1/hdfs-site.xml +++ /dev/null @@ -1,698 +0,0 @@ - - - - dfs.block.access.token.enable - true - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.failover.proxy.provider.b1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.client.retry.policy.enabled - false - - - - dfs.cluster.administrators - hdfs - - - - dfs.content-summary.limit - 5000 - - - - dfs.data.transfer.protection - authentication,privacy - - - - dfs.datanode.address - 0.0.0.0:1019 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - - dfs.datanode.du.reserved - 26405499904 - - - - dfs.datanode.failed.volumes.tolerated - 2 - true - - - - dfs.datanode.http.address - 0.0.0.0:1022 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.kerberos.principal - dn/_HOST@ECLD.COM - - - - dfs.datanode.keytab.file - /etc/security/keytabs/dn.service.keytab - - - - dfs.datanode.max.transfer.threads - 16384 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.encrypt.data.transfer.cipher.suites - AES/CTR/NoPadding - - - - dfs.ha.automatic-failover.enabled - true - - - - dfs.ha.fencing.methods - shell(/bin/true) - - - - dfs.ha.namenodes.b1 - nn1,nn2 - - - - dfs.ha.namenodes.b2 - nn3,nn4 - - - - dfs.heartbeat.interval - 3 - - - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude - - - - dfs.http.policy - HTTP_ONLY - - - - dfs.https.port - 50470 - - - - dfs.internal.nameservices - b1,b2 - - - - dfs.journalnode.edits.dir.b1 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.edits.dir.b2 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.journalnode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.journalnode.kerberos.principal - jn/_HOST@ECLD.COM - - - - dfs.journalnode.keytab.file - /etc/security/keytabs/jn.service.keytab - - - - dfs.namenode.accesstime.precision - 0 - - - - dfs.namenode.acls.enabled - true - - - - dfs.namenode.audit.log.async - true - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /data/hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.fslock.fair - false - - - - dfs.namenode.handler.count - 200 - - - - dfs.namenode.http-address.b1.nn1 - b1m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn2 - b1m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn3 - b1m5.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn4 - b1m6.hdp.dc:50070 - - - - dfs.namenode.https-address.b1.nn1 - b1m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn2 - b1m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn3 - b1m5.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn4 - b1m6.hdp.dc:50470 - - - - dfs.namenode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.namenode.kerberos.principal - nn/_HOST@ECLD.COM - - - - dfs.namenode.keytab.file - /etc/security/keytabs/nn.service.keytab - - - - dfs.namenode.max.extra.edits.segments.retained - 180 - - - - dfs.namenode.name.dir - /data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.num.extra.edits.retained - 18000 - - - - dfs.namenode.rpc-address.b1.nn1 - b1m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn2 - b1m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn3 - b1m5.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn4 - b1m6.hdp.dc:8020 - - - - dfs.namenode.safemode.threshold-pct - 0.99 - - - - dfs.namenode.shared.edits.dir.b1 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1 - - - - dfs.namenode.shared.edits.dir.b2 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2 - - - - dfs.namenode.stale.datanode.interval - 30000 - - - - dfs.namenode.startup.delay.block.deletion.sec - 3600 - - - - dfs.namenode.write.stale.datanode.ratio - 1.0f - - - - dfs.nameservices - b1,b2,b3,b4,a3,a4,f1,d2,e1 - - - - dfs.permissions.ContentSummary.subAccess - true - - - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs - - - - dfs.replication - 3 - - - - dfs.replication.max - 50 - - - - dfs.web.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - dfs.web.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - dfs.webhdfs.enabled - true - true - - - - fs.permissions.umask-mode - 022 - - - - hadoop.caller.context.enabled - true - - - - manage.include.files - false - - - - nfs.exports.allowed.hosts - * rw - - - - nfs.file.dump.dir - /tmp/.hdfs-nfs - - - - dfs.client.datanode-restart.timeout - 30 - - - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a3 - nn1,nn2 - - - dfs.namenode.http-address.a3.nn1 - a3m2.hdp.dc:50070 - - - dfs.namenode.http-address.a3.nn2 - a3m3.hdp.dc:50070 - - - dfs.namenode.https-address.a3.nn1 - a3m2.hdp.dc:50470 - - - dfs.namenode.https-address.a3.nn2 - a3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a3.nn1 - a3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a3.nn2 - a3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b3 - nn1,nn2 - - - dfs.namenode.http-address.b3.nn1 - b3m2.hdp.dc:50070 - - - dfs.namenode.http-address.b3.nn2 - b3m3.hdp.dc:50070 - - - dfs.namenode.https-address.b3.nn1 - b3m2.hdp.dc:50470 - - - dfs.namenode.https-address.b3.nn2 - b3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b3.nn1 - b3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b3.nn2 - b3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b4 - nn1,nn2 - - - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 - - - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 - - - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 - - - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.f1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.f1 - nn1,nn2 - - - dfs.namenode.http-address.f1.nn1 - f1m2.hdp.dc:50070 - - - dfs.namenode.http-address.f1.nn2 - f1m3.hdp.dc:50070 - - - dfs.namenode.https-address.f1.nn1 - f1m2.hdp.dc:50470 - - - dfs.namenode.https-address.f1.nn2 - f1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.f1.nn1 - f1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.f1.nn2 - f1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.d2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.d2 - nn1,nn2 - - - dfs.namenode.http-address.d2.nn1 - d2m2.hdp.dc:50070 - - - dfs.namenode.http-address.d2.nn2 - d2m3.hdp.dc:50070 - - - dfs.namenode.https-address.d2.nn1 - d2m2.hdp.dc:50470 - - - dfs.namenode.https-address.d2.nn2 - d2m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.d2.nn1 - d2m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.d2.nn2 - d2m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.e1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.e1 - nn1,nn2 - - - dfs.namenode.http-address.e1.nn1 - e1m2.hdp.dc:50070 - - - dfs.namenode.http-address.e1.nn2 - e1m3.hdp.dc:50070 - - - dfs.namenode.https-address.e1.nn1 - e1m2.hdp.dc:50470 - - - dfs.namenode.https-address.e1.nn2 - e1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.e1.nn1 - e1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.e1.nn2 - e1m3.hdp.dc:8020 - - - diff --git a/config/b1e1/yarn-site.xml b/config/b1e1/yarn-site.xml deleted file mode 100644 index 407d1f5..0000000 --- a/config/b1e1/yarn-site.xml +++ /dev/null @@ -1,1026 +0,0 @@ - - - - hadoop.http.cross-origin.allowed-origins - regex:.*[.]hdp[.]dc(:\d*)? - - - - hadoop.registry.client.auth - kerberos - - - - hadoop.registry.dns.bind-address - 0.0.0.0 - - - - hadoop.registry.dns.bind-port - 5354 - true - - - - hadoop.registry.dns.domain-name - ECLD.COM - - - - hadoop.registry.dns.enabled - true - - - - hadoop.registry.dns.zone-mask - 255.255.255.0 - - - - hadoop.registry.dns.zone-subnet - 172.17.0.0 - - - - hadoop.registry.jaas.context - Client - - - - hadoop.registry.secure - true - - - - hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b1,sasl:rm,sasl:hive,sasl:spark - - - - hadoop.registry.zk.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - manage.include.files - false - - - - yarn.acl.enable - true - - - - yarn.admin.acl - * - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/lib/edp/hadoop-3.2.2/share/hadoop/common/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/common/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 2592000 - - - - yarn.log.server.url - http://b1m4.hdp.dc:19888/jobhistory/logs - - - - yarn.log.server.web-service.url - http://b1m4.hdp.dc:8188/ws/v1/applicationhistory - - - - yarn.node-labels.enabled - false - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle,spark_shuffle,timeline_collector,sparkv2_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.aux-services.spark_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.spark_shuffle.classpath - /usr/lib/edp/spark-3.1.1-bin-hadoop3.2/yarn/* - - - - yarn.nodemanager.aux-services.timeline_collector.class - org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor - - - - yarn.nodemanager.container-metrics.unregister-delay-ms - 60000 - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 86400 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.keytab - /etc/security/keytabs/nm.service.keytab - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users - true - - - - yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - 3600 - - - - yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log - - - - yarn.nodemanager.log.retain-seconds - 604800 - - - - yarn.nodemanager.principal - nm/_HOST@ECLD.COM - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled - true - - - - yarn.nodemanager.recovery.supervised - true - - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource-plugins - - - - - yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - - - - - yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - - - - - yarn.nodemanager.resource.cpu-vcores - 39 - - - - yarn.nodemanager.resource.memory-mb - 184320 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 - - - - yarn.nodemanager.resourcemanager.connect.wait.secs - 1800 - - - - yarn.nodemanager.runtime.linux.allowed-runtimes - default,docker - - - - yarn.nodemanager.runtime.linux.docker.allowed-container-networks - host,none,bridge - - - - yarn.nodemanager.runtime.linux.docker.capabilities - - CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP, - SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE - - - - yarn.nodemanager.runtime.linux.docker.default-container-network - host - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed - false - - - - yarn.nodemanager.vmem-check-enabled - false - - - - yarn.nodemanager.vmem-pmem-ratio - 2.1 - - - - yarn.nodemanager.webapp.cross-origin.enabled - true - - - - yarn.nodemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.nodemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.address - b1m2.hdp.dc:8050 - - - - yarn.resourcemanager.admin.address - b1m2.hdp.dc:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.cluster-id - yarn-cluster - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.display.per-user-apps - true - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.automatic-failover.zk-base-path - /yarn-leader-election - - - - yarn.resourcemanager.ha.enabled - true - - - - yarn.resourcemanager.ha.rm-ids - rm1,rm2 - - - - yarn.resourcemanager.hostname - b1m2.hdp.dc - - - - yarn.resourcemanager.hostname.rm1 - b1m2.hdp.dc - - - - yarn.resourcemanager.hostname.rm2 - b1m3.hdp.dc - - - - yarn.resourcemanager.keytab - /etc/security/keytabs/rm.service.keytab - - - - yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled - true - - - - yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval - 15000 - - - - yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor - 1 - - - - yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round - 0.1 - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.placement-constraints.handler - scheduler - - - - yarn.resourcemanager.principal - rm/_HOST@ECLD.COM - - - - yarn.resourcemanager.proxy-user-privileges.enabled - true - - - - yarn.resourcemanager.proxyuser.*.groups - - - - - yarn.resourcemanager.proxyuser.*.hosts - - - - - yarn.resourcemanager.proxyuser.*.users - - - - - yarn.resourcemanager.recovery.enabled - true - - - - yarn.resourcemanager.resource-tracker.address - b1m2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm1 - b1m2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm2 - b1m3.hdp.dc:8025 - - - - yarn.resourcemanager.scheduler.address - b1m2.hdp.dc:8030 - - - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.scheduler.monitor.enable - true - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true - - - - yarn.resourcemanager.webapp.address - b1m2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm1 - b1m2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm2 - b1m3.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.cross-origin.enabled - true - - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - - - yarn.resourcemanager.webapp.https.address - b1m2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm1 - b1m2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm2 - b1m3.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.resourcemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.work-preserving-recovery.enabled - true - - - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 - - - - yarn.resourcemanager.zk-acl - sasl:rm:rwcda - - - - yarn.resourcemanager.zk-address - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - yarn.resourcemanager.zk-num-retries - 1000 - - - - yarn.resourcemanager.zk-retry-interval-ms - 1000 - - - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore - - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.rm.system-metricspublisher.emit-container-events - true - - - - yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled - true - - - - yarn.scheduler.maximum-allocation-mb - 40960 - - - - yarn.scheduler.maximum-allocation-vcores - 51 - - - - yarn.scheduler.minimum-allocation-mb - 1024 - - - - yarn.scheduler.minimum-allocation-vcores - 1 - - - - yarn.service.system-service.dir - /services - - - - yarn.system-metricspublisher.enabled - true - - - - yarn.timeline-service.address - b1m4.hdp.dc:10200 - - - - yarn.timeline-service.bind-host - 0.0.0.0 - - - - yarn.timeline-service.client.max-retries - 30 - - - - yarn.timeline-service.client.retry-interval-ms - 1000 - - - - yarn.timeline-service.enabled - false - - - - yarn.timeline-service.entity-group-fs-store.active-dir - /ats/active/ - - - - yarn.timeline-service.entity-group-fs-store.app-cache-size - 10 - - - - yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds - 3600 - - - - yarn.timeline-service.entity-group-fs-store.done-dir - /ats/done/ - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes - org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - - - - - yarn.timeline-service.entity-group-fs-store.retain-seconds - 604800 - - - - yarn.timeline-service.entity-group-fs-store.scan-interval-seconds - 60 - - - - yarn.timeline-service.entity-group-fs-store.summary-store - org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore - - - - yarn.timeline-service.generic-application-history.save-non-am-container-meta-info - false - - - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - - - yarn.timeline-service.hbase-schema.prefix - prod. - - - - yarn.timeline-service.hbase.configuration.file - file:///etc/hadoop/conf/embedded-yarn-ats-hbase/hbase-site.xml - - - - yarn.timeline-service.hbase.coprocessor.jar.hdfs.location - file:///usr/lib/edp/hadoop-3.2.2/lib/hadoop-yarn-server-timelineservice-3.2.2.jar - - - - yarn.timeline-service.http-authentication.cookie.domain - - - - - yarn.timeline-service.http-authentication.cookie.path - - - - - yarn.timeline-service.http-authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - yarn.timeline-service.http-authentication.kerberos.name.rules - - - - - yarn.timeline-service.http-authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - yarn.timeline-service.http-authentication.proxyuser.*.groups - - - - - yarn.timeline-service.http-authentication.proxyuser.*.hosts - - - - - yarn.timeline-service.http-authentication.proxyuser.*.users - - - - - yarn.timeline-service.http-authentication.signature.secret - - - - - yarn.timeline-service.http-authentication.signature.secret.file - - - - - yarn.timeline-service.http-authentication.signer.secret.provider - - - - - yarn.timeline-service.http-authentication.signer.secret.provider.object - - - - - yarn.timeline-service.http-authentication.simple.anonymous.allowed - true - - - - yarn.timeline-service.http-authentication.token.validity - - - - - yarn.timeline-service.http-authentication.type - simple - - - - yarn.timeline-service.http-cross-origin.enabled - true - - - - yarn.timeline-service.keytab - /etc/security/keytabs/yarn.service.keytab - - - - yarn.timeline-service.leveldb-state-store.path - /data/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.path - /data/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.principal - yarn/_HOST@ECLD.COM - - - - yarn.timeline-service.reader.webapp.address - b1m4.hdp.dc:8198 - - - - yarn.timeline-service.reader.webapp.https.address - b1m4.hdp.dc:8199 - - - - yarn.timeline-service.recovery.enabled - true - - - - yarn.timeline-service.state-store-class - org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.version - 2.0f - - - - yarn.timeline-service.versions - 1.5f,2.0f - - - - yarn.timeline-service.webapp.address - b1m4.hdp.dc:8188 - - - - yarn.timeline-service.webapp.https.address - b1m4.hdp.dc:8190 - - - - yarn.webapp.api-service.enable - true - - - - yarn.webapp.ui2.enable - true - - - - yarn.resourcemanager.max-completed-applications - 10000 - - - diff --git a/config/b1e11/core-site.xml b/config/b1e11/core-site.xml deleted file mode 100644 index 9d72427..0000000 --- a/config/b1e11/core-site.xml +++ /dev/null @@ -1,262 +0,0 @@ - - - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - - - fs.defaultFS - hdfs://b1 - true - - - - fs.s3a.fast.upload - true - - - - fs.s3a.fast.upload.buffer - disk - - - - fs.s3a.multipart.size - 67108864 - - - - fs.trash.interval - 360 - - - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 - - - - ha.zookeeper.acl - sasl:nn:rwcda - - - - ha.zookeeper.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - hadoop.http.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - hadoop.http.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - hadoop.http.authentication.signature.secret.file - /etc/security/http_secret - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - - - hadoop.http.authentication.type - simple - - - - hadoop.http.cross-origin.allowed-headers - X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding - - - - hadoop.http.cross-origin.allowed-methods - GET,PUT,POST,OPTIONS,HEAD,DELETE - - - - hadoop.http.cross-origin.allowed-origins - * - - - - hadoop.http.cross-origin.max-age - 1800 - - - - hadoop.http.filter.initializers - org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer - - - - hadoop.proxyuser.hdfs.groups - * - - - - hadoop.proxyuser.hdfs.hosts - * - - - - hadoop.proxyuser.hive.groups - * - - - - hadoop.proxyuser.hive.hosts - * - - - - hadoop.proxyuser.HTTP.groups - * - - - - hadoop.proxyuser.HTTP.hosts - * - - - - hadoop.proxyuser.iap.groups - * - - - - hadoop.proxyuser.iap.hosts - * - - - - hadoop.proxyuser.livy.groups - * - - - - hadoop.proxyuser.livy.hosts - * - - - - hadoop.proxyuser.yarn.groups - * - - - - hadoop.proxyuser.yarn.hosts - * - - - - hadoop.rpc.protection - authentication,privacy - - - - hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/ -RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// -RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ -RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/ -RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/ -RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/ -RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/ -RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/ -RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/ -RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/ -DEFAULT - - - - hadoop.security.authentication - kerberos - - - - hadoop.security.authorization - true - - - - hadoop.security.instrumentation.requires.admin - false - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec - - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.file.buffer.size - 131072 - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - - - - ipc.client.connect.max.retries - 50 - - - - ipc.client.connection.maxidletime - 30000 - - - - ipc.client.idlethreshold - 8000 - - - - ipc.server.tcpnodelay - true - - - - mapreduce.jobtracker.webinterface.trusted - false - - - - ipc.client.fallback-to-simple-auth-allowed - true - - - - fs.hdfs.impl.disable.cache - true - - - diff --git a/config/b1e11/hdfs-site.xml b/config/b1e11/hdfs-site.xml deleted file mode 100644 index 9b4eda4..0000000 --- a/config/b1e11/hdfs-site.xml +++ /dev/null @@ -1,698 +0,0 @@ - - - - dfs.block.access.token.enable - true - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.failover.proxy.provider.b1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.client.retry.policy.enabled - false - - - - dfs.cluster.administrators - hdfs - - - - dfs.content-summary.limit - 5000 - - - - dfs.data.transfer.protection - authentication,privacy - - - - dfs.datanode.address - 0.0.0.0:1019 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - - dfs.datanode.du.reserved - 26405499904 - - - - dfs.datanode.failed.volumes.tolerated - 2 - true - - - - dfs.datanode.http.address - 0.0.0.0:1022 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.kerberos.principal - dn/_HOST@ECLD.COM - - - - dfs.datanode.keytab.file - /etc/security/keytabs/dn.service.keytab - - - - dfs.datanode.max.transfer.threads - 16384 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.encrypt.data.transfer.cipher.suites - AES/CTR/NoPadding - - - - dfs.ha.automatic-failover.enabled - true - - - - dfs.ha.fencing.methods - shell(/bin/true) - - - - dfs.ha.namenodes.b1 - nn1,nn2 - - - - dfs.ha.namenodes.b2 - nn3,nn4 - - - - dfs.heartbeat.interval - 3 - - - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude - - - - dfs.http.policy - HTTP_ONLY - - - - dfs.https.port - 50470 - - - - dfs.internal.nameservices - b1,b2 - - - - dfs.journalnode.edits.dir.b1 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.edits.dir.b2 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.journalnode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.journalnode.kerberos.principal - jn/_HOST@ECLD.COM - - - - dfs.journalnode.keytab.file - /etc/security/keytabs/jn.service.keytab - - - - dfs.namenode.accesstime.precision - 0 - - - - dfs.namenode.acls.enabled - true - - - - dfs.namenode.audit.log.async - true - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /data/hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.fslock.fair - false - - - - dfs.namenode.handler.count - 200 - - - - dfs.namenode.http-address.b1.nn1 - b1m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn2 - b1m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn3 - b1m5.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn4 - b1m6.hdp.dc:50070 - - - - dfs.namenode.https-address.b1.nn1 - b1m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn2 - b1m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn3 - b1m5.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn4 - b1m6.hdp.dc:50470 - - - - dfs.namenode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.namenode.kerberos.principal - nn/_HOST@ECLD.COM - - - - dfs.namenode.keytab.file - /etc/security/keytabs/nn.service.keytab - - - - dfs.namenode.max.extra.edits.segments.retained - 180 - - - - dfs.namenode.name.dir - /data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.num.extra.edits.retained - 18000 - - - - dfs.namenode.rpc-address.b1.nn1 - b1m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn2 - b1m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn3 - b1m5.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn4 - b1m6.hdp.dc:8020 - - - - dfs.namenode.safemode.threshold-pct - 0.99 - - - - dfs.namenode.shared.edits.dir.b1 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1 - - - - dfs.namenode.shared.edits.dir.b2 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2 - - - - dfs.namenode.stale.datanode.interval - 30000 - - - - dfs.namenode.startup.delay.block.deletion.sec - 3600 - - - - dfs.namenode.write.stale.datanode.ratio - 1.0f - - - - dfs.nameservices - b1,b2,b3,b4,a3,a4,f1,d2,e1 - - - - dfs.permissions.ContentSummary.subAccess - true - - - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs - - - - dfs.replication - 3 - - - - dfs.replication.max - 50 - - - - dfs.web.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - dfs.web.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - dfs.webhdfs.enabled - true - true - - - - fs.permissions.umask-mode - 022 - - - - hadoop.caller.context.enabled - true - - - - manage.include.files - false - - - - nfs.exports.allowed.hosts - * rw - - - - nfs.file.dump.dir - /tmp/.hdfs-nfs - - - - dfs.client.datanode-restart.timeout - 30 - - - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a3 - nn1,nn2 - - - dfs.namenode.http-address.a3.nn1 - a3m2.hdp.dc:50070 - - - dfs.namenode.http-address.a3.nn2 - a3m3.hdp.dc:50070 - - - dfs.namenode.https-address.a3.nn1 - a3m2.hdp.dc:50470 - - - dfs.namenode.https-address.a3.nn2 - a3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a3.nn1 - a3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a3.nn2 - a3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b3 - nn1,nn2 - - - dfs.namenode.http-address.b3.nn1 - b3m2.hdp.dc:50070 - - - dfs.namenode.http-address.b3.nn2 - b3m3.hdp.dc:50070 - - - dfs.namenode.https-address.b3.nn1 - b3m2.hdp.dc:50470 - - - dfs.namenode.https-address.b3.nn2 - b3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b3.nn1 - b3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b3.nn2 - b3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b4 - nn1,nn2 - - - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 - - - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 - - - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 - - - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.f1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.f1 - nn1,nn2 - - - dfs.namenode.http-address.f1.nn1 - f1m2.hdp.dc:50070 - - - dfs.namenode.http-address.f1.nn2 - f1m3.hdp.dc:50070 - - - dfs.namenode.https-address.f1.nn1 - f1m2.hdp.dc:50470 - - - dfs.namenode.https-address.f1.nn2 - f1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.f1.nn1 - f1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.f1.nn2 - f1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.d2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.d2 - nn1,nn2 - - - dfs.namenode.http-address.d2.nn1 - d2m2.hdp.dc:50070 - - - dfs.namenode.http-address.d2.nn2 - d2m3.hdp.dc:50070 - - - dfs.namenode.https-address.d2.nn1 - d2m2.hdp.dc:50470 - - - dfs.namenode.https-address.d2.nn2 - d2m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.d2.nn1 - d2m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.d2.nn2 - d2m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.e1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.e1 - nn1,nn2 - - - dfs.namenode.http-address.e1.nn1 - e1m2.hdp.dc:50070 - - - dfs.namenode.http-address.e1.nn2 - e1m3.hdp.dc:50070 - - - dfs.namenode.https-address.e1.nn1 - e1m2.hdp.dc:50470 - - - dfs.namenode.https-address.e1.nn2 - e1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.e1.nn1 - e1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.e1.nn2 - e1m3.hdp.dc:8020 - - - diff --git a/config/b1e11/yarn-site.xml b/config/b1e11/yarn-site.xml deleted file mode 100644 index 01ee9e5..0000000 --- a/config/b1e11/yarn-site.xml +++ /dev/null @@ -1,1021 +0,0 @@ - - - - hadoop.http.cross-origin.allowed-origins - regex:.*[.]hdp[.]dc(:\d*)? - - - - hadoop.registry.client.auth - kerberos - - - - hadoop.registry.dns.bind-address - 0.0.0.0 - - - - hadoop.registry.dns.bind-port - 5354 - true - - - - hadoop.registry.dns.domain-name - ECLD.COM - - - - hadoop.registry.dns.enabled - true - - - - hadoop.registry.dns.zone-mask - 255.255.255.0 - - - - hadoop.registry.dns.zone-subnet - 172.17.0.0 - - - - hadoop.registry.jaas.context - Client - - - - hadoop.registry.secure - true - - - - hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b1,sasl:rm,sasl:hive,sasl:spark - - - - hadoop.registry.zk.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - manage.include.files - false - - - - yarn.acl.enable - true - - - - yarn.admin.acl - * - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/lib/edp/hadoop-3.2.2/share/hadoop/common/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/common/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 2592000 - - - - yarn.log.server.url - http://b1e1.hdp.dc:19888/jobhistory/logs - - - - yarn.log.server.web-service.url - http://b1e1.hdp.dc:8188/ws/v1/applicationhistory - - - - yarn.node-labels.enabled - false - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle,spark_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.aux-services.spark_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.spark_shuffle.classpath - /usr/lib/edp/spark-3.1.1-bin-hadoop3.2/yarn/* - - - - yarn.nodemanager.aux-services.timeline_collector.class - org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor - - - - yarn.nodemanager.container-metrics.unregister-delay-ms - 60000 - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 86400 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.keytab - /etc/security/keytabs/nm.service.keytab - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users - true - - - - yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - 3600 - - - - yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log - - - - yarn.nodemanager.log.retain-seconds - 604800 - - - - yarn.nodemanager.principal - nm/_HOST@ECLD.COM - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled - true - - - - yarn.nodemanager.recovery.supervised - true - - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource-plugins - - - - - yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - - - - - yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - - - - - yarn.nodemanager.resource.cpu-vcores - 39 - - - - yarn.nodemanager.resource.memory-mb - 102400 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 - - - - yarn.nodemanager.resourcemanager.connect.wait.secs - 1800 - - - - yarn.nodemanager.runtime.linux.allowed-runtimes - default,docker - - - - yarn.nodemanager.runtime.linux.docker.allowed-container-networks - host,none,bridge - - - - yarn.nodemanager.runtime.linux.docker.capabilities - - CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP, - SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE - - - - yarn.nodemanager.runtime.linux.docker.default-container-network - host - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed - false - - - - yarn.nodemanager.vmem-check-enabled - false - - - - yarn.nodemanager.vmem-pmem-ratio - 2.1 - - - - yarn.nodemanager.webapp.cross-origin.enabled - true - - - - yarn.nodemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.nodemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.address - b1e2.hdp.dc:8050 - - - - yarn.resourcemanager.admin.address - b1e2.hdp.dc:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.cluster-id - yarn-cluster - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.display.per-user-apps - true - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.automatic-failover.zk-base-path - /yarn-leader-election-b1e - - - - yarn.resourcemanager.ha.enabled - true - - - - yarn.resourcemanager.ha.rm-ids - rm1,rm2 - - - - yarn.resourcemanager.hostname - b1e2.hdp.dc - - - - yarn.resourcemanager.hostname.rm1 - b1e2.hdp.dc - - - - yarn.resourcemanager.hostname.rm2 - b1e3.hdp.dc - - - - yarn.resourcemanager.keytab - /etc/security/keytabs/rm.service.keytab - - - - yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled - true - - - - yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval - 15000 - - - - yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor - 1 - - - - yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round - 0.1 - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.placement-constraints.handler - scheduler - - - - yarn.resourcemanager.principal - rm/_HOST@ECLD.COM - - - - yarn.resourcemanager.proxy-user-privileges.enabled - true - - - - yarn.resourcemanager.proxyuser.*.groups - - - - - yarn.resourcemanager.proxyuser.*.hosts - - - - - yarn.resourcemanager.proxyuser.*.users - - - - - yarn.resourcemanager.recovery.enabled - true - - - - yarn.resourcemanager.resource-tracker.address - b1e2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm1 - b1e2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm2 - b1e3.hdp.dc:8025 - - - - yarn.resourcemanager.scheduler.address - b1e2.hdp.dc:8030 - - - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.scheduler.monitor.enable - true - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true - - - - yarn.resourcemanager.webapp.address - b1e2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm1 - b1e2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm2 - b1e3.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.cross-origin.enabled - true - - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - - - yarn.resourcemanager.webapp.https.address - b1e2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm1 - b1e2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm2 - b1e3.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.resourcemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.work-preserving-recovery.enabled - true - - - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 - - - - yarn.resourcemanager.zk-acl - sasl:rm:rwcda - - - - yarn.resourcemanager.zk-address - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - yarn.resourcemanager.zk-num-retries - 1000 - - - - yarn.resourcemanager.zk-retry-interval-ms - 1000 - - - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore-b1e - - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.rm.system-metricspublisher.emit-container-events - true - - - - yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled - true - - - - yarn.scheduler.maximum-allocation-mb - 2048000 - - - - yarn.scheduler.maximum-allocation-vcores - 39 - - - - yarn.scheduler.minimum-allocation-mb - 128 - - - - yarn.scheduler.minimum-allocation-vcores - 1 - - - - yarn.service.system-service.dir - /services - - - - yarn.system-metricspublisher.enabled - true - - - - yarn.timeline-service.address - b1e1.hdp.dc:10200 - - - - yarn.timeline-service.bind-host - 0.0.0.0 - - - - yarn.timeline-service.client.max-retries - 30 - - - - yarn.timeline-service.client.retry-interval-ms - 1000 - - - - yarn.timeline-service.enabled - false - - - - yarn.timeline-service.entity-group-fs-store.active-dir - /ats/active/ - - - - yarn.timeline-service.entity-group-fs-store.app-cache-size - 10 - - - - yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds - 3600 - - - - yarn.timeline-service.entity-group-fs-store.done-dir - /ats/done/ - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes - org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - - - - - yarn.timeline-service.entity-group-fs-store.retain-seconds - 604800 - - - - yarn.timeline-service.entity-group-fs-store.scan-interval-seconds - 60 - - - - yarn.timeline-service.entity-group-fs-store.summary-store - org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore - - - - yarn.timeline-service.generic-application-history.save-non-am-container-meta-info - false - - - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - - - yarn.timeline-service.hbase-schema.prefix - prod. - - - - yarn.timeline-service.hbase.configuration.file - file:///etc/hadoop/conf/embedded-yarn-ats-hbase/hbase-site.xml - - - - yarn.timeline-service.hbase.coprocessor.jar.hdfs.location - file:///usr/lib/edp/hadoop-3.2.2/lib/hadoop-yarn-server-timelineservice-3.2.2.jar - - - - yarn.timeline-service.http-authentication.cookie.domain - - - - - yarn.timeline-service.http-authentication.cookie.path - - - - - yarn.timeline-service.http-authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - yarn.timeline-service.http-authentication.kerberos.name.rules - - - - - yarn.timeline-service.http-authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - yarn.timeline-service.http-authentication.proxyuser.*.groups - - - - - yarn.timeline-service.http-authentication.proxyuser.*.hosts - - - - - yarn.timeline-service.http-authentication.proxyuser.*.users - - - - - yarn.timeline-service.http-authentication.signature.secret - - - - - yarn.timeline-service.http-authentication.signature.secret.file - - - - - yarn.timeline-service.http-authentication.signer.secret.provider - - - - - yarn.timeline-service.http-authentication.signer.secret.provider.object - - - - - yarn.timeline-service.http-authentication.simple.anonymous.allowed - true - - - - yarn.timeline-service.http-authentication.token.validity - - - - - yarn.timeline-service.http-authentication.type - simple - - - - yarn.timeline-service.http-cross-origin.enabled - true - - - - yarn.timeline-service.keytab - /etc/security/keytabs/yarn.service.keytab - - - - yarn.timeline-service.leveldb-state-store.path - /data/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.path - /data/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.principal - yarn/_HOST@ECLD.COM - - - - yarn.timeline-service.reader.webapp.address - b1e1.hdp.dc:8198 - - - - yarn.timeline-service.reader.webapp.https.address - b1e1.hdp.dc:8199 - - - - yarn.timeline-service.recovery.enabled - true - - - - yarn.timeline-service.state-store-class - org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.version - 2.0f - - - - yarn.timeline-service.versions - 1.5f,2.0f - - - - yarn.timeline-service.webapp.address - b1e1.hdp.dc:8188 - - - - yarn.timeline-service.webapp.https.address - b1e1.hdp.dc:8190 - - - - yarn.webapp.api-service.enable - true - - - - yarn.webapp.ui2.enable - true - - - diff --git a/config/b2s119/core-site.xml b/config/b2a4/core-site.xml similarity index 89% rename from config/b2s119/core-site.xml rename to config/b2a4/core-site.xml index fecd70c..655001f 100644 --- a/config/b2s119/core-site.xml +++ b/config/b2a4/core-site.xml @@ -1,12 +1,5 @@ - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - fs.defaultFS hdfs://b2 @@ -32,7 +25,7 @@ fs.trash.interval 4320 - + fs.trash.checkpoint.interval 360 @@ -50,7 +43,7 @@ ha.zookeeper.quorum - b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 + a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181 @@ -160,7 +153,7 @@ hadoop.proxyuser.yarn.hosts - * + a4m1.hdp.dc,a4m2.hdp.dc @@ -170,10 +163,10 @@ hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b5@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b5@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b5@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b5@ECLD.COM)s/.*/yarn-ats/ + RULE:[1:$1@$0](hbase-a4@ECLD.COM)s/.*/hbase/ +RULE:[1:$1@$0](hdfs-a4@ECLD.COM)s/.*/hdfs/ +RULE:[1:$1@$0](spark-a4@ECLD.COM)s/.*/spark/ +RULE:[1:$1@$0](yarn-ats-a4@ECLD.COM)s/.*/yarn-ats/ RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ @@ -254,14 +247,4 @@ DEFAULT false - - ipc.client.fallback-to-simple-auth-allowed - true - - - - fs.hdfs.impl.disable.cache - true - - diff --git a/config/b2s119/hdfs-site.xml b/config/b2a4/hdfs-site.xml similarity index 90% rename from config/b2s119/hdfs-site.xml rename to config/b2a4/hdfs-site.xml index 089155b..2ecfb0c 100644 --- a/config/b2s119/hdfs-site.xml +++ b/config/b2a4/hdfs-site.xml @@ -1,4 +1,5 @@ - + + dfs.block.access.token.enable @@ -16,7 +17,7 @@ - dfs.client.failover.proxy.provider.b5 + dfs.client.failover.proxy.provider.a4 org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider @@ -62,7 +63,7 @@ dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data,[DISK]file:///data13/hadoop/hdfs/data,[DISK]file:///data14/hadoop/hdfs/data,[DISK]file:///data15/hadoop/hdfs/data,[DISK]file:///data16/hadoop/hdfs/data,[DISK]file:///data17/hadoop/hdfs/data,[DISK]file:///data18/hadoop/hdfs/data,[DISK]file:///data19/hadoop/hdfs/data,[DISK]file:///data20/hadoop/hdfs/data,[DISK]file:///data21/hadoop/hdfs/data,[DISK]file:///data22/hadoop/hdfs/data,[DISK]file:///data23/hadoop/hdfs/data,[DISK]file:///data24/hadoop/hdfs/data + [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data true @@ -133,7 +134,7 @@ - dfs.ha.namenodes.b5 + dfs.ha.namenodes.a4 nn1,nn2 @@ -159,11 +160,11 @@ dfs.internal.nameservices - b5 + a4 - dfs.journalnode.edits.dir.b5 + dfs.journalnode.edits.dir.a4 /data2/hadoop/hdfs/journal @@ -248,23 +249,23 @@ - dfs.namenode.http-address.b5.nn1 - b5m2.hdp.dc:50070 + dfs.namenode.http-address.a4.nn1 + a4m1.hdp.dc:50070 - dfs.namenode.http-address.b5.nn2 - b5m3.hdp.dc:50070 - - - - dfs.namenode.https-address.b5.nn1 - b5m2.hdp.dc:50470 + dfs.namenode.http-address.a4.nn2 + a4m2.hdp.dc:50070 - dfs.namenode.https-address.b5.nn2 - b5m3.hdp.dc:50470 + dfs.namenode.https-address.a4.nn1 + a4m1.hdp.dc:50470 + + + + dfs.namenode.https-address.a4.nn2 + a4m2.hdp.dc:50470 @@ -304,13 +305,13 @@ - dfs.namenode.rpc-address.b5.nn1 - b5m2.hdp.dc:8020 + dfs.namenode.rpc-address.a4.nn1 + a4m1.hdp.dc:8020 - dfs.namenode.rpc-address.b5.nn2 - b5m3.hdp.dc:8020 + dfs.namenode.rpc-address.a4.nn2 + a4m2.hdp.dc:8020 @@ -319,8 +320,8 @@ - dfs.namenode.shared.edits.dir.b5 - qjournal://b5m1.hdp.dc:8485;b5m2.hdp.dc:8485;b5m3.hdp.dc:8485/b5 + dfs.namenode.shared.edits.dir.a4 + qjournal://a4m1.hdp.dc:8485;a4m2.hdp.dc:8485;a4m3.hdp.dc:8485/a4 @@ -340,7 +341,7 @@ dfs.nameservices - b5,b1,b2,b3,b4,a3,a4,f1,e1,d2 + a4,a3,b3,b1,b2,b4,f1,d2,b5,e1,a6 @@ -414,37 +415,13 @@ 30 - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + dfs.permissions + true - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 + dfs.namenode.inode.attributes.provider.class + org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer @@ -454,19 +431,19 @@ dfs.ha.namenodes.a3 nn1,nn2 - + dfs.namenode.http-address.a3.nn1 a3m2.hdp.dc:50070 - + dfs.namenode.http-address.a3.nn2 a3m3.hdp.dc:50070 - + dfs.namenode.https-address.a3.nn1 a3m2.hdp.dc:50470 - + dfs.namenode.https-address.a3.nn2 a3m3.hdp.dc:50470 @@ -487,19 +464,19 @@ dfs.ha.namenodes.b3 nn1,nn2 - + dfs.namenode.http-address.b3.nn1 b3m2.hdp.dc:50070 - + dfs.namenode.http-address.b3.nn2 b3m3.hdp.dc:50070 - + dfs.namenode.https-address.b3.nn1 b3m2.hdp.dc:50470 - + dfs.namenode.https-address.b3.nn2 b3m3.hdp.dc:50470 @@ -578,6 +555,39 @@ b1m6.hdp.dc:8020 + + dfs.client.failover.proxy.provider.b4 + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + dfs.ha.namenodes.b4 + nn1,nn2 + + + dfs.namenode.http-address.b4.nn1 + b4m2.hdp.dc:50070 + + + dfs.namenode.http-address.b4.nn2 + b4m3.hdp.dc:50070 + + + dfs.namenode.https-address.b4.nn1 + b4m2.hdp.dc:50470 + + + dfs.namenode.https-address.b4.nn2 + b4m3.hdp.dc:50470 + + + dfs.namenode.rpc-address.b4.nn1 + b4m2.hdp.dc:8020 + + + dfs.namenode.rpc-address.b4.nn2 + b4m3.hdp.dc:8020 + + dfs.client.failover.proxy.provider.f1 org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider @@ -644,6 +654,44 @@ d2m3.hdp.dc:8020 + + dfs.client.failover.proxy.provider.b5 + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + dfs.ha.namenodes.b5 + nn1,nn2 + + + dfs.namenode.http-address.b5.nn1 + b5m2.hdp.dc:50070 + + + dfs.namenode.http-address.b5.nn2 + b5m3.hdp.dc:50070 + + + dfs.namenode.https-address.b5.nn1 + b5m2.hdp.dc:50470 + + + dfs.namenode.https-address.b5.nn2 + b5m3.hdp.dc:50470 + + + dfs.namenode.rpc-address.b5.nn1 + b5m2.hdp.dc:8020 + + + dfs.namenode.rpc-address.b5.nn2 + b5m3.hdp.dc:8020 + + + + dfs.namenode.checkpoint.txns + 15000000 + + dfs.client.failover.proxy.provider.e1 org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider @@ -678,36 +726,36 @@ - dfs.client.failover.proxy.provider.b4 + dfs.client.failover.proxy.provider.a6 org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - dfs.ha.namenodes.b4 + dfs.ha.namenodes.a6 nn1,nn2 - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 + dfs.namenode.http-address.a6.nn1 + a6m2.hdp.dc:50070 - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 + dfs.namenode.http-address.a6.nn2 + a6m3.hdp.dc:50070 - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 + dfs.namenode.https-address.a6.nn1 + a6m2.hdp.dc:50470 - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 + dfs.namenode.https-address.a6.nn2 + a6m3.hdp.dc:50470 - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 + dfs.namenode.rpc-address.a6.nn1 + a6m2.hdp.dc:8020 - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 + dfs.namenode.rpc-address.a6.nn2 + a6m3.hdp.dc:8020 - + diff --git a/config/b2e1/yarn-site.xml b/config/b2a4/yarn-site.xml similarity index 91% rename from config/b2e1/yarn-site.xml rename to config/b2a4/yarn-site.xml index 407d1f5..81d8d95 100644 --- a/config/b2e1/yarn-site.xml +++ b/config/b2a4/yarn-site.xml @@ -1,4 +1,5 @@ - + + hadoop.http.cross-origin.allowed-origins @@ -53,12 +54,12 @@ hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b1,sasl:rm,sasl:hive,sasl:spark + sasl:yarn,sasl:jhs,sasl:hdfs-a4,sasl:rm,sasl:hive,sasl:spark hadoop.registry.zk.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 + a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181 @@ -105,15 +106,25 @@ yarn.log-aggregation.retain-seconds 2592000 + + + yarn.log-aggregation.file-formats + TFile + + + + yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + yarn.log.server.url - http://b1m4.hdp.dc:19888/jobhistory/logs + http://a4m3.hdp.dc:19888/jobhistory/logs yarn.log.server.web-service.url - http://b1m4.hdp.dc:8188/ws/v1/applicationhistory + http://a4m3.hdp.dc:8188/ws/v1/applicationhistory @@ -146,6 +157,16 @@ mapreduce_shuffle,spark_shuffle,timeline_collector,sparkv2_shuffle + + yarn.nodemanager.aux-services.sparkv2_shuffle.class + org.apache.spark.network.yarn.YarnShuffleService + + + + yarn.nodemanager.aux-services.sparkv2_shuffle.classpath + /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* + + yarn.nodemanager.aux-services.mapreduce_shuffle.class org.apache.hadoop.mapred.ShuffleHandler @@ -166,16 +187,6 @@ org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - yarn.nodemanager.bind-host 0.0.0.0 @@ -198,7 +209,7 @@ yarn.nodemanager.delete.debug-delay-sec - 86400 + 0 @@ -248,7 +259,7 @@ yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local + /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local @@ -273,7 +284,7 @@ yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log + /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log @@ -288,7 +299,7 @@ yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state + /data3/var/log/hadoop-yarn/nodemanager/recovery-state @@ -313,42 +324,42 @@ yarn.nodemanager.resource-plugins - + yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - + yarn.nodemanager.resource-plugins.gpu.docker-plugin - + yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - + yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - + yarn.nodemanager.resource.cpu-vcores - 39 + 96 yarn.nodemanager.resource.memory-mb - 184320 + 348160 yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 + 96 @@ -380,7 +391,7 @@ yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - + @@ -415,12 +426,12 @@ yarn.resourcemanager.address - b1m2.hdp.dc:8050 + a4m1.hdp.dc:8050 yarn.resourcemanager.admin.address - b1m2.hdp.dc:8141 + a4m1.hdp.dc:8141 @@ -480,17 +491,17 @@ yarn.resourcemanager.hostname - b1m2.hdp.dc + a4m1.hdp.dc yarn.resourcemanager.hostname.rm1 - b1m2.hdp.dc + a4m1.hdp.dc yarn.resourcemanager.hostname.rm2 - b1m3.hdp.dc + a4m2.hdp.dc @@ -540,17 +551,17 @@ yarn.resourcemanager.proxyuser.*.groups - + yarn.resourcemanager.proxyuser.*.hosts - + yarn.resourcemanager.proxyuser.*.users - + @@ -560,22 +571,22 @@ yarn.resourcemanager.resource-tracker.address - b1m2.hdp.dc:8025 + a4m1.hdp.dc:8025 yarn.resourcemanager.resource-tracker.address.rm1 - b1m2.hdp.dc:8025 + a4m1.hdp.dc:8025 yarn.resourcemanager.resource-tracker.address.rm2 - b1m3.hdp.dc:8025 + a4m2.hdp.dc:8025 yarn.resourcemanager.scheduler.address - b1m2.hdp.dc:8030 + a4m1.hdp.dc:8030 @@ -610,17 +621,17 @@ yarn.resourcemanager.webapp.address - b1m2.hdp.dc:8088 + a4m1.hdp.dc:8088 yarn.resourcemanager.webapp.address.rm1 - b1m2.hdp.dc:8088 + a4m1.hdp.dc:8088 yarn.resourcemanager.webapp.address.rm2 - b1m3.hdp.dc:8088 + a4m2.hdp.dc:8088 @@ -635,17 +646,17 @@ yarn.resourcemanager.webapp.https.address - b1m2.hdp.dc:8090 + a4m1.hdp.dc:8090 yarn.resourcemanager.webapp.https.address.rm1 - b1m2.hdp.dc:8090 + a4m1.hdp.dc:8090 yarn.resourcemanager.webapp.https.address.rm2 - b1m3.hdp.dc:8090 + a4m2.hdp.dc:8090 @@ -675,7 +686,7 @@ yarn.resourcemanager.zk-address - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 + a4m1.hdp.dc:2181,a4m2.hdp.dc:2181,a4m3.hdp.dc:2181 @@ -740,7 +751,7 @@ yarn.timeline-service.address - b1m4.hdp.dc:10200 + a4m3.hdp.dc:10200 @@ -750,7 +761,7 @@ yarn.timeline-service.client.max-retries - 30 + 180 @@ -760,7 +771,7 @@ yarn.timeline-service.enabled - false + true @@ -790,7 +801,7 @@ yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - + @@ -835,12 +846,12 @@ yarn.timeline-service.http-authentication.cookie.domain - + yarn.timeline-service.http-authentication.cookie.path - + @@ -850,7 +861,7 @@ yarn.timeline-service.http-authentication.kerberos.name.rules - + @@ -860,37 +871,37 @@ yarn.timeline-service.http-authentication.proxyuser.*.groups - + yarn.timeline-service.http-authentication.proxyuser.*.hosts - + yarn.timeline-service.http-authentication.proxyuser.*.users - + yarn.timeline-service.http-authentication.signature.secret - + yarn.timeline-service.http-authentication.signature.secret.file - + yarn.timeline-service.http-authentication.signer.secret.provider - + yarn.timeline-service.http-authentication.signer.secret.provider.object - + @@ -900,7 +911,7 @@ yarn.timeline-service.http-authentication.token.validity - + @@ -920,12 +931,12 @@ yarn.timeline-service.leveldb-state-store.path - /data/hadoop/yarn/timeline + /data2/hadoop/yarn/timeline yarn.timeline-service.leveldb-timeline-store.path - /data/hadoop/yarn/timeline + /data2/hadoop/yarn/timeline @@ -955,12 +966,12 @@ yarn.timeline-service.reader.webapp.address - b1m4.hdp.dc:8198 + a4m3.hdp.dc:8198 yarn.timeline-service.reader.webapp.https.address - b1m4.hdp.dc:8199 + a4m3.hdp.dc:8199 @@ -1000,12 +1011,12 @@ yarn.timeline-service.webapp.address - b1m4.hdp.dc:8188 + a4m3.hdp.dc:8188 yarn.timeline-service.webapp.https.address - b1m4.hdp.dc:8190 + a4m3.hdp.dc:8190 @@ -1018,9 +1029,18 @@ true + + yarn.authorization-provider + org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer + + - yarn.resourcemanager.max-completed-applications - 10000 + yarn.nodemanager.numa-awareness.enabled + true - + + yarn.nodemanager.numa-awareness.read-topology + true + + diff --git a/config/b2b1/core-site.xml b/config/b2b1/core-site.xml index 9c80e12..4d7c788 100644 --- a/config/b2b1/core-site.xml +++ b/config/b2b1/core-site.xml @@ -1,7 +1,5 @@ - - - + fs.azure.user.agent.prefix User-Agent: APN/1.0 Hortonworks/1.0 HDP/ @@ -32,7 +30,7 @@ fs.trash.interval 360 - + ha.failover-controller.active-standby-elector.zk.op.retries 120 diff --git a/config/b2b1/yarn-site.xml b/config/b2b1/yarn-site.xml index 407d1f5..c33a4eb 100644 --- a/config/b2b1/yarn-site.xml +++ b/config/b2b1/yarn-site.xml @@ -105,7 +105,17 @@ yarn.log-aggregation.retain-seconds 2592000 - + + + yarn.log-aggregation.file-formats + TFile + + + + yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + + yarn.log.server.url http://b1m4.hdp.dc:19888/jobhistory/logs diff --git a/config/b2b12/yarn-site.xml b/config/b2b12/yarn-site.xml index dd15923..8ab0ef0 100644 --- a/config/b2b12/yarn-site.xml +++ b/config/b2b12/yarn-site.xml @@ -105,6 +105,16 @@ yarn.log-aggregation.retain-seconds 172800 + + + yarn.log-aggregation.file-formats + TFile + + + + yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + yarn.log.server.url diff --git a/config/b2b5/core-site.xml b/config/b2b5/core-site.xml index fecd70c..4796983 100644 --- a/config/b2b5/core-site.xml +++ b/config/b2b5/core-site.xml @@ -1,6 +1,4 @@ - - fs.azure.user.agent.prefix diff --git a/config/b2b5/yarn-site.xml b/config/b2b5/yarn-site.xml index 86efdfb..358e05a 100644 --- a/config/b2b5/yarn-site.xml +++ b/config/b2b5/yarn-site.xml @@ -105,7 +105,17 @@ yarn.log-aggregation.retain-seconds 604800 - + + + yarn.log-aggregation.file-formats + TFile + + + + yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + + yarn.log.server.url http://b5m1.hdp.dc:19888/jobhistory/logs diff --git a/config/b2e1/core-site.xml b/config/b2e1/core-site.xml deleted file mode 100644 index dbdb62b..0000000 --- a/config/b2e1/core-site.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - - - fs.defaultFS - hdfs://b2 - true - - - - fs.s3a.fast.upload - true - - - - fs.s3a.fast.upload.buffer - disk - - - - fs.s3a.multipart.size - 67108864 - - - - fs.trash.interval - 4320 - - - - fs.trash.checkpoint.interval - 360 - - - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 - - - - ha.zookeeper.acl - sasl:nn:rwcda - - - - ha.zookeeper.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - hadoop.http.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - hadoop.http.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - hadoop.http.authentication.signature.secret.file - /etc/security/http_secret - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - - - hadoop.http.authentication.type - simple - - - - hadoop.http.cross-origin.allowed-headers - X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding - - - - hadoop.http.cross-origin.allowed-methods - GET,PUT,POST,OPTIONS,HEAD,DELETE - - - - hadoop.http.cross-origin.allowed-origins - * - - - - hadoop.http.cross-origin.max-age - 1800 - - - - hadoop.http.filter.initializers - org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer - - - - hadoop.proxyuser.hdfs.groups - * - - - - hadoop.proxyuser.hdfs.hosts - * - - - - hadoop.proxyuser.hive.groups - * - - - - hadoop.proxyuser.hive.hosts - * - - - - hadoop.proxyuser.HTTP.groups - * - - - - hadoop.proxyuser.HTTP.hosts - * - - - - hadoop.proxyuser.iap.groups - * - - - - hadoop.proxyuser.iap.hosts - * - - - - hadoop.proxyuser.livy.groups - * - - - - hadoop.proxyuser.livy.hosts - * - - - - hadoop.proxyuser.yarn.groups - * - - - - hadoop.proxyuser.yarn.hosts - * - - - - hadoop.rpc.protection - authentication,privacy - - - - hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/ -RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// -RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ -RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/ -RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/ -RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/ -RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/ -RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/ -RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/ -RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/ -DEFAULT - - - - hadoop.security.authentication - kerberos - - - - hadoop.security.authorization - true - - - - hadoop.security.instrumentation.requires.admin - false - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec - - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.file.buffer.size - 131072 - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - - - - ipc.client.connect.max.retries - 50 - - - - ipc.client.connection.maxidletime - 30000 - - - - ipc.client.idlethreshold - 8000 - - - - ipc.server.tcpnodelay - true - - - - mapreduce.jobtracker.webinterface.trusted - false - - - - ipc.client.fallback-to-simple-auth-allowed - true - - - - fs.hdfs.impl.disable.cache - true - - - diff --git a/config/b2e1/hdfs-site.xml b/config/b2e1/hdfs-site.xml deleted file mode 100644 index 9b4eda4..0000000 --- a/config/b2e1/hdfs-site.xml +++ /dev/null @@ -1,698 +0,0 @@ - - - - dfs.block.access.token.enable - true - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.failover.proxy.provider.b1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.client.retry.policy.enabled - false - - - - dfs.cluster.administrators - hdfs - - - - dfs.content-summary.limit - 5000 - - - - dfs.data.transfer.protection - authentication,privacy - - - - dfs.datanode.address - 0.0.0.0:1019 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - - dfs.datanode.du.reserved - 26405499904 - - - - dfs.datanode.failed.volumes.tolerated - 2 - true - - - - dfs.datanode.http.address - 0.0.0.0:1022 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.kerberos.principal - dn/_HOST@ECLD.COM - - - - dfs.datanode.keytab.file - /etc/security/keytabs/dn.service.keytab - - - - dfs.datanode.max.transfer.threads - 16384 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.encrypt.data.transfer.cipher.suites - AES/CTR/NoPadding - - - - dfs.ha.automatic-failover.enabled - true - - - - dfs.ha.fencing.methods - shell(/bin/true) - - - - dfs.ha.namenodes.b1 - nn1,nn2 - - - - dfs.ha.namenodes.b2 - nn3,nn4 - - - - dfs.heartbeat.interval - 3 - - - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude - - - - dfs.http.policy - HTTP_ONLY - - - - dfs.https.port - 50470 - - - - dfs.internal.nameservices - b1,b2 - - - - dfs.journalnode.edits.dir.b1 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.edits.dir.b2 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.journalnode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.journalnode.kerberos.principal - jn/_HOST@ECLD.COM - - - - dfs.journalnode.keytab.file - /etc/security/keytabs/jn.service.keytab - - - - dfs.namenode.accesstime.precision - 0 - - - - dfs.namenode.acls.enabled - true - - - - dfs.namenode.audit.log.async - true - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /data/hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.fslock.fair - false - - - - dfs.namenode.handler.count - 200 - - - - dfs.namenode.http-address.b1.nn1 - b1m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn2 - b1m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn3 - b1m5.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn4 - b1m6.hdp.dc:50070 - - - - dfs.namenode.https-address.b1.nn1 - b1m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn2 - b1m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn3 - b1m5.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn4 - b1m6.hdp.dc:50470 - - - - dfs.namenode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.namenode.kerberos.principal - nn/_HOST@ECLD.COM - - - - dfs.namenode.keytab.file - /etc/security/keytabs/nn.service.keytab - - - - dfs.namenode.max.extra.edits.segments.retained - 180 - - - - dfs.namenode.name.dir - /data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.num.extra.edits.retained - 18000 - - - - dfs.namenode.rpc-address.b1.nn1 - b1m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn2 - b1m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn3 - b1m5.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn4 - b1m6.hdp.dc:8020 - - - - dfs.namenode.safemode.threshold-pct - 0.99 - - - - dfs.namenode.shared.edits.dir.b1 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1 - - - - dfs.namenode.shared.edits.dir.b2 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2 - - - - dfs.namenode.stale.datanode.interval - 30000 - - - - dfs.namenode.startup.delay.block.deletion.sec - 3600 - - - - dfs.namenode.write.stale.datanode.ratio - 1.0f - - - - dfs.nameservices - b1,b2,b3,b4,a3,a4,f1,d2,e1 - - - - dfs.permissions.ContentSummary.subAccess - true - - - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs - - - - dfs.replication - 3 - - - - dfs.replication.max - 50 - - - - dfs.web.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - dfs.web.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - dfs.webhdfs.enabled - true - true - - - - fs.permissions.umask-mode - 022 - - - - hadoop.caller.context.enabled - true - - - - manage.include.files - false - - - - nfs.exports.allowed.hosts - * rw - - - - nfs.file.dump.dir - /tmp/.hdfs-nfs - - - - dfs.client.datanode-restart.timeout - 30 - - - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a3 - nn1,nn2 - - - dfs.namenode.http-address.a3.nn1 - a3m2.hdp.dc:50070 - - - dfs.namenode.http-address.a3.nn2 - a3m3.hdp.dc:50070 - - - dfs.namenode.https-address.a3.nn1 - a3m2.hdp.dc:50470 - - - dfs.namenode.https-address.a3.nn2 - a3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a3.nn1 - a3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a3.nn2 - a3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b3 - nn1,nn2 - - - dfs.namenode.http-address.b3.nn1 - b3m2.hdp.dc:50070 - - - dfs.namenode.http-address.b3.nn2 - b3m3.hdp.dc:50070 - - - dfs.namenode.https-address.b3.nn1 - b3m2.hdp.dc:50470 - - - dfs.namenode.https-address.b3.nn2 - b3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b3.nn1 - b3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b3.nn2 - b3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b4 - nn1,nn2 - - - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 - - - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 - - - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 - - - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.f1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.f1 - nn1,nn2 - - - dfs.namenode.http-address.f1.nn1 - f1m2.hdp.dc:50070 - - - dfs.namenode.http-address.f1.nn2 - f1m3.hdp.dc:50070 - - - dfs.namenode.https-address.f1.nn1 - f1m2.hdp.dc:50470 - - - dfs.namenode.https-address.f1.nn2 - f1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.f1.nn1 - f1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.f1.nn2 - f1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.d2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.d2 - nn1,nn2 - - - dfs.namenode.http-address.d2.nn1 - d2m2.hdp.dc:50070 - - - dfs.namenode.http-address.d2.nn2 - d2m3.hdp.dc:50070 - - - dfs.namenode.https-address.d2.nn1 - d2m2.hdp.dc:50470 - - - dfs.namenode.https-address.d2.nn2 - d2m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.d2.nn1 - d2m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.d2.nn2 - d2m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.e1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.e1 - nn1,nn2 - - - dfs.namenode.http-address.e1.nn1 - e1m2.hdp.dc:50070 - - - dfs.namenode.http-address.e1.nn2 - e1m3.hdp.dc:50070 - - - dfs.namenode.https-address.e1.nn1 - e1m2.hdp.dc:50470 - - - dfs.namenode.https-address.e1.nn2 - e1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.e1.nn1 - e1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.e1.nn2 - e1m3.hdp.dc:8020 - - - diff --git a/config/b2s119/yarn-site.xml b/config/b2s119/yarn-site.xml deleted file mode 100644 index e2ab884..0000000 --- a/config/b2s119/yarn-site.xml +++ /dev/null @@ -1,1026 +0,0 @@ - - - - hadoop.http.cross-origin.allowed-origins - regex:.*[.]hdp[.]dc(:\d*)? - - - - hadoop.registry.client.auth - kerberos - - - - hadoop.registry.dns.bind-address - 0.0.0.0 - - - - hadoop.registry.dns.bind-port - 5354 - true - - - - hadoop.registry.dns.domain-name - ECLD.COM - - - - hadoop.registry.dns.enabled - true - - - - hadoop.registry.dns.zone-mask - 255.255.255.0 - - - - hadoop.registry.dns.zone-subnet - 172.17.0.0 - - - - hadoop.registry.jaas.context - Client - - - - hadoop.registry.secure - true - - - - hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b5,sasl:rm,sasl:hive,sasl:spark - - - - hadoop.registry.zk.quorum - b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 - - - - manage.include.files - false - - - - yarn.acl.enable - true - - - - yarn.admin.acl - * - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/lib/edp/hadoop-3.2.2/share/hadoop/common/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/common/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 2592000 - - - - yarn.log.server.url - http://b5s119.hdp.dc:19888/jobhistory/logs - - - - yarn.log.server.web-service.url - http://b5s119.hdp.dc:8188/ws/v1/applicationhistory - - - - yarn.node-labels.enabled - false - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle,spark_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.aux-services.spark_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.spark_shuffle.classpath - /usr/lib/edp/spark-3.1.1-bin-hadoop3.2/yarn/* - - - - yarn.nodemanager.aux-services.timeline_collector.class - org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor - - - - yarn.nodemanager.container-metrics.unregister-delay-ms - 60000 - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 86400 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.keytab - /etc/security/keytabs/nm.service.keytab - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users - true - - - - yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local,/data13/hadoop/yarn/local,/data14/hadoop/yarn/local,/data15/hadoop/yarn/local,/data16/hadoop/yarn/local,/data17/hadoop/yarn/local,/data18/hadoop/yarn/local,/data19/hadoop/yarn/local,/data20/hadoop/yarn/local,/data21/hadoop/yarn/local,/data22/hadoop/yarn/local,/data23/hadoop/yarn/local,/data24/hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - 3600 - - - - yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log,/data13/hadoop/yarn/log,/data14/hadoop/yarn/log,/data15/hadoop/yarn/log,/data16/hadoop/yarn/log,/data17/hadoop/yarn/log,/data18/hadoop/yarn/log,/data19/hadoop/yarn/log,/data20/hadoop/yarn/log,/data21/hadoop/yarn/log,/data22/hadoop/yarn/log,/data23/hadoop/yarn/log,/data24/hadoop/yarn/log - - - - yarn.nodemanager.log.retain-seconds - 604800 - - - - yarn.nodemanager.principal - nm/_HOST@ECLD.COM - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled - true - - - - yarn.nodemanager.recovery.supervised - true - - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource-plugins - - - - - yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - - - - - yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - - - - - yarn.nodemanager.resource.cpu-vcores - 70 - - - - yarn.nodemanager.resource.memory-mb - 348160 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 - - - - yarn.nodemanager.resourcemanager.connect.wait.secs - 1800 - - - - yarn.nodemanager.runtime.linux.allowed-runtimes - default,docker - - - - yarn.nodemanager.runtime.linux.docker.allowed-container-networks - host,none,bridge - - - - yarn.nodemanager.runtime.linux.docker.capabilities - - CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP, - SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE - - - - yarn.nodemanager.runtime.linux.docker.default-container-network - host - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed - false - - - - yarn.nodemanager.vmem-check-enabled - false - - - - yarn.nodemanager.vmem-pmem-ratio - 2.1 - - - - yarn.nodemanager.webapp.cross-origin.enabled - true - - - - yarn.nodemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.nodemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.address - b5s120.hdp.dc:8050 - - - - yarn.resourcemanager.admin.address - b5s120.hdp.dc:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.cluster-id - yarn-cluster - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.display.per-user-apps - true - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.automatic-failover.zk-base-path - /yarn-leader-election-b5hudi - - - - yarn.resourcemanager.ha.enabled - true - - - - yarn.resourcemanager.ha.rm-ids - rm1,rm2 - - - - yarn.resourcemanager.hostname - b5s120.hdp.dc - - - - yarn.resourcemanager.hostname.rm1 - b5s120.hdp.dc - - - - yarn.resourcemanager.hostname.rm2 - b5s121.hdp.dc - - - - yarn.resourcemanager.keytab - /etc/security/keytabs/rm.service.keytab - - - - yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled - true - - - - yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval - 15000 - - - - yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor - 1 - - - - yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round - 0.1 - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.placement-constraints.handler - scheduler - - - - yarn.resourcemanager.principal - rm/_HOST@ECLD.COM - - - - yarn.resourcemanager.proxy-user-privileges.enabled - true - - - - yarn.resourcemanager.proxyuser.*.groups - - - - - yarn.resourcemanager.proxyuser.*.hosts - - - - - yarn.resourcemanager.proxyuser.*.users - - - - - yarn.resourcemanager.recovery.enabled - true - - - - yarn.resourcemanager.resource-tracker.address - b5s120.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm1 - b5s120.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm2 - b5s121.hdp.dc:8025 - - - - yarn.resourcemanager.scheduler.address - b5s120.hdp.dc:8030 - - - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.scheduler.monitor.enable - true - - - - yarn.resourcemanager.max-completed-applications - 10000 - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true - - - - yarn.resourcemanager.webapp.address - b5s120.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm1 - b5s120.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm2 - b5s121.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.cross-origin.enabled - true - - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - - - yarn.resourcemanager.webapp.https.address - b5s120.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm1 - b5s120.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm2 - b5s121.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.resourcemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.work-preserving-recovery.enabled - true - - - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 - - - - yarn.resourcemanager.zk-acl - sasl:rm:rwcda - - - - yarn.resourcemanager.zk-address - b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 - - - - yarn.resourcemanager.zk-num-retries - 1000 - - - - yarn.resourcemanager.zk-retry-interval-ms - 1000 - - - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore-b5hudi - - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.rm.system-metricspublisher.emit-container-events - true - - - - yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled - true - - - - yarn.scheduler.maximum-allocation-mb - 81920 - - - - yarn.scheduler.maximum-allocation-vcores - 70 - - - - yarn.scheduler.minimum-allocation-mb - 1024 - - - - yarn.scheduler.minimum-allocation-vcores - 1 - - - - yarn.service.system-service.dir - /services - - - - yarn.system-metricspublisher.enabled - true - - - - yarn.timeline-service.address - b5s119.hdp.dc:10200 - - - - yarn.timeline-service.bind-host - 0.0.0.0 - - - - yarn.timeline-service.client.max-retries - 30 - - - - yarn.timeline-service.client.retry-interval-ms - 1000 - - - - yarn.timeline-service.enabled - false - - - - yarn.timeline-service.entity-group-fs-store.active-dir - /ats/active/ - - - - yarn.timeline-service.entity-group-fs-store.app-cache-size - 10 - - - - yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds - 3600 - - - - yarn.timeline-service.entity-group-fs-store.done-dir - /ats/done/ - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes - org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - - - - - yarn.timeline-service.entity-group-fs-store.retain-seconds - 604800 - - - - yarn.timeline-service.entity-group-fs-store.scan-interval-seconds - 60 - - - - yarn.timeline-service.entity-group-fs-store.summary-store - org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore - - - - yarn.timeline-service.generic-application-history.save-non-am-container-meta-info - false - - - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - - - yarn.timeline-service.hbase-schema.prefix - prod. - - - - yarn.timeline-service.hbase.configuration.file - file:///etc/hadoop/conf/embedded-yarn-ats-hbase/hbase-site.xml - - - - yarn.timeline-service.hbase.coprocessor.jar.hdfs.location - file:///usr/lib/edp/hadoop-3.2.2/lib/hadoop-yarn-server-timelineservice-3.2.2.jar - - - - yarn.timeline-service.http-authentication.cookie.domain - - - - - yarn.timeline-service.http-authentication.cookie.path - - - - - yarn.timeline-service.http-authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - yarn.timeline-service.http-authentication.kerberos.name.rules - - - - - yarn.timeline-service.http-authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - yarn.timeline-service.http-authentication.proxyuser.*.groups - - - - - yarn.timeline-service.http-authentication.proxyuser.*.hosts - - - - - yarn.timeline-service.http-authentication.proxyuser.*.users - - - - - yarn.timeline-service.http-authentication.signature.secret - - - - - yarn.timeline-service.http-authentication.signature.secret.file - - - - - yarn.timeline-service.http-authentication.signer.secret.provider - - - - - yarn.timeline-service.http-authentication.signer.secret.provider.object - - - - - yarn.timeline-service.http-authentication.simple.anonymous.allowed - true - - - - yarn.timeline-service.http-authentication.token.validity - - - - - yarn.timeline-service.http-authentication.type - simple - - - - yarn.timeline-service.http-cross-origin.enabled - true - - - - yarn.timeline-service.keytab - /etc/security/keytabs/yarn.service.keytab - - - - yarn.timeline-service.leveldb-state-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.principal - yarn/_HOST@ECLD.COM - - - - yarn.timeline-service.reader.webapp.address - b5s119.hdp.dc:8198 - - - - yarn.timeline-service.reader.webapp.https.address - b5s119.hdp.dc:8199 - - - - yarn.timeline-service.recovery.enabled - true - - - - yarn.timeline-service.state-store-class - org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.version - 2.0f - - - - yarn.timeline-service.versions - 1.5f,2.0f - - - - yarn.timeline-service.webapp.address - b5s119.hdp.dc:8188 - - - - yarn.timeline-service.webapp.https.address - b5s119.hdp.dc:8190 - - - - yarn.webapp.api-service.enable - true - - - - yarn.webapp.ui2.enable - true - - - diff --git a/config/b5s119/core-site.xml b/config/b5s119/core-site.xml deleted file mode 100644 index 9d72427..0000000 --- a/config/b5s119/core-site.xml +++ /dev/null @@ -1,262 +0,0 @@ - - - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - - - fs.defaultFS - hdfs://b1 - true - - - - fs.s3a.fast.upload - true - - - - fs.s3a.fast.upload.buffer - disk - - - - fs.s3a.multipart.size - 67108864 - - - - fs.trash.interval - 360 - - - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 - - - - ha.zookeeper.acl - sasl:nn:rwcda - - - - ha.zookeeper.quorum - b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 - - - - hadoop.http.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - hadoop.http.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - hadoop.http.authentication.signature.secret.file - /etc/security/http_secret - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - - - hadoop.http.authentication.type - simple - - - - hadoop.http.cross-origin.allowed-headers - X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding - - - - hadoop.http.cross-origin.allowed-methods - GET,PUT,POST,OPTIONS,HEAD,DELETE - - - - hadoop.http.cross-origin.allowed-origins - * - - - - hadoop.http.cross-origin.max-age - 1800 - - - - hadoop.http.filter.initializers - org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer - - - - hadoop.proxyuser.hdfs.groups - * - - - - hadoop.proxyuser.hdfs.hosts - * - - - - hadoop.proxyuser.hive.groups - * - - - - hadoop.proxyuser.hive.hosts - * - - - - hadoop.proxyuser.HTTP.groups - * - - - - hadoop.proxyuser.HTTP.hosts - * - - - - hadoop.proxyuser.iap.groups - * - - - - hadoop.proxyuser.iap.hosts - * - - - - hadoop.proxyuser.livy.groups - * - - - - hadoop.proxyuser.livy.hosts - * - - - - hadoop.proxyuser.yarn.groups - * - - - - hadoop.proxyuser.yarn.hosts - * - - - - hadoop.rpc.protection - authentication,privacy - - - - hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/ -RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// -RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ -RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/ -RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/ -RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/ -RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/ -RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/ -RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/ -RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/ -DEFAULT - - - - hadoop.security.authentication - kerberos - - - - hadoop.security.authorization - true - - - - hadoop.security.instrumentation.requires.admin - false - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec - - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.file.buffer.size - 131072 - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - - - - ipc.client.connect.max.retries - 50 - - - - ipc.client.connection.maxidletime - 30000 - - - - ipc.client.idlethreshold - 8000 - - - - ipc.server.tcpnodelay - true - - - - mapreduce.jobtracker.webinterface.trusted - false - - - - ipc.client.fallback-to-simple-auth-allowed - true - - - - fs.hdfs.impl.disable.cache - true - - - diff --git a/config/b5s119/hdfs-site.xml b/config/b5s119/hdfs-site.xml deleted file mode 100644 index 9b4eda4..0000000 --- a/config/b5s119/hdfs-site.xml +++ /dev/null @@ -1,698 +0,0 @@ - - - - dfs.block.access.token.enable - true - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.failover.proxy.provider.b1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.client.retry.policy.enabled - false - - - - dfs.cluster.administrators - hdfs - - - - dfs.content-summary.limit - 5000 - - - - dfs.data.transfer.protection - authentication,privacy - - - - dfs.datanode.address - 0.0.0.0:1019 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - - dfs.datanode.du.reserved - 26405499904 - - - - dfs.datanode.failed.volumes.tolerated - 2 - true - - - - dfs.datanode.http.address - 0.0.0.0:1022 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.kerberos.principal - dn/_HOST@ECLD.COM - - - - dfs.datanode.keytab.file - /etc/security/keytabs/dn.service.keytab - - - - dfs.datanode.max.transfer.threads - 16384 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.encrypt.data.transfer.cipher.suites - AES/CTR/NoPadding - - - - dfs.ha.automatic-failover.enabled - true - - - - dfs.ha.fencing.methods - shell(/bin/true) - - - - dfs.ha.namenodes.b1 - nn1,nn2 - - - - dfs.ha.namenodes.b2 - nn3,nn4 - - - - dfs.heartbeat.interval - 3 - - - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude - - - - dfs.http.policy - HTTP_ONLY - - - - dfs.https.port - 50470 - - - - dfs.internal.nameservices - b1,b2 - - - - dfs.journalnode.edits.dir.b1 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.edits.dir.b2 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.journalnode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.journalnode.kerberos.principal - jn/_HOST@ECLD.COM - - - - dfs.journalnode.keytab.file - /etc/security/keytabs/jn.service.keytab - - - - dfs.namenode.accesstime.precision - 0 - - - - dfs.namenode.acls.enabled - true - - - - dfs.namenode.audit.log.async - true - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /data/hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.fslock.fair - false - - - - dfs.namenode.handler.count - 200 - - - - dfs.namenode.http-address.b1.nn1 - b1m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn2 - b1m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn3 - b1m5.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn4 - b1m6.hdp.dc:50070 - - - - dfs.namenode.https-address.b1.nn1 - b1m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn2 - b1m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn3 - b1m5.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn4 - b1m6.hdp.dc:50470 - - - - dfs.namenode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.namenode.kerberos.principal - nn/_HOST@ECLD.COM - - - - dfs.namenode.keytab.file - /etc/security/keytabs/nn.service.keytab - - - - dfs.namenode.max.extra.edits.segments.retained - 180 - - - - dfs.namenode.name.dir - /data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.num.extra.edits.retained - 18000 - - - - dfs.namenode.rpc-address.b1.nn1 - b1m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn2 - b1m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn3 - b1m5.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn4 - b1m6.hdp.dc:8020 - - - - dfs.namenode.safemode.threshold-pct - 0.99 - - - - dfs.namenode.shared.edits.dir.b1 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1 - - - - dfs.namenode.shared.edits.dir.b2 - qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2 - - - - dfs.namenode.stale.datanode.interval - 30000 - - - - dfs.namenode.startup.delay.block.deletion.sec - 3600 - - - - dfs.namenode.write.stale.datanode.ratio - 1.0f - - - - dfs.nameservices - b1,b2,b3,b4,a3,a4,f1,d2,e1 - - - - dfs.permissions.ContentSummary.subAccess - true - - - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs - - - - dfs.replication - 3 - - - - dfs.replication.max - 50 - - - - dfs.web.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - dfs.web.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - dfs.webhdfs.enabled - true - true - - - - fs.permissions.umask-mode - 022 - - - - hadoop.caller.context.enabled - true - - - - manage.include.files - false - - - - nfs.exports.allowed.hosts - * rw - - - - nfs.file.dump.dir - /tmp/.hdfs-nfs - - - - dfs.client.datanode-restart.timeout - 30 - - - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a3 - nn1,nn2 - - - dfs.namenode.http-address.a3.nn1 - a3m2.hdp.dc:50070 - - - dfs.namenode.http-address.a3.nn2 - a3m3.hdp.dc:50070 - - - dfs.namenode.https-address.a3.nn1 - a3m2.hdp.dc:50470 - - - dfs.namenode.https-address.a3.nn2 - a3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a3.nn1 - a3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a3.nn2 - a3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b3 - nn1,nn2 - - - dfs.namenode.http-address.b3.nn1 - b3m2.hdp.dc:50070 - - - dfs.namenode.http-address.b3.nn2 - b3m3.hdp.dc:50070 - - - dfs.namenode.https-address.b3.nn1 - b3m2.hdp.dc:50470 - - - dfs.namenode.https-address.b3.nn2 - b3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b3.nn1 - b3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b3.nn2 - b3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b4 - nn1,nn2 - - - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 - - - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 - - - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 - - - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.f1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.f1 - nn1,nn2 - - - dfs.namenode.http-address.f1.nn1 - f1m2.hdp.dc:50070 - - - dfs.namenode.http-address.f1.nn2 - f1m3.hdp.dc:50070 - - - dfs.namenode.https-address.f1.nn1 - f1m2.hdp.dc:50470 - - - dfs.namenode.https-address.f1.nn2 - f1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.f1.nn1 - f1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.f1.nn2 - f1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.d2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.d2 - nn1,nn2 - - - dfs.namenode.http-address.d2.nn1 - d2m2.hdp.dc:50070 - - - dfs.namenode.http-address.d2.nn2 - d2m3.hdp.dc:50070 - - - dfs.namenode.https-address.d2.nn1 - d2m2.hdp.dc:50470 - - - dfs.namenode.https-address.d2.nn2 - d2m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.d2.nn1 - d2m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.d2.nn2 - d2m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.e1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.e1 - nn1,nn2 - - - dfs.namenode.http-address.e1.nn1 - e1m2.hdp.dc:50070 - - - dfs.namenode.http-address.e1.nn2 - e1m3.hdp.dc:50070 - - - dfs.namenode.https-address.e1.nn1 - e1m2.hdp.dc:50470 - - - dfs.namenode.https-address.e1.nn2 - e1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.e1.nn1 - e1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.e1.nn2 - e1m3.hdp.dc:8020 - - - diff --git a/config/b5s119/yarn-site.xml b/config/b5s119/yarn-site.xml deleted file mode 100644 index e2ab884..0000000 --- a/config/b5s119/yarn-site.xml +++ /dev/null @@ -1,1026 +0,0 @@ - - - - hadoop.http.cross-origin.allowed-origins - regex:.*[.]hdp[.]dc(:\d*)? - - - - hadoop.registry.client.auth - kerberos - - - - hadoop.registry.dns.bind-address - 0.0.0.0 - - - - hadoop.registry.dns.bind-port - 5354 - true - - - - hadoop.registry.dns.domain-name - ECLD.COM - - - - hadoop.registry.dns.enabled - true - - - - hadoop.registry.dns.zone-mask - 255.255.255.0 - - - - hadoop.registry.dns.zone-subnet - 172.17.0.0 - - - - hadoop.registry.jaas.context - Client - - - - hadoop.registry.secure - true - - - - hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b5,sasl:rm,sasl:hive,sasl:spark - - - - hadoop.registry.zk.quorum - b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 - - - - manage.include.files - false - - - - yarn.acl.enable - true - - - - yarn.admin.acl - * - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/lib/edp/hadoop-3.2.2/share/hadoop/common/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/common/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 2592000 - - - - yarn.log.server.url - http://b5s119.hdp.dc:19888/jobhistory/logs - - - - yarn.log.server.web-service.url - http://b5s119.hdp.dc:8188/ws/v1/applicationhistory - - - - yarn.node-labels.enabled - false - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle,spark_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.aux-services.spark_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.spark_shuffle.classpath - /usr/lib/edp/spark-3.1.1-bin-hadoop3.2/yarn/* - - - - yarn.nodemanager.aux-services.timeline_collector.class - org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor - - - - yarn.nodemanager.container-metrics.unregister-delay-ms - 60000 - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 86400 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.keytab - /etc/security/keytabs/nm.service.keytab - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users - true - - - - yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local,/data13/hadoop/yarn/local,/data14/hadoop/yarn/local,/data15/hadoop/yarn/local,/data16/hadoop/yarn/local,/data17/hadoop/yarn/local,/data18/hadoop/yarn/local,/data19/hadoop/yarn/local,/data20/hadoop/yarn/local,/data21/hadoop/yarn/local,/data22/hadoop/yarn/local,/data23/hadoop/yarn/local,/data24/hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - 3600 - - - - yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log,/data13/hadoop/yarn/log,/data14/hadoop/yarn/log,/data15/hadoop/yarn/log,/data16/hadoop/yarn/log,/data17/hadoop/yarn/log,/data18/hadoop/yarn/log,/data19/hadoop/yarn/log,/data20/hadoop/yarn/log,/data21/hadoop/yarn/log,/data22/hadoop/yarn/log,/data23/hadoop/yarn/log,/data24/hadoop/yarn/log - - - - yarn.nodemanager.log.retain-seconds - 604800 - - - - yarn.nodemanager.principal - nm/_HOST@ECLD.COM - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled - true - - - - yarn.nodemanager.recovery.supervised - true - - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource-plugins - - - - - yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - - - - - yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - - - - - yarn.nodemanager.resource.cpu-vcores - 70 - - - - yarn.nodemanager.resource.memory-mb - 348160 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 - - - - yarn.nodemanager.resourcemanager.connect.wait.secs - 1800 - - - - yarn.nodemanager.runtime.linux.allowed-runtimes - default,docker - - - - yarn.nodemanager.runtime.linux.docker.allowed-container-networks - host,none,bridge - - - - yarn.nodemanager.runtime.linux.docker.capabilities - - CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP, - SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE - - - - yarn.nodemanager.runtime.linux.docker.default-container-network - host - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed - false - - - - yarn.nodemanager.vmem-check-enabled - false - - - - yarn.nodemanager.vmem-pmem-ratio - 2.1 - - - - yarn.nodemanager.webapp.cross-origin.enabled - true - - - - yarn.nodemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.nodemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.address - b5s120.hdp.dc:8050 - - - - yarn.resourcemanager.admin.address - b5s120.hdp.dc:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.cluster-id - yarn-cluster - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.display.per-user-apps - true - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.automatic-failover.zk-base-path - /yarn-leader-election-b5hudi - - - - yarn.resourcemanager.ha.enabled - true - - - - yarn.resourcemanager.ha.rm-ids - rm1,rm2 - - - - yarn.resourcemanager.hostname - b5s120.hdp.dc - - - - yarn.resourcemanager.hostname.rm1 - b5s120.hdp.dc - - - - yarn.resourcemanager.hostname.rm2 - b5s121.hdp.dc - - - - yarn.resourcemanager.keytab - /etc/security/keytabs/rm.service.keytab - - - - yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled - true - - - - yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval - 15000 - - - - yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor - 1 - - - - yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round - 0.1 - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.placement-constraints.handler - scheduler - - - - yarn.resourcemanager.principal - rm/_HOST@ECLD.COM - - - - yarn.resourcemanager.proxy-user-privileges.enabled - true - - - - yarn.resourcemanager.proxyuser.*.groups - - - - - yarn.resourcemanager.proxyuser.*.hosts - - - - - yarn.resourcemanager.proxyuser.*.users - - - - - yarn.resourcemanager.recovery.enabled - true - - - - yarn.resourcemanager.resource-tracker.address - b5s120.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm1 - b5s120.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm2 - b5s121.hdp.dc:8025 - - - - yarn.resourcemanager.scheduler.address - b5s120.hdp.dc:8030 - - - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.scheduler.monitor.enable - true - - - - yarn.resourcemanager.max-completed-applications - 10000 - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true - - - - yarn.resourcemanager.webapp.address - b5s120.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm1 - b5s120.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm2 - b5s121.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.cross-origin.enabled - true - - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - - - yarn.resourcemanager.webapp.https.address - b5s120.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm1 - b5s120.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm2 - b5s121.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.resourcemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.work-preserving-recovery.enabled - true - - - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 - - - - yarn.resourcemanager.zk-acl - sasl:rm:rwcda - - - - yarn.resourcemanager.zk-address - b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 - - - - yarn.resourcemanager.zk-num-retries - 1000 - - - - yarn.resourcemanager.zk-retry-interval-ms - 1000 - - - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore-b5hudi - - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.rm.system-metricspublisher.emit-container-events - true - - - - yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled - true - - - - yarn.scheduler.maximum-allocation-mb - 81920 - - - - yarn.scheduler.maximum-allocation-vcores - 70 - - - - yarn.scheduler.minimum-allocation-mb - 1024 - - - - yarn.scheduler.minimum-allocation-vcores - 1 - - - - yarn.service.system-service.dir - /services - - - - yarn.system-metricspublisher.enabled - true - - - - yarn.timeline-service.address - b5s119.hdp.dc:10200 - - - - yarn.timeline-service.bind-host - 0.0.0.0 - - - - yarn.timeline-service.client.max-retries - 30 - - - - yarn.timeline-service.client.retry-interval-ms - 1000 - - - - yarn.timeline-service.enabled - false - - - - yarn.timeline-service.entity-group-fs-store.active-dir - /ats/active/ - - - - yarn.timeline-service.entity-group-fs-store.app-cache-size - 10 - - - - yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds - 3600 - - - - yarn.timeline-service.entity-group-fs-store.done-dir - /ats/done/ - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes - org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - - - - - yarn.timeline-service.entity-group-fs-store.retain-seconds - 604800 - - - - yarn.timeline-service.entity-group-fs-store.scan-interval-seconds - 60 - - - - yarn.timeline-service.entity-group-fs-store.summary-store - org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore - - - - yarn.timeline-service.generic-application-history.save-non-am-container-meta-info - false - - - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - - - yarn.timeline-service.hbase-schema.prefix - prod. - - - - yarn.timeline-service.hbase.configuration.file - file:///etc/hadoop/conf/embedded-yarn-ats-hbase/hbase-site.xml - - - - yarn.timeline-service.hbase.coprocessor.jar.hdfs.location - file:///usr/lib/edp/hadoop-3.2.2/lib/hadoop-yarn-server-timelineservice-3.2.2.jar - - - - yarn.timeline-service.http-authentication.cookie.domain - - - - - yarn.timeline-service.http-authentication.cookie.path - - - - - yarn.timeline-service.http-authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - yarn.timeline-service.http-authentication.kerberos.name.rules - - - - - yarn.timeline-service.http-authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - yarn.timeline-service.http-authentication.proxyuser.*.groups - - - - - yarn.timeline-service.http-authentication.proxyuser.*.hosts - - - - - yarn.timeline-service.http-authentication.proxyuser.*.users - - - - - yarn.timeline-service.http-authentication.signature.secret - - - - - yarn.timeline-service.http-authentication.signature.secret.file - - - - - yarn.timeline-service.http-authentication.signer.secret.provider - - - - - yarn.timeline-service.http-authentication.signer.secret.provider.object - - - - - yarn.timeline-service.http-authentication.simple.anonymous.allowed - true - - - - yarn.timeline-service.http-authentication.token.validity - - - - - yarn.timeline-service.http-authentication.type - simple - - - - yarn.timeline-service.http-cross-origin.enabled - true - - - - yarn.timeline-service.keytab - /etc/security/keytabs/yarn.service.keytab - - - - yarn.timeline-service.leveldb-state-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.principal - yarn/_HOST@ECLD.COM - - - - yarn.timeline-service.reader.webapp.address - b5s119.hdp.dc:8198 - - - - yarn.timeline-service.reader.webapp.https.address - b5s119.hdp.dc:8199 - - - - yarn.timeline-service.recovery.enabled - true - - - - yarn.timeline-service.state-store-class - org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.version - 2.0f - - - - yarn.timeline-service.versions - 1.5f,2.0f - - - - yarn.timeline-service.webapp.address - b5s119.hdp.dc:8188 - - - - yarn.timeline-service.webapp.https.address - b5s119.hdp.dc:8190 - - - - yarn.webapp.api-service.enable - true - - - - yarn.webapp.ui2.enable - true - - - diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java index f61ba5d..6c766a4 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java @@ -11,6 +11,7 @@ public class RuntimeInfo { private String jarPath; private String jdkPath; private String logPath; + private String confPath; private String dataPath; private String kerberosKeytabPath; private LokiInfo loki; @@ -52,6 +53,14 @@ public class RuntimeInfo { this.logPath = logPath; } + public String getConfPath() { + return confPath; + } + + public void setConfPath(String confPath) { + this.confPath = confPath; + } + public String getDataPath() { return dataPath; } @@ -123,6 +132,7 @@ public class RuntimeInfo { ", jarPath='" + jarPath + '\'' + ", jdkPath='" + jdkPath + '\'' + ", logPath='" + logPath + '\'' + + ", confPath='" + confPath + '\'' + ", dataPath='" + dataPath + '\'' + ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' + ", loki=" + loki + @@ -130,6 +140,7 @@ public class RuntimeInfo { ", connectorZkUrl='" + connectorZkUrl + '\'' + ", hudi=" + hudi + ", security=" + security + + ", yarn=" + yarn + '}'; } diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java index 1c25a2e..07a0442 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java @@ -17,6 +17,7 @@ public class ServiceInfo { private List groups = new ArrayList<>(); private Integer replicas = 0; private String sourceJar; + private List classpath = new ArrayList<>(); private Map environments = new HashMap<>(); private Map arguments = new HashMap<>(); @@ -60,6 +61,14 @@ public class ServiceInfo { this.sourceJar = sourceJar; } + public List getClasspath() { + return classpath; + } + + public void setClasspath(List classpath) { + this.classpath = classpath; + } + public Map getEnvironments() { return environments; } @@ -84,6 +93,7 @@ public class ServiceInfo { ", groups=" + groups + ", replicas=" + replicas + ", sourceJar='" + sourceJar + '\'' + + ", classpath=" + classpath + ", environments=" + environments + ", arguments=" + arguments + '}'; diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 88a6a07..d43260a 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -17,10 +17,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.MutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -120,13 +126,12 @@ public class RunnerApplication implements ApplicationRunner { String deployScript = deployTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) .put("hosts", hostInfoList - .stream() - .map(HostInfoWrapper::getHostnameIp) - .sorted((o1, o2) -> Comparator.naturalOrder().compare(o1.getIp(), o2.getIp())) - .collect(Collectors.toList())) + .collect(HostInfoWrapper::getHostnameIp) + .toSortedList((o1, o2) -> Comparator.naturalOrder().compare(o1.getIp(), o2.getIp()))) .put("selectedHosts", selectedHosts) .put("runtime", runtimeInfo) .put("info", serviceInfo) + .put("classpath", String.join(":", serviceInfo.getClasspath())) .put("arguments", serviceInfo.getArguments()) .put("environments", serviceInfo.getEnvironments()) .build()); @@ -141,10 +146,8 @@ public class RunnerApplication implements ApplicationRunner { String stopScript = stopTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) .put("hosts", hostInfoList - .stream() - .map(HostInfoWrapper::getIp) - .sorted(Comparator.naturalOrder()) - .collect(Collectors.toList())) + .collect(HostInfoWrapper::getIp) + .toSortedList(Comparator.naturalOrder())) .put("runtime", runtimeInfo) .put("info", serviceInfo) .put("arguments", serviceInfo.getArguments()) @@ -161,10 +164,8 @@ public class RunnerApplication implements ApplicationRunner { String logScript = logTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) .put("hosts", hostInfoList - .stream() - .map(HostInfoWrapper::getIp) - .sorted(Comparator.naturalOrder()) - .collect(Collectors.toList())) + .collect(HostInfoWrapper::getIp) + .toSortedList(Comparator.naturalOrder())) .put("selectedHosts", selectedHosts) .put("runtime", runtimeInfo) .put("info", serviceInfo) @@ -179,29 +180,56 @@ public class RunnerApplication implements ApplicationRunner { Files.write(logScriptFile, logScript.getBytes()); } + ImmutableList clusters = Lists.mutable.of(runtimeInfo.getYarn().getSyncClusters().split(",")) + .withAll(Lists.mutable.of(runtimeInfo.getYarn().getCompactionClusters().split(","))) + .collect(n -> runtimeInfo.getYarn().getStoreCluster() + n) + .distinct() + .toImmutable(); + Template downloadConfigTemplate = engine.getTemplate("cloud/download-config.ftl"); + String downloadConfigScript = downloadConfigTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .put("classpath", clusters) + .build()); + Path downloadConfigScriptFile = Paths.get(root.toString(), "download-config.sh"); + Files.deleteIfExists(downloadConfigScriptFile); + Files.write(downloadConfigScriptFile, downloadConfigScript.getBytes()); + + Template deployConfigTemplate = engine.getTemplate("cloud/deploy-config.ftl"); + String deployConfigScript = deployConfigTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .put("hosts", hostInfoList + .collect(HostInfoWrapper::getHostnameIp) + .toSortedList((o1, o2) -> Comparator.naturalOrder().compare(o1.getIp(), o2.getIp()))) + .build()); + Path deployConfigScriptFile = Paths.get(root.toString(), "deploy-config.sh"); + Files.deleteIfExists(deployConfigScriptFile); + Files.write(deployConfigScriptFile, deployConfigScript.getBytes()); + ClassPathResource stopFile = new ClassPathResource("script/stop.sh"); String stopScript = new String(IoUtil.readBytes(stopFile.getInputStream())); Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); - Map> groups = new HashMap<>(); + MutableMap> groups = Maps.mutable.empty(); for (ServiceInfoWrapper service : serviceInfoList) { service.getGroups().add(0, "all"); for (String group : service.getGroups()) { if (!groups.containsKey(group)) { - groups.put(group, new ArrayList<>()); + groups.put(group, Lists.mutable.empty()); } List list = groups.get(group); list.add(service); } } - for (Map.Entry> entry : groups.entrySet()) { + for (Map.Entry> entry : groups.entrySet()) { String group = entry.getKey(); - List infos = entry.getValue(); + MutableList infos = entry.getValue(); Template batchDeployTemplate = engine.getTemplate("cloud/batch-deploy.ftl"); String batchDeployScript = batchDeployTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList())) + .put("services", infos.collect(ServiceInfoWrapper::getName)) .build()); Path batchDeployScriptFile = Paths.get( root.toString(), @@ -213,7 +241,7 @@ public class RunnerApplication implements ApplicationRunner { Template batchStopTemplate = engine.getTemplate("cloud/batch-stop.ftl"); String batchStopScript = batchStopTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList())) + .put("services", infos.collect(ServiceInfoWrapper::getName)) .build()); Path batchStopScriptFile = Paths.get( root.toString(), diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java index edd0ae0..49f07ee 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java @@ -49,6 +49,10 @@ public class ServiceInfoWrapper { return serviceInfo.getSourceJar(); } + public List getClasspath() { + return serviceInfo.getClasspath(); + } + public Map getEnvironments() { return environments; } diff --git a/service-cli/service-cli-runner/src/main/resources/application-b12.yml b/service-cli/service-cli-runner/src/main/resources/application-b12.yml index 3d35ab7..f5ae385 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml @@ -4,6 +4,7 @@ deploy: jar-path: /data/datalake/jars jdk-path: /opt/jdk8u252-b09/bin/java log-path: /data/datalake/logs + conf-path: /data/datalake/conf data-path: /data/datalake/data kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab loki: diff --git a/service-cli/service-cli-runner/src/main/resources/application-b5.yml b/service-cli/service-cli-runner/src/main/resources/application-b5.yml index 27baf62..d0f61a5 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml @@ -4,6 +4,7 @@ deploy: jar-path: /apps/datalake/hudi/jars jdk-path: /opt/jdk1.8.0_162/bin/java log-path: /apps/datalake/hudi/logs + conf-path: /apps/datalake/hudi/conf data-path: /apps/datalake/hudi/data kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab loki: diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index e2b9fdc..ad12d2e 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -28,6 +28,8 @@ deploy: - "service-hudi-launcher" source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar replicas: 6 + classpath: + - ${deploy.runtime.conf-path}/b2b1 environments: connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} @@ -49,6 +51,8 @@ deploy: - "service-hudi-launcher" source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar replicas: 6 + classpath: + - ${deploy.runtime.conf-path}/b2b5 environments: connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} @@ -70,6 +74,8 @@ deploy: - "service-hudi-launcher" source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar replicas: 6 + classpath: + - ${deploy.runtime.conf-path}/b2a4 environments: connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} @@ -91,6 +97,8 @@ deploy: - "service-hudi-launcher" source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar replicas: 6 + classpath: + - ${deploy.runtime.conf-path}/b2b12 environments: connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} @@ -154,6 +162,8 @@ deploy: - "service" source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar replicas: 5 + classpath: + - ${deploy.runtime.conf-path}/b2b12 environments: connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} diff --git a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy-config.ftl b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy-config.ftl new file mode 100644 index 0000000..7af9fe0 --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy-config.ftl @@ -0,0 +1,5 @@ +#!/bin/bash +<#list hosts as host> +echo ${host.ip} ${host.hostname} +ssh ${host.ip} 'bash -s' < ${currentPath}/download-config.sh + diff --git a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl index eff7841..c0ae93c 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl @@ -24,7 +24,7 @@ hostname=`ssh $host 'hostname'` hostname_full=`ssh $host 'hostname -f'` ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" start_time=`date +%Y%m%d%H%M%S` -ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters} <#list arguments?keys as key>--${key}=${arguments[key]?string} > /dev/null 2>&1 &" +ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -Xbootclasspath/a:${classpath} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters} <#list arguments?keys as key>--${key}=${arguments[key]?string} > /dev/null 2>&1 &" echo '' diff --git a/service-cli/service-cli-runner/src/main/resources/template/cloud/download-config.ftl b/service-cli/service-cli-runner/src/main/resources/template/cloud/download-config.ftl new file mode 100644 index 0000000..cb34b95 --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/download-config.ftl @@ -0,0 +1,18 @@ +#!/bin/bash + +config_names=( +<#list classpath as cp> + ${cp} + +) + +for config_name in <#noparse>${config_names[@]} +do + conf_path=${runtime.confPath}/<#noparse>${config_name} + tar_path=<#noparse>${conf_path}.tar.gz + rm -rf <#noparse>${conf_path} + mkdir -p <#noparse>${conf_path} + curl -s ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/<#noparse>${config_name}.tar.gz -o <#noparse>${tar_path} + tar -zxf <#noparse>${tar_path} -C <#noparse>${conf_path} + rm -rf <#noparse>${tar_path} +done diff --git a/service-cli/service-cli-runner/src/test/java/com/lanyuanxiaoyao/service/cli/runner/UrlTest.java b/service-cli/service-cli-runner/src/test/java/com/lanyuanxiaoyao/service/cli/runner/UrlTest.java new file mode 100644 index 0000000..2e77bcb --- /dev/null +++ b/service-cli/service-cli-runner/src/test/java/com/lanyuanxiaoyao/service/cli/runner/UrlTest.java @@ -0,0 +1,13 @@ +package com.lanyuanxiaoyao.service.cli.runner; + +import cn.hutool.core.util.URLUtil; + +/** + * @author lanyuanxiaoyao + * @date 2024-02-27 + */ +public class UrlTest { + public static void main(String[] args) { + System.out.println(URLUtil.toURI("ftp://yyy:QeY!68)4nH1@132.121.122.15:2222/b9b9.tar.gz").getPath()); + } +}