feature(cli,yarn-query): 增加b12集群配置

This commit is contained in:
2023-12-05 15:45:33 +08:00
parent 62bfc08fc3
commit 9905c2c856
8 changed files with 1926 additions and 21 deletions

View File

@@ -1,23 +1,16 @@
#!/bin/bash
mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
mvn -pl service-dependencies,service-configuration,service-forest,service-executor,service-executor/service-executor-core clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
mvn -pl service-flink-query,service-info-query,service-loki-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml -P b2b12
mvn -pl service-hudi-query,service-executor/service-executor-manager,service-executor/service-executor-task clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
mvn -pl service-flink-query,service-hudi-query,service-info-query,service-loki-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml -P b2e1
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-web/target/service-web-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-flink-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-hudi-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-info-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-loki-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-pulsar-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-yarn-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-zookeeper-query-1.0.0-SNAPSHOT.jar'
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-web-1.0.0-SNAPSHOT.jar'
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-web/target/service-web-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar
ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar

260
config/b2b12/core-site.xml Normal file
View File

@@ -0,0 +1,260 @@
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<property>
<name>fs.azure.user.agent.prefix</name>
<value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://b2</value>
<final>true</final>
</property>
<property>
<name>fs.s3a.fast.upload</name>
<value>true</value>
</property>
<property>
<name>fs.s3a.fast.upload.buffer</name>
<value>disk</value>
</property>
<property>
<name>fs.s3a.multipart.size</name>
<value>67108864</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>360</value>
</property>
<property>
<name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
<value>120</value>
</property>
<property>
<name>ha.zookeeper.acl</name>
<value>sasl:nn:rwcda</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181</value>
</property>
<property>
<name>hadoop.http.authentication.kerberos.keytab</name>
<value>/etc/security/keytabs/spnego.service.keytab</value>
</property>
<property>
<name>hadoop.http.authentication.kerberos.principal</name>
<value>HTTP/_HOST@ECLD.COM</value>
</property>
<property>
<name>hadoop.http.authentication.signature.secret.file</name>
<value>/etc/security/http_secret</value>
</property>
<property>
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
<value>true</value>
</property>
<property>
<name>hadoop.http.authentication.type</name>
<value>simple</value>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-headers</name>
<value>X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding</value>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-methods</name>
<value>GET,PUT,POST,OPTIONS,HEAD,DELETE</value>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-origins</name>
<value>*</value>
</property>
<property>
<name>hadoop.http.cross-origin.max-age</name>
<value>1800</value>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.iap.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.iap.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.livy.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.livy.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.hosts</name>
<value>b1m2.hdp.dc,b1m3.hdp.dc</value>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication,privacy</value>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value>RULE:[1:$1@$0](hbase-b1@ECLD.COM)s/.*/hbase/
RULE:[1:$1@$0](hdfs-b1@ECLD.COM)s/.*/hdfs/
RULE:[1:$1@$0](spark-b1@ECLD.COM)s/.*/spark/
RULE:[1:$1@$0](yarn-ats-b1@ECLD.COM)s/.*/yarn-ats/
RULE:[1:$1@$0](.*@ECLD.COM)s/@.*//
RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/
RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/
RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/
RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/
RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/
RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/
RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/
RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/
RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/
RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/
RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/
RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/
RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/
RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/
RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/
RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/
DEFAULT</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>kerberos</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>true</value>
</property>
<property>
<name>hadoop.security.instrumentation.requires.admin</name>
<value>false</value>
</property>
<property>
<name>io.compression.codec.lzo.class</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>io.serializations</name>
<value>org.apache.hadoop.io.serializer.WritableSerialization</value>
</property>
<property>
<name>ipc.client.connect.max.retries</name>
<value>50</value>
</property>
<property>
<name>ipc.client.connection.maxidletime</name>
<value>30000</value>
</property>
<property>
<name>ipc.client.idlethreshold</name>
<value>8000</value>
</property>
<property>
<name>ipc.server.tcpnodelay</name>
<value>true</value>
</property>
<property>
<name>mapreduce.jobtracker.webinterface.trusted</name>
<value>false</value>
</property>
<property>
<name>ipc.client.fallback-to-simple-auth-allowed</name>
<value>true</value>
</property>
<property>
<name>fs.hdfs.impl.disable.cache</name>
<value>true</value>
</property>
</configuration>

500
config/b2b12/hdfs-site.xml Normal file
View File

@@ -0,0 +1,500 @@
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<property>
<name>dfs.block.access.token.enable</name>
<value>true</value>
</property>
<property>
<name>dfs.blockreport.initialDelay</name>
<value>120</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.b1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.b2</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>true</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
<value>4096</value>
</property>
<property>
<name>dfs.client.retry.policy.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.cluster.administrators</name>
<value> hdfs</value>
</property>
<property>
<name>dfs.content-summary.limit</name>
<value>5000</value>
</property>
<property>
<name>dfs.data.transfer.protection</name>
<value>authentication,privacy</value>
</property>
<property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:1019</value>
</property>
<property>
<name>dfs.datanode.balance.bandwidthPerSec</name>
<value>6250000</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>[DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data</value>
<final>true</final>
</property>
<property>
<name>dfs.datanode.data.dir.perm</name>
<value>750</value>
</property>
<property>
<name>dfs.datanode.du.reserved</name>
<value>26405499904</value>
</property>
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>2</value>
<final>true</final>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>0.0.0.0:1022</value>
</property>
<property>
<name>dfs.datanode.https.address</name>
<value>0.0.0.0:50475</value>
</property>
<property>
<name>dfs.datanode.ipc.address</name>
<value>0.0.0.0:8010</value>
</property>
<property>
<name>dfs.datanode.kerberos.principal</name>
<value>dn/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
<value>/etc/security/keytabs/dn.service.keytab</value>
</property>
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>16384</value>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
<property>
<name>dfs.encrypt.data.transfer.cipher.suites</name>
<value>AES/CTR/NoPadding</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.ha.namenodes.b1</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.ha.namenodes.b2</name>
<value>nn3,nn4</value>
</property>
<property>
<name>dfs.heartbeat.interval</name>
<value>3</value>
</property>
<property>
<name>dfs.hosts.exclude</name>
<value>/etc/hadoop/conf/dfs.exclude</value>
</property>
<property>
<name>dfs.http.policy</name>
<value>HTTP_ONLY</value>
</property>
<property>
<name>dfs.https.port</name>
<value>50470</value>
</property>
<property>
<name>dfs.internal.nameservices</name>
<value>b1,b2</value>
</property>
<property>
<name>dfs.journalnode.edits.dir.b1</name>
<value>/data2/hadoop/hdfs/journal</value>
</property>
<property>
<name>dfs.journalnode.edits.dir.b2</name>
<value>/data2/hadoop/hdfs/journal</value>
</property>
<property>
<name>dfs.journalnode.http-address</name>
<value>0.0.0.0:8480</value>
</property>
<property>
<name>dfs.journalnode.https-address</name>
<value>0.0.0.0:8481</value>
</property>
<property>
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
<value>HTTP/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.journalnode.kerberos.principal</name>
<value>jn/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.journalnode.keytab.file</name>
<value>/etc/security/keytabs/jn.service.keytab</value>
</property>
<property>
<name>dfs.namenode.accesstime.precision</name>
<value>0</value>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.audit.log.async</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.avoid.read.stale.datanode</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.avoid.write.stale.datanode</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/data/hadoop/hdfs/namesecondary</value>
</property>
<property>
<name>dfs.namenode.checkpoint.edits.dir</name>
<value>${dfs.namenode.checkpoint.dir}</value>
</property>
<property>
<name>dfs.namenode.checkpoint.period</name>
<value>21600</value>
</property>
<property>
<name>dfs.namenode.checkpoint.txns</name>
<value>1000000</value>
</property>
<property>
<name>dfs.namenode.fslock.fair</name>
<value>false</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
<property>
<name>dfs.namenode.http-address.b1.nn1</name>
<value>b1m2.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.b1.nn2</name>
<value>b1m3.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.b2.nn3</name>
<value>b1m5.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.b2.nn4</name>
<value>b1m6.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.b1.nn1</name>
<value>b1m2.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.https-address.b1.nn2</name>
<value>b1m3.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.https-address.b2.nn3</name>
<value>b1m5.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.https-address.b2.nn4</name>
<value>b1m6.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
<value>HTTP/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.namenode.kerberos.principal</name>
<value>nn/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.namenode.keytab.file</name>
<value>/etc/security/keytabs/nn.service.keytab</value>
</property>
<property>
<name>dfs.namenode.max.extra.edits.segments.retained</name>
<value>180</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode</value>
<final>true</final>
</property>
<property>
<name>dfs.namenode.name.dir.restore</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.num.extra.edits.retained</name>
<value>18000</value>
</property>
<property>
<name>dfs.namenode.rpc-address.b1.nn1</name>
<value>b1m2.hdp.dc:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.b1.nn2</name>
<value>b1m3.hdp.dc:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.b2.nn3</name>
<value>b1m5.hdp.dc:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.b2.nn4</name>
<value>b1m6.hdp.dc:8020</value>
</property>
<property>
<name>dfs.namenode.safemode.threshold-pct</name>
<value>0.99</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir.b1</name>
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b1</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir.b2</name>
<value>qjournal://b1m2.hdp.dc:8485;b1m3.hdp.dc:8485;b1m5.hdp.dc:8485/b2</value>
</property>
<property>
<name>dfs.namenode.stale.datanode.interval</name>
<value>30000</value>
</property>
<property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>3600</value>
</property>
<property>
<name>dfs.namenode.write.stale.datanode.ratio</name>
<value>1.0f</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>b1,b2,f2</value>
</property>
<property>
<name>dfs.permissions.ContentSummary.subAccess</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>hdfs</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.replication.max</name>
<value>50</value>
</property>
<property>
<name>dfs.web.authentication.kerberos.keytab</name>
<value>/etc/security/keytabs/spnego.service.keytab</value>
</property>
<property>
<name>dfs.web.authentication.kerberos.principal</name>
<value>HTTP/_HOST@ECLD.COM</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
<final>true</final>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>022</value>
</property>
<property>
<name>hadoop.caller.context.enabled</name>
<value>true</value>
</property>
<property>
<name>manage.include.files</name>
<value>false</value>
</property>
<property>
<name>nfs.exports.allowed.hosts</name>
<value>* rw</value>
</property>
<property>
<name>nfs.file.dump.dir</name>
<value>/tmp/.hdfs-nfs</value>
</property>
<property>
<name>dfs.client.datanode-restart.timeout</name>
<value>30</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.f2</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.namenodes.f2</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.http-address.f2.nn1</name>
<value>f2m2.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.f2.nn2</name>
<value>f2m3.hdp.dc:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.f2.nn1</name>
<value>f2m2.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.https-address.f2.nn2</name>
<value>f2m3.hdp.dc:50470</value>
</property>
<property>
<name>dfs.namenode.rpc-address.f2.nn1</name>
<value>f2m2.hdp.dc:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.f2.nn2</name>
<value>f2m3.hdp.dc:8020</value>
</property>
</configuration>

1026
config/b2b12/yarn-site.xml Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -71,6 +71,12 @@
<build-tag>b2s119</build-tag>
</properties>
</profile>
<profile>
<id>b2b12</id>
<properties>
<build-tag>b2b12</build-tag>
</properties>
</profile>
</profiles>
<dependencyManagement>

View File

@@ -0,0 +1,118 @@
deploy:
runtime:
user: datalake
jar-path: /data/datalake/jars
jdk-path: /opt/jdk1.8.0_162/bin/java
log-path: /data/datalake/logs
data-path: /data/datalake/data
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
loki-base-url: http://132.126.207.125:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
connector-zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
eureka-url: ENC(1gFdUy0/FYxDMK910ZS65qNqdULnnSOPrR8EkqrZ3zYXsTP9kk8y55C7WbcxM5vaa1KIDNEqilEIA2sOLX3920gu3Woq805pf8dzxl9/velQWdaTFUxn9XqypuP+p/bSyY3upCYehCZFWgPPSn8ag0m3JKSofkk9V4mO+WswxFDlWiM1P5Om7Ev7WsWSIYMqhN2viRjCscctrIkxh3trIgsb9qfw5snNYZvh+gHRQJ1GA0VYlLmqJLvasKcHSzKGabu8iRAnXMHaAv7jwgdMpHBz7uBeeQh+BIFqurSEotjRG+TlmidLA8X0BWiRTGiGp9X9mV/5Zm5QkBp1em8L1D9+wz8R9rn8eFPXMSHhlqocxKl7ZJgnzSfu9CS67y5k7PzSIjkViN01HTPzg36zyv9xzqVt5W+7rXbQJpmXLXfUAPsjRP/ns0a6ly0jkHoIOHqapZcJ8gVCVH46D0e5C59h57Q6iX/BW0f8VIsH4HfbSGi2doRqNVqbIRwdVnkzUMDihhKNvLFTh70H20v1uw6wNZclYF1wh9Lkn28TKqz/y45FbxhRwjBCvWu55R8/qSkfR2K/QyLw4WvE3raG23fssN2BYlNA/x9OaGBWPXVUZcphGl6rPUZeAqXVKms/qcRuLOzZrZMv42OeKFk/NzR8BrwiXN7NafhXrvH1CNN0b69t7jGqS5+sIwkBjNtD+J17G9G8qUB9ETH6C+uRO4MWJ+KesERdI3E0PIqzimTQiV60lzVL4kYB5KJuDnw0LAsxpu3a5VgKWIZzF/zWXZcBQTeea0IKRBEeujwTCecvaNTjd+x53jwTSobCyL4ADt2hcGHOGMcynuryZYh++XTMEjx0P/cMxXqHDXqb21liJftWmMzqyQ5HHQZCY6Mg1Lh8FsLaV3vyl142rbXan0NwolIGJ96QZ7PoNa9zeQOgcwXvGjHnpzZWsOwq/90Bnl1JDcm0MJpjAq2LnZN5r/7aa74TvC54Pu4jMMzZMdyrUS0bWAxF9NS2erfP2BPywI2AwSGgvl7qh2Sx4d/nh1984TTwtK5jDd+LIdml8SfVIcBfNG3dPvlrJN6TmvbBxtRRFo1lXX47axfr6kL6DNDUPTFbnZRjK0RLRuNdIueLhcUCLAJfVrP0jz4axcg12ZdRU4oPUTQg7m6blN5IjJu90GcHXia2kPXudSQidQLBIrE6RrZxUYO3DPwUXzVvOwYmuGzu2TJWS4vLYR00jDMCEv/Gosy0zurzs/zLCA/awGlR4c3InQo8IahyDh2faSNljCnfUPMMgfl/vE8BBODCKX18m2a+xvyHn+out4sB6MwBqRwINn5/QWKw8qqMOPS5rmLHvPUi8BdZZnetFk2najdRib7o1SdUG1Sn3pDOYRHXVXxoHxebSyAl5ybzis9PAlc9MTJnxM1qVEEdzwUBPElDuPOE8Zr2lYhz4/RKWnwd7BG2WUP3YYVyQmDL/H00xQCagUBoQ6CD3sKmTZSFkJUAKX3r1t+K1LqqjkyYP6XSvgp7ha8CsW4ssPhCFKgieUMR5T8cFVV2tSxLEJTlm15GLVFgiWtmTsHlZ32CU388BUYqVvtv4gNvgxOxfr9suFNa4V/xRTxlpsLjQ+GSSBMHznVykxf9Buk05CuuqfW8U4Ulgc+9LxXpKFcKAHjXe3Hj5RghtuOz+Nb+Iyi3sZibeLjqYoS0pWOE5g44azVPyeSunOWCNmlQI6lwEm/st6FoupwkI70X+V5Mthd90lnFpUtp49GlC81jJZ2zSiKV7ZFGsZfLEDQRU12S)
hudi:
app-hdfs-path: hdfs://b2/apps/datalake/jars/app
archive-hdfs-path: hdfs://b2/apps/datalake/flink/completed-jobs-hudi
victoria-push-url: http://132.126.207.125:35710/api/v1/import/prometheus
security:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
hosts:
- host: b12s4.hdp.dc
ip: 132.126.207.130
- host: b12s5.hdp.dc
ip: 132.126.207.131
- host: b12s6.hdp.dc
ip: 132.126.207.132
- host: b12s7.hdp.dc
ip: 132.126.207.133
- host: b12s8.hdp.dc
ip: 132.126.207.134
- host: b12s9.hdp.dc
ip: 132.126.207.135
- host: b12s10.hdp.dc
ip: 132.126.207.136
- host: b12s11.hdp.dc
ip: 132.126.207.137
- host: b12s12.hdp.dc
ip: 132.126.207.138
- host: b12s13.hdp.dc
ip: 132.126.207.139
- host: b12s14.hdp.dc
ip: 132.126.207.140
- host: b12s15.hdp.dc
ip: 132.126.207.141
- host: b12s16.hdp.dc
ip: 132.126.207.142
- host: b12s17.hdp.dc
ip: 132.126.207.143
- host: b12s18.hdp.dc
ip: 132.126.207.144
- host: b12s19.hdp.dc
ip: 132.126.207.145
- host: b12s20.hdp.dc
ip: 132.126.207.146
- host: b12s21.hdp.dc
ip: 132.126.207.147
- host: b12s22.hdp.dc
ip: 132.126.207.148
- host: b12s23.hdp.dc
ip: 132.126.207.149
- host: b12s24.hdp.dc
ip: 132.126.207.150
- host: b12s25.hdp.dc
ip: 132.126.207.151
- host: b12s26.hdp.dc
ip: 132.126.207.152
- host: b12s27.hdp.dc
ip: 132.126.207.153
- host: b12s28.hdp.dc
ip: 132.126.207.154
- host: b12s29.hdp.dc
ip: 132.126.207.155
- host: b12s30.hdp.dc
ip: 132.126.207.156
- host: b12s31.hdp.dc
ip: 132.126.207.157
- host: b12s32.hdp.dc
ip: 132.126.207.158
- host: b12s33.hdp.dc
ip: 132.126.207.159
- host: b12s34.hdp.dc
ip: 132.126.207.160
- host: b12s35.hdp.dc
ip: 132.126.207.161
- host: b12s36.hdp.dc
ip: 132.126.207.162
- host: b12s37.hdp.dc
ip: 132.126.207.163
- host: b12s38.hdp.dc
ip: 132.126.207.164
- host: b12s39.hdp.dc
ip: 132.126.207.165
- host: b12s40.hdp.dc
ip: 132.126.207.166
- host: b12s41.hdp.dc
ip: 132.126.207.167
- host: b12s42.hdp.dc
ip: 132.126.207.168
- host: b12s43.hdp.dc
ip: 132.126.207.169
- host: b12s44.hdp.dc
ip: 132.126.207.170
- host: b12s45.hdp.dc
ip: 132.126.207.171
- host: b12s46.hdp.dc
ip: 132.126.207.172
- host: b12s47.hdp.dc
ip: 132.126.207.173
- host: b12s48.hdp.dc
ip: 132.126.207.174
- host: b12s49.hdp.dc
ip: 132.126.207.175
- host: b12s50.hdp.dc
ip: 132.126.207.176
- host: b12s51.hdp.dc
ip: 132.126.207.177

View File

@@ -11,3 +11,4 @@ yarn:
b5: http://132.122.116.12:8088
b5-sync: http://132.122.116.143:8088
t5: http://132.121.126.84:8088
b12: http://132.126.207.125:8088

View File

@@ -11,7 +11,7 @@ function yarnClusterTab() {
{
title: '总体情况',
tab: [
yarnQueueCrud('b1,b5,b5-sync,a4,b4')
yarnQueueCrud('b1,b5,b5-sync,a4,b4,b12')
],
},
yarnTab('b1', 'b1'),
@@ -20,6 +20,7 @@ function yarnClusterTab() {
yarnTab('b5', 'b5'),
yarnTab('b5-sync', 'b5-sync'),
yarnTab('t5', 't5'),
yarnTab('b12', 'b12'),
]
}
]