diff --git a/bin/build-all.sh b/bin/build-all.sh index 9c88d34..9e454de 100755 --- a/bin/build-all.sh +++ b/bin/build-all.sh @@ -19,9 +19,6 @@ ${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12 ${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar -mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b9b9 -${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b9b9-1.0.0-SNAPSHOT.jar - ${current_path}/bin/upload-rm.sh ${current_path}/service-api/target/service-api-1.0.0-SNAPSHOT.jar ${current_path}/bin/upload-rm.sh ${current_path}/service-check/target/service-check-1.0.0-SNAPSHOT.jar ${current_path}/bin/upload-rm.sh ${current_path}/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar diff --git a/bin/upload-config.sh b/bin/upload-config.sh index 257d838..26041a9 100755 --- a/bin/upload-config.sh +++ b/bin/upload-config.sh @@ -7,7 +7,6 @@ clusters=( b2b1 b2b5 b2b12 - b9b9 ) for cluster in ${clusters[*]}; diff --git a/config/b9b9/core-site.xml b/config/b9b9/core-site.xml deleted file mode 100644 index 6612d3f..0000000 --- a/config/b9b9/core-site.xml +++ /dev/null @@ -1,333 +0,0 @@ - - - - - - fs.azure.user.agent.prefix - User-Agent: APN/1.0 Hortonworks/1.0 HDP/ - - - - fs.defaultFS - - jfs://tdsc - true - - - - fs.s3a.fast.upload - true - - - - fs.s3a.fast.upload.buffer - disk - - - - fs.s3a.multipart.size - 67108864 - - - - fs.trash.interval - 4320 - - - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 - - - - ha.zookeeper.acl - sasl:nn:rwcda - - - - ha.zookeeper.quorum - b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181 - - - - hadoop.http.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - hadoop.http.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - hadoop.http.authentication.signature.secret.file - /etc/security/http_secret - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - - - hadoop.http.authentication.type - simple - - - - hadoop.http.cross-origin.allowed-headers - X-Requested-With,Content-Type,Accept,Origin,WWW-Authenticate,Accept-Encoding,Transfer-Encoding - - - - hadoop.http.cross-origin.allowed-methods - GET,PUT,POST,OPTIONS,HEAD,DELETE - - - - hadoop.http.cross-origin.allowed-origins - * - - - - hadoop.http.cross-origin.max-age - 1800 - - - - hadoop.http.filter.initializers - org.apache.hadoop.security.AuthenticationFilterInitializer,org.apache.hadoop.security.HttpCrossOriginFilterInitializer - - - - hadoop.proxyuser.hdfs.groups - * - - - - hadoop.proxyuser.hdfs.hosts - * - - - - hadoop.proxyuser.hive.groups - * - - - - hadoop.proxyuser.hive.hosts - * - - - - hadoop.proxyuser.HTTP.groups - * - - - - hadoop.proxyuser.HTTP.hosts - * - - - - hadoop.proxyuser.iap.groups - * - - - - hadoop.proxyuser.iap.hosts - * - - - - hadoop.proxyuser.livy.groups - * - - - - hadoop.proxyuser.livy.hosts - * - - - - hadoop.proxyuser.yarn.groups - * - - - - hadoop.proxyuser.yarn.hosts - b9m2.hdp.dc,b9m3.hdp.dc - - - - hadoop.rpc.protection - authentication,privacy - - - - hadoop.security.auth_to_local - RULE:[1:$1@$0](hbase-b9@ECLD.COM)s/.*/hbase/ -RULE:[1:$1@$0](hdfs-b9@ECLD.COM)s/.*/hdfs/ -RULE:[1:$1@$0](spark-b9@ECLD.COM)s/.*/spark/ -RULE:[1:$1@$0](yarn-ats-b9@ECLD.COM)s/.*/yarn-ats/ -RULE:[1:$1@$0](.*@ECLD.COM)s/@.*// -RULE:[2:$1@$0](dn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](hbase@ECLD.COM)s/.*/hbase/ -RULE:[2:$1@$0](hive@ECLD.COM)s/.*/hive/ -RULE:[2:$1@$0](jhs@ECLD.COM)s/.*/mapred/ -RULE:[2:$1@$0](jn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](livy@ECLD.COM)s/.*/livy/ -RULE:[2:$1@$0](nm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](nn@ECLD.COM)s/.*/hdfs/ -RULE:[2:$1@$0](rangeradmin@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangerlookup@ECLD.COM)s/.*/ranger/ -RULE:[2:$1@$0](rangertagsync@ECLD.COM)s/.*/rangertagsync/ -RULE:[2:$1@$0](rangerusersync@ECLD.COM)s/.*/rangerusersync/ -RULE:[2:$1@$0](rm@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](spark@ECLD.COM)s/.*/spark/ -RULE:[2:$1@$0](yarn@ECLD.COM)s/.*/yarn/ -RULE:[2:$1@$0](yarn-ats-hbase@ECLD.COM)s/.*/yarn-ats/ -DEFAULT - - - - hadoop.security.authentication - kerberos - - - - hadoop.security.authorization - true - - - - hadoop.security.instrumentation.requires.admin - false - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec - - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.file.buffer.size - 131072 - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - - - - ipc.client.connect.max.retries - 50 - - - - ipc.client.connection.maxidletime - 30000 - - - - ipc.client.idlethreshold - 8000 - - - - ipc.server.tcpnodelay - true - - - - mapreduce.jobtracker.webinterface.trusted - false - - - - fs.jfs.impl - io.juicefs.JuiceFileSystem - - - fs.AbstractFileSystem.jfs.impl - io.juicefs.JuiceFS - - - juicefs.cache-dir - /var/jfsCache - - - juicefs.cache-size - 0 - - - juicefs.cache-full-block - true - - - juicefs.free-space - 0.05 - - - juicefs.attr-cache - 86400 - - - juicefs.entry-cache - 86400 - - - juicefs.dir-entry-cache - 86400 - - - juicefs.get-timeout - 120 - - - juicefs.put-timeout - 120 - - - juicefs.no-usage-report - true - - - juicefs.tdsc.meta - tikv://132.126.207.85:2379,132.126.207.86:2379,132.126.207.87:2379/tdsc?ca=/etc/tikv/ca.crt&cert=/etc/tikv/client.crt&key=/etc/tikv/client.pem - - - juicefs.backup-meta - 0 - - - juicefs.block.size - 268435456 - - - juicefs.prefetch - 0 - - - juicefs.debug - false - - - juicefs.no-bgjob - true - - - jfs.kerberos.token-servers - hdfs://b9 - - - mapreduce.job.hdfs-servers - hdfs://b9 - - diff --git a/config/b9b9/hdfs-site.xml b/config/b9b9/hdfs-site.xml deleted file mode 100644 index 8e3e27c..0000000 --- a/config/b9b9/hdfs-site.xml +++ /dev/null @@ -1,847 +0,0 @@ - - - - - dfs.block.access.token.enable - true - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.failover.proxy.provider.b9 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.failover.proxy.provider.b1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.client.retry.policy.enabled - false - - - - dfs.cluster.administrators - hdfs - - - - dfs.content-summary.limit - 5000 - - - - dfs.data.transfer.protection - authentication,privacy - - - - dfs.datanode.address - 0.0.0.0:1019 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - [DISK]file:///data1/hadoop/hdfs/data,[DISK]file:///data2/hadoop/hdfs/data,[DISK]file:///data3/hadoop/hdfs/data,[DISK]file:///data4/hadoop/hdfs/data,[DISK]file:///data5/hadoop/hdfs/data,[DISK]file:///data6/hadoop/hdfs/data,[DISK]file:///data7/hadoop/hdfs/data,[DISK]file:///data8/hadoop/hdfs/data,[DISK]file:///data9/hadoop/hdfs/data,[DISK]file:///data10/hadoop/hdfs/data,[DISK]file:///data11/hadoop/hdfs/data,[DISK]file:///data12/hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - - dfs.datanode.du.reserved - 26405499904 - - - - dfs.datanode.failed.volumes.tolerated - 2 - true - - - - dfs.datanode.http.address - 0.0.0.0:1022 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.kerberos.principal - dn/_HOST@ECLD.COM - - - - dfs.datanode.keytab.file - /etc/security/keytabs/dn.service.keytab - - - - dfs.datanode.max.transfer.threads - 16384 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.encrypt.data.transfer.cipher.suites - AES/CTR/NoPadding - - - - dfs.ha.automatic-failover.enabled - true - - - - dfs.ha.fencing.methods - shell(/bin/true) - - - - dfs.ha.namenodes.b9 - nn1,nn2 - - - - dfs.ha.namenodes.b1 - nn1,nn2 - - - - dfs.ha.namenodes.b2 - nn3,nn4 - - - - dfs.heartbeat.interval - 3 - - - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude - - - - dfs.http.policy - HTTP_ONLY - - - - dfs.https.port - 50470 - - - - dfs.internal.nameservices - b9 - - - - dfs.journalnode.edits.dir - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.edits.dir.b9 - /data2/hadoop/hdfs/journal - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.journalnode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.journalnode.kerberos.principal - jn/_HOST@ECLD.COM - - - - dfs.journalnode.keytab.file - /etc/security/keytabs/jn.service.keytab - - - - dfs.namenode.accesstime.precision - 0 - - - - dfs.namenode.acls.enabled - true - - - - dfs.namenode.audit.log.async - true - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /data/hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.fslock.fair - false - - - - dfs.namenode.handler.count - 100 - - - - dfs.namenode.http-address.b9.nn1 - b9m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b9.nn2 - b9m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn1 - b1m2.hdp.dc:50070 - - - - dfs.namenode.http-address.b1.nn2 - b1m3.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn3 - b1m5.hdp.dc:50070 - - - - dfs.namenode.http-address.b2.nn4 - b1m6.hdp.dc:50070 - - - - dfs.namenode.https-address.b9.nn1 - b9m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b9.nn2 - b9m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn1 - b1m2.hdp.dc:50470 - - - - dfs.namenode.https-address.b1.nn2 - b1m3.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn3 - b1m5.hdp.dc:50470 - - - - dfs.namenode.https-address.b2.nn4 - b1m6.hdp.dc:50470 - - - - dfs.namenode.kerberos.internal.spnego.principal - HTTP/_HOST@ECLD.COM - - - - dfs.namenode.kerberos.principal - nn/_HOST@ECLD.COM - - - - dfs.namenode.keytab.file - /etc/security/keytabs/nn.service.keytab - - - - dfs.namenode.max.extra.edits.segments.retained - 180 - - - - dfs.namenode.name.dir - /data1/hadoop/hdfs/namenode,/data2/hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.num.extra.edits.retained - 18000 - - - - dfs.namenode.rpc-address.b9.nn1 - b9m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b9.nn2 - b9m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn1 - b1m2.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b1.nn2 - b1m3.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn3 - b1m5.hdp.dc:8020 - - - - dfs.namenode.rpc-address.b2.nn4 - b1m6.hdp.dc:8020 - - - - dfs.namenode.safemode.threshold-pct - 0.99 - - - - dfs.namenode.shared.edits.dir - qjournal://b9m1.hdp.dc:8485;b9m2.hdp.dc:8485;b9m3.hdp.dc:8485/b9 - - - - dfs.namenode.shared.edits.dir.b9 - qjournal://b9m1.hdp.dc:8485;b9m2.hdp.dc:8485;b9m3.hdp.dc:8485/b9 - - - - dfs.namenode.stale.datanode.interval - 30000 - - - - dfs.namenode.startup.delay.block.deletion.sec - 3600 - - - - dfs.namenode.write.stale.datanode.ratio - 1.0f - - - - dfs.nameservices - b9,b7,b5,b1,b2,b3,b4,a3,a4,a6,f1,e1,d2 - - - - dfs.permissions.ContentSummary.subAccess - true - - - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs - - - - dfs.replication - 3 - - - - dfs.replication.max - 50 - - - - dfs.web.authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - dfs.web.authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - dfs.webhdfs.enabled - true - true - - - - fs.permissions.umask-mode - 022 - - - - hadoop.caller.context.enabled - true - - - - manage.include.files - false - - - - nfs.exports.allowed.hosts - * rw - - - - nfs.file.dump.dir - /tmp/.hdfs-nfs - - - - dfs.client.datanode-restart.timeout - 30 - - - - dfs.client.failover.proxy.provider.f1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.f1 - nn1,nn2 - - - dfs.namenode.http-address.f1.nn1 - f1m2.hdp.dc:50070 - - - dfs.namenode.http-address.f1.nn2 - f1m3.hdp.dc:50070 - - - dfs.namenode.https-address.f1.nn1 - f1m2.hdp.dc:50470 - - - dfs.namenode.https-address.f1.nn2 - f1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.f1.nn1 - f1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.f1.nn2 - f1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.d2 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.d2 - nn1,nn2 - - - dfs.namenode.http-address.d2.nn1 - d2m2.hdp.dc:50070 - - - dfs.namenode.http-address.d2.nn2 - d2m3.hdp.dc:50070 - - - dfs.namenode.https-address.d2.nn1 - d2m2.hdp.dc:50470 - - - dfs.namenode.https-address.d2.nn2 - d2m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.d2.nn1 - d2m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.d2.nn2 - d2m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.e1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.e1 - nn1,nn2 - - - dfs.namenode.http-address.e1.nn1 - e1m2.hdp.dc:50070 - - - dfs.namenode.http-address.e1.nn2 - e1m3.hdp.dc:50070 - - - dfs.namenode.https-address.e1.nn1 - e1m2.hdp.dc:50470 - - - dfs.namenode.https-address.e1.nn2 - e1m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.e1.nn1 - e1m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.e1.nn2 - e1m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b4 - nn1,nn2 - - - dfs.namenode.http-address.b4.nn1 - b4m2.hdp.dc:50070 - - - dfs.namenode.http-address.b4.nn2 - b4m3.hdp.dc:50070 - - - dfs.namenode.https-address.b4.nn1 - b4m2.hdp.dc:50470 - - - dfs.namenode.https-address.b4.nn2 - b4m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b4.nn1 - b4m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b4.nn2 - b4m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a6 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a6 - nn1,nn2 - - - dfs.namenode.http-address.a6.nn1 - a6m2.hdp.dc:50070 - - - dfs.namenode.http-address.a6.nn2 - a6m3.hdp.dc:50070 - - - dfs.namenode.https-address.a6.nn1 - a6m2.hdp.dc:50470 - - - dfs.namenode.https-address.a6.nn2 - a6m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a6.nn1 - a6m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a6.nn2 - a6m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b5 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b5 - nn1,nn2 - - - dfs.namenode.http-address.b5.nn1 - b5m1.hdp.dc:50070 - - - dfs.namenode.http-address.b5.nn2 - b5m2.hdp.dc:50070 - - - dfs.namenode.https-address.b5.nn1 - b5m1.hdp.dc:50470 - - - dfs.namenode.https-address.b5.nn2 - b5m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.b5.nn1 - b5m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.b5.nn2 - b5m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a4 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a4 - nn1,nn2 - - - dfs.namenode.http-address.a4.nn1 - a4m1.hdp.dc:50070 - - - dfs.namenode.http-address.a4.nn2 - a4m2.hdp.dc:50070 - - - dfs.namenode.https-address.a4.nn1 - a4m1.hdp.dc:50470 - - - dfs.namenode.https-address.a4.nn2 - a4m2.hdp.dc:50470 - - - dfs.namenode.rpc-address.a4.nn1 - a4m1.hdp.dc:8020 - - - dfs.namenode.rpc-address.a4.nn2 - a4m2.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.a3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.a3 - nn1,nn2 - - - dfs.namenode.http-address.a3.nn1 - a3m2.hdp.dc:50070 - - - dfs.namenode.http-address.a3.nn2 - a3m3.hdp.dc:50070 - - - dfs.namenode.https-address.a3.nn1 - a3m2.hdp.dc:50470 - - - dfs.namenode.https-address.a3.nn2 - a3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.a3.nn1 - a3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.a3.nn2 - a3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b3 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b3 - nn1,nn2 - - - dfs.namenode.http-address.b3.nn1 - b3m2.hdp.dc:50070 - - - dfs.namenode.http-address.b3.nn2 - b3m3.hdp.dc:50070 - - - dfs.namenode.https-address.b3.nn1 - b3m2.hdp.dc:50470 - - - dfs.namenode.https-address.b3.nn2 - b3m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b3.nn1 - b3m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b3.nn2 - b3m3.hdp.dc:8020 - - - - dfs.client.failover.proxy.provider.b7 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - dfs.ha.namenodes.b7 - nn1,nn2 - - - dfs.namenode.http-address.b7.nn1 - b7m2.hdp.dc:50070 - - - dfs.namenode.http-address.b7.nn2 - b7m3.hdp.dc:50070 - - - dfs.namenode.https-address.b7.nn1 - b7m2.hdp.dc:50470 - - - dfs.namenode.https-address.b7.nn2 - b7m3.hdp.dc:50470 - - - dfs.namenode.rpc-address.b7.nn1 - b7m2.hdp.dc:8020 - - - dfs.namenode.rpc-address.b7.nn2 - b7m3.hdp.dc:8020 - - - - dfs.permissions - true - - - dfs.namenode.inode.attributes.provider.class - org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer - - - diff --git a/config/b9b9/viewfs-mount-table.xml b/config/b9b9/viewfs-mount-table.xml deleted file mode 100644 index 9f6e902..0000000 --- a/config/b9b9/viewfs-mount-table.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - fs.viewfs.mounttable.datalake.link./app-logs - hdfs://b9/app-logs - - - fs.viewfs.mounttable.datalake.link./apps - hdfs://b9/apps - - - fs.viewfs.mounttable.datalake.link./ats - hdfs://b9/ats - - - fs.viewfs.mounttable.datalake.link./atsv2 - hdfs://b9/atsv2 - - - fs.viewfs.mounttable.datalake.link./mr-history - hdfs://b9/mr-history - - - fs.viewfs.mounttable.datalake.link./spark2-history - hdfs://b9/spark2-history - - - fs.viewfs.mounttable.datalake.link./tmp - hdfs://b9/tmp - - - fs.viewfs.mounttable.datalake.link./user - hdfs://b9/user - - - fs.viewfs.mounttable.datalake.link./warehouse - hdfs://b9/warehouse - - - fs.viewfs.mounttable.datalake.link./b1tmp - hdfs://b1/tmp - - - fs.viewfs.mounttable.datalake.link./b2tmp - hdfs://b2/tmp - - diff --git a/config/b9b9/yarn-site.xml b/config/b9b9/yarn-site.xml deleted file mode 100644 index e722216..0000000 --- a/config/b9b9/yarn-site.xml +++ /dev/null @@ -1,1021 +0,0 @@ - - - - hadoop.http.cross-origin.allowed-origins - regex:.*[.]hdp[.]dc(:\d*)? - - - - hadoop.registry.client.auth - kerberos - - - - hadoop.registry.dns.bind-address - 0.0.0.0 - - - - hadoop.registry.dns.bind-port - 5354 - true - - - - hadoop.registry.dns.domain-name - ECLD.COM - - - - hadoop.registry.dns.enabled - true - - - - hadoop.registry.dns.zone-mask - 255.255.255.0 - - - - hadoop.registry.dns.zone-subnet - 172.17.0.0 - - - - hadoop.registry.jaas.context - Client - - - - hadoop.registry.secure - true - - - - hadoop.registry.system.accounts - sasl:yarn,sasl:jhs,sasl:hdfs-b9,sasl:rm,sasl:hive,sasl:spark - - - - hadoop.registry.zk.quorum - b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181 - - - - manage.include.files - false - - - - yarn.acl.enable - true - - - - yarn.admin.acl - * - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/lib/edp/hadoop-3.2.2/share/hadoop/common/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/common/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/hdfs/lib/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/*, /usr/lib/edp/hadoop-3.2.2/share/hadoop/yarn/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 172800 - - - - yarn.log.server.url - http://b9m1.hdp.dc:19888/jobhistory/logs - - - - yarn.log.server.web-service.url - http://b9m1.hdp.dc:8188/ws/v1/applicationhistory - - - - yarn.node-labels.enabled - false - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle,spark_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.aux-services.spark_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.spark_shuffle.classpath - /usr/lib/edp/spark-3.1.1-bin-hadoop3.2/yarn/* - - - - yarn.nodemanager.aux-services.timeline_collector.class - org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.class - org.apache.spark.network.yarn.YarnShuffleService - - - - yarn.nodemanager.aux-services.sparkv2_shuffle.classpath - /usr/lib/edp/spark-2.4.7-bin-hadoop-3.1.2/yarn/* - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor - - - - yarn.nodemanager.container-metrics.unregister-delay-ms - 60000 - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 0 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.keytab - /etc/security/keytabs/nm.service.keytab - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users - true - - - - yarn.nodemanager.local-dirs - /data1/hadoop/yarn/local,/data2/hadoop/yarn/local,/data3/hadoop/yarn/local,/data4/hadoop/yarn/local,/data5/hadoop/yarn/local,/data6/hadoop/yarn/local,/data7/hadoop/yarn/local,/data8/hadoop/yarn/local,/data9/hadoop/yarn/local,/data10/hadoop/yarn/local,/data11/hadoop/yarn/local,/data12/hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - 3600 - - - - yarn.nodemanager.log-dirs - /data1/hadoop/yarn/log,/data2/hadoop/yarn/log,/data3/hadoop/yarn/log,/data4/hadoop/yarn/log,/data5/hadoop/yarn/log,/data6/hadoop/yarn/log,/data7/hadoop/yarn/log,/data8/hadoop/yarn/log,/data9/hadoop/yarn/log,/data10/hadoop/yarn/log,/data11/hadoop/yarn/log,/data12/hadoop/yarn/log - - - - yarn.nodemanager.log.retain-seconds - 604800 - - - - yarn.nodemanager.principal - nm/_HOST@ECLD.COM - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled - true - - - - yarn.nodemanager.recovery.supervised - true - - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource-plugins - - - - - yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin - - - - - yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint - - - - - yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables - - - - - yarn.nodemanager.resource.cpu-vcores - 51 - - - - yarn.nodemanager.resource.memory-mb - 348160 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 80 - - - - yarn.nodemanager.resourcemanager.connect.wait.secs - 1800 - - - - yarn.nodemanager.runtime.linux.allowed-runtimes - default,docker - - - - yarn.nodemanager.runtime.linux.docker.allowed-container-networks - host,none,bridge - - - - yarn.nodemanager.runtime.linux.docker.capabilities - - CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP, - SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE - - - - yarn.nodemanager.runtime.linux.docker.default-container-network - host - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.acl - - - - - yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed - false - - - - yarn.nodemanager.vmem-check-enabled - false - - - - yarn.nodemanager.vmem-pmem-ratio - 2.1 - - - - yarn.nodemanager.webapp.cross-origin.enabled - true - - - - yarn.nodemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.nodemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.address - b9m2.hdp.dc:8050 - - - - yarn.resourcemanager.admin.address - b9m2.hdp.dc:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.cluster-id - yarn-cluster - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.display.per-user-apps - true - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.automatic-failover.zk-base-path - /yarn-leader-election-b9 - - - - yarn.resourcemanager.ha.enabled - true - - - - yarn.resourcemanager.ha.rm-ids - rm1,rm2 - - - - yarn.resourcemanager.hostname - b9m2.hdp.dc - - - - yarn.resourcemanager.hostname.rm1 - b9m2.hdp.dc - - - - yarn.resourcemanager.hostname.rm2 - b9m3.hdp.dc - - - - yarn.resourcemanager.keytab - /etc/security/keytabs/rm.service.keytab - - - - yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled - true - - - - yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval - 15000 - - - - yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor - 1 - - - - yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round - 0.1 - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.placement-constraints.handler - scheduler - - - - yarn.resourcemanager.principal - rm/_HOST@ECLD.COM - - - - yarn.resourcemanager.proxy-user-privileges.enabled - true - - - - yarn.resourcemanager.proxyuser.*.groups - - - - - yarn.resourcemanager.proxyuser.*.hosts - - - - - yarn.resourcemanager.proxyuser.*.users - - - - - yarn.resourcemanager.recovery.enabled - true - - - - yarn.resourcemanager.resource-tracker.address - b9m2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm1 - b9m2.hdp.dc:8025 - - - - yarn.resourcemanager.resource-tracker.address.rm2 - b9m3.hdp.dc:8025 - - - - yarn.resourcemanager.scheduler.address - b9m2.hdp.dc:8030 - - - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.scheduler.monitor.enable - true - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true - - - - yarn.resourcemanager.webapp.address - b9m2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm1 - b9m2.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.address.rm2 - b9m3.hdp.dc:8088 - - - - yarn.resourcemanager.webapp.cross-origin.enabled - true - - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - - - yarn.resourcemanager.webapp.https.address - b9m2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm1 - b9m2.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.https.address.rm2 - b9m3.hdp.dc:8090 - - - - yarn.resourcemanager.webapp.spnego-keytab-file - /etc/security/keytabs/spnego.service.keytab - - - - yarn.resourcemanager.webapp.spnego-principal - HTTP/_HOST@ECLD.COM - - - - yarn.resourcemanager.work-preserving-recovery.enabled - true - - - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 - - - - yarn.resourcemanager.zk-acl - sasl:rm:rwcda - - - - yarn.resourcemanager.zk-address - b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181 - - - - yarn.resourcemanager.zk-num-retries - 1000 - - - - yarn.resourcemanager.zk-retry-interval-ms - 1000 - - - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore-b9 - - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.rm.system-metricspublisher.emit-container-events - true - - - - yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled - true - - - - yarn.scheduler.maximum-allocation-mb - 102400 - - - - yarn.scheduler.maximum-allocation-vcores - 51 - - - - yarn.scheduler.minimum-allocation-mb - 1024 - - - - yarn.scheduler.minimum-allocation-vcores - 1 - - - - yarn.service.system-service.dir - /services - - - - yarn.system-metricspublisher.enabled - true - - - - yarn.timeline-service.address - b9m1.hdp.dc:10200 - - - - yarn.timeline-service.bind-host - 0.0.0.0 - - - - yarn.timeline-service.client.max-retries - 30 - - - - yarn.timeline-service.client.retry-interval-ms - 1000 - - - - yarn.timeline-service.enabled - false - - - - yarn.timeline-service.entity-group-fs-store.active-dir - /ats/active/ - - - - yarn.timeline-service.entity-group-fs-store.app-cache-size - 10 - - - - yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds - 3600 - - - - yarn.timeline-service.entity-group-fs-store.done-dir - /ats/done/ - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes - org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin - - - - yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath - - - - - yarn.timeline-service.entity-group-fs-store.retain-seconds - 604800 - - - - yarn.timeline-service.entity-group-fs-store.scan-interval-seconds - 60 - - - - yarn.timeline-service.entity-group-fs-store.summary-store - org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore - - - - yarn.timeline-service.generic-application-history.save-non-am-container-meta-info - false - - - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore - - - - yarn.timeline-service.hbase-schema.prefix - prod. - - - - yarn.timeline-service.hbase.configuration.file - file:///etc/hadoop/conf/embedded-yarn-ats-hbase/hbase-site.xml - - - - yarn.timeline-service.hbase.coprocessor.jar.hdfs.location - file:///usr/lib/edp/hadoop-3.2.2/lib/hadoop-yarn-server-timelineservice-3.2.2.jar - - - - yarn.timeline-service.http-authentication.cookie.domain - - - - - yarn.timeline-service.http-authentication.cookie.path - - - - - yarn.timeline-service.http-authentication.kerberos.keytab - /etc/security/keytabs/spnego.service.keytab - - - - yarn.timeline-service.http-authentication.kerberos.name.rules - - - - - yarn.timeline-service.http-authentication.kerberos.principal - HTTP/_HOST@ECLD.COM - - - - yarn.timeline-service.http-authentication.proxyuser.*.groups - - - - - yarn.timeline-service.http-authentication.proxyuser.*.hosts - - - - - yarn.timeline-service.http-authentication.proxyuser.*.users - - - - - yarn.timeline-service.http-authentication.signature.secret - - - - - yarn.timeline-service.http-authentication.signature.secret.file - - - - - yarn.timeline-service.http-authentication.signer.secret.provider - - - - - yarn.timeline-service.http-authentication.signer.secret.provider.object - - - - - yarn.timeline-service.http-authentication.simple.anonymous.allowed - true - - - - yarn.timeline-service.http-authentication.token.validity - - - - - yarn.timeline-service.http-authentication.type - simple - - - - yarn.timeline-service.http-cross-origin.enabled - true - - - - yarn.timeline-service.keytab - /etc/security/keytabs/yarn.service.keytab - - - - yarn.timeline-service.leveldb-state-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.path - /data1/hadoop/yarn/timeline - - - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size - 10000 - - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.principal - yarn/_HOST@ECLD.COM - - - - yarn.timeline-service.reader.webapp.address - b9m1.hdp.dc:8198 - - - - yarn.timeline-service.reader.webapp.https.address - b9m1.hdp.dc:8199 - - - - yarn.timeline-service.recovery.enabled - true - - - - yarn.timeline-service.state-store-class - org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.version - 2.0f - - - - yarn.timeline-service.versions - 1.5f,2.0f - - - - yarn.timeline-service.webapp.address - b9m1.hdp.dc:8188 - - - - yarn.timeline-service.webapp.https.address - b9m1.hdp.dc:8190 - - - - yarn.webapp.api-service.enable - true - - - - yarn.webapp.ui2.enable - true - - - diff --git a/service-cli/service-cli-runner/src/main/resources/application-b9.yml b/service-cli/service-cli-runner/src/main/resources/application-b9.yml deleted file mode 100644 index 754b41f..0000000 --- a/service-cli/service-cli-runner/src/main/resources/application-b9.yml +++ /dev/null @@ -1,36 +0,0 @@ -deploy: - runtime: - user: zone_scfp - jar-path: /apps/zone_scfp/hudi/jars - jdk-path: /opt/jdk8u252-b09/bin/java - log-path: /apps/zone_scfp/hudi/logs - data-path: /apps/zone_scfp/hudi/data - kerberos-keytab-path: /etc/security/keytabs/zone_scfp.app.keytab - loki: - service-url: http://132.126.207.34:33100 - service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push - hudi-url: http://132.126.207.34:33100 - hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push - zk-url: b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181 - connector-zk-url: b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181 - hudi: - app-hdfs-path: jfs://tdsc/user/zone_scfp/jars/b9 - archive-hdfs-path: jfs://tdsc/user/zone_scfp/flink/completed-jobs-hudi - victoria-push-url: http://132.126.207.34:35710/api/v1/import/prometheus - loki-push-url: ${deploy.runtime.loki.hudi-push-url} - executor: - staging-path: jfs://tdsc/user/zone_scfp/yarn - archive-hdfs-path: jfs://tdsc/user/zone_scfp/flink/completed-jobs/ - task-jar-path: jfs://tdsc/user/zone_scfp/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar - task-result-path: jfs://tdsc/user/zone_scfp/task-results - security: - authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) - username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) - darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) - yarn: - store-cluster: b9 - sync-clusters: b9 - compaction-clusters: b9 - hosts: - b9s1_hdp_dc: - ip: 132.126.207.34 diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 08047f9..2768a08 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -100,25 +100,6 @@ deploy: connector_cluster_sync-queue-name: sync-queue-b12 connector_cluster_compaction-queue-name: compaction-queue-b12 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - service-launcher-b9: - groups: - - "service" - - "service-hudi" - - "service-hudi-launcher" - source-jar: service-launcher-b9b9-1.0.0-SNAPSHOT.jar - replicas: 6 - environments: - connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM - connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} - connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path} - connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url} - connector_hudi_loki-push-url: ${deploy.runtime.hudi.loki-push-url} - arguments: - spring_application_name: service-launcher-b9 - connector_cluster_cluster: b9 - connector_cluster_sync-queue-name: sync-queue-b9 - connector_cluster_compaction-queue-name: compaction-queue-b9 - connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} service-info-query: order: 1 groups: diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java index dee2c4f..3ef059f 100644 --- a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java +++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java @@ -196,7 +196,6 @@ public interface Constants { String CLUSTER_B5 = "b5"; String CLUSTER_A4 = "a4"; String CLUSTER_B12 = "b12"; - String CLUSTER_B9 = "b9"; String SCHEDULE_JOB_FAIL_COUNT = "SCHEDULE_JOB_FAIL_COUNT"; diff --git a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java index 74ca02f..ad49354 100644 --- a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java +++ b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java @@ -24,7 +24,6 @@ public class YarnClusters { .put("b1", new Cluster("http://132.122.98.13:8088")) .put("b5", new Cluster("http://132.122.116.12:8088")) .put("b12", new Cluster("http://132.126.207.125:8088")) - .put("b9", new Cluster("http://132.126.207.32:8088")) .build() ); diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B9LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B9LauncherService.java deleted file mode 100644 index 133e50d..0000000 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B9LauncherService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher.impl; - -import com.dtflys.forest.annotation.BaseRequest; -import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService; - -/** - * @author lanyuanxiaoyao - * @date 2023-06-06 - */ -@BaseRequest(baseURL = "http://service-launcher-b9") -public interface B9LauncherService extends LauncherService { -} diff --git a/service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/quartz/distribute/cluster/B9Cluster.java b/service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/quartz/distribute/cluster/B9Cluster.java deleted file mode 100644 index f9cc348..0000000 --- a/service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/quartz/distribute/cluster/B9Cluster.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.lanyuanxiaoyao.service.scheduler.quartz.distribute.cluster; - -import com.lanyuanxiaoyao.service.common.Constants; -import com.lanyuanxiaoyao.service.forest.service.YarnService; -import org.springframework.cloud.client.discovery.DiscoveryClient; -import org.springframework.stereotype.Component; - -/** - * A4 - * - * @author lanyuanxiaoyao - * @date 2023-06-08 - */ -@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") -@Component -public class B9Cluster extends Cluster { - public B9Cluster(DiscoveryClient client, YarnService yarnService) { - super( - Constants.CLUSTER_B9, - Constants.COMPACTION_QUEUE_B9 - ); - } -} diff --git a/service-web/src/main/resources/static/common/info.js b/service-web/src/main/resources/static/common/info.js index ebd304a..3415083 100644 --- a/service-web/src/main/resources/static/common/info.js +++ b/service-web/src/main/resources/static/common/info.js @@ -19,12 +19,12 @@ const commonInfo = { }, }, loki: { - grafanaUrl: 'http://132.126.207.34:35700', + grafanaUrl: 'http://132.126.207.125:35700', hudi: { - datasource: 'e2477661-3c3d-4e3d-a3f7-1d398a5b4b7b', + datasource: 'f648174e-7593-45cf-8fe8-8f8d5cf0fdde', }, service: { - datasource: 'fbbc4d0c-16d1-45a3-8ceb-4039b1b9a353', + datasource: 'b6fee51c-facd-4261-a0eb-8c69a975fba3', } }, } diff --git a/service-web/src/main/resources/static/components/overview-tab.js b/service-web/src/main/resources/static/components/overview-tab.js index 0045425..1ce3631 100644 --- a/service-web/src/main/resources/static/components/overview-tab.js +++ b/service-web/src/main/resources/static/components/overview-tab.js @@ -346,7 +346,7 @@ function overviewTab() { ] }, '集群 压缩队列任务数(集群总资源使用,队列资源使用)(调度中任务数,运行中任务数)', - overviewYarnJob(commonInfo.clusters.compaction_names(), 'Compaction', 'compaction-queue-b9', 'default'), + ...Object.keys(commonInfo.clusters.compaction).map(name => overviewYarnJob(name, 'Compaction', `compaction-queue-${name}`, commonInfo.clusters.compaction[name])), {type: 'divider'}, { type: 'service',