diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 863cbab..ae5338b 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -101,6 +101,13 @@ deploy: replicas: 3 arguments: loki_host: ${deploy.runtime.loki-base-url} + - name: service-yarn-query-a4 + source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar + replicas: 4 + arguments: + yarn_cluster: a4 + yarn_web-url: http://132.121.107.91:8088 + spring_application_name: service-yarn-query-a4 - name: service-yarn-query-b1 source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnA4Service.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnA4Service.java new file mode 100644 index 0000000..eb254e7 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnA4Service.java @@ -0,0 +1,13 @@ +package com.lanyuanxiaoyao.service.forest.service; + +import com.dtflys.forest.annotation.BaseRequest; + +/** + * Yarn 信息 + * + * @author lanyuanxiaoyao + * @date 2023-04-21 + */ +@BaseRequest(baseURL = "http://service-yarn-query-a4") +public interface YarnA4Service extends YarnService { +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java index 526d0b1..0835c1b 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java @@ -39,13 +39,15 @@ public class YarnClusterService { YarnB1Service yarnB1Service, YarnB5Service yarnB5Service, YarnB5SyncService yarnB5SyncService, - YarnB4Service yarnB4Service + YarnB4Service yarnB4Service, + YarnA4Service yarnA4Service ) { MutableMap servicesMap = Maps.mutable.empty(); servicesMap.put(Constants.CLUSTER_B1, yarnB1Service); servicesMap.put(Constants.CLUSTER_B5, yarnB5Service); servicesMap.put(Constants.CLUSTER_B5_SYNC, yarnB5SyncService); servicesMap.put(Constants.CLUSTER_B4, yarnB4Service); + servicesMap.put(Constants.CLUSTER_A4, yarnA4Service); this.servicesMap = servicesMap.toImmutable(); } diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java index cb857f4..5617e0b 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java @@ -7,7 +7,8 @@ import com.lanyuanxiaoyao.service.configuration.entity.AmisResponse; import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication; import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue; import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnRootQueue; -import com.lanyuanxiaoyao.service.forest.service.*; +import com.lanyuanxiaoyao.service.forest.service.YarnClusterService; +import com.lanyuanxiaoyao.service.forest.service.YarnService; import com.lanyuanxiaoyao.service.web.entity.YarnApplicationVO; import com.lanyuanxiaoyao.service.web.entity.YarnClusterVO; import com.lanyuanxiaoyao.service.web.utils.ComparatorUtil; @@ -47,14 +48,8 @@ public class YarnController extends BaseController { ); private final ImmutableMap yarnServices; - @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") - public YarnController(YarnB1Service yarnB1Service, YarnB5SyncService yarnB5SyncService, YarnB4Service yarnB4Service, YarnB5Service yarnB5Service) { - yarnServices = Maps.immutable.ofAll(MapUtil.builder() - .put("b1", yarnB1Service) - .put("b5-sync", yarnB5SyncService) - .put("b4", yarnB4Service) - .put("b5", yarnB5Service) - .build()); + public YarnController(YarnClusterService yarnClusterService) { + this.yarnServices = yarnClusterService.servicesMap(); } private YarnService getService(String cluster) { diff --git a/web/components/common.js b/web/components/common.js index 9b6621b..98e7917 100644 --- a/web/components/common.js +++ b/web/components/common.js @@ -44,6 +44,125 @@ function applicationLogDialog() { } } +function yarnQueueCrud(clusters = undefined, queueNames = undefined) { + return { + type: 'crud', + api: { + method: 'get', + url: '${base}/yarn/queue_list', + data: { + clusters: '${cluster|default:undefined}', + names: '${queueName|default:undefined}' + }, + }, + affixHeader: false, + defaultParams: { + ...(clusters ? {cluster: clusters} : {}), + ...(queueNames ? {queueName: queueNames} : {}), + }, + interval: 10000, + syncLocation: false, + silentPolling: true, + headerToolbar: [ + "reload", + ], + columns: [ + { + name: 'queueName', + label: '队列名称', + width: 130, + type: 'tooltip-wrapper', + body: '${TRUNCATE(queueName, 20)}', + content: '${queueName}', + }, + { + label: '当前容量', + type: 'progress', + value: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}', + stripe: true, + animate: true, + showLabel: false, + map: [ + { + value: 30, + color: "#28a745" + }, { + value: 90, + color: "#007bff" + }, + { + value: 100, + color: "#dc3545" + } + ] + }, + { + label: '进度', + width: 35, + align: 'center', + type: 'tpl', + tpl: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}%', + }, + { + type: "operation", + label: "操作", + width: 100, + fixed: 'right', + buttons: [ + { + label: "详情", + type: "button", + level: "link", + tooltip: '查看队列详情', + visibleOn: '${!root}', + actionType: 'dialog', + dialog: { + closeOnEsc: true, + closeOnOutside: true, + size: 'md', + close: true, + title: '队列详情', + body: { + type: 'property', + items: [ + {label: 'CPU', content: '${resourcesUsed.vCores}'}, + // 有空看看这个值的单位 + {label: '内存', content: '${resourcesUsed.memory}', span: 2}, + {label: '容量(%)', content: '${capacity}'}, + {label: '最大容量(%)', content: '${maxCapacity}'}, + {label: '已用容量(%)', content: '${usedCapacity}'}, + {label: '绝对容量(%)', content: '${absoluteCapacity}'}, + {label: '绝对最大容量(%)', content: '${absoluteMaxCapacity}'}, + {label: '绝对已用容量(%)', content: '${absoluteUsedCapacity}'}, + {label: '应用数量', content: '${numApplications}', span: 3}, + {label: '最大应用数量', content: '${maxApplications}'}, + {label: '活跃应用数量', content: '${numActiveApplications}'}, + {label: '等待应用数量', content: '${numPendingApplications}'}, + {label: '容器数量', content: '${numContainers}', span: 3}, + {label: '已分配容器数量', content: '${numApplications}'}, + {label: '预留容器数量数量', content: '${numApplications}'}, + {label: '等待容器数量', content: '${numApplications}'}, + ] + }, + actions: [] + } + }, + { + visibleOn: "${webUrl}", + label: "管理页面", + type: "action", + level: "link", + tooltip: '打开管理页面', + actionType: 'url', + url: '${webUrl}', + blank: true, + } + ] + } + ] + } +} + function yarnCrudColumns() { return [ { @@ -263,7 +382,11 @@ function simpleYarnDialog(cluster, title, filterField) { {label: '已结束', content: '${detail.jobsFinished}'}, {label: '已失败', content: '${detail.jobsFailed}'}, {label: '被取消', content: '${detail.jobsCanceled}'}, - {label: 'Slot (可用/总数)', content: '${detail.slotsAvailable}/${detail.slotsTotal}', span: 4}, + { + label: 'Slot (可用/总数)', + content: '${detail.slotsAvailable}/${detail.slotsTotal}', + span: 4 + }, ] }, ] @@ -700,16 +823,40 @@ function tableMetaDialog() { column: 4, items: [ {label: '入队列消息速率', content: '${detail.state.messageRateIn}'}, - {label: '出队列消息速率', content: '${detail.state.messageRateOut}'}, - {label: '入队列消息吞吐量', content: '${detail.state.messageThroughputIn}'}, - {label: '出队列消息吞吐量', content: '${detail.state.messageThroughputOut}'}, - {label: '入队列消息数量', content: '${detail.state.messageInCounter}'}, - {label: '出队列消息数量', content: '${detail.state.messageOutCounter}'}, - {label: '入队列消息字节数', content: '${detail.state.byteInCounter}'}, - {label: '出队列消息字节数', content: '${detail.state.byteOutCounter}'}, + { + label: '出队列消息速率', + content: '${detail.state.messageRateOut}' + }, + { + label: '入队列消息吞吐量', + content: '${detail.state.messageThroughputIn}' + }, + { + label: '出队列消息吞吐量', + content: '${detail.state.messageThroughputOut}' + }, + { + label: '入队列消息数量', + content: '${detail.state.messageInCounter}' + }, + { + label: '出队列消息数量', + content: '${detail.state.messageOutCounter}' + }, + { + label: '入队列消息字节数', + content: '${detail.state.byteInCounter}' + }, + { + label: '出队列消息字节数', + content: '${detail.state.byteOutCounter}' + }, {label: '存储消息大小', content: '${detail.state. storageSize}'}, {label: '积压消息大小', content: '${detail.state.backlogSize}'}, - {label: '平均消息大小', content: '${detail.state.averageMessageSize}'}, + { + label: '平均消息大小', + content: '${detail.state.averageMessageSize}' + }, ], }, {type: 'divider'}, @@ -732,20 +879,62 @@ function tableMetaDialog() { type: 'property', column: 1, items: [ - {label: '在此订阅上传递的消息的总速率(msg/s)', content: '${messageRateOut}'}, - {label: '此订阅提供的总吞吐量(字节/秒)', content: '${messageThroughputOut}'}, - {label: '传送给消费者的总字节数(字节)', content: '${bytesOutCounter}'}, - {label: '传递给消费者的消息总数(msg)', content: '${messageOutCounter}'}, - {label: '此订阅上重新传递的消息的总速率(msg/s)', content: '${messageRateRedeliver}'}, - {label: '分块消息调度速率', content: '${chunkedMessageRate}'}, - {label: '积压的大小(字节)', content: '${backlogSize}'}, - {label: '积压中不包含延迟消息的消息数', content: '${messageBacklogNoDelayed}'}, - {label: '验证订阅是否由于达到未确认消息的阈值而被阻止', content: '${blockedSubscriptionOnUnackedMessages}'}, - {label: '当前正在跟踪的延迟消息数', content: '${messageDelayed}'}, - {label: '订阅的未确认消息数', content: '${unackedMessages}'}, - {label: '单个活跃消费者订阅处于活跃状态的消费者名称(故障转移、独占)', content: '${activeConsumerName}'}, - {label: '此订阅上过期的消息总速率(msg/s)', content: '${messageRateExpired}'}, - {label: '此订阅上过期的消息总数', content: '${totalMessageExpired}'}, + { + label: '在此订阅上传递的消息的总速率(msg/s)', + content: '${messageRateOut}' + }, + { + label: '此订阅提供的总吞吐量(字节/秒)', + content: '${messageThroughputOut}' + }, + { + label: '传送给消费者的总字节数(字节)', + content: '${bytesOutCounter}' + }, + { + label: '传递给消费者的消息总数(msg)', + content: '${messageOutCounter}' + }, + { + label: '此订阅上重新传递的消息的总速率(msg/s)', + content: '${messageRateRedeliver}' + }, + { + label: '分块消息调度速率', + content: '${chunkedMessageRate}' + }, + { + label: '积压的大小(字节)', + content: '${backlogSize}' + }, + { + label: '积压中不包含延迟消息的消息数', + content: '${messageBacklogNoDelayed}' + }, + { + label: '验证订阅是否由于达到未确认消息的阈值而被阻止', + content: '${blockedSubscriptionOnUnackedMessages}' + }, + { + label: '当前正在跟踪的延迟消息数', + content: '${messageDelayed}' + }, + { + label: '订阅的未确认消息数', + content: '${unackedMessages}' + }, + { + label: '单个活跃消费者订阅处于活跃状态的消费者名称(故障转移、独占)', + content: '${activeConsumerName}' + }, + { + label: '此订阅上过期的消息总速率(msg/s)', + content: '${messageRateExpired}' + }, + { + label: '此订阅上过期的消息总数', + content: '${totalMessageExpired}' + }, { label: '最后一条消息过期时间', content: { @@ -774,8 +963,14 @@ function tableMetaDialog() { tpl: '${lastAckedTimestamp|date:YYYY-MM-DD HH\\:mm\\:ss:x}', } }, - {label: '此订阅是持久订阅还是临时订阅', content: '${durable}'}, - {label: '标记订阅状态在不同区域之间保持同步', content: '${replicated}'}, + { + label: '此订阅是持久订阅还是临时订阅', + content: '${durable}' + }, + { + label: '标记订阅状态在不同区域之间保持同步', + content: '${replicated}' + }, ] }, ] @@ -816,10 +1011,22 @@ function tableMetaDialog() { type: 'property', column: 1, items: [ - {label: '发布消息速率(msg/s)', content: '${messageRateIn}'}, - {label: '发布消息吞吐量(字节/秒)', content: '${messageThroughputIn}'}, - {label: '消息平均大小(字节)', content: '${averageMessageSize}'}, - {label: '接收到的分块消息总数(msg)', content: '${chunkedMessageRate}'}, + { + label: '发布消息速率(msg/s)', + content: '${messageRateIn}' + }, + { + label: '发布消息吞吐量(字节/秒)', + content: '${messageThroughputIn}' + }, + { + label: '消息平均大小(字节)', + content: '${averageMessageSize}' + }, + { + label: '接收到的分块消息总数(msg)', + content: '${chunkedMessageRate}' + }, {label: '生产者地址', content: '${address}'}, {label: '客户端版本', content: '${clientVersion}'}, ] @@ -868,7 +1075,7 @@ function tableMetaDialog() { {type: 'divider', visibleOn: '${compactionRuntime}'}, { visibleOn: '${compactionRuntime}', - ...runMetaProperty('compaction'), + ...runMetaProperty('compaction'), }, {type: 'divider'}, flinkJobProperty('flinkJobId', 'flinkJob.name', 'flinkJob.runMode'), diff --git a/web/components/yarn-cluster-tab.js b/web/components/yarn-cluster-tab.js new file mode 100644 index 0000000..51eb1bb --- /dev/null +++ b/web/components/yarn-cluster-tab.js @@ -0,0 +1,8 @@ +function yarnClusterTab() { + return { + title: '集群资源', + tab: [ + yarnQueueCrud('b1,b5,b5-sync,a4,b4') + ] + } +} \ No newline at end of file diff --git a/web/components/yarn-tab.js b/web/components/yarn-tab.js index 48ec2e8..5ef1a74 100644 --- a/web/components/yarn-tab.js +++ b/web/components/yarn-tab.js @@ -11,121 +11,7 @@ function yarnTab(cluster, title, queueNames = 'default', searchName = undefined) type: 'tpl', tpl: '集群资源', }, - { - type: 'crud', - api: { - method: 'get', - url: '${base}/yarn/queue_list', - data: { - clusters: `${cluster}`, - names: '${queueName|default:undefined}' - }, - }, - affixHeader: false, - defaultParams: { - queueName: queueNames, - }, - interval: 10000, - syncLocation: false, - silentPolling: true, - headerToolbar: [ - "reload", - ], - columns: [ - { - name: 'queueName', - label: '队列名称', - width: 130, - type: 'tooltip-wrapper', - body: '${TRUNCATE(queueName, 20)}', - content: '${queueName}', - }, - { - label: '当前容量', - type: 'progress', - value: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}', - stripe: true, - animate: true, - showLabel: false, - map: [ - { - value: 30, - color: "#28a745" - }, { - value: 90, - color: "#007bff" - }, - { - value: 100, - color: "#dc3545" - } - ] - }, - { - label: '进度', - width: 35, - align: 'center', - type: 'tpl', - tpl: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}%', - }, - { - type: "operation", - label: "操作", - width: 100, - fixed: 'right', - buttons: [ - { - label: "详情", - type: "button", - level: "link", - tooltip: '查看队列详情', - visibleOn: '${!root}', - actionType: 'dialog', - dialog: { - closeOnEsc: true, - closeOnOutside: true, - size: 'md', - close: true, - title: '队列详情', - body: { - type: 'property', - items: [ - {label: 'CPU', content: '${resourcesUsed.vCores}'}, - // 有空看看这个值的单位 - {label: '内存', content: '${resourcesUsed.memory}', span: 2}, - {label: '容量(%)', content: '${capacity}'}, - {label: '最大容量(%)', content: '${maxCapacity}'}, - {label: '已用容量(%)', content: '${usedCapacity}'}, - {label: '绝对容量(%)', content: '${absoluteCapacity}'}, - {label: '绝对最大容量(%)', content: '${absoluteMaxCapacity}'}, - {label: '绝对已用容量(%)', content: '${absoluteUsedCapacity}'}, - {label: '应用数量', content: '${numApplications}', span: 3}, - {label: '最大应用数量', content: '${maxApplications}'}, - {label: '活跃应用数量', content: '${numActiveApplications}'}, - {label: '等待应用数量', content: '${numPendingApplications}'}, - {label: '容器数量', content: '${numContainers}', span: 3}, - {label: '已分配容器数量', content: '${numApplications}'}, - {label: '预留容器数量数量', content: '${numApplications}'}, - {label: '等待容器数量', content: '${numApplications}'}, - ] - }, - actions: [] - } - }, - { - visibleOn: "${webUrl}", - label: "管理页面", - type: "action", - level: "link", - tooltip: '打开管理页面', - actionType: 'url', - url: '${webUrl}', - blank: true, - } - ] - } - ] - }, + yarnQueueCrud(cluster, queueNames), { type: 'tpl', tpl: '集群任务', diff --git a/web/index.html b/web/index.html index a3de0e7..2e856f2 100644 --- a/web/index.html +++ b/web/index.html @@ -37,6 +37,7 @@ + @@ -56,11 +57,13 @@ tabs: [ // logTab(), // runningTab(), + // yarnClusterTab(), tableTab(), queueTab(), yarnTab('b5-sync', '同步 b5', undefined, 'Sync'), yarnTab('b1,b5', '压缩 b1 b5', 'datalake,ten_iap.datalake', 'Compaction'), yarnTab('b1', '压缩 b1', 'datalake,tyly'), + yarnTab('a4', '压缩 a4'), yarnTab('b4', '压缩 b4'), yarnTab('b5', '压缩 b5', 'ten_iap.datalake'), cloudTab(),