lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(yarn-query): 增加 yarn cluster 页面的入口

Browse Source
This commit is contained in:
兰缘小妖
2023-05-30 10:36:04 +08:00
parent 2d95a3824a
commit 58c945fa55
8 changed files with 52 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnQueue.java
View File

@@ -41,6 +41,7 @@ public final class YarnQueue {
private ResourcesUsed usedAMResource;
private ResourcesUsed userAMResourceLimit;
private Queues queues;
private String webUrl;
public String getCluster() {
return cluster;
@@ -180,6 +181,15 @@ public final class YarnQueue {
return queues;
}
public String getWebUrl() {
return webUrl;
}
public YarnQueue setWebUrl(String webUrl) {
this.webUrl = webUrl;
return this;
}
@Override
public String toString() {
return "YarnQueue{" +
@@ -215,6 +225,7 @@ public final class YarnQueue {
", usedAMResource=" + usedAMResource +
", userAMResourceLimit=" + userAMResourceLimit +
", queues=" + queues +
", webUrl='" + webUrl + '\'' +
'}';
}

11
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnRootQueue.java
View File

@@ -13,6 +13,7 @@ public final class YarnRootQueue {
private Float capacity;
private Float usedCapacity;
private Float maxCapacity;
private String webUrl;
public String getCluster() {
return cluster;
@@ -43,6 +44,15 @@ public final class YarnRootQueue {
return maxCapacity;
}
public String getWebUrl() {
return webUrl;
}
public YarnRootQueue setWebUrl(String webUrl) {
this.webUrl = webUrl;
return this;
}
@Override
public String toString() {
return "YarnRootQueue{" +
@@ -52,6 +62,7 @@ public final class YarnRootQueue {
", capacity=" + capacity +
", usedCapacity=" + usedCapacity +
", maxCapacity=" + maxCapacity +
", webUrl='" + webUrl + '\'' +
'}';
}
}

4
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnClusterVO.java
View File

@@ -48,6 +48,10 @@ public class YarnClusterVO {
return yarnRootQueue.getMaxCapacity();
}
public String getWebUrl() {
return yarnRootQueue.getWebUrl();
}
public ImmutableList<YarnQueue> getChildren() {
return children;
}

4
service-web/src/main/resources/static/components/yarn-tab.js
View File

@@ -112,13 +112,13 @@ function yarnTab(cluster, title, queueNames = 'default', searchName = undefined)
}
},
{
disabled: true,
visibleOn: "${webUrl}",
label: "管理页面",
type: "action",
level: "link",
tooltip: '打开管理页面',
actionType: 'url',
url: '',
url: '${webUrl}',
blank: true,
}
]

17
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/QueueService.java
View File

@@ -1,8 +1,13 @@
package com.lanyuanxiaoyao.service.yarn.service;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
/**
* Queue
@@ -14,4 +19,16 @@ public interface QueueService {
ImmutableList<YarnQueue> list() throws JsonProcessingException;
YarnQueue detail(String name) throws Exception;
default Function<YarnQueue, Iterable<YarnQueue>> flatChildren() {
return queue -> {
MutableList<YarnQueue> children = Lists.mutable.of(queue);
if (ObjectUtil.isNotNull(queue.getQueues())) {
queue.getQueues().getQueue()
.tap(child -> child.setQueueName(StrUtil.format("{}.{}", queue.getQueueName(), child.getQueueName())))
.forEach(children::add);
}
return children;
};
}
}

4
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/ClusterServiceImpl.java
View File

@@ -40,6 +40,8 @@ public class ClusterServiceImpl implements ClusterService {
public YarnRootQueue info() throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/scheduler");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo().setCluster(yarnConfiguration.getCluster());
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo()
.setCluster(yarnConfiguration.getCluster())
.setWebUrl(URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/cluster/scheduler"));
}
}

3
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueAutoRefreshServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.lanyuanxiaoyao.service.yarn.service.impl;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpUtil;
@@ -12,6 +13,7 @@ import com.lanyuanxiaoyao.service.yarn.service.QueueService;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
@@ -43,6 +45,7 @@ public class QueueAutoRefreshServiceImpl implements QueueService {
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
ImmutableList<YarnQueue> queues = response.getScheduler().getSchedulerInfo().getQueues().getQueue()
.flatCollect(flatChildren())
.tap(q -> q.setCluster(yarnConfiguration.getCluster()));
CACHE.set(queues);
}

13
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueServiceImpl.java
View File

@@ -1,6 +1,5 @@
package com.lanyuanxiaoyao.service.yarn.service.impl;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpUtil;
@@ -10,9 +9,7 @@ import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
import com.lanyuanxiaoyao.service.yarn.configuration.YarnConfiguration;
import com.lanyuanxiaoyao.service.yarn.response.QueueListResponse;
import com.lanyuanxiaoyao.service.yarn.service.QueueService;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheConfig;
@@ -47,15 +44,7 @@ public class QueueServiceImpl implements QueueService {
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
return response.getScheduler().getSchedulerInfo().getQueues().getQueue()
.flatCollect(q -> {
MutableList<YarnQueue> queues = Lists.mutable.of(q);
if (ObjectUtil.isNotNull(q.getQueues())) {
q.getQueues().getQueue()
.tap(q1 -> q1.setQueueName(StrUtil.format("{}.{}", q.getQueueName(), q1.getQueueName())))
.forEach(queues::add);
}
return queues;
})
.flatCollect(flatChildren())
.tap(q -> q.setCluster(yarnConfiguration.getCluster()));
}