lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(yarn-query): 增加 yarn cluster 页面的入口

Browse Source
This commit is contained in:
兰缘小妖
2023-05-30 10:36:04 +08:00
parent 2d95a3824a
commit 58c945fa55
8 changed files with 52 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/QueueService.java
View File

@@ -1,8 +1,13 @@
package com.lanyuanxiaoyao.service.yarn.service;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
/**
* Queue
@@ -14,4 +19,16 @@ public interface QueueService {
ImmutableList<YarnQueue> list() throws JsonProcessingException;
YarnQueue detail(String name) throws Exception;
default Function<YarnQueue, Iterable<YarnQueue>> flatChildren() {
return queue -> {
MutableList<YarnQueue> children = Lists.mutable.of(queue);
if (ObjectUtil.isNotNull(queue.getQueues())) {
queue.getQueues().getQueue()
.tap(child -> child.setQueueName(StrUtil.format("{}.{}", queue.getQueueName(), child.getQueueName())))
.forEach(children::add);
}
return children;
};
}
}

4
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/ClusterServiceImpl.java
View File

@@ -40,6 +40,8 @@ public class ClusterServiceImpl implements ClusterService {
public YarnRootQueue info() throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/scheduler");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo().setCluster(yarnConfiguration.getCluster());
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo()
.setCluster(yarnConfiguration.getCluster())
.setWebUrl(URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/cluster/scheduler"));
}
}

3
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueAutoRefreshServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.lanyuanxiaoyao.service.yarn.service.impl;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpUtil;
@@ -12,6 +13,7 @@ import com.lanyuanxiaoyao.service.yarn.service.QueueService;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
@@ -43,6 +45,7 @@ public class QueueAutoRefreshServiceImpl implements QueueService {
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
ImmutableList<YarnQueue> queues = response.getScheduler().getSchedulerInfo().getQueues().getQueue()
.flatCollect(flatChildren())
.tap(q -> q.setCluster(yarnConfiguration.getCluster()));
CACHE.set(queues);
}

13
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueServiceImpl.java
View File

@@ -1,6 +1,5 @@
package com.lanyuanxiaoyao.service.yarn.service.impl;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpUtil;
@@ -10,9 +9,7 @@ import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
import com.lanyuanxiaoyao.service.yarn.configuration.YarnConfiguration;
import com.lanyuanxiaoyao.service.yarn.response.QueueListResponse;
import com.lanyuanxiaoyao.service.yarn.service.QueueService;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheConfig;
@@ -47,15 +44,7 @@ public class QueueServiceImpl implements QueueService {
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
return response.getScheduler().getSchedulerInfo().getQueues().getQueue()
.flatCollect(q -> {
MutableList<YarnQueue> queues = Lists.mutable.of(q);
if (ObjectUtil.isNotNull(q.getQueues())) {
q.getQueues().getQueue()
.tap(q1 -> q1.setQueueName(StrUtil.format("{}.{}", q.getQueueName(), q1.getQueueName())))
.forEach(queues::add);
}
return queues;
})
.flatCollect(flatChildren())
.tap(q -> q.setCluster(yarnConfiguration.getCluster()));
}