lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(yarn-query): 优化 yarn-query 为集群共享查询

Browse Source 
原本一个 yarn-query 对应一个集群，集群多了之后，比较浪费，改为一个组件可以查询任意集群的信息，减少部署的复杂性
This commit is contained in:
兰缘小妖
2023-06-02 09:09:26 +08:00
parent 7768b427aa
commit cd676367c6
22 changed files with 112 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnA4Service.java
View File

@@ -1,13 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
/**
* Yarn 信息
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@BaseRequest(baseURL = "http://service-yarn-query-a4")
public interface YarnA4Service extends YarnService {
}

13
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnB1Service.java
View File

@@ -1,13 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
/**
* Yarn 信息
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@BaseRequest(baseURL = "http://service-yarn-query-b1")
public interface YarnB1Service extends YarnService {
}

13
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnB4Service.java
View File

@@ -1,13 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
/**
* Yarn 信息
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@BaseRequest(baseURL = "http://service-yarn-query-b4")
public interface YarnB4Service extends YarnService {
}

13
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnB5Service.java
View File

@@ -1,13 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
/**
* Yarn 信息
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@BaseRequest(baseURL = "http://service-yarn-query-b5")
public interface YarnB5Service extends YarnService {
}

13
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnB5SyncService.java
View File

@@ -1,13 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
/**
* Yarn 信息
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@BaseRequest(baseURL = "http://service-yarn-query-b5-sync")
public interface YarnB5SyncService extends YarnService {
}

158
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java
View File

@@ -1,158 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service;
import cn.hutool.core.util.ObjectUtil;
import com.eshore.odcp.hudi.connector.Constants;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnRootQueue;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Function;
import javax.annotation.Nullable;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
import org.eclipse.collections.api.map.MutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
/**
* 聚合 Yarn 查询
*
* @author lanyuanxiaoyao
* @date 2023-05-11
*/
@Service
public class YarnClusterService {
private static final Logger logger = LoggerFactory.getLogger(YarnClusterService.class);
private static final ExecutorService EXECUTOR = Executors.newWorkStealingPool(20);
private final ImmutableMap<String, YarnService> servicesMap;
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
public YarnClusterService(
YarnB1Service yarnB1Service,
YarnB5Service yarnB5Service,
YarnB5SyncService yarnB5SyncService,
YarnB4Service yarnB4Service,
YarnA4Service yarnA4Service
) {
MutableMap<String, YarnService> servicesMap = Maps.mutable.empty();
servicesMap.put(Constants.CLUSTER_B1, yarnB1Service);
servicesMap.put(Constants.CLUSTER_B5, yarnB5Service);
servicesMap.put(Constants.CLUSTER_B5_SYNC, yarnB5SyncService);
servicesMap.put(Constants.CLUSTER_B4, yarnB4Service);
servicesMap.put(Constants.CLUSTER_A4, yarnA4Service);
this.servicesMap = servicesMap.toImmutable();
}
public ImmutableList<String> clusters() {
return servicesMap.keysView().toList().toImmutable();
}
public ImmutableList<YarnService> services() {
return servicesMap.valuesView().toList().toImmutable();
}
public ImmutableList<YarnService> services(ImmutableList<String> clusters) {
return clusters.collect(this::service);
}
public ImmutableMap<String, YarnService> servicesMap() {
return servicesMap;
}
public @Nullable YarnService service(String cluster) {
return servicesMap.get(cluster);
}
public ImmutableList<YarnApplication> jobList(String cluster) {
return jobList(Lists.immutable.of(cluster));
}
public ImmutableList<YarnApplication> jobList(ImmutableList<String> clusters) {
return list(clusters, YarnService::jobList);
}
public ImmutableList<YarnApplication> jobListEquals(String cluster, String name) {
return jobListEquals(Lists.immutable.of(cluster), name);
}
public ImmutableList<YarnApplication> jobListEquals(ImmutableList<String> clusters, String name) {
return list(clusters, yarnService -> yarnService.jobListEquals(name));
}
public ImmutableList<YarnApplication> jobListLike(String cluster, String text) {
return jobListLike(Lists.immutable.of(cluster), text);
}
public ImmutableList<YarnApplication> jobListLike(ImmutableList<String> clusters, String text) {
return list(clusters, yarnService -> yarnService.jobListLike(text));
}
private ImmutableList<YarnApplication> list(ImmutableList<String> clusters, Function<YarnService, ImmutableList<YarnApplication>> getter) {
if (ObjectUtil.isEmpty(clusters)) {
return Lists.immutable.empty();
}
return clusters.toList()
.collect(this::service)
.reject(Objects::isNull)
.asParallel(EXECUTOR, 1)
.flatCollect(getter::apply)
.toList()
.toImmutable();
}
public YarnApplication jobDetail(String cluster, String applicationId) {
YarnService service = service(cluster);
return ObjectUtil.isNull(service) ? null : service.jobDetail(applicationId);
}
@SuppressWarnings("DataFlowIssue")
public ImmutableList<YarnQueue> queueList(String cluster) {
return servicesMap.containsKey(cluster) ? service(cluster).queueList() : Lists.immutable.empty();
}
public ImmutableList<YarnQueue> queueList(ImmutableList<String> clusters) {
//noinspection DataFlowIssue
return clusters.toList()
.select(servicesMap::containsKey)
.asParallel(EXECUTOR, 1)
.flatCollect(cluster -> service(cluster).queueList())
.toList()
.toImmutable();
}
@SuppressWarnings("DataFlowIssue")
public YarnQueue queueDetail(String cluster, String name) {
return servicesMap.containsKey(cluster) ? service(cluster).queueDetail(name) : null;
}
public ImmutableList<YarnQueue> queueDetail(ImmutableList<String> clusters, String name) {
//noinspection DataFlowIssue
return clusters.toList()
.select(servicesMap::containsKey)
.asParallel(EXECUTOR, 1)
.collect(cluster -> service(cluster).queueDetail(name))
.toList()
.toImmutable();
}
@SuppressWarnings("DataFlowIssue")
public YarnRootQueue cluster(String cluster) {
return servicesMap.containsKey(cluster) ? service(cluster).cluster() : null;
}
public ImmutableList<YarnRootQueue> cluster(ImmutableList<String> clusters) {
//noinspection DataFlowIssue
return clusters.toList()
.select(servicesMap::containsKey)
.asParallel(EXECUTOR, 1)
.collect(cluster -> service(cluster).cluster())
.toList()
.toImmutable();
}
}

19
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnService.java
View File

@@ -1,5 +1,6 @@
package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Query;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
@@ -11,28 +12,26 @@ import org.eclipse.collections.api.list.ImmutableList;
* @author lanyuanxiaoyao
* @date 2023-04-23
*/
@BaseRequest(baseURL = "http://service-yarn-query")
public interface YarnService {
@Get("/job/list")
ImmutableList<YarnApplication> jobList();
ImmutableList<YarnApplication> jobList(@Query("cluster") String cluster);
@Get("/job/list")
ImmutableList<YarnApplication> jobListEquals(@Query("name") String name);
ImmutableList<YarnApplication> jobListEquals(@Query("cluster") String cluster, @Query("name") String name);
@Get("/job/list")
ImmutableList<YarnApplication> jobListLike(@Query("text") String text);
@Get("/job/kill")
void jobKill(@Query("application_id") String applicationId);
ImmutableList<YarnApplication> jobListLike(@Query("cluster") String cluster, @Query("text") String text);
@Get("/job/detail")
YarnApplication jobDetail(@Query("application_id") String applicationId);
YarnApplication jobDetail(@Query("cluster") String cluster, @Query("application_id") String applicationId);
@Get("/queue/list")
ImmutableList<YarnQueue> queueList();
ImmutableList<YarnQueue> queueList(@Query("cluster") String cluster);
@Get("/queue/detail")
YarnQueue queueDetail(@Query("name") String name);
YarnQueue queueDetail(@Query("cluster") String cluster, @Query("name") String name);
@Get("/cluster/info")
YarnRootQueue cluster();
YarnRootQueue cluster(@Query("cluster") String cluster);
}