lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(forest): 修复 yarn 查询集群信息顺序不对

Browse Source 
Set 这个结构不保顺序，疏忽了，改回使用 List，不然查询的顺序和返回结果的顺序不一致
This commit is contained in:
兰缘小妖
2023-06-01 10:31:10 +08:00
parent 15eb84e993
commit 17f957a4bb
2 changed files with 17 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/YarnClusterService.java
View File

@@ -1,6 +1,5 @@
package com.lanyuanxiaoyao.service.forest.service; package com.lanyuanxiaoyao.service.forest.service;
import cn.hutool.core.lang.Pair;
import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.ObjectUtil;
import com.eshore.odcp.hudi.connector.Constants; import com.eshore.odcp.hudi.connector.Constants;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication; import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
@@ -13,11 +12,9 @@ import java.util.function.Function;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap; import org.eclipse.collections.api.map.ImmutableMap;
import org.eclipse.collections.api.map.MutableMap; import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.set.ImmutableSet;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -60,8 +57,8 @@ public class YarnClusterService {
return servicesMap.valuesView().toList().toImmutable(); return servicesMap.valuesView().toList().toImmutable();
} }
public ImmutableList<YarnService> services(ImmutableSet<String> clusters) { public ImmutableList<YarnService> services(ImmutableList<String> clusters) {
return clusters.toList().collect(this::service).toImmutable(); return clusters.collect(this::service);
} }
public ImmutableMap<String, YarnService> servicesMap() { public ImmutableMap<String, YarnService> servicesMap() {
@@ -73,30 +70,30 @@ public class YarnClusterService {
} }
public ImmutableList<YarnApplication> jobList(String cluster) { public ImmutableList<YarnApplication> jobList(String cluster) {
return jobList(Sets.immutable.of(cluster)); return jobList(Lists.immutable.of(cluster));
} }
public ImmutableList<YarnApplication> jobList(ImmutableSet<String> clusters) { public ImmutableList<YarnApplication> jobList(ImmutableList<String> clusters) {
return list(clusters, YarnService::jobList); return list(clusters, YarnService::jobList);
} }
public ImmutableList<YarnApplication> jobListEquals(String cluster, String name) { public ImmutableList<YarnApplication> jobListEquals(String cluster, String name) {
return jobListEquals(Sets.immutable.of(cluster), name); return jobListEquals(Lists.immutable.of(cluster), name);
} }
public ImmutableList<YarnApplication> jobListEquals(ImmutableSet<String> clusters, String name) { public ImmutableList<YarnApplication> jobListEquals(ImmutableList<String> clusters, String name) {
return list(clusters, yarnService -> yarnService.jobListEquals(name)); return list(clusters, yarnService -> yarnService.jobListEquals(name));
} }
public ImmutableList<YarnApplication> jobListLike(String cluster, String text) { public ImmutableList<YarnApplication> jobListLike(String cluster, String text) {
return jobListLike(Sets.immutable.of(cluster), text); return jobListLike(Lists.immutable.of(cluster), text);
} }
public ImmutableList<YarnApplication> jobListLike(ImmutableSet<String> clusters, String text) { public ImmutableList<YarnApplication> jobListLike(ImmutableList<String> clusters, String text) {
return list(clusters, yarnService -> yarnService.jobListLike(text)); return list(clusters, yarnService -> yarnService.jobListLike(text));
} }
private ImmutableList<YarnApplication> list(ImmutableSet<String> clusters, Function<YarnService, ImmutableList<YarnApplication>> getter) { private ImmutableList<YarnApplication> list(ImmutableList<String> clusters, Function<YarnService, ImmutableList<YarnApplication>> getter) {
if (ObjectUtil.isEmpty(clusters)) { if (ObjectUtil.isEmpty(clusters)) {
return Lists.immutable.empty(); return Lists.immutable.empty();
} }
@@ -119,7 +116,7 @@ public class YarnClusterService {
return servicesMap.containsKey(cluster) ? service(cluster).queueList() : Lists.immutable.empty(); return servicesMap.containsKey(cluster) ? service(cluster).queueList() : Lists.immutable.empty();
} }
public ImmutableList<YarnQueue> queueList(ImmutableSet<String> clusters) { public ImmutableList<YarnQueue> queueList(ImmutableList<String> clusters) {
//noinspection DataFlowIssue //noinspection DataFlowIssue
return clusters.toList() return clusters.toList()
.select(servicesMap::containsKey) .select(servicesMap::containsKey)
@@ -134,7 +131,7 @@ public class YarnClusterService {
return servicesMap.containsKey(cluster) ? service(cluster).queueDetail(name) : null; return servicesMap.containsKey(cluster) ? service(cluster).queueDetail(name) : null;
} }
public ImmutableList<YarnQueue> queueDetail(ImmutableSet<String> clusters, String name) { public ImmutableList<YarnQueue> queueDetail(ImmutableList<String> clusters, String name) {
//noinspection DataFlowIssue //noinspection DataFlowIssue
return clusters.toList() return clusters.toList()
.select(servicesMap::containsKey) .select(servicesMap::containsKey)
@@ -149,7 +146,7 @@ public class YarnClusterService {
return servicesMap.containsKey(cluster) ? service(cluster).cluster() : null; return servicesMap.containsKey(cluster) ? service(cluster).cluster() : null;
} }
public ImmutableList<YarnRootQueue> cluster(ImmutableSet<String> clusters) { public ImmutableList<YarnRootQueue> cluster(ImmutableList<String> clusters) {
//noinspection DataFlowIssue //noinspection DataFlowIssue
return clusters.toList() return clusters.toList()
.select(servicesMap::containsKey) .select(servicesMap::containsKey)

11
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java
View File

@@ -19,7 +19,6 @@ import java.util.concurrent.Executors;
import java.util.function.Function; import java.util.function.Function;
import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap; import org.eclipse.collections.api.map.ImmutableMap;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -70,7 +69,7 @@ public class YarnController extends BaseController {
boolean isSearchId = StrUtil.isNotBlank(searchId); boolean isSearchId = StrUtil.isNotBlank(searchId);
boolean isSearchName = StrUtil.isNotBlank(searchName); boolean isSearchName = StrUtil.isNotBlank(searchName);
Comparator<YarnApplication> comparator = ComparatorUtil.longComparator(order, direction, SORT_MAP); Comparator<YarnApplication> comparator = ComparatorUtil.longComparator(order, direction, SORT_MAP);
ImmutableList<YarnApplication> applications = yarnClusterService.jobList(Sets.immutable.ofAll(clusters)) ImmutableList<YarnApplication> applications = yarnClusterService.jobList(Lists.immutable.ofAll(clusters))
.asParallel(EXECUTOR, 1) .asParallel(EXECUTOR, 1)
.select(app -> !isFilterState || ObjectUtil.contains(filterState, app.getState())) .select(app -> !isFilterState || ObjectUtil.contains(filterState, app.getState()))
.select(app -> !isFilterFinalStatus || ObjectUtil.contains(filterFinalStatus, app.getFinalStatus())) .select(app -> !isFilterFinalStatus || ObjectUtil.contains(filterFinalStatus, app.getFinalStatus()))
@@ -91,7 +90,7 @@ public class YarnController extends BaseController {
@GetMapping("job_current") @GetMapping("job_current")
public AmisResponse jobCurrent(@RequestParam("clusters") List<String> clusters, @RequestParam("name") String name) { public AmisResponse jobCurrent(@RequestParam("clusters") List<String> clusters, @RequestParam("name") String name) {
Optional<YarnApplication> currentApp = yarnClusterService.jobListEquals(Sets.immutable.ofAll(clusters), name) Optional<YarnApplication> currentApp = yarnClusterService.jobListEquals(Lists.immutable.ofAll(clusters), name)
.asParallel(EXECUTOR, 1) .asParallel(EXECUTOR, 1)
.select(app -> ObjectUtil.equals(app.getState(), "RUNNING")) .select(app -> ObjectUtil.equals(app.getState(), "RUNNING"))
.toSortedList(ComparatorUtil.longComparator("startedTime", ComparatorUtil.DESC, SORT_MAP)) .toSortedList(ComparatorUtil.longComparator("startedTime", ComparatorUtil.DESC, SORT_MAP))
@@ -109,7 +108,7 @@ public class YarnController extends BaseController {
public AmisResponse queueList(@RequestParam("clusters") List<String> clusters, @RequestParam(value = "names", defaultValue = "") String names) { public AmisResponse queueList(@RequestParam("clusters") List<String> clusters, @RequestParam(value = "names", defaultValue = "") String names) {
boolean isFilterNames = StrUtil.isNotBlank(names); boolean isFilterNames = StrUtil.isNotBlank(names);
ImmutableList<String> filterNames = Lists.immutable.of(names.split(",")); ImmutableList<String> filterNames = Lists.immutable.of(names.split(","));
ImmutableList<YarnClusterVO> results = yarnClusterService.services(Sets.immutable.ofAll(clusters)) ImmutableList<YarnClusterVO> results = yarnClusterService.services(Lists.immutable.ofAll(clusters))
.asParallel(EXECUTOR, 1) .asParallel(EXECUTOR, 1)
.collect(yarnService -> { .collect(yarnService -> {
YarnRootQueue cluster = yarnService.cluster(); YarnRootQueue cluster = yarnService.cluster();
@@ -123,11 +122,11 @@ public class YarnController extends BaseController {
@GetMapping("queue_names") @GetMapping("queue_names")
public AmisResponse queueNames(@RequestParam("clusters") List<String> clusters) { public AmisResponse queueNames(@RequestParam("clusters") List<String> clusters) {
return responseData(MapUtil.of("queueNames", yarnClusterService.queueList(Sets.immutable.ofAll(clusters)).collect(YarnQueue::getQueueName))); return responseData(MapUtil.of("queueNames", yarnClusterService.queueList(Lists.immutable.ofAll(clusters)).collect(YarnQueue::getQueueName)));
} }
@GetMapping("clusters") @GetMapping("clusters")
public AmisResponse clusters(@RequestParam("clusters") List<String> clusters) { public AmisResponse clusters(@RequestParam("clusters") List<String> clusters) {
return responseData(MapUtil.of("cluster", yarnClusterService.cluster(Sets.immutable.ofAll(clusters)))); return responseData(MapUtil.of("cluster", yarnClusterService.cluster(Lists.immutable.ofAll(clusters))));
} }
} }