lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(yarn-query,web): 新增 yarn 多集群查询

Browse Source 
可以联合或单独查询 yarn 集群，并且聚合过滤排序等数据筛选内容
This commit is contained in:
兰缘小妖
2023-05-05 11:52:09 +08:00
parent 49a77a3795
commit 027fa10d50
24 changed files with 348 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/configuration/YarnConfiguration.java
View File

@@ -12,8 +12,17 @@ import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties("yarn")
public class YarnConfiguration {
private String cluster;
private String webUrl;
public String getCluster() {
return cluster;
}
public void setCluster(String cluster) {
this.cluster = cluster;
}
public String getWebUrl() {
return webUrl;
}
@@ -25,7 +34,8 @@ public class YarnConfiguration {
@Override
public String toString() {
return "YarnConfiguration{" +
"webUrl='" + webUrl + '\'' +
"cluster='" + cluster + '\'' +
", webUrl='" + webUrl + '\'' +
'}';
}
}

2
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/ClusterServiceImpl.java
View File

@@ -40,6 +40,6 @@ public class ClusterServiceImpl implements ClusterService {
public YarnRootQueue info() throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/scheduler");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo();
return mapper.readValue(body, ClusterInfoResponse.class).getScheduler().getSchedulerInfo().setCluster(yarnConfiguration.getCluster());
}
}

2
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/JobAutoRefreshServiceImpl.java
View File

@@ -41,7 +41,7 @@ public class JobAutoRefreshServiceImpl implements JobService {
public void refresh() throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/apps");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
ImmutableList<YarnApplication> apps = mapper.readValue(body, ApplicationsListResponse.class).getApps().getApp();
ImmutableList<YarnApplication> apps = mapper.readValue(body, ApplicationsListResponse.class).getApps().getApp().tap(app -> app.setCluster(yarnConfiguration.getCluster()));
CACHE.set(apps);
}

8
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/JobServiceImpl.java
View File

@@ -44,7 +44,7 @@ public class JobServiceImpl implements JobService {
public ImmutableList<YarnApplication> list() throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/apps");
try (HttpResponse response = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute()) {
return mapper.readValue(response.body(), ApplicationsListResponse.class).getApps().getApp();
return mapper.readValue(response.body(), ApplicationsListResponse.class).getApps().getApp().tap(app -> app.setCluster(yarnConfiguration.getCluster()));
}
}
@@ -52,14 +52,14 @@ public class JobServiceImpl implements JobService {
@Retryable(Throwable.class)
@Override
public ImmutableList<YarnApplication> listEquals(String name) throws JsonProcessingException {
return list().select(app -> StrUtil.equals(app.getName(), name));
return list().select(app -> StrUtil.equals(app.getName(), name)).tap(app -> app.setCluster(yarnConfiguration.getCluster()));
}
@Cacheable(value = "job-list", sync = true, key = "#{methodName+name}")
@Retryable(Throwable.class)
@Override
public ImmutableList<YarnApplication> listLike(String name) throws JsonProcessingException {
return list().select(app -> StrUtil.contains(app.getName(), name));
return list().select(app -> StrUtil.contains(app.getName(), name)).tap(app -> app.setCluster(yarnConfiguration.getCluster()));
}
@Override
@@ -72,7 +72,7 @@ public class JobServiceImpl implements JobService {
public YarnApplication detail(String applicationId) throws JsonProcessingException {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/apps/" + applicationId);
try (HttpResponse response = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute()) {
return mapper.readValue(response.body(), ApplicationDetailResponse.class).getApp();
return mapper.readValue(response.body(), ApplicationDetailResponse.class).getApp().setCluster(yarnConfiguration.getCluster());
}
}
}

3
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueAutoRefreshServiceImpl.java
View File

@@ -42,7 +42,8 @@ public class QueueAutoRefreshServiceImpl implements QueueService {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/scheduler");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
ImmutableList<YarnQueue> queues = Lists.immutable.ofAll(response.getScheduler().getSchedulerInfo().getQueues().getQueue());
ImmutableList<YarnQueue> queues = Lists.immutable.ofAll(response.getScheduler().getSchedulerInfo().getQueues().getQueue())
.tap(q -> q.setCluster(yarnConfiguration.getCluster()));
CACHE.set(queues);
}

5
service-yarn-query/src/main/java/com/lanyuanxiaoyao/service/yarn/service/impl/QueueServiceImpl.java
View File

@@ -44,7 +44,7 @@ public class QueueServiceImpl implements QueueService {
String queryUrl = URLUtil.completeUrl(yarnConfiguration.getWebUrl(), "/ws/v1/cluster/scheduler");
String body = HttpUtil.createGet(queryUrl).setMaxRedirectCount(10).execute().body();
QueueListResponse response = mapper.readValue(body, QueueListResponse.class);
return Lists.immutable.ofAll(response.getScheduler().getSchedulerInfo().getQueues().getQueue());
return Lists.immutable.ofAll(response.getScheduler().getSchedulerInfo().getQueues().getQueue()).tap(q -> q.setCluster(yarnConfiguration.getCluster()));
}
@Cacheable(value = "queue-detail", sync = true, key = "#name")
@@ -54,6 +54,7 @@ public class QueueServiceImpl implements QueueService {
return list()
.select(q -> StrUtil.equals(q.getQueueName(), name))
.getFirstOptional()
.orElseThrow(() -> new Exception("cannot found " + name));
.orElseThrow(() -> new Exception("cannot found " + name))
.setCluster(yarnConfiguration.getCluster());
}
}