diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java index ea1d039..9f29cb6 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java @@ -10,9 +10,10 @@ import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas; import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias; import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated; -import java.util.Map; import org.eclipse.collections.api.list.ImmutableList; +import java.util.Map; + /** * Info 接口 * @@ -21,6 +22,30 @@ import org.eclipse.collections.api.list.ImmutableList; */ @BaseRequest(baseURL = "http://service-info-query/info") public interface InfoService { + @Get("/table_total") + Long tableTotal(); + + @Get("/hudi_total") + Long hudiTotal(); + + @Get("/focus_count") + Long focusCount(); + + @Get("/normal_count") + Long normalCount(); + + @Get("/un_scheduled_normal_table_count") + Long unScheduledNormalTableCount(@Query("version") String version); + + @Get("/un_scheduled_focus_table_count") + Long unScheduledFocusTableCount(@Query("version") String version); + + @Get("/un_receive_version_normal_table_count") + Long unReceiveVersionNormalTableCount(@Query("version") String version); + + @Get("/un_receive_version_focus_table_count") + Long unReceiveVersionFocusTableCount(@Query("version") String version); + @Get("/job_id_alias") PageResponse jobIdAndAlias(@Query Map queryMap); diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/QueueService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/QueueService.java index 92c6828..e08f2cc 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/QueueService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/QueueService.java @@ -20,4 +20,7 @@ public interface QueueService { @Get("/all/{name}") ImmutableList> all(@Var("name") String name); + + @Get("/size/{name}") + Integer size(@Var("name") String name); } diff --git a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java index b50642c..60dfcd6 100644 --- a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java +++ b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java @@ -9,7 +9,6 @@ import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas; import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias; import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated; import com.lanyuanxiaoyao.service.info.service.InfoService; -import java.util.List; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; import org.slf4j.Logger; @@ -19,6 +18,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + /** * 信息接口 * @@ -36,6 +37,26 @@ public class InfoController { this.infoService = infoService; } + @GetMapping("table_total") + public Long tableTotal() { + return infoService.tableTotal(); + } + + @GetMapping("hudi_total") + public Long hudiTotal() { + return infoService.hudiTotal(); + } + + @GetMapping("focus_count") + public Long focusCount() { + return infoService.focusCount(); + } + + @GetMapping("normal_count") + public Long normalCount() { + return infoService.normalCount(); + } + @GetMapping("/job_id_alias") public PageResponse jobIdAndAlias( @RequestParam(value = "page", defaultValue = "1") Integer page, @@ -123,4 +144,24 @@ public class InfoController { public ImmutableList updatedVersionTables() { return infoService.nonUpdatedVersionTables(); } + + @GetMapping("/un_scheduled_normal_table_count") + public Long unScheduledNormalTableCount(@RequestParam("version") String version) { + return infoService.unScheduledNormalTableCount(version); + } + + @GetMapping("/un_scheduled_focus_table_count") + public Long unScheduledFocusTableCount(@RequestParam("version") String version) { + return infoService.unScheduledFocusTableCount(version); + } + + @GetMapping("/un_receive_version_normal_table_count") + public Long unReceiveVersionNormalTableCount(@RequestParam("version") String version) { + return infoService.unReceiveVersionNormalTableCount(version); + } + + @GetMapping("/un_receive_version_focus_table_count") + public Long unReceiveVersionFocusTableCount(@RequestParam("version") String version) { + return infoService.unReceiveVersionFocusTableCount(version); + } } diff --git a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java index 8cb8d81..600fad4 100644 --- a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java +++ b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java @@ -16,7 +16,6 @@ import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas; import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias; import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated; -import java.util.List; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.map.ImmutableMap; @@ -29,6 +28,8 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; +import java.util.List; + import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME; /** @@ -224,6 +225,8 @@ public class InfoService { private static final Alias TABLE_INFO = Alias.of(StrUtil.format("{}.tb_app_collect_table_info", DATABASE_NAME), "tacti"); private static final String TABLE_INFO_FLINK_JOB_ID = column(TABLE_INFO, "flink_job_id"); private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias"); + private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority"); + private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status"); private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss"); private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id"); @@ -247,6 +250,7 @@ public class InfoService { .join(TABLE_VERSION) .onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID)) .andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS)) + .andEq(TABLE_INFO_STATUS, "y") .join(TABLE_SYNC_STATE) .on(StrUtil.format("{} = CONCAT({}, '-', {})", TABLE_SYNC_STATE_ID, TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS)) .whereLike(ObjectUtil.isNotNull(flinkJobId), TABLE_INFO_FLINK_JOB_ID, flinkJobId) @@ -338,4 +342,120 @@ public class InfoService { .withMetadata("unScheduled", scheduleCount.getOrDefault(false, 0)); }); } + + @Cacheable(value = "un-receive-version-normal-table-count", sync = true) + @Retryable(Throwable.class) + public Long unReceiveVersionNormalTableCount(String version) { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select(COUNT) + .from(TABLE_INFO) + .whereLt(TABLE_INFO_PRIORITY, 10000) + .andEq(TABLE_INFO_STATUS, "y") + .andNotIn( + StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS), + SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS)) + .from(TABLE_VERSION) + .whereEq(TABLE_VERSION_VERSION, version) + ) + .build(), + Long.class + ); + } + + @Cacheable(value = "un-receive-version-focus-table-count", sync = true) + @Retryable(Throwable.class) + public Long unReceiveVersionFocusTableCount(String version) { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select(COUNT) + .from(TABLE_INFO) + .whereGe(TABLE_INFO_PRIORITY, 10000) + .andEq(TABLE_INFO_STATUS, "y") + .andNotIn( + StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS), + SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS)) + .from(TABLE_VERSION) + .whereEq(TABLE_VERSION_VERSION, version) + ) + .build(), + Long.class + ); + } + + @Cacheable(value = "un_scheduled_normal_table_count", sync = true) + @Retryable(Throwable.class) + public Long unScheduledNormalTableCount(String version) { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select(COUNT) + .from(TABLE_INFO) + .join(TABLE_VERSION) + .onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID)) + .andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS)) + .whereLt(TABLE_INFO_PRIORITY, 10000) + .andEq(TABLE_VERSION_VERSION, version) + .andEq(TABLE_INFO_STATUS, "y") + .build(), + Long.class + ); + } + + @Cacheable(value = "un_scheduled_focus_table_count", sync = true) + @Retryable(Throwable.class) + public Long unScheduledFocusTableCount(String version) { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select(COUNT) + .from(TABLE_INFO) + .join(TABLE_VERSION) + .onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID)) + .andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS)) + .whereGe(TABLE_INFO_PRIORITY, 10000) + .andEq(TABLE_VERSION_VERSION, version) + .andEq(TABLE_INFO_STATUS, "y") + .build(), + Long.class + ); + } + + @Cacheable(value = "table-total", sync = true) + @Retryable(Throwable.class) + public Long tableTotal() { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select("count(distinct concat(src_schema, src_table))").from(TABLE_INFO).whereEq(TABLE_INFO_STATUS, "y").build(), + Long.class + ); + } + + @Cacheable(value = "hudi-total", sync = true) + @Retryable(Throwable.class) + public Long hudiTotal() { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select("count(distinct tgt_hdfs_path) as count").from(TABLE_INFO).whereEq(TABLE_INFO_STATUS, "y").build(), + Long.class + ); + } + + @Cacheable(value = "focus-count", sync = true) + @Retryable(Throwable.class) + public Long focusCount() { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select("count(distinct concat(src_schema, src_table))") + .from(TABLE_INFO) + .whereGe(TABLE_INFO_PRIORITY, 1000) + .andEq(TABLE_INFO_STATUS, "y") + .build(), + Long.class + ); + } + + @Cacheable(value = "normal-count", sync = true) + @Retryable(Throwable.class) + public Long normalCount() { + return mysqlJdbcTemplate.queryForObject( + SqlBuilder.select("count(distinct concat(src_schema, src_table))") + .from(TABLE_INFO) + .whereLt(TABLE_INFO_PRIORITY, 1000) + .andEq(TABLE_INFO_STATUS, "y") + .build(), + Long.class + ); + } } diff --git a/service-info-query/src/test/java/SqlBuilderTests.java b/service-info-query/src/test/java/SqlBuilderTests.java index bf45359..c99b9f0 100644 --- a/service-info-query/src/test/java/SqlBuilderTests.java +++ b/service-info-query/src/test/java/SqlBuilderTests.java @@ -1,9 +1,7 @@ import club.kingon.sql.builder.SqlBuilder; import club.kingon.sql.builder.entry.Alias; -import club.kingon.sql.builder.entry.Column; import cn.hutool.core.util.StrUtil; import cn.hutool.db.sql.SqlFormatter; -import org.eclipse.collections.api.factory.Lists; import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME; @@ -12,8 +10,21 @@ import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME; * @date 2023-06-07 */ public class SqlBuilderTests { + private static final String COUNT = "count(*)"; + private static final Alias TABLE_VERSION = Alias.of(StrUtil.format("{}.tb_app_collect_table_version", DATABASE_NAME), "tactv"); + private static final String TABLE_VERSION_FLINK_JOB_ID = column(TABLE_VERSION, "flink_job_id"); + private static final String TABLE_VERSION_ALIAS = column(TABLE_VERSION, "alias"); + private static final String TABLE_VERSION_VERSION = column(TABLE_VERSION, "version"); + private static final String TABLE_VERSION_SCHEDULED = column(TABLE_VERSION, "scheduled"); + private static final Alias TABLE_INFO = Alias.of(StrUtil.format("{}.tb_app_collect_table_info", DATABASE_NAME), "tacti"); + private static final String TABLE_INFO_FLINK_JOB_ID = column(TABLE_INFO, "flink_job_id"); + private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias"); + private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority"); + + private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss"); + private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id"); private static String column(Alias table, String column) { return StrUtil.format("{}.{}", table.getAlias(), column); @@ -21,15 +32,15 @@ public class SqlBuilderTests { public static void main(String[] args) { System.out.println(SqlFormatter.format( - SqlBuilder - .select("count(*)") + SqlBuilder.select(COUNT) .from(TABLE_INFO) - .join(TABLE_VERSION) - .onEq(column(TABLE_INFO, "flink_job_id"), Column.as(column(TABLE_VERSION, "flink_job_id"))) - .andEq(column(TABLE_INFO, "alias"), Column.as(column(TABLE_VERSION, "alias"))) - .whereEq(false, "a", "b") - .andEq("b", "c") - .andIn("d", Lists.immutable.empty()) + .whereGe(TABLE_INFO_PRIORITY, 10000) + .andNotIn( + StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS), + SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS)) + .from(TABLE_VERSION) + .whereEq(TABLE_VERSION_VERSION, "20230611") + ) .build() )); } diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/OverviewController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/OverviewController.java new file mode 100644 index 0000000..7c13b80 --- /dev/null +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/OverviewController.java @@ -0,0 +1,137 @@ +package com.lanyuanxiaoyao.service.web.controller; + +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; +import com.lanyuanxiaoyao.service.configuration.ExecutorProvider; +import com.lanyuanxiaoyao.service.configuration.entity.AmisResponse; +import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication; +import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnRootQueue; +import com.lanyuanxiaoyao.service.forest.service.InfoService; +import com.lanyuanxiaoyao.service.forest.service.QueueService; +import com.lanyuanxiaoyao.service.forest.service.YarnService; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +/** + * 概览 + * + * @author lanyuanxiaoyao + * @date 2023-06-12 + */ +@RestController +@RequestMapping("overview") +public class OverviewController extends BaseController { + private static final Logger logger = LoggerFactory.getLogger(OverviewController.class); + + private final InfoService infoService; + private final YarnService yarnService; + private final QueueService queueService; + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + public OverviewController(InfoService infoService, YarnService yarnService, QueueService queueService) { + this.infoService = infoService; + this.yarnService = yarnService; + this.queueService = queueService; + } + + @GetMapping("") + public AmisResponse overview() throws ExecutionException, InterruptedException { + CompletableFuture tableTotalFuture = CompletableFuture.supplyAsync(infoService::tableTotal, ExecutorProvider.EXECUTORS); + CompletableFuture hudiTotalFuture = CompletableFuture.supplyAsync(infoService::hudiTotal, ExecutorProvider.EXECUTORS); + CompletableFuture focusCountFuture = CompletableFuture.supplyAsync(infoService::focusCount, ExecutorProvider.EXECUTORS); + CompletableFuture normalCountFuture = CompletableFuture.supplyAsync(infoService::normalCount, ExecutorProvider.EXECUTORS); + CompletableFuture.allOf( + tableTotalFuture, + hudiTotalFuture, + focusCountFuture, + normalCountFuture + ); + return responseData() + .withData("table_total", tableTotalFuture.get()) + .withData("hudi_total", hudiTotalFuture.get()) + .withData("focus_count", focusCountFuture.get()) + .withData("normal_count", normalCountFuture.get()); + } + + @GetMapping("yarn-job") + private ImmutableMap yarnOverview(@RequestParam("cluster") String cluster, @RequestParam("search") String text) { + boolean isSearch = StrUtil.isNotBlank(text); + ImmutableList applications = yarnService.jobList(cluster).select(app -> !isSearch || StrUtil.contains(app.getName(), text)); + return Maps.immutable.ofAll(MapUtil.builder() + .put("name", cluster) + .put("total", applications.size()) + .put("running", applications.count(app -> StrUtil.equals(app.getState(), "RUNNING"))) + .put("scheduling", applications.count(app -> StrUtil.equals(app.getState(), "ACCEPTED"))) + .put("failure", applications.count(app -> StrUtil.equals(app.getState(), "FAILED"))) + .build()); + } + + @GetMapping("yarn") + public AmisResponse yarnOverview(@RequestParam("clusters") List clusters, @RequestParam("search") String text) { + ImmutableList> maps = Lists.immutable.ofAll(clusters) + .asParallel(ExecutorProvider.EXECUTORS, 1) + .collect(cluster -> yarnOverview(cluster, text)) + .toList() + .toImmutable(); + long total = maps.sumOfInt(m -> (int) m.get("total")); + long running = maps.sumOfInt(m -> (int) m.get("running")); + long scheduling = maps.sumOfInt(m -> (int) m.get("scheduling")); + long failure = maps.sumOfInt(m -> (int) m.get("failure")); + return responseData() + .withData("total", total) + .withData("running", running) + .withData("scheduling", scheduling) + .withData("failure", failure) + .withData("items", maps); + } + + @GetMapping("yarn-cluster") + public AmisResponse yarnClusterOverview(@RequestParam("cluster") String cluster, @RequestParam("queue") String queue) { + AmisResponse response = responseData(); + YarnRootQueue root = yarnService.cluster(cluster); + response.withData("root", root); + if (StrUtil.isNotBlank(queue)) { + yarnService.queueList(cluster) + .select(q -> StrUtil.equals(q.getQueueName(), queue)) + .getFirstOptional() + .ifPresent(yarnQueue -> response.withData("target", yarnQueue)); + } + return response; + } + + @GetMapping("queue") + public AmisResponse queueOverview(@RequestParam("queue") String queue) { + return responseData() + .withData("size", queueService.size(queue)); + } + + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + + @GetMapping("version") + public AmisResponse versionOverview() throws ExecutionException, InterruptedException { + String version = LocalDateTime.now().minusDays(1).format(FORMATTER); + CompletableFuture unReceiveNormalTableCount = CompletableFuture.supplyAsync(() -> infoService.unReceiveVersionNormalTableCount(version), ExecutorProvider.EXECUTORS); + CompletableFuture unReceiveFocusCount = CompletableFuture.supplyAsync(() -> infoService.unReceiveVersionFocusTableCount(version), ExecutorProvider.EXECUTORS); + CompletableFuture unScheduledNormalTableCount = CompletableFuture.supplyAsync(() -> infoService.unScheduledNormalTableCount(version), ExecutorProvider.EXECUTORS); + CompletableFuture unScheduledFocusTableCount = CompletableFuture.supplyAsync(() -> infoService.unScheduledFocusTableCount(version), ExecutorProvider.EXECUTORS); + CompletableFuture.allOf(unReceiveNormalTableCount, unReceiveFocusCount, unScheduledNormalTableCount, unScheduledFocusTableCount); + return responseData() + .withData("version", version) + .withData("unReceive", Maps.immutable.of("normal", unReceiveNormalTableCount.get(), "focus", unReceiveFocusCount.get())) + .withData("unSchedule", Maps.immutable.of("normal", unScheduledNormalTableCount.get(), "focus", unScheduledFocusTableCount.get())); + } +} diff --git a/web/components/overview-tab.js b/web/components/overview-tab.js new file mode 100644 index 0000000..6f9e706 --- /dev/null +++ b/web/components/overview-tab.js @@ -0,0 +1,198 @@ +function overviewYarnJob(cluster, search, queue, yarnQueue) { + return { + className: 'font-mono', + type: 'service', + api: `\${base}/overview/yarn-job?cluster=${cluster}&search=${search}`, + interval: 10000, + silentPolling: true, + body: [ + { + type: 'tpl', + className: 'mr-1 font-bold', + tpl: `${cluster}`, + }, + queue === undefined ? {} : { + type: 'service', + className: 'inline ml-2', + api: `\${base}/overview/queue?queue=compaction-queue-${cluster}`, + body: [ + { + type: 'tpl', + tpl: '${PADSTART(size, 2)}', + } + ] + }, + overviewYarnCluster(cluster, yarnQueue), + '(', + { + type: 'tpl', + tpl: '${PADSTART(total, 3)}', + }, + ',', + { + type: 'tpl', + tpl: '${PADSTART(scheduling, 3)}', + }, + ',', + { + type: 'tpl', + tpl: '${PADSTART(running, 3)}', + }, + ',', + { + type: 'tpl', + tpl: '${PADSTART(failure, 3)}', + }, + ')', + ] + } +} + +function color(number) { + let color = 'text-success' + if (number > 30) { + color = 'text-primary' + } + if (number > 80) { + color = 'text-danger' + } + return color +} + +function overviewYarnCluster(cluster, queue) { + return { + type: 'service', + className: 'inline', + api: { + method: 'get', + url: `\${base}/overview/yarn-cluster`, + data: { + cluster: cluster, + queue: queue, + }, + adaptor: function (payload, response) { + let rootUsed = (payload['data']['root']['usedCapacity'] * 100 / payload['data']['root']['capacity']).toFixed(0) + let targetUsed = (payload['data']['target']['absoluteUsedCapacity'] * 100 / payload['data']['target']['absoluteMaxCapacity']).toFixed(0) + return { + ...payload, + data: { + ...payload.data, + rootUsed: rootUsed, + rootUsedColor: color(rootUsed), + targetUsed: targetUsed, + targetUsedColor: color(targetUsed), + } + } + } + }, + interval: 10000, + silentPolling: true, + body: [ + '(', + { + type: 'tpl', + className: 'font-bold', + tpl: '${PADSTART(ROUND(rootUsed), 3)}%', + }, + ',', + { + type: 'tpl', + tpl: '${PADSTART(ROUND(targetUsed), 3)}%', + }, + ')' + ], + } +} + +function overviewTab() { + return { + title: '概览', + tab: [ + { + type: 'service', + // language=JavaScript + dataProvider: 'const timer = setInterval(() => {\n setData({date: new Date().toLocaleString()})\n}, 1000)\nreturn () => {\n clearInterval(timer)\n}', + body: [ + '当前时间：', + { + type: 'tpl', + className: 'font-bold', + tpl: '${date}' + } + ] + }, + {type: 'divider'}, + { + type: 'service', + api: '${base}/overview', + interval: 60000, + silentPolling: true, + body: [ + { + type: 'tpl', + tpl: '逻辑表：${table_total}', + }, + '
', + { + type: 'tpl', + tpl: 'Hudi表：${hudi_total}', + }, + '
', + { + type: 'tpl', + tpl: '重点表：${focus_count}', + }, + '
', + { + type: 'tpl', + tpl: '普通表：${normal_count}', + }, + ] + }, + {type: 'divider'}, + overviewYarnJob('b5-sync', 'Sync', undefined, 'default'), + {type: 'divider'}, + { + type: 'service', + api: `\${base}/overview/queue?queue=compaction-queue-pre`, + body: [ + { + type: 'tpl', + tpl: 'Pre：${size}', + } + ] + }, + '集群 压缩队列任务数(集群总资源使用,队列资源使用)(总压缩任务数,调度中任务数,运行中任务数,失败任务数)', + overviewYarnJob('b1', 'Compaction', 'compaction-queue-b1', 'datalake'), + overviewYarnJob('b5', 'Compaction', 'compaction-queue-b5', 'ten_iap.datalake'), + overviewYarnJob('a4', 'Compaction', 'compaction-queue-a4', 'ten_iap.datalake'), + {type: 'divider'}, + { + type: 'service', + api: '${base}/overview/version', + body: [ + '版本：', + { + type: 'tpl', + className: 'font-bold', + tpl: '${version}', + }, + '
', + '未接收,未跨天', + '
', + '重点表：', + { + type: 'tpl', + tpl: '${PADSTART(unReceive.focus, 3)},${PADSTART(unSchedule.focus, 3)}', + }, + '
', + '普通表：', + { + type: 'tpl', + tpl: '${PADSTART(unReceive.normal, 3)},${PADSTART(unSchedule.normal, 3)}', + }, + ] + } + ] + } +} \ No newline at end of file diff --git a/web/index.html b/web/index.html index e322d76..e132296 100644 --- a/web/index.html +++ b/web/index.html @@ -43,6 +43,7 @@ +