From b13e70d9cbd9968b546083da9db2b199aee9ed71 Mon Sep 17 00:00:00 2001 From: lanyuanxiaoyao Date: Fri, 7 Jul 2023 15:55:27 +0800 Subject: [PATCH] =?UTF-8?q?feature(web):=20=E5=A2=9E=E5=8A=A0=20hdfs=20?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=A1=A5=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/forest/service/InfoService.java | 9 ++++ .../info/controller/InfoController.java | 15 ++++++ .../service/info/service/InfoService.java | 41 ++++++++++++++++ .../src/test/java/SqlBuilderTests.java | 4 +- .../web/controller/TableController.java | 47 +++++++++++++++++++ web/components/tool-tab.js | 23 ++------- 6 files changed, 117 insertions(+), 22 deletions(-) diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java index 662d48a..fe13da8 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java @@ -105,4 +105,13 @@ public interface InfoService { @Get("/non_exists_table") Boolean nonExistsTable(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias); + + @Get("/all_flink_job_id") + ImmutableList allFlinkJobId(); + + @Get("/all_alias") + ImmutableList allAlias(); + + @Get("/all_hdfs") + ImmutableList allHdfs(); } diff --git a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java index 5f49a67..763611b 100644 --- a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java +++ b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java @@ -225,4 +225,19 @@ public class InfoController { public Boolean nonExistsTable(@RequestParam(value = "flink_job_id") Long flinkJobId, @RequestParam(value = "alias") String alias) { return !infoService.existsTable(flinkJobId, alias); } + + @GetMapping("/all_flink_job_id") + public ImmutableList allFlinkJobId() { + return infoService.allFlinkJobId(); + } + + @GetMapping("/all_alias") + public ImmutableList allAlias() { + return infoService.allAlias(); + } + + @GetMapping("/all_hdfs") + public ImmutableList allHdfs() { + return infoService.allHdfs(); + } } diff --git a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java index 6f2d1a7..bb4fa9b 100644 --- a/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java +++ b/service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java @@ -242,6 +242,7 @@ public class InfoService { private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias"); private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority"); private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status"); + private static final String TABLE_INFO_TARGET_HDFS = column(TABLE_INFO, "tgt_hdfs_path"); private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss"); private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id"); @@ -745,4 +746,44 @@ public class InfoService { Boolean.class ); } + + @Cacheable(value = "all-flink-job-id", sync = true, cacheManager = "long-cache") + @Retryable(Throwable.class) + public ImmutableList allFlinkJobId() { + return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList( + SqlBuilder.select(StrUtil.format("distinct {}", TABLE_FLINK_JOB_ID)) + .from(TABLE_FLINK_JOB) + .whereEq(TABLE_FLINK_JOB_STATUS, "y") + .build(), + Long.class + )); + } + + @Cacheable(value = "all-alias", sync = true, cacheManager = "long-cache") + @Retryable(Throwable.class) + public ImmutableList allAlias() { + return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList( + SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_ALIAS)) + .from(TABLE_INFO, TABLE_FLINK_JOB) + .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID)) + .andEq(TABLE_FLINK_JOB_STATUS, "y") + .andEq(TABLE_INFO_STATUS, "y") + .build(), + String.class + )); + } + + @Cacheable(value = "all-hdfs", sync = true, cacheManager = "long-cache") + @Retryable(Throwable.class) + public ImmutableList allHdfs() { + return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList( + SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_TARGET_HDFS)) + .from(TABLE_INFO, TABLE_FLINK_JOB) + .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID)) + .andEq(TABLE_FLINK_JOB_STATUS, "y") + .andEq(TABLE_INFO_STATUS, "y") + .build(), + String.class + )); + } } diff --git a/service-info-query/src/test/java/SqlBuilderTests.java b/service-info-query/src/test/java/SqlBuilderTests.java index dc4561b..aae65e4 100644 --- a/service-info-query/src/test/java/SqlBuilderTests.java +++ b/service-info-query/src/test/java/SqlBuilderTests.java @@ -43,11 +43,9 @@ public class SqlBuilderTests { public static void main(String[] args) { System.out.println(SqlFormatter.format( - SqlBuilder.select("count(*) > 0") + SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_ALIAS)) .from(TABLE_INFO, TABLE_FLINK_JOB) .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID)) - .andEq(TABLE_FLINK_JOB_ID, 1552408244680593408L) - .andEq(TABLE_INFO_ALIAS, "crm_order_ord_prod_inst_sub_his") .andEq(TABLE_FLINK_JOB_STATUS, "y") .andEq(TABLE_INFO_STATUS, "y") .build() diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java index c76e541..c0433af 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java @@ -8,6 +8,8 @@ import com.eshore.odcp.hudi.connector.entity.SyncState; import com.eshore.odcp.hudi.connector.entity.TableMeta; import com.eshore.odcp.hudi.connector.utils.NameHelper; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.lanyuanxiaoyao.service.configuration.ExecutorProvider; import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; import com.lanyuanxiaoyao.service.configuration.entity.info.CompactionMetrics; @@ -22,7 +24,9 @@ import com.lanyuanxiaoyao.service.web.entity.CompactionMetricsVO; import com.lanyuanxiaoyao.service.web.entity.FlinkJobVO; import com.lanyuanxiaoyao.service.web.entity.SyncStateVO; import com.lanyuanxiaoyao.service.web.entity.TableVO; +import java.time.Duration; import java.util.List; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.eclipse.collections.api.factory.Lists; @@ -55,6 +59,20 @@ public class TableController extends BaseController { this.infoService = infoService; this.zookeeperService = zookeeperService; this.mapper = jackson2ObjectMapperBuilder.build(); + this.cache = Caffeine.newBuilder() + .expireAfterAccess(Duration.ofMinutes(5)) + .build(key -> { + switch (key) { + case "flink_job_id": + return this.infoService.allFlinkJobId().collect(Objects::toString); + case "alias": + return this.infoService.allAlias(); + case "hdfs": + return this.infoService.allHdfs(); + default: + return Lists.immutable.empty(); + } + }); } @GetMapping("list") @@ -185,4 +203,33 @@ public class TableController extends BaseController { ImmutableList vos = Lists.immutable.ofAll(pageResponse.getData()).collect(CompactionMetricsVO::new); return AmisResponse.responseCrudData(vos, pageResponse.getTotal()); } + + private final LoadingCache> cache; + + @SuppressWarnings("DataFlowIssue") + @GetMapping("all_flink_job_id") + public AmisCrudResponse allFlinkJobId(@RequestParam(value = "key", required = false) String key) { + if (StrUtil.isBlank(key)) { + return AmisResponse.responseCrudData(Lists.immutable.empty()); + } + return AmisResponse.responseCrudData(cache.get("flink_job_id").select(id -> StrUtil.contains(id, key))); + } + + @SuppressWarnings("DataFlowIssue") + @GetMapping("all_alias") + public AmisCrudResponse allAlias(@RequestParam(value = "key", required = false) String key) { + if (StrUtil.isBlank(key)) { + return AmisResponse.responseCrudData(Lists.immutable.empty()); + } + return AmisResponse.responseCrudData(cache.get("alias").select(id -> StrUtil.contains(id, key))); + } + + @SuppressWarnings("DataFlowIssue") + @GetMapping("all_hdfs") + public AmisCrudResponse allHdfs(@RequestParam(value = "key", required = false) String key) { + if (StrUtil.isBlank(key)) { + return AmisResponse.responseCrudData(Lists.immutable.empty()); + } + return AmisResponse.responseCrudData(cache.get("hdfs").select(id -> StrUtil.contains(id, key))); + } } diff --git a/web/components/tool-tab.js b/web/components/tool-tab.js index 1c1a310..ff68122 100644 --- a/web/components/tool-tab.js +++ b/web/components/tool-tab.js @@ -8,7 +8,7 @@ function toolTab() { actions: [ { type: 'submit', - label: '查询', + label: '查询时间线', actionType: 'dialog', dialog: { title: 'Hudi 表时间线', @@ -42,26 +42,10 @@ function toolTab() { columns: timelineColumns(), } } - } - ], - body: [ - { - type: 'input-text', - name: 'hdfs', - label: 'HDFS路经', - required: true, - clearable: true, - description: '输入表HDFS路径', - } - ] - }, - { - type: 'form', - title: '查询表结构', - actions: [ + }, { type: 'submit', - label: '查询', + label: '查询表结构', actionType: 'dialog', dialog: { title: 'Hudi 表结构', @@ -96,6 +80,7 @@ function toolTab() { required: true, clearable: true, description: '输入表HDFS路径', + autoComplete: '${base}/table/all_hdfs?key=$term', } ] },