feature(web): 优化flink job id和alias的自动匹配获取

service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/info/TableInfoSearchCache.java

@@ -0,0 +1,55 @@
+package com.lanyuanxiaoyao.service.configuration.entity.info;
+
+/**
+ * Table Info 表查询条件缓存
+ *
+ * @author lanyuanxiaoyao
+ * @date 2023-07-14
+ */
+public class TableInfoSearchCache {
+    private Long flinkJobId;
+    private String alias;
+    private String hdfs;
+
+    public TableInfoSearchCache() {
+    }
+
+    public TableInfoSearchCache(Long flinkJobId, String alias, String hdfs) {
+        this.flinkJobId = flinkJobId;
+        this.alias = alias;
+        this.hdfs = hdfs;
+    }
+
+    public Long getFlinkJobId() {
+        return flinkJobId;
+    }
+
+    public void setFlinkJobId(Long flinkJobId) {
+        this.flinkJobId = flinkJobId;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public void setAlias(String alias) {
+        this.alias = alias;
+    }
+
+    public String getHdfs() {
+        return hdfs;
+    }
+
+    public void setHdfs(String hdfs) {
+        this.hdfs = hdfs;
+    }
+
+    @Override
+    public String toString() {
+        return "TableInfoSearchCache{" +
+                "flinkJobId=" + flinkJobId +
+                ", alias='" + alias + '\'' +
+                ", hdfs='" + hdfs + '\'' +
+                '}';
+    }
+}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/InfoService.java

@@ -109,9 +109,24 @@ public interface InfoService {
     @Get("/info/all_flink_job_id")
     ImmutableList<Long> allFlinkJobId();
 
+    @Get("/info/all_flink_job_id")
+    ImmutableList<Long> allFlinkJobId(@Query("key") String key);
+
+    @Get("/info/all_flink_job_id")
+    ImmutableList<Long> allFlinkJobId(@Query("key") String key, @Query("alias") String alias);
+
     @Get("/info/all_alias")
     ImmutableList<String> allAlias();
 
+    @Get("/info/all_alias")
+    ImmutableList<String> allAlias(@Query("key") String key);
+
+    @Get("/info/all_alias")
+    ImmutableList<String> allAlias(@Query("key") String key, @Query("flink_job_id") String flinkJobId);
+
     @Get("/info/all_hdfs")
     ImmutableList<String> allHdfs();
+
+    @Get("/info/all_hdfs")
+    ImmutableList<String> allHdfs(@Query("key") String key);
 }

service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java

@@ -1,14 +1,12 @@
 package com.lanyuanxiaoyao.service.info.controller;
 
 import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
 import com.eshore.odcp.hudi.connector.entity.FlinkJob;
 import com.eshore.odcp.hudi.connector.entity.SyncState;
 import com.eshore.odcp.hudi.connector.entity.TableMeta;
 import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
-import com.lanyuanxiaoyao.service.configuration.entity.info.CompactionMetrics;
+import com.lanyuanxiaoyao.service.configuration.entity.info.*;
-import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas;
-import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias;
-import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated;
 import com.lanyuanxiaoyao.service.info.service.InfoService;
 import java.util.List;
 import org.eclipse.collections.api.factory.Lists;
@@ -229,17 +227,34 @@ public class InfoController {
     }
 
     @GetMapping("/all_flink_job_id")
-    public ImmutableList<Long> allFlinkJobId() {
+    public ImmutableList<Long> allFlinkJobId(
-        return infoService.allFlinkJobId();
+            @RequestParam(value = "key", required = false) String key,
+            @RequestParam(value = "alias", required = false) String alias
+    ) {
+        return infoService.allTableInfoSearchCache()
+                .select(cache -> StrUtil.isBlank(key) || StrUtil.contains(cache.getFlinkJobId().toString(), key))
+                .select(cache -> StrUtil.isBlank(alias) || StrUtil.contains(cache.getAlias(), alias))
+                .collect(TableInfoSearchCache::getFlinkJobId)
+                .distinct();
     }
 
     @GetMapping("/all_alias")
-    public ImmutableList<String> allAlias() {
+    public ImmutableList<String> allAlias(
-        return infoService.allAlias();
+            @RequestParam(value = "key", required = false) String key,
+            @RequestParam(value = "flink_job_id", required = false) String flinkJobId
+    ) {
+        return infoService.allTableInfoSearchCache()
+                .select(cache -> StrUtil.isBlank(key) || StrUtil.contains(cache.getAlias(), key))
+                .select(cache -> StrUtil.isBlank(flinkJobId) || StrUtil.contains(cache.getFlinkJobId().toString(), flinkJobId))
+                .collect(TableInfoSearchCache::getAlias)
+                .distinct();
     }
 
     @GetMapping("/all_hdfs")
-    public ImmutableList<String> allHdfs() {
+    public ImmutableList<String> allHdfs(@RequestParam(value = "key", required = false) String key) {
-        return infoService.allHdfs();
+        return infoService.allTableInfoSearchCache()
+                .select(cache -> StrUtil.isBlank(key) || StrUtil.contains(cache.getAlias(), key))
+                .collect(TableInfoSearchCache::getHdfs)
+                .distinct();
     }
 }

service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java

@@ -8,16 +8,16 @@ import club.kingon.sql.builder.entry.Column;
 import cn.hutool.core.lang.Pair;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
+import com.eshore.odcp.hudi.connector.SQLConstants;
 import com.eshore.odcp.hudi.connector.entity.FlinkJob;
 import com.eshore.odcp.hudi.connector.entity.SyncState;
 import com.eshore.odcp.hudi.connector.entity.TableMeta;
 import com.eshore.odcp.hudi.connector.utils.database.DatabaseService;
 import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
-import com.lanyuanxiaoyao.service.configuration.entity.info.CompactionMetrics;
+import com.lanyuanxiaoyao.service.configuration.entity.info.*;
-import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas;
-import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias;
-import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated;
 import com.lanyuanxiaoyao.service.info.configuration.SQLLoggerProvider;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Timestamp;
 import java.util.List;
 import org.eclipse.collections.api.factory.Lists;
@@ -28,11 +28,15 @@ import org.slf4j.LoggerFactory;
 import org.springframework.cache.annotation.CacheConfig;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowMapper;
 import org.springframework.retry.annotation.Retryable;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.support.TransactionTemplate;
 
 import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME;
+import static com.eshore.odcp.hudi.connector.SQLConstants.HudiCollectBuild.*;
+import static com.eshore.odcp.hudi.connector.SQLConstants.HudiCollectBuild.TbAppCollectTableInfo;
+import static com.eshore.odcp.hudi.connector.SQLConstants.HudiCollectBuild.TbAppFlinkJobConfig;
 
 /**
  * @author lanyuanxiaoyao
@@ -622,18 +626,18 @@ public class InfoService {
         int offset = limit * Math.max(page - 1, 0);
         Alias m1 = Alias.of(
                 SqlBuilder.selectAll()
-                        .from(TABLE_COMPACTION_METRICS)
+                        .from(TbAppHudiCompactionMetrics._alias_)
-                        .whereEq(TABLE_COMPACTION_METRICS_TYPE, "pre")
+                        .whereEq(TbAppHudiCompactionMetrics.TYPE_A, "pre")
-                        .andEq(TABLE_COMPACTION_METRICS_FLINK_JOB_ID, flinkJobId)
+                        .andEq(TbAppHudiCompactionMetrics.FLINK_JOB_ID_A, flinkJobId)
-                        .andEq(TABLE_COMPACTION_METRICS_ALIAS, alias),
+                        .andEq(TbAppHudiCompactionMetrics.ALIAS_A, alias),
                 "m1"
         );
         Alias m2 = Alias.of(
                 SqlBuilder.selectAll()
-                        .from(TABLE_COMPACTION_METRICS)
+                        .from(TbAppHudiCompactionMetrics._alias_)
-                        .whereEq(TABLE_COMPACTION_METRICS_TYPE, "complete")
+                        .whereEq(TbAppHudiCompactionMetrics.TYPE_A, "complete")
-                        .andEq(TABLE_COMPACTION_METRICS_FLINK_JOB_ID, flinkJobId)
+                        .andEq(TbAppHudiCompactionMetrics.FLINK_JOB_ID_A, flinkJobId)
-                        .andEq(TABLE_COMPACTION_METRICS_ALIAS, alias),
+                        .andEq(TbAppHudiCompactionMetrics.ALIAS_A, alias),
                 "m2"
         );
         return builder
@@ -762,54 +766,28 @@ public class InfoService {
     public Boolean existsTable(Long flinkJobId, String alias) {
         return mysqlJdbcTemplate.queryForObject(
                 SqlBuilder.select("count(*) > 0")
-                        .from(TABLE_INFO, TABLE_FLINK_JOB)
+                        .from(TbAppCollectTableInfo._alias_, TbAppFlinkJobConfig._alias_)
-                        .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID))
+                        .whereEq(TbAppCollectTableInfo.FLINK_JOB_ID_A, Column.as(TbAppFlinkJobConfig.ID_A))
-                        .andEq(TABLE_FLINK_JOB_ID, flinkJobId)
+                        .andEq(TbAppFlinkJobConfig.ID_A, flinkJobId)
-                        .andEq(TABLE_INFO_ALIAS, alias)
+                        .andEq(TbAppCollectTableInfo.ALIAS_A, alias)
-                        .andEq(TABLE_FLINK_JOB_STATUS, "y")
+                        .andEq(TbAppFlinkJobConfig.STATUS_A, "y")
-                        .andEq(TABLE_INFO_STATUS, "y")
+                        .andEq(TbAppCollectTableInfo.STATUS_A, "y")
                         .build(),
                 Boolean.class
         );
     }
 
-    @Cacheable(value = "all-flink-job-id", sync = true, cacheManager = "long-cache")
+    @Cacheable(value = "all-table-info-search-cache", sync = true, cacheManager = "long-cache")
     @Retryable(Throwable.class)
-    public ImmutableList<Long> allFlinkJobId() {
+    public ImmutableList<TableInfoSearchCache> allTableInfoSearchCache() {
-        return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList(
+        return Lists.immutable.ofAll(mysqlJdbcTemplate.query(
-                SqlBuilder.select(StrUtil.format("distinct {}", TABLE_FLINK_JOB_ID))
+                SqlBuilder.select(TbAppCollectTableInfo.FLINK_JOB_ID_A, TbAppCollectTableInfo.ALIAS_A, TbAppCollectTableInfo.TGT_HDFS_PATH_A)
-                        .from(TABLE_FLINK_JOB)
+                        .from(TbAppCollectTableInfo._alias_, TbAppFlinkJobConfig._alias_)
-                        .whereEq(TABLE_FLINK_JOB_STATUS, "y")
+                        .whereEq(TbAppCollectTableInfo.FLINK_JOB_ID_A, Column.as(TbAppFlinkJobConfig.ID_A))
+                        .andEq(TbAppFlinkJobConfig.STATUS_A, "y")
+                        .andEq(TbAppCollectTableInfo.STATUS_A, "y")
                         .build(),
-                Long.class
+                (rs, row) -> new TableInfoSearchCache(rs.getLong(1), rs.getString(2), rs.getString(3))
-        ));
-    }
-
-    @Cacheable(value = "all-alias", sync = true, cacheManager = "long-cache")
-    @Retryable(Throwable.class)
-    public ImmutableList<String> allAlias() {
-        return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList(
-                SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_ALIAS))
-                        .from(TABLE_INFO, TABLE_FLINK_JOB)
-                        .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID))
-                        .andEq(TABLE_FLINK_JOB_STATUS, "y")
-                        .andEq(TABLE_INFO_STATUS, "y")
-                        .build(),
-                String.class
-        ));
-    }
-
-    @Cacheable(value = "all-hdfs", sync = true, cacheManager = "long-cache")
-    @Retryable(Throwable.class)
-    public ImmutableList<String> allHdfs() {
-        return Lists.immutable.ofAll(mysqlJdbcTemplate.queryForList(
-                SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_TARGET_HDFS))
-                        .from(TABLE_INFO, TABLE_FLINK_JOB)
-                        .whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID))
-                        .andEq(TABLE_FLINK_JOB_STATUS, "y")
-                        .andEq(TABLE_INFO_STATUS, "y")
-                        .build(),
-                String.class
         ));
     }
 }

service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java

@@ -60,20 +60,6 @@ public class TableController extends BaseController {
         this.infoService = infoService;
         this.zookeeperService = zookeeperService;
         this.mapper = jackson2ObjectMapperBuilder.build();
-        this.cache = Caffeine.newBuilder()
-                .expireAfterAccess(Duration.ofMinutes(5))
-                .build(key -> {
-                    switch (key) {
-                        case "flink_job_id":
-                            return this.infoService.allFlinkJobId().collect(Objects::toString);
-                        case "alias":
-                            return this.infoService.allAlias();
-                        case "hdfs":
-                            return this.infoService.allHdfs();
-                        default:
-                            return Lists.immutable.empty();
-                    }
-                });
     }
 
     @GetMapping("list")
@@ -209,24 +195,36 @@ public class TableController extends BaseController {
         return AmisResponse.responseCrudData(vos, pageResponse.getTotal());
     }
 
-    private final LoadingCache<String, ImmutableList<String>> cache;
-
     @SuppressWarnings("DataFlowIssue")
     @GetMapping("all_flink_job_id")
-    public AmisCrudResponse allFlinkJobId(@RequestParam(value = "key", required = false) String key) {
+    public AmisCrudResponse allFlinkJobId(
+            @RequestParam(value = "key", required = false) String key,
+            @RequestParam(value = "alias", required = false) String alias
+    ) {
         if (StrUtil.isBlank(key)) {
             return AmisResponse.responseCrudData(Lists.immutable.empty());
         }
-        return AmisResponse.responseCrudData(cache.get("flink_job_id").select(id -> StrUtil.contains(id, key)));
+        if (StrUtil.isBlank(alias)) {
+            return AmisResponse.responseCrudData(infoService.allFlinkJobId(key).collect(Objects::toString));
+        } else {
+            return AmisResponse.responseCrudData(infoService.allFlinkJobId(key, alias).collect(Objects::toString));
+        }
     }
 
     @SuppressWarnings("DataFlowIssue")
     @GetMapping("all_alias")
-    public AmisCrudResponse allAlias(@RequestParam(value = "key", required = false) String key) {
+    public AmisCrudResponse allAlias(
-        if (StrUtil.isBlank(key)) {
+            @RequestParam(value = "key", required = false) String key,
+            @RequestParam(value = "flink_job_id", required = false) String flinkJobId
+    ) {
+        if (StrUtil.isBlank(key) && StrUtil.isBlank(flinkJobId)) {
             return AmisResponse.responseCrudData(Lists.immutable.empty());
         }
-        return AmisResponse.responseCrudData(cache.get("alias").select(id -> StrUtil.contains(id, key)));
+        if (StrUtil.isBlank(flinkJobId)) {
+            return AmisResponse.responseCrudData(infoService.allAlias(key));
+        } else {
+            return AmisResponse.responseCrudData(infoService.allAlias(key, flinkJobId));
+        }
     }
 
     @SuppressWarnings("DataFlowIssue")
@@ -235,6 +233,6 @@ public class TableController extends BaseController {
         if (StrUtil.isBlank(key)) {
             return AmisResponse.responseCrudData(Lists.immutable.empty());
         }
-        return AmisResponse.responseCrudData(cache.get("hdfs").select(id -> StrUtil.contains(id, key)));
+        return AmisResponse.responseCrudData(infoService.allHdfs(key));
     }
 }

web/components/common.js

@@ -1934,7 +1934,13 @@ function flinkJobIdTextInput(require = false) {
         placeholder: '通过 ID 搜索',
         clearable: true,
         required: require,
-        autoComplete: '${base}/table/all_flink_job_id?key=$term',
+        autoComplete: {
+            method: 'get',
+            url: '${base}/table/all_flink_job_id?key=$term',
+            data: {
+                alias: '${alias|default:undefined}'
+            }
+        },
     }
 }
 
@@ -1946,6 +1952,13 @@ function aliasTextInput(require = false) {
         placeholder: '通过别名搜索',
         clearable: true,
         required: require,
-        autoComplete: '${base}/table/all_alias?key=$term',
+        creatable: false,
+        autoComplete: {
+            method: 'get',
+            url: '${base}/table/all_alias?key=$term',
+            data: {
+                flink_job_id: '${flinkJobId|default:undefined}'
+            }
+        },
     }
 }