feat(scheduler): 使用简化的table_meta加速查询

service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java

@@ -6,7 +6,13 @@ import com.lanyuanxiaoyao.service.common.entity.TableMeta;
 import com.lanyuanxiaoyao.service.common.exception.ConfigException;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -17,7 +23,7 @@ import java.util.stream.Collectors;
  * @date 2021-12-01
  */
 public class TableMetaHelper {
-    //private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
+    // private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
 
     public static String tableMetaSql(String database) {
         return tableMetaSql(database, true, false);
@@ -227,7 +233,7 @@ public class TableMetaHelper {
     public static List<TableMeta> from(ResultSet rs) throws SQLException {
         List<TableMeta> results = new ArrayList<>();
         List<TableMeta.RowMeta> metaList = new ArrayList<>();
-        while (rs.next( )) {
+        while (rs.next()) {
             metaList.add(
                     TableMeta.RowMeta.builder()
                             .dsName(rs.getString(1))
@@ -571,6 +577,14 @@ public class TableMetaHelper {
     }
 
     public static boolean existsTag(TableMeta meta, String tag) {
-        return meta.getTags() != null && meta.getTags().contains(tag);
+        return existsTag(meta.getTags(), tag);
+    }
+
+    public static boolean existsTag(String sourceTags, String tag) {
+        return existsTag(Arrays.asList(sourceTags.split(",")), tag);
+    }
+
+    public static boolean existsTag(List<String> sourceTags, String tag) {
+        return sourceTags != null && sourceTags.contains(tag);
     }
 }

service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/info/SimpleTableMeta.java

@@ -12,22 +12,30 @@ public class SimpleTableMeta {
     private String alias;
     private String schema;
     private String table;
+    private Integer priority;
+    private String targetTableType;
     private String targetDb;
     private String targetTable;
     private String targetHdfs;
+    private String tags;
+    private Integer bucketNumber;
 
     public SimpleTableMeta() {
     }
 
-    public SimpleTableMeta(Long flinkJobId, String flinkJobName, String alias, String schema, String table, String targetDb, String targetTable, String targetHdfs) {
+    public SimpleTableMeta(Long flinkJobId, String flinkJobName, String alias, String schema, String table, Integer priority, String targetTableType, String targetDb, String targetTable, String targetHdfs, String tags, Integer bucketNumber) {
         this.flinkJobId = flinkJobId;
         this.flinkJobName = flinkJobName;
         this.alias = alias;
         this.schema = schema;
         this.table = table;
+        this.priority = priority;
+        this.targetTableType = targetTableType;
         this.targetDb = targetDb;
         this.targetTable = targetTable;
         this.targetHdfs = targetHdfs;
+        this.tags = tags;
+        this.bucketNumber = bucketNumber;
     }
 
     public Long getFlinkJobId() {
@@ -50,6 +58,14 @@ public class SimpleTableMeta {
         return table;
     }
 
+    public Integer getPriority() {
+        return priority;
+    }
+
+    public String getTargetTableType() {
+        return targetTableType;
+    }
+
     public String getTargetDb() {
         return targetDb;
     }
@@ -62,6 +78,14 @@ public class SimpleTableMeta {
         return targetHdfs;
     }
 
+    public String getTags() {
+        return tags;
+    }
+
+    public Integer getBucketNumber() {
+        return bucketNumber;
+    }
+
     @Override
     public String toString() {
         return "SimpleTableMeta{" +
@@ -70,9 +94,13 @@ public class SimpleTableMeta {
                ", alias='" + alias + '\'' +
                ", schema='" + schema + '\'' +
                ", table='" + table + '\'' +
+               ", priority=" + priority +
+               ", targetTableType='" + targetTableType + '\'' +
                ", targetDb='" + targetDb + '\'' +
                ", targetTable='" + targetTable + '\'' +
                ", targetHdfs='" + targetHdfs + '\'' +
+               ", tags='" + tags + '\'' +
+               ", bucketNumber=" + bucketNumber +
                '}';
     }
 }

service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/TableMetaService.java

@@ -593,9 +593,13 @@ public class TableMetaService extends BaseService {
                                         TbAppCollectTableInfo.ALIAS_A,
                                         TbAppCollectTableInfo.SRC_SCHEMA_A,
                                         TbAppCollectTableInfo.SRC_TABLE_A,
+                                        TbAppCollectTableInfo.PRIORITY_A,
+                                        TbAppCollectTableInfo.TGT_TABLE_TYPE_A,
                                         TbAppCollectTableInfo.TGT_DB_A,
                                         TbAppCollectTableInfo.TGT_TABLE_A,
-                                        TbAppCollectTableInfo.TGT_HDFS_PATH_A
+                                        TbAppCollectTableInfo.TGT_HDFS_PATH_A,
+                                        TbAppCollectTableInfo.TAGS_A,
+                                        TbAppCollectTableInfo.BUCKET_NUMBER_A
                                 ),
                                 flinkJobId,
                                 alias
@@ -606,9 +610,13 @@ public class TableMetaService extends BaseService {
                                 rs.getString(3),
                                 rs.getString(4),
                                 rs.getString(5),
-                                rs.getString(6),
+                                rs.getInt(6),
                                 rs.getString(7),
-                                rs.getString(8)
+                                rs.getString(8),
+                                rs.getString(9),
+                                rs.getString(10),
+                                rs.getString(11),
+                                rs.getInt(12)
                         )
                 )
         );

service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/controller/ScheduleController.java

@@ -138,7 +138,7 @@ public class ScheduleController {
                 infoService,
                 hudiService,
                 mapper,
-                meta -> StrUtil.equals(meta.getAlias(), alias) && NumberUtil.equals(meta.getJob().getId(), flinkJobId),
+                meta -> StrUtil.equals(meta.getAlias(), alias) && NumberUtil.equals(meta.getFlinkJobId(), flinkJobId),
                 "Schedule manually",
                 metadata.toImmutable()
         );
@@ -155,7 +155,7 @@ public class ScheduleController {
                 infoService,
                 hudiService,
                 mapper,
-                meta -> keys.contains(StrUtil.format("{}-{}", meta.getJob().getId(), meta.getAlias())),
+                meta -> keys.contains(StrUtil.format("{}-{}", meta.getFlinkJobId(), meta.getAlias())),
                 "Schedule manually"
         );
     }

service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/quartz/compaction/FocusUnVersionUpdateScheduleJob.java

@@ -48,7 +48,7 @@ public class FocusUnVersionUpdateScheduleJob extends BaseScheduleJob {
                 hudiService,
                 mapper,
                 meta -> meta.getPriority() >= 10000
-                        && unUpdateVersionTableIds.contains(StrUtil.format("{}-{}", meta.getJob().getId(), meta.getAlias())),
+                        && unUpdateVersionTableIds.contains(StrUtil.format("{}-{}", meta.getFlinkJobId(), meta.getAlias())),
                 comment
         );
     }

service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/quartz/compaction/OdsFocusScheduleJob.java

@@ -46,7 +46,7 @@ public class OdsFocusScheduleJob extends BaseScheduleJob {
                 infoService,
                 hudiService,
                 mapper,
-                meta -> TableMetaHelper.existsTag(meta, Constants.TAGS_ODS_FOCUS),
+                meta -> TableMetaHelper.existsTag(meta.getTags(), Constants.TAGS_ODS_FOCUS),
                 comment
         );
     }

service-scheduler/src/main/java/com/lanyuanxiaoyao/service/scheduler/utils/ScheduleHelper.java

@@ -7,10 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.loki4j.slf4j.marker.LabelMarker;
 import com.lanyuanxiaoyao.service.common.Constants;
 import com.lanyuanxiaoyao.service.common.entity.SyncState;
-import com.lanyuanxiaoyao.service.common.entity.TableMeta;
 import com.lanyuanxiaoyao.service.common.entity.compaction.ScheduleJob;
 import com.lanyuanxiaoyao.service.common.utils.TableMetaHelper;
 import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
+import com.lanyuanxiaoyao.service.configuration.entity.info.SimpleTableMeta;
 import com.lanyuanxiaoyao.service.configuration.entity.queue.QueueItem;
 import com.lanyuanxiaoyao.service.configuration.utils.QueueUtil;
 import com.lanyuanxiaoyao.service.forest.service.HudiService;
@@ -42,7 +42,7 @@ public class ScheduleHelper {
             InfoService infoService,
             HudiService hudiService,
             ObjectMapper mapper,
-            Predicate<TableMeta> predicate,
+            Predicate<SimpleTableMeta> predicate,
             String comment
     ) {
         schedule(discoveryClient, infoService, hudiService, mapper, predicate, comment, Maps.immutable.empty());
@@ -53,57 +53,57 @@ public class ScheduleHelper {
             InfoService infoService,
             HudiService hudiService,
             ObjectMapper mapper,
-            Predicate<TableMeta> predicate,
+            Predicate<SimpleTableMeta> predicate,
             String comment,
             ImmutableMap<String, String> metadata
     ) {
         String batchId = IdUtil.nanoId(10);
-        infoService.tableMetaList()
+        infoService.simpleTableMetas()
                 // 只调度 MOR 表
-                .select(meta -> StrUtil.equals(Constants.MOR, meta.getHudi().getTargetTableType()))
+                .select(meta -> StrUtil.equals(Constants.MOR, meta.getTargetTableType()))
                 .asParallel(ExecutorProvider.EXECUTORS, 1)
                 .select(predicate::test)
                 // 没有 Hudi 表的过滤掉
                 .select(meta -> {
                     try {
-                        return hudiService.existsHudiTable(meta.getJob().getId(), meta.getAlias());
+                        return hudiService.existsHudiTable(meta.getFlinkJobId(), meta.getAlias());
                     } catch (Throwable throwable) {
-                        logger.error(makeMarker(meta.getJob().getId(), meta.getAlias()), "Get hudi status failure", throwable);
+                        logger.error(makeMarker(meta.getFlinkJobId(), meta.getAlias()), "Get hudi status failure", throwable);
                     }
                     return false;
                 })
                 // 没有压缩计划的过滤掉
                 .select(meta -> {
                     try {
-                        return hudiService.existsCompactionPlan(meta.getJob().getId(), meta.getAlias());
+                        return hudiService.existsCompactionPlan(meta.getFlinkJobId(), meta.getAlias());
                     } catch (Throwable throwable) {
-                        logger.error(makeMarker(meta.getJob().getId(), meta.getAlias()), "Get compaction status failure", throwable);
+                        logger.error(makeMarker(meta.getFlinkJobId(), meta.getAlias()), "Get compaction status failure", throwable);
                     }
                     return false;
                 })
                 // 拒绝不压缩标志的任务
-                .reject(meta -> TableMetaHelper.existsTag(meta, Constants.TAGS_NO_COMPACT))
+                .reject(meta -> TableMetaHelper.existsTag(meta.getTags(), Constants.TAGS_NO_COMPACT))
                 // 拒绝不调度压缩标志的任务
-                .reject(meta -> TableMetaHelper.existsTag(meta, Constants.TAGS_NO_SCHEDULE_COMPACT))
+                .reject(meta -> TableMetaHelper.existsTag(meta.getTags(), Constants.TAGS_NO_SCHEDULE_COMPACT))
                 .collect(meta -> {
                     long compactionDuration = 0L;
                     try {
                         // 计算压缩耗时
-                        SyncState syncState = infoService.syncStateDetail(meta.getJob().getId(), meta.getAlias());
+                        SyncState syncState = infoService.syncStateDetail(meta.getFlinkJobId(), meta.getAlias());
                         if (ObjectUtil.isNotNull(syncState)
                             && ObjectUtil.isNotNull(syncState.getCompactionFinishTime())
                             && ObjectUtil.isNotNull(syncState.getCompactionStartTime())) {
                             compactionDuration = syncState.getCompactionFinishTime() - syncState.getCompactionStartTime();
                         }
                     } catch (Throwable e) {
-                        logger.error(makeMarker(meta.getJob().getId(), meta.getAlias()), "Get sync state failure for {} {}", meta.getJob().getId(), meta.getAlias());
+                        logger.error(makeMarker(meta.getFlinkJobId(), meta.getAlias()), "Get sync state failure for {} {}", meta.getFlinkJobId(), meta.getAlias());
                     }
                     return new TableMetaWrapper(meta, compactionDuration);
                 })
                 .toSortedList(
                         Comparator
                                 // 比较 Bucket 数，数量大的在前面
-                                .comparing(TableMetaWrapper::getBucketIndexNumber, Comparator.reverseOrder())
+                                .comparing(TableMetaWrapper::getBucketNumber, Comparator.reverseOrder())
                                 // 比较压缩耗时，压缩耗时长的在前面
                                 .thenComparing(TableMetaWrapper::getCompactionDuration, Comparator.reverseOrder()))
                 .collect(meta -> new QueueItem<>(
@@ -131,28 +131,28 @@ public class ScheduleHelper {
     }
 
     private static final class TableMetaWrapper {
-        private final TableMeta tableMeta;
+        private final SimpleTableMeta meta;
         private final Long compactionDuration;
 
-        private TableMetaWrapper(TableMeta tableMeta, Long compactionDuration) {
+        private TableMetaWrapper(SimpleTableMeta meta, Long compactionDuration) {
-            this.tableMeta = tableMeta;
+            this.meta = meta;
             this.compactionDuration = compactionDuration;
         }
 
         public Long getFlinkJobId() {
-            return tableMeta.getJob().getId();
+            return meta.getFlinkJobId();
         }
 
         public String getAlias() {
-            return tableMeta.getAlias();
+            return meta.getAlias();
         }
 
-        public Integer getBucketIndexNumber() {
+        public Integer getBucketNumber() {
-            return tableMeta.getHudi().getBucketIndexNumber();
+            return meta.getBucketNumber();
         }
 
         public Integer getPriority() {
-            return tableMeta.getPriority();
+            return meta.getPriority();
         }
 
         public Long getCompactionDuration() {