perf(api): 优化api指标存储和传输

统一命名格式，减少不必要的时间字段传输，统一字段命名，减少默认值的传输
2024-05-09 18:12:25 +08:00
parent 7fb5710cb3
commit def035fa90
8 changed files with 238 additions and 204 deletions
--- a/service-api/src/main/java/com/lanyuanxiaoyao/service/api/controller/ApiController.java
+++ b/service-api/src/main/java/com/lanyuanxiaoyao/service/api/controller/ApiController.java
@@ -11,7 +11,13 @@ import org.eclipse.collections.api.factory.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;

 /**
 * Rest 接口
@@ -47,45 +53,65 @@ public class ApiController {
    public void syncStart(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam(value = "cluster", required = false) String cluster,
-            @RequestParam(value = "application_id", required = false) String applicationId
+            @RequestParam("database") String database,
+            @RequestParam("schema") String schema,
+            @RequestParam("table") String table,
+            @RequestParam("cluster") String cluster,
+            @RequestParam("application_id") String applicationId
    ) {
        logger.info(makeMarker(flinkJobId, alias), "Sync start: {} {} {} {}", flinkJobId, alias, cluster, applicationId);
-        syncStateService.saveSyncStartTime(flinkJobId, alias, cluster, applicationId);
+        syncStateService.syncStart(flinkJobId, alias, database, schema, table, cluster, applicationId);
    }

-    @GetMapping("sync_checkpoint_state")
-    public void syncCheckpointState(
+    @GetMapping("sync_checkpoint")
+    public void syncCheckpoint(
+            @RequestParam("flink_job_id") Long flinkJobId,
+            @RequestParam("alias") String alias,
+            @RequestParam("message_id") String messageId
+    ) {
+        logger.info(makeMarker(flinkJobId, alias), "Sync checkpoint: {} {} {}", flinkJobId, alias, messageId);
+        syncStateService.syncCheckpoint(flinkJobId, alias, messageId);
+    }
+
+    @GetMapping("sync_latest_publish_time")
+    public void syncLatestPublishTime(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam("message_id") String messageId,
            @RequestParam("publish_time") Long publishTime
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Sync checkpoint state: {} {} {} {}", flinkJobId, alias, messageId, publishTime);
-        syncStateService.saveSyncCheckpointState(flinkJobId, alias, messageId, publishTime);
+        logger.info(makeMarker(flinkJobId, alias), "Sync latest publish time: {} {} {}", flinkJobId, alias, publishTime);
+        syncStateService.syncPublishTime(flinkJobId, alias, publishTime);
    }

-    @GetMapping("sync_operation_state")
-    public void syncOperationState(
+    @GetMapping("sync_latest_receive_time")
+    public void syncLatestReceiveTime(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam(value = "operation_time",
-                    required = false) Long operationTime
+            @RequestParam("receive_time") Long receiveTime
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Sync operation state: {} {} {}", flinkJobId, alias, operationTime);
-        syncStateService.saveSyncOperationState(flinkJobId, alias, operationTime);
+        logger.info(makeMarker(flinkJobId, alias), "Sync latest receive time: {} {} {}", flinkJobId, alias, receiveTime);
+        syncStateService.syncLatestReceiveTime(flinkJobId, alias, receiveTime);
+    }
+
+    @GetMapping("sync_latest_operation_time")
+    public void syncLatestOperationTime(
+            @RequestParam("flink_job_id") Long flinkJobId,
+            @RequestParam("alias") String alias,
+            @RequestParam("operation_time") Long operationTime
+    ) {
+        logger.info(makeMarker(flinkJobId, alias), "Sync latest operation time: {} {} {}", flinkJobId, alias, operationTime);
+        syncStateService.syncLatestOperationTime(flinkJobId, alias, operationTime);
    }

    @GetMapping("compaction_start")
    public void compactionStart(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam(value = "type", required = false) String type,
-            @RequestParam(value = "cluster", required = false) String cluster,
-            @RequestParam(value = "application_id", required = false) String applicationId
+            @RequestParam("cluster") String cluster,
+            @RequestParam("application_id") String applicationId
    ) {
        logger.info(makeMarker(flinkJobId, alias), "Compaction start: {} {} {} {}", flinkJobId, alias, cluster, applicationId);
-        syncStateService.saveCompactionStartTime(flinkJobId, alias, cluster, applicationId, null);
+        syncStateService.compactionStart(flinkJobId, alias, cluster, applicationId);
    }

    @PostMapping("compaction_pre_commit")
@@ -93,12 +119,12 @@ public class ApiController {
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
            @RequestParam("instant") String instant,
-            @RequestParam(value = "cluster", required = false) String cluster,
-            @RequestParam(value = "application_id", required = false) String applicationId,
+            @RequestParam("cluster") String cluster,
+            @RequestParam("application_id") String applicationId,
            @RequestBody HoodieCommitMetadata metadata
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Compaction state: {} {} {} {} {}", flinkJobId, alias, instant, cluster, applicationId);
-        syncStateService.saveCompactionMetrics(
+        logger.info(makeMarker(flinkJobId, alias), "Compaction pre commit: {} {} {} {} {}", flinkJobId, alias, instant, cluster, applicationId);
+        syncStateService.compactionCommit(
                flinkJobId,
                alias,
                "pre",
@@ -114,13 +140,13 @@ public class ApiController {
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
            @RequestParam("instant") String instant,
-            @RequestParam(value = "cluster", required = false) String cluster,
-            @RequestParam(value = "application_id", required = false) String applicationId,
+            @RequestParam("cluster") String cluster,
+            @RequestParam("application_id") String applicationId,
            @RequestBody HoodieCommitMetadata metadata
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Compaction state: {} {} {} {} {}", flinkJobId, alias, instant, cluster, applicationId);
-        syncStateService.saveCompactionFinishTime(flinkJobId, alias);
-        syncStateService.saveCompactionMetrics(
+        logger.info(makeMarker(flinkJobId, alias), "Compaction commit: {} {} {} {} {}", flinkJobId, alias, instant, cluster, applicationId);
+        syncStateService.compactionFinish(flinkJobId, alias);
+        syncStateService.compactionCommit(
                flinkJobId,
                alias,
                "complete",
@@ -135,12 +161,11 @@ public class ApiController {
    public void compactionFinish(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam("time") Long timestamp,
            @RequestParam("state") Boolean success,
            @RequestBody(required = false) String exception
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Compaction finish: {} {} {} {}", flinkJobId, alias, timestamp, success);
-        syncStateService.saveCompactionFinishTime(flinkJobId, alias, timestamp, StrUtil.isBlank(exception) ? null : exception);
+        logger.info(makeMarker(flinkJobId, alias), "Compaction finish: {} {} {}", flinkJobId, alias, success);
+        syncStateService.compactionFinish(flinkJobId, alias, StrUtil.isBlank(exception) ? null : exception);
    }

    @GetMapping("version_update")
@@ -148,29 +173,19 @@ public class ApiController {
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
            @RequestParam("version") String version,
-            @RequestParam(value = "opts", defaultValue = "") String opTs
+            @RequestParam("operation_time") String operationTime
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Version update: {} {} {}", flinkJobId, alias, version);
-        versionUpdateService.saveUpdateVersion(flinkJobId, alias, version, opTs);
+        logger.info(makeMarker(flinkJobId, alias), "Version update: {} {} {} {}", flinkJobId, alias, version, operationTime);
+        versionUpdateService.versionUpdate(flinkJobId, alias, version, operationTime);
    }

    @GetMapping("compaction_latest_operation_time")
    public void compactionLatestOperationTime(
            @RequestParam("flink_job_id") Long flinkJobId,
            @RequestParam("alias") String alias,
-            @RequestParam("latest_op_ts") Long latestOpTs
+            @RequestParam("operation_time") Long operationTime
    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Compaction latest operation time: {} {} {}", flinkJobId, alias, latestOpTs);
-        syncStateService.saveCompactionLatestOperationTime(flinkJobId, alias, latestOpTs);
-    }
-
-    @GetMapping("sync_receive_time")
-    public void syncReceiveTime(
-            @RequestParam("flink_job_id") Long flinkJobId,
-            @RequestParam("alias") String alias,
-            @RequestParam("receive_time") Long receiveTime
-    ) {
-        logger.info(makeMarker(flinkJobId, alias), "Source receive time: {} {} {}", flinkJobId, alias, receiveTime);
-        syncStateService.saveSyncReceiveTime(flinkJobId, alias, receiveTime);
+        logger.info(makeMarker(flinkJobId, alias), "Compaction latest operation time: {} {} {}", flinkJobId, alias, operationTime);
+        syncStateService.compactionLatestOperationTime(flinkJobId, alias, operationTime);
    }
 }
--- a/service-api/src/main/java/com/lanyuanxiaoyao/service/api/service/SyncStateService.java
+++ b/service-api/src/main/java/com/lanyuanxiaoyao/service/api/service/SyncStateService.java
@@ -3,7 +3,6 @@ package com.lanyuanxiaoyao.service.api.service;
 import club.kingon.sql.builder.SqlBuilder;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.util.ObjectUtil;
-import cn.hutool.core.util.StrUtil;
 import com.lanyuanxiaoyao.service.api.entity.HoodieCommitMetadata;
 import com.lanyuanxiaoyao.service.common.Constants;
 import java.time.Instant;
@@ -57,54 +56,95 @@ public class SyncStateService {
        return "";
    }

-    public void saveSyncStartTime(Long flinkJobId, String alias, String cluster, String applicationId) {
+    public void syncStart(Long flinkJobId, String alias, String database, String schema, String table, String cluster, String applicationId) {
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
                                TbAppHudiSyncState._origin_,
                                TbAppHudiSyncState.ID_O,
+                                TbAppHudiSyncState.SOURCE_DATABASE_O,
+                                TbAppHudiSyncState.SOURCE_SCHEMA_O,
+                                TbAppHudiSyncState.SOURCE_TABLE_O,
                                TbAppHudiSyncState.SOURCE_START_TIME_O,
                                TbAppHudiSyncState.SOURCE_CLUSTER_O,
                                TbAppHudiSyncState.SOURCE_APPLICATION_ID_O
                        )
                        .values()
-                        .addValue(null, null, null, null)
-                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.SOURCE_START_TIME_O)
+                        .addValue(null, null, null, null, null, null, null)
+                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.SOURCE_DATABASE_O)
+                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_SCHEMA_O)
+                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_TABLE_O)
+                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_START_TIME_O)
                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_CLUSTER_O)
                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_APPLICATION_ID_O)
                        .precompileSql(),
                syncStateId(flinkJobId, alias),
+                database,
+                schema,
+                table,
                now(),
                cluster,
                applicationId
        );
    }

-    public void saveSyncCheckpointState(Long flinkJobId, String alias, String messageId, Long publishTime) {
-        Date publishDate = Date.from(Instant.ofEpochMilli(publishTime));
+    public void syncCheckpoint(Long flinkJobId, String alias, String messageId) {
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
                                TbAppHudiSyncState._origin_,
                                TbAppHudiSyncState.ID_O,
                                TbAppHudiSyncState.MESSAGE_ID_O,
-                                TbAppHudiSyncState.SOURCE_PUBLISH_TIME_O,
                                TbAppHudiSyncState.SOURCE_CHECKPOINT_TIME_O
                        )
                        .values()
-                        .addValue(null, null, null, null)
+                        .addValue(null, null, null)
                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.MESSAGE_ID_O)
-                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_PUBLISH_TIME_O)
                        .addUpdateColumn(TbAppHudiSyncState.SOURCE_CHECKPOINT_TIME_O)
                        .precompileSql(),
                syncStateId(flinkJobId, alias),
                messageId,
-                publishDate,
                now()
        );
    }

-    public void saveSyncOperationState(Long flinkJobId, String alias, Long operationTime) {
+    public void syncPublishTime(Long flinkJobId, String alias, Long publishTime) {
+        Date publishDate = ObjectUtil.isNull(publishTime) ? null : Date.from(Instant.ofEpochMilli(publishTime));
+        jdbcTemplate.update(
+                SqlBuilder
+                        .insertInto(
+                                TbAppHudiSyncState._origin_,
+                                TbAppHudiSyncState.ID_O,
+                                TbAppHudiSyncState.SOURCE_PUBLISH_TIME_O
+                        )
+                        .values()
+                        .addValue(null, null)
+                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.SOURCE_PUBLISH_TIME_O)
+                        .precompileSql(),
+                syncStateId(flinkJobId, alias),
+                publishDate
+        );
+    }
+
+    public void syncLatestReceiveTime(Long flinkJobId, String alias, Long receiveTime) {
+        Date receiveDate = ObjectUtil.isNull(receiveTime) ? null : Date.from(Instant.ofEpochMilli(receiveTime));
+        jdbcTemplate.update(
+                SqlBuilder
+                        .insertInto(
+                                TbAppHudiSyncState._origin_,
+                                TbAppHudiSyncState.ID_O,
+                                TbAppHudiSyncState.SOURCE_RECEIVE_TIME_O
+                        )
+                        .values()
+                        .addValue(null, null)
+                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.SOURCE_RECEIVE_TIME_O)
+                        .precompileSql(),
+                syncStateId(flinkJobId, alias),
+                receiveDate
+        );
+    }
+
+    public void syncLatestOperationTime(Long flinkJobId, String alias, Long operationTime) {
        Date operationDate = ObjectUtil.isNull(operationTime) ? null : Date.from(Instant.ofEpochMilli(operationTime));
        jdbcTemplate.update(
                SqlBuilder
@@ -122,7 +162,7 @@ public class SyncStateService {
        );
    }

-    public void saveCompactionStartTime(Long flinkJobId, String alias, String cluster, String applicationId, String message) {
+    public void compactionStart(Long flinkJobId, String alias, String cluster, String applicationId) {
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
@@ -146,7 +186,7 @@ public class SyncStateService {
                        .precompileSql(),
                syncStateId(flinkJobId, alias),
                now(),
-                message,
+                null,
                Constants.COMPACTION_STATUS_START,
                now(),
                cluster,
@@ -154,12 +194,11 @@ public class SyncStateService {
        );
    }

-    public void saveCompactionFinishTime(Long flinkJobId, String alias) {
-        saveCompactionFinishTime(flinkJobId, alias, Instant.now().toEpochMilli(), "");
+    public void compactionFinish(Long flinkJobId, String alias) {
+        compactionFinish(flinkJobId, alias, "");
    }

-    public void saveCompactionFinishTime(Long flinkJobId, String alias, Long timestamp, String message) {
-        Date finishDate = Date.from(Instant.ofEpochMilli(timestamp));
+    public void compactionFinish(Long flinkJobId, String alias, String message) {
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
@@ -178,14 +217,14 @@ public class SyncStateService {
                        .addUpdateColumn(TbAppHudiSyncState.COMPACTION_STATUS_TIME_O)
                        .precompileSql(),
                syncStateId(flinkJobId, alias),
-                finishDate,
+                now(),
                message,
                Constants.COMPACTION_STATUS_FINISH,
                now()
        );
    }

-    public void saveCompactionMetrics(Long flinkJobId, String alias, String type, String instant, String cluster, String applicationId, HoodieCommitMetadata metadata) {
+    public void compactionCommit(Long flinkJobId, String alias, String type, String instant, String cluster, String applicationId, HoodieCommitMetadata metadata) {
        if (ObjectUtil.isNull(metadata)) {
            jdbcTemplate.update(
                    SqlBuilder
@@ -249,10 +288,8 @@ public class SyncStateService {
        }
    }

-    public void saveCompactionLatestOperationTime(Long flinkJobId, String alias, Long latestOperationTime) {
-        Date operationDate = (ObjectUtil.isNull(latestOperationTime) || latestOperationTime == 0)
-                ? null
-                : Date.from(Instant.ofEpochMilli(latestOperationTime));
+    public void compactionLatestOperationTime(Long flinkJobId, String alias, Long operationTime) {
+        Date operationDate = ObjectUtil.isNull(operationTime) ? null : Date.from(Instant.ofEpochMilli(operationTime));
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
@@ -262,39 +299,10 @@ public class SyncStateService {
                        )
                        .values()
                        .addValue(null, null)
-                        .onDuplicateKeyUpdateSetter(
-                                StrUtil.format(
-                                        "{} = if({} is null, ?, greatest({}, ?))",
-                                        TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O,
-                                        TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O,
-                                        TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O
-                                )
-                        )
+                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O)
                        .precompileSql(),
                syncStateId(flinkJobId, alias),
-                operationDate,
-                operationDate,
                operationDate
        );
    }
-
-    public void saveSyncReceiveTime(Long flinkJobId, String alias, Long sourceReceiveTime) {
-        Date sourceReceiveDate = (ObjectUtil.isNull(sourceReceiveTime) || sourceReceiveTime == 0)
-                ? null
-                : Date.from(Instant.ofEpochMilli(sourceReceiveTime));
-        jdbcTemplate.update(
-                SqlBuilder
-                        .insertInto(
-                                TbAppHudiSyncState._origin_,
-                                TbAppHudiSyncState.ID_O,
-                                TbAppHudiSyncState.SOURCE_RECEIVE_TIME_O
-                        )
-                        .values()
-                        .addValue(null, null)
-                        .onDuplicateKeyUpdateColumn(TbAppHudiSyncState.SOURCE_RECEIVE_TIME_O)
-                        .precompileSql(),
-                syncStateId(flinkJobId, alias),
-                sourceReceiveDate
-        );
-    }
 }
--- a/service-api/src/main/java/com/lanyuanxiaoyao/service/api/service/VersionUpdateService.java
+++ b/service-api/src/main/java/com/lanyuanxiaoyao/service/api/service/VersionUpdateService.java
@@ -30,7 +30,7 @@ public class VersionUpdateService {
        return Date.from(Instant.now());
    }

-    public void saveUpdateVersion(Long flinkJobId, String alias, String version, String opTs) {
+    public void versionUpdate(Long flinkJobId, String alias, String version, String operationTime) {
        jdbcTemplate.update(
                SqlBuilder
                        .insertInto(
@@ -47,7 +47,7 @@ public class VersionUpdateService {
                flinkJobId,
                alias,
                version,
-                opTs,
+                operationTime,
                now()
        );
    }