lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitor): 增加table_version表和compaction_metrics表的定时清理

Browse Source
This commit is contained in:
v-zhangjc9
2024-05-23 11:52:21 +08:00
parent 496b09d2de
commit 960b020052
7 changed files with 162 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/CleanController.java Normal file
View File

@@ -0,0 +1,37 @@
package com.lanyuanxiaoyao.service.info.controller;
import com.lanyuanxiaoyao.service.info.service.CleanService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* Version
*
* @author lanyuanxiaoyao
* @date 2024-01-03
*/
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
@RestController
@RequestMapping("/info")
public class CleanController {
private static final Logger logger = LoggerFactory.getLogger(CleanController.class);
private final CleanService cleanService;
public CleanController(CleanService cleanService) {
this.cleanService = cleanService;
}
@GetMapping("/clean_compaction_metrics")
public Integer cleanCompactionMetrics() {
return cleanService.cleanCompactionMetrics();
}
@GetMapping("/clean_table_version")
public Integer cleanTableVersion() {
return cleanService.cleanTableVersion();
}
}

43
service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/CleanService.java Normal file
View File

@@ -0,0 +1,43 @@
package com.lanyuanxiaoyao.service.info.service;
import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.entry.Column;
import com.lanyuanxiaoyao.service.common.SQLConstants.HudiCollectBuild.TbAppCollectTableVersion;
import com.lanyuanxiaoyao.service.common.SQLConstants.HudiCollectBuild.TbAppHudiCompactionMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
/**
* 清理数据库
*
* @author lanyuanxiaoyao
* @date 2024-05-23
*/
public class CleanService extends BaseService {
private static final Logger logger = LoggerFactory.getLogger(CleanService.class);
private static final Column before7Days = Column.as("DATE_SUB(CURDATE(), INTERVAL 7 DAY)");
private static final Column before14Days = Column.as("DATE_SUB(CURDATE(), INTERVAL 14 DAY)");
private final JdbcTemplate mysqlJdbcTemplate;
public CleanService(JdbcTemplate mysqlJdbcTemplate) {
this.mysqlJdbcTemplate = mysqlJdbcTemplate;
}
public Integer cleanCompactionMetrics() {
return mysqlJdbcTemplate.update(
SqlBuilder.delete(TbAppHudiCompactionMetrics._origin_)
.whereLe(TbAppHudiCompactionMetrics.UPDATE_TIME_O, before7Days)
.build()
);
}
public Integer cleanTableVersion() {
return mysqlJdbcTemplate.update(
SqlBuilder.delete(TbAppCollectTableVersion._origin_)
.whereLe(TbAppCollectTableVersion.UPDATE_TIME_O, before14Days)
.build()
);
}
}

20
service-info-query/src/test/java/com/test/SqlBuilderTests.java
View File

@@ -69,23 +69,9 @@ public class SqlBuilderTests {
public static void main(String[] args) {
System.out.println(SqlUtil.formatSql(
SqlBuilder
.insertInto(
TbAppHudiSyncState._origin_,
TbAppHudiSyncState.ID_O,
TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O
)
.values()
.addValue(null, null)
.onDuplicateKeyUpdateSetter(
StrUtil.format(
"{} = if({} is null, ?, greatest({}, ?))",
TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O,
TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O,
TbAppHudiSyncState.COMPACTION_LATEST_OP_TS_O
)
)
.precompileSql()
SqlBuilder.delete(SQLConstants.HudiCollectBuild.TbAppCollectTableVersion._origin_)
.whereLe(SQLConstants.HudiCollectBuild.TbAppCollectTableVersion.UPDATE_TIME_O, Column.as("DATE_SUB(CURDATE(), INTERVAL 14 DAY)"))
.build()
));
}
}