lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(web): 增加 Overview 页面

Browse Source 
方便总览全局情况，跨页面查看信息多有不便
This commit is contained in:
兰缘小妖
2023-06-12 18:29:28 +08:00
parent d85c3a4864
commit 4e963fa537
8 changed files with 550 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/controller/InfoController.java
View File

@@ -9,7 +9,6 @@ import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas;
import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias;
import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated;
import com.lanyuanxiaoyao.service.info.service.InfoService;
import java.util.List;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
@@ -19,6 +18,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* 信息接口
*
@@ -36,6 +37,26 @@ public class InfoController {
this.infoService = infoService;
}
@GetMapping("table_total")
public Long tableTotal() {
return infoService.tableTotal();
}
@GetMapping("hudi_total")
public Long hudiTotal() {
return infoService.hudiTotal();
}
@GetMapping("focus_count")
public Long focusCount() {
return infoService.focusCount();
}
@GetMapping("normal_count")
public Long normalCount() {
return infoService.normalCount();
}
@GetMapping("/job_id_alias")
public PageResponse<JobIdAndAlias> jobIdAndAlias(
@RequestParam(value = "page", defaultValue = "1") Integer page,
@@ -123,4 +144,24 @@ public class InfoController {
public ImmutableList<String> updatedVersionTables() {
return infoService.nonUpdatedVersionTables();
}
@GetMapping("/un_scheduled_normal_table_count")
public Long unScheduledNormalTableCount(@RequestParam("version") String version) {
return infoService.unScheduledNormalTableCount(version);
}
@GetMapping("/un_scheduled_focus_table_count")
public Long unScheduledFocusTableCount(@RequestParam("version") String version) {
return infoService.unScheduledFocusTableCount(version);
}
@GetMapping("/un_receive_version_normal_table_count")
public Long unReceiveVersionNormalTableCount(@RequestParam("version") String version) {
return infoService.unReceiveVersionNormalTableCount(version);
}
@GetMapping("/un_receive_version_focus_table_count")
public Long unReceiveVersionFocusTableCount(@RequestParam("version") String version) {
return infoService.unReceiveVersionFocusTableCount(version);
}
}

122
service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java
View File

@@ -16,7 +16,6 @@ import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas;
import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias;
import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated;
import java.util.List;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
@@ -29,6 +28,8 @@ import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import java.util.List;
import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME;
/**
@@ -224,6 +225,8 @@ public class InfoService {
private static final Alias TABLE_INFO = Alias.of(StrUtil.format("{}.tb_app_collect_table_info", DATABASE_NAME), "tacti");
private static final String TABLE_INFO_FLINK_JOB_ID = column(TABLE_INFO, "flink_job_id");
private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias");
private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority");
private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status");
private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss");
private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id");
@@ -247,6 +250,7 @@ public class InfoService {
.join(TABLE_VERSION)
.onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID))
.andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS))
.andEq(TABLE_INFO_STATUS, "y")
.join(TABLE_SYNC_STATE)
.on(StrUtil.format("{} = CONCAT({}, '-', {})", TABLE_SYNC_STATE_ID, TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS))
.whereLike(ObjectUtil.isNotNull(flinkJobId), TABLE_INFO_FLINK_JOB_ID, flinkJobId)
@@ -338,4 +342,120 @@ public class InfoService {
.withMetadata("unScheduled", scheduleCount.getOrDefault(false, 0));
});
}
@Cacheable(value = "un-receive-version-normal-table-count", sync = true)
@Retryable(Throwable.class)
public Long unReceiveVersionNormalTableCount(String version) {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select(COUNT)
.from(TABLE_INFO)
.whereLt(TABLE_INFO_PRIORITY, 10000)
.andEq(TABLE_INFO_STATUS, "y")
.andNotIn(
StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS),
SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS))
.from(TABLE_VERSION)
.whereEq(TABLE_VERSION_VERSION, version)
)
.build(),
Long.class
);
}
@Cacheable(value = "un-receive-version-focus-table-count", sync = true)
@Retryable(Throwable.class)
public Long unReceiveVersionFocusTableCount(String version) {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select(COUNT)
.from(TABLE_INFO)
.whereGe(TABLE_INFO_PRIORITY, 10000)
.andEq(TABLE_INFO_STATUS, "y")
.andNotIn(
StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS),
SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS))
.from(TABLE_VERSION)
.whereEq(TABLE_VERSION_VERSION, version)
)
.build(),
Long.class
);
}
@Cacheable(value = "un_scheduled_normal_table_count", sync = true)
@Retryable(Throwable.class)
public Long unScheduledNormalTableCount(String version) {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select(COUNT)
.from(TABLE_INFO)
.join(TABLE_VERSION)
.onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID))
.andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS))
.whereLt(TABLE_INFO_PRIORITY, 10000)
.andEq(TABLE_VERSION_VERSION, version)
.andEq(TABLE_INFO_STATUS, "y")
.build(),
Long.class
);
}
@Cacheable(value = "un_scheduled_focus_table_count", sync = true)
@Retryable(Throwable.class)
public Long unScheduledFocusTableCount(String version) {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select(COUNT)
.from(TABLE_INFO)
.join(TABLE_VERSION)
.onEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_VERSION_FLINK_JOB_ID))
.andEq(TABLE_INFO_ALIAS, Column.as(TABLE_VERSION_ALIAS))
.whereGe(TABLE_INFO_PRIORITY, 10000)
.andEq(TABLE_VERSION_VERSION, version)
.andEq(TABLE_INFO_STATUS, "y")
.build(),
Long.class
);
}
@Cacheable(value = "table-total", sync = true)
@Retryable(Throwable.class)
public Long tableTotal() {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select("count(distinct concat(src_schema, src_table))").from(TABLE_INFO).whereEq(TABLE_INFO_STATUS, "y").build(),
Long.class
);
}
@Cacheable(value = "hudi-total", sync = true)
@Retryable(Throwable.class)
public Long hudiTotal() {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select("count(distinct tgt_hdfs_path) as count").from(TABLE_INFO).whereEq(TABLE_INFO_STATUS, "y").build(),
Long.class
);
}
@Cacheable(value = "focus-count", sync = true)
@Retryable(Throwable.class)
public Long focusCount() {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select("count(distinct concat(src_schema, src_table))")
.from(TABLE_INFO)
.whereGe(TABLE_INFO_PRIORITY, 1000)
.andEq(TABLE_INFO_STATUS, "y")
.build(),
Long.class
);
}
@Cacheable(value = "normal-count", sync = true)
@Retryable(Throwable.class)
public Long normalCount() {
return mysqlJdbcTemplate.queryForObject(
SqlBuilder.select("count(distinct concat(src_schema, src_table))")
.from(TABLE_INFO)
.whereLt(TABLE_INFO_PRIORITY, 1000)
.andEq(TABLE_INFO_STATUS, "y")
.build(),
Long.class
);
}
}

31
service-info-query/src/test/java/SqlBuilderTests.java
View File

@@ -1,9 +1,7 @@
import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.entry.Alias;
import club.kingon.sql.builder.entry.Column;
import cn.hutool.core.util.StrUtil;
import cn.hutool.db.sql.SqlFormatter;
import org.eclipse.collections.api.factory.Lists;
import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME;
@@ -12,8 +10,21 @@ import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME;
* @date 2023-06-07
*/
public class SqlBuilderTests {
private static final String COUNT = "count(*)";
private static final Alias TABLE_VERSION = Alias.of(StrUtil.format("{}.tb_app_collect_table_version", DATABASE_NAME), "tactv");
private static final String TABLE_VERSION_FLINK_JOB_ID = column(TABLE_VERSION, "flink_job_id");
private static final String TABLE_VERSION_ALIAS = column(TABLE_VERSION, "alias");
private static final String TABLE_VERSION_VERSION = column(TABLE_VERSION, "version");
private static final String TABLE_VERSION_SCHEDULED = column(TABLE_VERSION, "scheduled");
private static final Alias TABLE_INFO = Alias.of(StrUtil.format("{}.tb_app_collect_table_info", DATABASE_NAME), "tacti");
private static final String TABLE_INFO_FLINK_JOB_ID = column(TABLE_INFO, "flink_job_id");
private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias");
private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority");
private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss");
private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id");
private static String column(Alias table, String column) {
return StrUtil.format("{}.{}", table.getAlias(), column);
@@ -21,15 +32,15 @@ public class SqlBuilderTests {
public static void main(String[] args) {
System.out.println(SqlFormatter.format(
SqlBuilder
.select("count(*)")
SqlBuilder.select(COUNT)
.from(TABLE_INFO)
.join(TABLE_VERSION)
.onEq(column(TABLE_INFO, "flink_job_id"), Column.as(column(TABLE_VERSION, "flink_job_id")))
.andEq(column(TABLE_INFO, "alias"), Column.as(column(TABLE_VERSION, "alias")))
.whereEq(false, "a", "b")
.andEq("b", "c")
.andIn("d", Lists.immutable.empty())
.whereGe(TABLE_INFO_PRIORITY, 10000)
.andNotIn(
StrUtil.format("concat({}, {})", TABLE_INFO_FLINK_JOB_ID, TABLE_INFO_ALIAS),
SqlBuilder.select(StrUtil.format("concat({}, {})", TABLE_VERSION_FLINK_JOB_ID, TABLE_VERSION_ALIAS))
.from(TABLE_VERSION)
.whereEq(TABLE_VERSION_VERSION, "20230611")
)
.build()
));
}