lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(web): 增加压缩结果显示

Browse Source
This commit is contained in:
兰缘小妖
2023-06-14 17:20:47 +08:00
parent 6abbd3d46a
commit 29c328a899
6 changed files with 212 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/info/CompactionMetrics.java
View File

@@ -1,7 +1,5 @@
package com.lanyuanxiaoyao.service.configuration.entity.info;
import java.time.LocalDateTime;
/**
* @author lanyuanxiaoyao
* @date 2023-06-14
@@ -15,11 +13,13 @@ public class CompactionMetrics {
private Boolean complete;
private Long startedTime;
private Long finishedTime;
private Statistics before;
private Statistics after;
public CompactionMetrics() {
}
public CompactionMetrics(Long flinkJobId, String alias, String applicationId, String cluster, String compactionPlanInstant, Boolean complete, Long startedTime, Long finishedTime) {
public CompactionMetrics(Long flinkJobId, String alias, String applicationId, String cluster, String compactionPlanInstant, Boolean complete, Long startedTime, Long finishedTime, Statistics before, Statistics after) {
this.flinkJobId = flinkJobId;
this.alias = alias;
this.applicationId = applicationId;
@@ -28,6 +28,8 @@ public class CompactionMetrics {
this.complete = complete;
this.startedTime = startedTime;
this.finishedTime = finishedTime;
this.before = before;
this.after = after;
}
public Long getFlinkJobId() {
@@ -62,6 +64,14 @@ public class CompactionMetrics {
return finishedTime;
}
public Statistics getBefore() {
return before;
}
public Statistics getAfter() {
return after;
}
@Override
public String toString() {
return "CompactionMetrics{" +
@@ -70,9 +80,56 @@ public class CompactionMetrics {
", applicationId='" + applicationId + '\'' +
", cluster='" + cluster + '\'' +
", compactionPlanInstant='" + compactionPlanInstant + '\'' +
", isComplete=" + complete +
", complete=" + complete +
", startedTime=" + startedTime +
", finishedTime=" + finishedTime +
", before=" + before +
", after=" + after +
'}';
}
public static class Statistics {
private Long totalScanTime;
private Long totalLogFilesCompacted;
private Long totalLogFilesSize;
private Long totalRecordsDeleted;
private Long totalRecordsUpdated;
private Long totalRecordsCompacted;
public Statistics() {
}
public Statistics(Long totalScanTime, Long totalLogFilesCompacted, Long totalLogFilesSize, Long totalRecordsDeleted, Long totalRecordsUpdated, Long totalRecordsCompacted) {
this.totalScanTime = totalScanTime;
this.totalLogFilesCompacted = totalLogFilesCompacted;
this.totalLogFilesSize = totalLogFilesSize;
this.totalRecordsDeleted = totalRecordsDeleted;
this.totalRecordsUpdated = totalRecordsUpdated;
this.totalRecordsCompacted = totalRecordsCompacted;
}
public Long getTotalScanTime() {
return totalScanTime;
}
public Long getTotalLogFilesCompacted() {
return totalLogFilesCompacted;
}
public Long getTotalLogFilesSize() {
return totalLogFilesSize;
}
public Long getTotalRecordsDeleted() {
return totalRecordsDeleted;
}
public Long getTotalRecordsUpdated() {
return totalRecordsUpdated;
}
public Long getTotalRecordsCompacted() {
return totalRecordsCompacted;
}
}
}

34
service-info-query/src/main/java/com/lanyuanxiaoyao/service/info/service/InfoService.java
View File

@@ -640,7 +640,19 @@ public class InfoService {
"m1.compaction_plan_instant",
"m2.type is not null as is_complete",
"m1.update_time as started_time",
"m2.update_time as finished_time"
"m2.update_time as finished_time",
"m1.total_scan_time as pre_total_scan_time",
"m1.total_log_files_compacted as pre_total_log_files_compacted",
"m1.total_log_files_size as pre_total_log_files_size",
"m1.total_records_deleted as pre_total_records_deleted",
"m1.total_records_updated as pre_total_records_updated",
"m1.total_records_compacted as pre_total_records_compacted",
"m2.total_scan_time as complete_total_scan_time",
"m2.total_log_files_compacted as complete_total_log_files_compacted",
"m2.total_log_files_size as complete_total_log_files_size",
"m2.total_records_deleted as complete_total_records_deleted",
"m2.total_records_updated as complete_total_records_updated",
"m2.total_records_compacted as complete_total_records_compacted"
),
page,
count,
@@ -666,6 +678,22 @@ public class InfoService {
if (ObjectUtil.isNotNull(finishedTimestamp)) {
finishedTime = finishedTimestamp.getTime();
}
CompactionMetrics.Statistics before = new CompactionMetrics.Statistics(
rs.getLong(9),
rs.getLong(10),
rs.getLong(11),
rs.getLong(12),
rs.getLong(13),
rs.getLong(14)
);
CompactionMetrics.Statistics after = new CompactionMetrics.Statistics(
rs.getLong(15),
rs.getLong(16),
rs.getLong(17),
rs.getLong(18),
rs.getLong(19),
rs.getLong(20)
);
return new CompactionMetrics(
rs.getLong(1),
rs.getString(2),
@@ -674,7 +702,9 @@ public class InfoService {
rs.getString(5),
isComplete,
startedTime,
finishedTime
finishedTime,
before,
after
);
}
);

18
service-info-query/src/test/java/SqlBuilderTests.java
View File

@@ -48,7 +48,8 @@ public class SqlBuilderTests {
String alias,
String order,
String direction,
ImmutableList<Boolean> filterCompletes
ImmutableList<Boolean> filterCompletes,
boolean limited
) {
int limit = Math.max(count, 1);
int offset = limit * Math.max(page - 1, 0);
@@ -78,7 +79,7 @@ public class SqlBuilderTests {
.whereNotNull(filterCompletes.anySatisfy(b -> b), column(m2, "type"))
.orNull(filterCompletes.anySatisfy(b -> !b), column(m2, "type"))
.orderBy(StrUtil.isNotBlank(order) && StrUtil.isNotBlank(direction), StrUtil.format("m1.{} {}", order, direction))
.limit(offset, count);
.limit(limited, offset, count);
}
public static void main(String[] args) {
@@ -95,12 +96,13 @@ public class SqlBuilderTests {
"m2.update_time as finished_time"
),
1,
100,
1542097996099055616L,
"acct_acct_item_fs",
"update_time",
"asc",
Lists.immutable.of(false, true)
10,
1542097983881048064L,
"conf_center_balance_type",
null,
null,
Lists.immutable.of(false),
true
).build()
));
}

64
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/CompactionMetricsVO.java
View File

@@ -18,7 +18,8 @@ public class CompactionMetricsVO {
private final Boolean complete;
private final Long startedTime;
private final Long finishedTime;
private final StatisticsVO before;
private final StatisticsVO after;
private String startedTimeFromNow;
private String finishedTimeFromNow;
@@ -32,6 +33,8 @@ public class CompactionMetricsVO {
this.complete = metrics.getComplete();
this.startedTime = metrics.getStartedTime();
this.finishedTime = metrics.getFinishedTime();
this.before = new StatisticsVO(metrics.getBefore());
this.after = new StatisticsVO(metrics.getAfter());
long now = Instant.now().toEpochMilli();
if (ObjectUtil.isNotNull(metrics.getStartedTime()) && metrics.getStartedTime() != 0) {
@@ -74,6 +77,14 @@ public class CompactionMetricsVO {
return finishedTime;
}
public StatisticsVO getBefore() {
return before;
}
public StatisticsVO getAfter() {
return after;
}
public String getStartedTimeFromNow() {
return startedTimeFromNow;
}
@@ -81,4 +92,55 @@ public class CompactionMetricsVO {
public String getFinishedTimeFromNow() {
return finishedTimeFromNow;
}
public static class StatisticsVO {
private final String totalScanTime;
private final String totalLogFilesCompacted;
private final String totalLogFilesSize;
private final String totalRecordsDeleted;
private final String totalRecordsUpdated;
private final String totalRecordsCompacted;
public StatisticsVO(CompactionMetrics.Statistics statistics) {
if (statistics == null) {
this.totalScanTime = null;
this.totalLogFilesCompacted = null;
this.totalLogFilesSize = null;
this.totalRecordsDeleted = null;
this.totalRecordsUpdated = null;
this.totalRecordsCompacted = null;
return;
}
this.totalScanTime = statistics.getTotalScanTime() == null ? null : statistics.getTotalScanTime().toString();
this.totalLogFilesCompacted = statistics.getTotalLogFilesCompacted() == null ? null : statistics.getTotalLogFilesCompacted().toString();
this.totalLogFilesSize = statistics.getTotalLogFilesSize() == null ? null : statistics.getTotalLogFilesSize().toString();
this.totalRecordsDeleted = statistics.getTotalRecordsDeleted() == null ? null : statistics.getTotalRecordsDeleted().toString();
this.totalRecordsUpdated = statistics.getTotalRecordsUpdated() == null ? null : statistics.getTotalRecordsUpdated().toString();
this.totalRecordsCompacted = statistics.getTotalRecordsCompacted() == null ? null : statistics.getTotalRecordsCompacted().toString();
}
public String getTotalScanTime() {
return totalScanTime;
}
public String getTotalLogFilesCompacted() {
return totalLogFilesCompacted;
}
public String getTotalLogFilesSize() {
return totalLogFilesSize;
}
public String getTotalRecordsDeleted() {
return totalRecordsDeleted;
}
public String getTotalRecordsUpdated() {
return totalRecordsUpdated;
}
public String getTotalRecordsCompacted() {
return totalRecordsCompacted;
}
}
}

4
test/test.http
View File

@@ -38,7 +38,7 @@ Content-Type: application/json
]
### Info
GET http://{{username}}:{{password}}@132.122.116.146:25638/info/compaction_metrics?flink_job_id=1542097996099055616&alias=acct_acct_item_fs&filter_completes=false
GET http://{{username}}:{{password}}@132.122.116.146:18166/info/compaction_metrics?flink_job_id=1542097996099055616&alias=acct_acct_item_fs&filter_completes=true
### Info
GET http://{{username}}:{{password}}@132.122.116.150:16883/table/list_compaction_metrics?search_flink_job_id=1542097996099055616&search_alias=acct_acct_item_fs
GET http://{{username}}:{{password}}@132.122.116.150:27510/table/list_compaction_metrics?search_flink_job_id=1542097996099055616&search_alias=acct_acct_item_fs

44
web/components/common.js
View File

@@ -627,6 +627,22 @@ function runMetaProperty(runMode) {
}
}
function statisticsProperty(title, statistic) {
return {
type: 'property',
title: title,
column: 3,
items: [
{label: '扫描总时间', content: `\${${statistic}.totalScanTime}`},
{label: '压缩日志文件总数', content: `\${${statistic}.totalLogFilesCompacted}`},
{label: '压缩日志文件大小', content: `\${${statistic}.totalLogFilesSize}`},
{label: '删除记录数', content: `\${${statistic}.totalRecordsDeleted}`},
{label: '更新记录数', content: `\${${statistic}.totalRecordsUpdated}`},
{label: '压缩日志数', content: `\${${statistic}.totalRecordsCompacted}`},
],
}
}
function flinkJobDialog() {
return {
title: 'Flink job 详情',
@@ -828,6 +844,34 @@ function tableMetaDialog() {
align: 'center',
canAccessSuperData: false,
},
{
type: "operation",
width: 50,
label: "操作",
fixed: 'right',
className: 'nowrap',
buttons: [
{
label: '详情',
type: 'action',
level: 'link',
actionType: 'dialog',
dialog: {
title: '压缩详情',
size: 'xl',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
body: [
statisticsProperty('压缩预扫描', 'before'),
{type: 'divider'},
statisticsProperty('压缩成果', 'after'),
]
}
}
]
}
],
}
}