diff --git a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/TimelineController.java b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/TimelineController.java index 3c50929..ac94095 100644 --- a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/TimelineController.java +++ b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/TimelineController.java @@ -34,31 +34,16 @@ public class TimelineController { public ImmutableList allInstants( @RequestParam("flink_job_id") Long flinkJobId, @RequestParam("alias") String alias, - @RequestParam(value = "filter_type", required = false) List filterType, - @RequestParam(value = "filter_action", required = false) List filterAction, - @RequestParam(value = "filter_state", required = false) List filterState + @RequestParam(value = "filter_type", required = false) List filterType ) throws IOException { - return timelineService.timeline( - flinkJobId, - alias, - Lists.immutable.ofAll(filterType), - Lists.immutable.ofAll(filterAction), - Lists.immutable.ofAll(filterState) - ); + return timelineService.timeline(flinkJobId, alias, Lists.immutable.ofAll(filterType)); } @GetMapping("list_hdfs") public ImmutableList allInstants( @RequestParam("hdfs") String hdfs, - @RequestParam(value = "filter_type", required = false) List filterType, - @RequestParam(value = "filter_action", required = false) List filterAction, - @RequestParam(value = "filter_state", required = false) List filterState + @RequestParam(value = "filter_type", required = false) List filterType ) throws IOException { - return timelineService.timeline( - hdfs, - Lists.immutable.ofAll(filterType), - Lists.immutable.ofAll(filterAction), - Lists.immutable.ofAll(filterState) - ); + return timelineService.timeline(hdfs, Lists.immutable.ofAll(filterType)); } } diff --git a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/TimelineService.java b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/TimelineService.java index 1caefb1..236593a 100644 --- a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/TimelineService.java +++ b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/TimelineService.java @@ -37,9 +37,9 @@ public class TimelineService { @Cacheable(value = "timeline", sync = true, key = "#flinkJobId.toString()+#alias") @Retryable(Throwable.class) - public ImmutableList timeline(Long flinkJobId, String alias, ImmutableList filterType, ImmutableList filterAction, ImmutableList filterState) throws IOException { + public ImmutableList timeline(Long flinkJobId, String alias, ImmutableList filterType) throws IOException { TableMeta meta = infoService.tableMetaDetail(flinkJobId, alias); - return timeline(meta.getHudi().getTargetHdfsPath(), filterType, filterAction, filterState); + return timeline(meta.getHudi().getTargetHdfsPath(), filterType); } private static final String INSTANT_TYPE_ACTIVE = "active"; @@ -47,7 +47,7 @@ public class TimelineService { @Cacheable(value = "timeline", sync = true, key = "#hdfs") @Retryable(Throwable.class) - public ImmutableList timeline(String hdfs, ImmutableList filterType, ImmutableList filterAction, ImmutableList filterState) throws IOException { + public ImmutableList timeline(String hdfs, ImmutableList filterType) throws IOException { HoodieTableMetaClient client = HoodieTableMetaClient.builder() .setConf(new Configuration()) .setBasePath(hdfs) @@ -59,15 +59,11 @@ public class TimelineService { if (filterType.contains(INSTANT_TYPE_ARCHIVE)) { HoodieUtils.getAllInstants(client, HoodieTableMetaClient::getArchivedTimeline) .collect(instant -> covert(INSTANT_TYPE_ARCHIVE, instant)) - .select(instant -> ObjectUtil.isEmpty(filterAction) || filterAction.contains(instant.getAction())) - .select(instant -> ObjectUtil.isEmpty(filterState) || filterState.contains(instant.getState())) .forEach(instants::add); } if (filterType.contains(INSTANT_TYPE_ACTIVE)) { HoodieUtils.getAllInstants(client, HoodieTableMetaClient::getActiveTimeline) .collect(instant -> covert(INSTANT_TYPE_ACTIVE, instant)) - .select(instant -> ObjectUtil.isEmpty(filterAction) || filterAction.contains(instant.getAction())) - .select(instant -> ObjectUtil.isEmpty(filterState) || filterState.contains(instant.getState())) .forEach(instants::add); } return instants diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java index 9bd9b26..2b83c92 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java @@ -2,9 +2,14 @@ package com.lanyuanxiaoyao.service.web.controller; import cn.hutool.core.util.ObjectUtil; import com.lanyuanxiaoyao.service.configuration.entity.AmisResponse; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant; import com.lanyuanxiaoyao.service.forest.service.HudiService; +import com.lanyuanxiaoyao.service.web.utils.ComparatorUtil; import java.util.List; +import java.util.function.Function; import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; import org.eclipse.collections.api.map.MutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,8 +36,17 @@ public class HudiController extends BaseController { this.hudiService = hudiService; } + private static final ImmutableMap> TIMELINE_SORT_MAP = Maps.immutable.of( + "timestamp", + HudiInstant::getTimestamp + ); + @GetMapping("/timeline/list") public AmisResponse timeline( + @RequestParam(value = "page", defaultValue = "1") Integer page, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "order", defaultValue = "timestamp") String order, + @RequestParam(value = "direction", defaultValue = "desc") String direction, @RequestParam("flink_job_id") Long flinkJobId, @RequestParam("alias") String alias, @RequestParam(value = "filter_type", required = false) List filterType, @@ -45,12 +59,14 @@ public class HudiController extends BaseController { if (ObjectUtil.isNotEmpty(filterType)) { queryMap.put("filter_type", filterType); } - if (ObjectUtil.isNotEmpty(filterAction)) { - queryMap.put("filter_action", filterAction); - } - if (ObjectUtil.isNotEmpty(filterAction)) { - queryMap.put("filter_state", filterState); - } - return responseCrudData(hudiService.timelineList(queryMap)); + ImmutableList hudiInstants = hudiService.timelineList(queryMap) + .select(instant -> ObjectUtil.isEmpty(filterAction) || filterAction.contains(instant.getAction())) + .select(instant -> ObjectUtil.isEmpty(filterState) || filterState.contains(instant.getState())) + .toSortedList(ComparatorUtil.stringComparator(order, direction, TIMELINE_SORT_MAP)) + .toImmutable(); + ImmutableList result = hudiInstants + .drop(Math.max(page - 1, 0) * count) + .take(count); + return responseCrudData(result, hudiInstants.size()); } } diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/ComparatorUtil.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/ComparatorUtil.java index ca3d775..9cffb2d 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/ComparatorUtil.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/ComparatorUtil.java @@ -28,4 +28,18 @@ public class ComparatorUtil { return (o1, o2) -> 0; } } + + public static Comparator stringComparator(String order, String direction, ImmutableMap> getters) { + if (StrUtil.isBlank(order) || StrUtil.isBlank(direction) || !getters.containsKey(order)) { + return (o1, o2) -> 0; + } + Function getter = getters.get(order); + if (DESC.equalsIgnoreCase(direction)) { + return (o1, o2) -> StrUtil.compare(getter.apply(o2), getter.apply(o1), true); + } else if (ASC.equalsIgnoreCase(direction)) { + return (o1, o2) -> StrUtil.compare(getter.apply(o1), getter.apply(o2), true); + } else { + return (o1, o2) -> 0; + } + } } diff --git a/service-web/src/main/resources/static/components/table-tab.js b/service-web/src/main/resources/static/components/table-tab.js index d4f71dd..f01da15 100644 --- a/service-web/src/main/resources/static/components/table-tab.js +++ b/service-web/src/main/resources/static/components/table-tab.js @@ -262,6 +262,10 @@ function tableTab() { method: 'get', url: '${base}/hudi/timeline/list', data: { + page: '${page|default:undefined}', + count: '${perPage|default:undefined}', + order: '${orderBy|default:undefined}', + direction: '${orderDir|default:undefined}', flink_job_id: '${flinkJobId|default:undefined}', alias: '${tableMeta.alias|default:undefined}', filter_type: '${filter_type|default:undefined}' @@ -271,11 +275,33 @@ function tableTab() { }, }, syncLocation: false, + silentPolling: true, + stopAutoRefreshWhenModalIsOpen: true, + resizable: false, + perPage: 10, + headerToolbar: [ + "reload", + { + type: 'pagination', + layout: ['pager', 'perPage'], + maxButtons: 8, + showPageInput: false, + }, + ], + footerToolbar: [ + { + type: 'pagination', + layout: ['pager', 'perPage'], + maxButtons: 8, + showPageInput: false, + }, + ], columns: [ { name: 'timestamp', label: '时间点', width: 150, + sortable: true, }, { name: 'filter_action', @@ -287,7 +313,7 @@ function tableTab() { { name: 'filter_state', label: ' 状态', - width: 100, + width: 80, align: 'center', ...mappingField('state', hudiTimelineStateMapping), filterable: filterableField(hudiTimelineStateMapping, true),