lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(hudi-query): Hudi 时间线查询增加 instant 文件创建时间

Browse Source 
Hudi 时间线查询增加分页参数下推，增加性能
This commit is contained in:
兰缘小妖
2023-06-28 16:53:03 +08:00
parent 1ed1b8105a
commit 1ce0b74e2d
14 changed files with 154 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java
View File

@@ -2,9 +2,10 @@ package com.lanyuanxiaoyao.service.web.controller;
import cn.hutool.core.util.ObjectUtil;
import com.lanyuanxiaoyao.service.configuration.entity.AmisResponse;
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
import com.lanyuanxiaoyao.service.forest.service.HudiService;
import com.lanyuanxiaoyao.service.web.utils.ComparatorUtil;
import com.lanyuanxiaoyao.service.configuration.utils.ComparatorUtil;
import java.util.List;
import java.util.function.Function;
import org.eclipse.collections.api.factory.Maps;
@@ -53,20 +54,17 @@ public class HudiController extends BaseController {
@RequestParam(value = "filter_action", required = false) List<String> filterAction,
@RequestParam(value = "filter_state", required = false) List<String> filterState
) {
MutableMap<String, Object> queryMap = Maps.mutable.empty();
queryMap.put("flink_job_id", flinkJobId);
queryMap.put("alias", alias);
MutableMap<String, Object> queryMap = buildQueryMap(page, count, order, direction, flinkJobId.toString(), alias);
if (ObjectUtil.isNotEmpty(filterType)) {
queryMap.put("filter_type", filterType);
}
ImmutableList<HudiInstant> hudiInstants = hudiService.timelineList(queryMap)
.select(instant -> ObjectUtil.isEmpty(filterAction) || filterAction.contains(instant.getAction()))
.select(instant -> ObjectUtil.isEmpty(filterState) || filterState.contains(instant.getState()))
.toSortedList(ComparatorUtil.stringComparator(order, direction, TIMELINE_SORT_MAP))
.toImmutable();
ImmutableList<HudiInstant> result = hudiInstants
.drop(Math.max(page - 1, 0) * count)
.take(count);
return responseCrudData(result, hudiInstants.size());
if (ObjectUtil.isNotEmpty(filterAction)) {
queryMap.put("filter_action", filterAction);
}
if (ObjectUtil.isNotEmpty(filterState)) {
queryMap.put("filter_state", filterState);
}
PageResponse<HudiInstant> response = hudiService.timelineList(queryMap);
return responseCrudData(response.getData(), response.getTotal());
}
}

2
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/YarnController.java
View File

@@ -14,7 +14,7 @@ import com.lanyuanxiaoyao.service.forest.service.FlinkService;
import com.lanyuanxiaoyao.service.forest.service.YarnService;
import com.lanyuanxiaoyao.service.web.entity.YarnApplicationVO;
import com.lanyuanxiaoyao.service.web.entity.YarnClusterVO;
import com.lanyuanxiaoyao.service.web.utils.ComparatorUtil;
import com.lanyuanxiaoyao.service.configuration.utils.ComparatorUtil;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;

2
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/CompactionMetricsVO.java
View File

@@ -2,7 +2,7 @@ package com.lanyuanxiaoyao.service.web.entity;
import cn.hutool.core.util.ObjectUtil;
import com.lanyuanxiaoyao.service.configuration.entity.info.CompactionMetrics;
import com.lanyuanxiaoyao.service.web.utils.DatetimeUtil;
import com.lanyuanxiaoyao.service.configuration.utils.DatetimeUtil;
import java.time.Instant;
/**

2
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/SyncStateVO.java
View File

@@ -2,7 +2,7 @@ package com.lanyuanxiaoyao.service.web.entity;
import com.eshore.odcp.hudi.connector.entity.SyncState;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.lanyuanxiaoyao.service.web.utils.DatetimeUtil;
import com.lanyuanxiaoyao.service.configuration.utils.DatetimeUtil;
import java.time.Instant;
/**

2
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnApplicationVO.java
View File

@@ -5,7 +5,7 @@ import cn.hutool.core.util.ReUtil;
import com.eshore.odcp.hudi.connector.utils.NameHelper;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
import com.lanyuanxiaoyao.service.web.utils.DatetimeUtil;
import com.lanyuanxiaoyao.service.configuration.utils.DatetimeUtil;
import java.time.Instant;
/**

2
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/ZookeeperNodeVO.java
View File

@@ -4,7 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
import com.eshore.odcp.hudi.connector.entity.RunMeta;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.lanyuanxiaoyao.service.configuration.entity.zookeeper.ZookeeperNode;
import com.lanyuanxiaoyao.service.web.utils.DatetimeUtil;
import com.lanyuanxiaoyao.service.configuration.utils.DatetimeUtil;
import java.time.Instant;
import java.util.Optional;

45
service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/ComparatorUtil.java
View File

@@ -1,45 +0,0 @@
package com.lanyuanxiaoyao.service.web.utils;
import cn.hutool.core.util.StrUtil;
import java.util.Comparator;
import java.util.function.Function;
import org.eclipse.collections.api.map.ImmutableMap;
/**
* 生成字段比较器
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
public class ComparatorUtil {
public static final String ASC = "asc";
public static final String DESC = "desc";
public static <T> Comparator<T> longComparator(String order, String direction, ImmutableMap<String, Function<T, Long>> getters) {
if (StrUtil.isBlank(order) || StrUtil.isBlank(direction) || !getters.containsKey(order)) {
return (o1, o2) -> 0;
}
Function<T, Long> getter = getters.get(order);
if (DESC.equalsIgnoreCase(direction)) {
return (o1, o2) -> Long.compare(getter.apply(o2), getter.apply(o1));
} else if (ASC.equalsIgnoreCase(direction)) {
return Comparator.comparingLong(getter::apply);
} else {
return (o1, o2) -> 0;
}
}
public static <T> Comparator<T> stringComparator(String order, String direction, ImmutableMap<String, Function<T, String>> getters) {
if (StrUtil.isBlank(order) || StrUtil.isBlank(direction) || !getters.containsKey(order)) {
return (o1, o2) -> 0;
}
Function<T, String> getter = getters.get(order);
if (DESC.equalsIgnoreCase(direction)) {
return (o1, o2) -> StrUtil.compare(getter.apply(o2), getter.apply(o1), true);
} else if (ASC.equalsIgnoreCase(direction)) {
return (o1, o2) -> StrUtil.compare(getter.apply(o1), getter.apply(o2), true);
} else {
return (o1, o2) -> 0;
}
}
}

45
service-web/src/main/java/com/lanyuanxiaoyao/service/web/utils/DatetimeUtil.java
View File

@@ -1,45 +0,0 @@
package com.lanyuanxiaoyao.service.web.utils;
import cn.hutool.core.util.ObjectUtil;
import java.util.function.Supplier;
/**
* 日期工具
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
public class DatetimeUtil {
public static String fromNow(long now, Supplier<Long> supplier) {
if (ObjectUtil.isNotNull(supplier.get()) && supplier.get() != 0) {
return fromNow(now, supplier.get());
}
return "";
}
public static String fromNow(long now, long target) {
long delta = now - target;
if (delta == 0) {
return "就是现在";
}
String suffix = delta > 0 ? "前" : "后";
delta = Math.abs(delta);
if (delta > 31536000000L) {
return (int) (delta / 31536000000L) + " 年" + suffix;
} else if (delta > 2592000000L) {
return (int) (delta / 2592000000L) + " 个月" + suffix;
} else if (delta > 604800000L) {
return (int) (delta / 604800000L) + " 周" + suffix;
} else if (delta > 86400000L) {
return (int) (delta / 86400000L) + " 天" + suffix;
} else if (delta > 3600000L) {
return (int) (delta / 3600000L) + " 小时" + suffix;
} else if (delta > 60000L) {
return (int) (delta / 60000L) + " 分钟" + suffix;
} else if (delta > 1000L) {
return (delta / 1000L) + "秒" + suffix;
} else {
return "就是现在";
}
}
}