lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hudi-query): 修复获取待压缩列表会拖慢性能

Browse Source
This commit is contained in:
v-zhangjc9
2024-05-28 11:52:42 +08:00
parent 9a1afbcceb
commit 095347cd1a
2 changed files with 15 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/TimelineService.java
View File

@@ -13,7 +13,9 @@ import com.lanyuanxiaoyao.service.configuration.utils.ComparatorUtil;
import com.lanyuanxiaoyao.service.forest.service.InfoService;
import com.lanyuanxiaoyao.service.hudi.utils.HoodieUtils;
import java.io.IOException;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -273,8 +275,15 @@ public class TimelineService {
.setConf(new Configuration())
.setBasePath(meta.getHudi().getTargetHdfsPath())
.build();
return HoodieUtils.getAllActiveInstants(client)
.select(instant -> StrUtil.equals(instant.getAction(), HoodieTimeline.COMPACTION_ACTION))
.reject(instant -> StrUtil.equals(instant.getState(), HoodieInstant.State.COMPLETED.name()));
// 使用默认的方式取待压缩列表，不然获取压缩完成状态需要再扫描一次时间线目录，会降低性能
List<HudiInstant> list = client.getActiveTimeline()
.filterPendingCompactionTimeline()
.getInstants()
.map(instant -> HoodieUtils.convert(instant, "active"))
.collect(Collectors.toList());
if (ObjectUtil.isNotEmpty(list)) {
return Lists.immutable.ofAll(list);
}
return Lists.immutable.empty();
}
}

6
service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/utils/HoodieUtils.java
View File

@@ -88,7 +88,7 @@ public class HoodieUtils {
.toImmutable();
}
private static HudiInstant convert(HoodieInstant instant, String type) {
public static HudiInstant convert(HoodieInstant instant, String type) {
return new HudiInstant(
instant.getAction(),
instant.getState().name(),
@@ -99,7 +99,7 @@ public class HoodieUtils {
);
}
private static ImmutableList<HudiInstant> activeInstants(HoodieTableMetaClient client) throws IOException {
public static ImmutableList<HudiInstant> activeInstants(HoodieTableMetaClient client) throws IOException {
HoodieActiveTimeline timeline = client.getActiveTimeline();
Set<String> committedTimestamps = timeline.getCommitsTimeline()
.filterCompletedInstants()
@@ -131,7 +131,7 @@ public class HoodieUtils {
return Lists.immutable.ofAll(instants);
}
private static ImmutableList<HudiInstant> archiveInstants(HoodieTableMetaClient client) {
public static ImmutableList<HudiInstant> archiveInstants(HoodieTableMetaClient client) {
HoodieArchivedTimeline timeline = client.getArchivedTimeline();
MutableList<HoodieInstant> instants = Lists.mutable.ofAll(timeline.getInstants().collect(Collectors.toList()));
instants.forEach(instant -> logger.info(instant.toString()));