lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-task): 增加一个mini hudi的功能

Browse Source 
从pulsar到hdfs落盘
This commit is contained in:
v-zhangjc9
2024-05-28 18:16:09 +08:00
parent 095347cd1a
commit deae4fd294
11 changed files with 507 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
service-command-pro/src/main/java/com/lanyuanxiaoyao/service/command/pro/commands/HudiCommand.java
View File

@@ -3,6 +3,8 @@ package com.lanyuanxiaoyao.service.command.pro.commands;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
import com.lanyuanxiaoyao.service.forest.service.HudiService;
import com.lanyuanxiaoyao.service.forest.service.InfoService;
import java.io.IOException;
@@ -23,6 +25,7 @@ import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
@@ -41,6 +44,7 @@ import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,6 +210,22 @@ public class HudiCommand {
hudiService.timelineHdfsAllActive(hdfs).forEach(instant -> logger.info(instant.toString()));
}
@ShellMethod("Check compaction")
public void checkCompaction(@ShellOption(help = "root hdfs path") String hdfs) {
logger.info("{}", hudiService.timelineHdfsAllActive(hdfs));
ImmutableList<HudiInstant> finishCompactionInstants = hudiService.timelineHdfsAllActive(hdfs)
.select(instant -> StrUtil.equals(instant.getAction(), HoodieTimeline.COMPACTION_ACTION))
.select(instant -> StrUtil.equals(instant.getState(), HoodieInstant.State.COMPLETED.name()));
for (HudiInstant instant : finishCompactionInstants) {
logger.info("Compaction: {}", instant);
HudiCompactionPlan compactionPlan = hudiService.readCompactionPlanHdfs(hdfs, instant.getTimestamp());
for (HudiCompactionPlan.Operation operation : compactionPlan.getOperations()) {
logger.info("Base: {}", operation.getDataFilePath());
operation.getDeltaFilePaths().forEach(path -> logger.info(" Delta: {}", path));
}
}
}
public interface Runnable {
void run(LongAdder counter);
}