lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-manager): 增加表数据查询10条采样任务

Browse Source
This commit is contained in:
v-zhangjc9
2024-06-03 16:17:41 +08:00
parent 8aba2475be
commit 6f2fce4359
6 changed files with 121 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/controller/ExecutorTaskController.java
View File

@@ -53,13 +53,21 @@ public class ExecutorTaskController {
}
@GetMapping("table_summary")
public String latestOpTs(@RequestParam("hdfs") String hdfs) throws Exception {
public String tableSummary(@RequestParam("hdfs") String hdfs) throws Exception {
if (StrUtil.isBlank(hdfs)) {
throw new RuntimeException("Hdfs path cannot be empty");
}
return executorTaskService.tableSummary(hdfs);
}
@GetMapping("table_sampling")
public String tableSampling(@RequestParam("hdfs") String hdfs) throws Exception {
if (StrUtil.isBlank(hdfs)) {
throw new RuntimeException("Hdfs path cannot be empty");
}
return executorTaskService.tableSampling(hdfs);
}
@GetMapping("law_enforcement")
public String lawEnforcement(
@RequestParam("pulsar_url") String pulsarUrl,

25
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java
View File

@@ -236,6 +236,31 @@ public class ExecutorTaskService {
return applicationId.toString();
}
public String tableSampling(String hdfs) throws Exception {
String taskId = taskId();
Configuration configuration = generateConfiguration(taskId, StrUtil.format("sampling {}", hdfs));
configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("1024m"));
MapBuilder<String, Object> builder = MapUtil.builder();
builder.put("hdfs", hdfs);
ApplicationId applicationId = Runner.run(
configuration,
"com.lanyuanxiaoyao.service.executor.task.TableSampling",
new String[]{
TaskConstants.TASK_CONTEXT_OPTION,
mapper.writeValueAsString(
new TaskContext(
taskId,
executorConfiguration.getTaskResultPath(),
Maps.mutable.ofMap(builder.build())
)
)
}
);
return applicationId.toString();
}
public String lawEnforcement(
String pulsarUrl,
String pulsarTopic,