lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-task): 使用Flink SQL查询表总数、业务最后操作时间

Browse Source
This commit is contained in:
v-zhangjc9
2024-05-17 17:32:07 +08:00
parent 592d27ad6f
commit f398b8cdc3
14 changed files with 249 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/controller/ExecutorTaskController.java
View File

@@ -52,12 +52,12 @@ public class ExecutorTaskController {
return executorTaskService.scanAvro(key, hdfs, pulsar, pulsarTopic, scanSource, scanQueue, scanLog, scanBase, scanTarget, filterFields);
}
@GetMapping("latest_op_ts")
@GetMapping("table_summary")
public String latestOpTs(@RequestParam("hdfs") String hdfs) throws Exception {
if (StrUtil.isBlank(hdfs)) {
throw new RuntimeException("Hdfs path cannot be empty");
}
return executorTaskService.scanLatestOpTs(hdfs);
return executorTaskService.tableSummary(hdfs);
}
@GetMapping("results")

6
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java
View File

@@ -209,9 +209,9 @@ public class ExecutorTaskService {
return applicationId.toString();
}
public String scanLatestOpTs(String hdfs) throws Exception {
public String tableSummary(String hdfs) throws Exception {
String taskId = taskId();
Configuration configuration = generateConfiguration(taskId, StrUtil.format("latest_op_ts {}", hdfs));
Configuration configuration = generateConfiguration(taskId, StrUtil.format("summary {}", hdfs));
configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("1024m"));
MapBuilder<String, Object> builder = MapUtil.builder();
@@ -219,7 +219,7 @@ public class ExecutorTaskService {
ApplicationId applicationId = Runner.run(
configuration,
"com.lanyuanxiaoyao.service.executor.task.LatestOperationTimeScan",
"com.lanyuanxiaoyao.service.executor.task.TableSummary",
new String[]{
TaskConstants.TASK_CONTEXT_OPTION,
mapper.writeValueAsString(