lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-task): 数据扫描增加pulsar队列读取

Browse Source
This commit is contained in:
兰缘小妖
2024-01-19 14:36:41 +08:00
parent 521e82104f
commit 9140a39bf1
22 changed files with 796 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TableController.java
View File

@@ -220,4 +220,22 @@ public class TableController extends BaseController {
}
return AmisResponse.responseSuccess(infoService.allHdfs(key).collect(Item::new));
}
@SuppressWarnings("DataFlowIssue")
@GetMapping("all_pulsar")
public AmisResponse<ImmutableList<Item>> allPulsar(@RequestParam(value = "key", required = false) String key) {
if (StrUtil.isBlank(key)) {
return AmisResponse.responseSuccess(infoService.allPulsar().collect(Item::new));
}
return AmisResponse.responseSuccess(infoService.allPulsar(key).collect(Item::new));
}
@SuppressWarnings("DataFlowIssue")
@GetMapping("all_pulsar_topic")
public AmisResponse<ImmutableList<Item>> allPulsarTopic(@RequestParam(value = "key", required = false) String key) {
if (StrUtil.isBlank(key)) {
return AmisResponse.responseSuccess(infoService.allPulsarTopic().collect(Item::new));
}
return AmisResponse.responseSuccess(infoService.allPulsarTopic(key).collect(Item::new));
}
}

26
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/TaskController.java
View File

@@ -2,6 +2,7 @@ package com.lanyuanxiaoyao.service.web.controller;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import com.lanyuanxiaoyao.service.forest.service.PulsarService;
import com.lanyuanxiaoyao.service.forest.service.TaskService;
import com.lanyuanxiaoyao.service.web.controller.base.AmisMapResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
@@ -25,24 +26,35 @@ public class TaskController {
private static final Logger logger = LoggerFactory.getLogger(TaskController.class);
private final TaskService taskService;
private final PulsarService pulsarService;
public TaskController(TaskService taskService) {
public TaskController(TaskService taskService, PulsarService pulsarService) {
this.taskService = taskService;
this.pulsarService = pulsarService;
}
@GetMapping("scan")
public AmisResponse<Object> scan(
@RequestParam("hdfs") String hdfs,
@RequestParam("key") String key,
@RequestParam(value = "hdfs", required = false) String hdfs,
@RequestParam(value = "pulsar", required = false) String pulsar,
@RequestParam(value = "topic", required = false) String topic,
@RequestParam(value = "mode", defaultValue = "") String mode
) {
if (StrUtil.isBlank(hdfs) || StrUtil.isBlank(key)) {
throw new RuntimeException("Argument cannot be blank");
if (StrUtil.isBlank(key)) {
throw new RuntimeException("Key cannot be blank");
}
boolean scanQueue = StrUtil.contains(mode, "queue");
boolean scanLog = StrUtil.contains(mode, "log");
boolean scanBase = StrUtil.contains(mode, "base");
if (scanQueue && (StrUtil.isBlank(topic) || StrUtil.isBlank(pulsar))) {
throw new RuntimeException("Pulsar topic or url cannot be empty");
}
if ((scanLog || scanBase) && StrUtil.isBlank(hdfs)) {
throw new RuntimeException("Hdfs path cannot be empty");
}
ExecutorProvider.EXECUTORS.submit(() -> {
boolean scanLog = StrUtil.contains(mode, "log");
boolean scanData = StrUtil.contains(mode, "data");
String applicationId = taskService.scan(hdfs, key, scanLog, scanData);
String applicationId = taskService.scan(key, hdfs, pulsar, topic, scanQueue, scanLog, scanBase);
logger.info("Task: {}", applicationId);
});
return AmisResponse.responseSuccess();

47
service-web/src/main/resources/static/components/task-tab.js
View File

@@ -15,8 +15,10 @@ function taskTab() {
method: 'get',
url: '${base}/task/scan',
data: {
hdfs: '${hdfs|default:undefined}',
key: '${key|default:undefined}',
hdfs: '${hdfs|default:undefined}',
pulsar: '${pulsar|default:undefined}',
topic: '${topic|default:undefined}',
mode: '${scan_mode|default:undefined}',
}
}
@@ -31,31 +33,52 @@ function taskTab() {
required: true,
value: 'log',
options: [
{label: '消息队列', value: 'queue'},
{label: '日志文件', value: 'log'},
{label: '数据文件', value: 'data'},
{label: '数据文件', value: 'base'},
]
},
{
type: 'input-text',
name: 'key',
label: '检索字段',
required: true,
clearable: true,
description: '检索带有该字符的记录',
},
{
type: 'input-text',
name: 'hdfs',
label: 'HDFS路经',
requiredOn: '${CONTAINS(scan_mode, \'log\') || CONTAINS(scan_mode, \'base\')}',
visibleOn: '${CONTAINS(scan_mode, \'log\') || CONTAINS(scan_mode, \'base\')}',
clearable: true,
description: '输入表HDFS路径',
autoComplete: '${base}/table/all_hdfs?key=$term',
},
{
type: 'group',
body: [
{
type: 'input-text',
name: 'key',
label: '检索字段',
required: true,
name: 'topic',
label: 'Pulsar主题',
requiredOn: '${CONTAINS(scan_mode, \'queue\')}',
visibleOn: '${CONTAINS(scan_mode, \'queue\')}',
clearable: true,
description: '检索带有该字符的记录',
description: '输入Pulsar主题',
autoComplete: '${base}/table/all_pulsar_topic?key=$term',
columnRatio: 4,
},
{
type: 'input-text',
name: 'hdfs',
label: 'HDFS路经',
required: true,
name: 'pulsar',
label: 'Pulsar地址',
requiredOn: '${CONTAINS(scan_mode, \'queue\')}',
visibleOn: '${CONTAINS(scan_mode, \'queue\')}',
clearable: true,
description: '输入表HDFS路径',
autoComplete: '${base}/table/all_hdfs?key=$term',
columnRatio: 8,
description: '输入Pulsar地址',
autoComplete: '${base}/table/all_pulsar?key=$term',
},
]
}