lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-task): 增加一个mini hudi的功能

Browse Source 
从pulsar到hdfs落盘
This commit is contained in:
v-zhangjc9
2024-05-28 18:16:09 +08:00
parent 095347cd1a
commit deae4fd294
11 changed files with 507 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/controller/ExecutorTaskController.java
View File

@@ -60,6 +60,30 @@ public class ExecutorTaskController {
return executorTaskService.tableSummary(hdfs);
}
@GetMapping("law_enforcement")
public String lawEnforcement(
@RequestParam("pulsar_url") String pulsarUrl,
@RequestParam("pulsar_topic") String pulsarTopic,
@RequestParam("start_time") Long startTime,
@RequestParam("end_time") Long endTime,
@RequestParam("primary_keys") String primaryKeys,
@RequestParam("partition_keys") String partitionKeys
) throws Exception {
if (StrUtil.isBlank(pulsarUrl)) {
throw new RuntimeException("pulsar_url cannot be empty");
}
if (StrUtil.isBlank(pulsarTopic)) {
throw new RuntimeException("pulsar_topic cannot be empty");
}
if (StrUtil.isBlank(primaryKeys)) {
throw new RuntimeException("primary_keys cannot be empty");
}
if (StrUtil.isBlank(partitionKeys)) {
throw new RuntimeException("partition_keys cannot be empty");
}
return executorTaskService.lawEnforcement(pulsarUrl, pulsarTopic, startTime, endTime, primaryKeys, partitionKeys);
}
@GetMapping("results")
public ImmutableList<String> results(
@RequestParam("task_id") String taskId,

37
service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java
View File

@@ -236,6 +236,43 @@ public class ExecutorTaskService {
return applicationId.toString();
}
public String lawEnforcement(
String pulsarUrl,
String pulsarTopic,
Long startTime,
Long endTime,
String primaryKeys,
String partitionKeys
) throws Exception {
String taskId = taskId();
Configuration configuration = generateConfiguration(taskId, StrUtil.format("law_enforcement"));
configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("1024m"));
MapBuilder<String, Object> builder = MapUtil.builder();
builder.put("pulsar_url", pulsarUrl);
builder.put("pulsar_topic", pulsarTopic);
builder.put("start_time", startTime);
builder.put("end_time", endTime);
builder.put("primary_keys", primaryKeys);
builder.put("partition_keys", partitionKeys);
ApplicationId applicationId = Runner.run(
configuration,
"com.lanyuanxiaoyao.service.executor.task.HoodiePolice",
new String[]{
TaskConstants.TASK_CONTEXT_OPTION,
mapper.writeValueAsString(
new TaskContext(
taskId,
executorConfiguration.getTaskResultPath(),
Maps.mutable.ofMap(builder.build())
)
)
}
);
return applicationId.toString();
}
@Cacheable(value = "results", sync = true)
@Retryable(Throwable.class)
public ImmutableList<String> taskResult(String taskId, Integer limit) throws IOException {