diff --git a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java index 8528412..6dd7c79 100644 --- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java +++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java @@ -97,7 +97,7 @@ public class ExecutorTaskService { configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("0m")); configuration.set(TaskManagerOptions.JVM_METASPACE, MemorySize.parse("128m")); configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 10); - configuration.setString(YarnConfigOptions.APPLICATION_NAME, "Service_Task_" + name + "_" + taskId); + configuration.setString(YarnConfigOptions.APPLICATION_NAME, StrUtil.format("Service_Task {} #{}", name, taskId)); configuration.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, executorConfiguration.getHistoryServerArchiveDir()); configuration.setLong(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_REFRESH_INTERVAL, 10000); @@ -124,7 +124,7 @@ public class ExecutorTaskService { Boolean scanTarget ) throws Exception { String taskId = taskId(); - Configuration configuration = generateConfiguration(taskId, "scan"); + Configuration configuration = generateConfiguration(taskId, "scan " + key); MapBuilder builder = MapUtil.builder(); setEnvironment(configuration, "key", key); @@ -163,7 +163,7 @@ public class ExecutorTaskService { public String scanLatestOpTs(String hdfs) throws Exception { String taskId = taskId(); - Configuration configuration = generateConfiguration(taskId, "latest_op_ts"); + Configuration configuration = generateConfiguration(taskId, StrUtil.format("latest_op_ts {}", hdfs)); configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("1024m")); MapBuilder builder = MapUtil.builder(); diff --git a/service-executor/service-executor-task/src/main/resources/log4j.properties b/service-executor/service-executor-task/src/main/resources/log4j.properties deleted file mode 100644 index 25483d5..0000000 --- a/service-executor/service-executor-task/src/main/resources/log4j.properties +++ /dev/null @@ -1,51 +0,0 @@ -# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds. -monitorInterval=30 - -# This affects logging for both user code and Flink -rootLogger.level = INFO -rootLogger.appenderRef.file.ref = MainAppender - -# Uncomment this if you want to _only_ change Flink's logging -logger.flink.name = org.apache.flink -logger.flink.level = INFO - -# The following lines keep the log level of common libraries/connectors on -# log level INFO. The root logger does not override this. You have to manually -# change the log levels here. -logger.akka.name = akka -logger.akka.level = INFO -logger.kafka.name= org.apache.kafka -logger.kafka.level = INFO -logger.hadoop.name = org.apache.hadoop -logger.hadoop.level = INFO -logger.zookeeper.name = org.apache.zookeeper -logger.zookeeper.level = INFO -logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3 -logger.shaded_zookeeper.level = INFO -logger.hudi.name=org.apache.hudi -logger.hudi.level=INFO - -# Log all infos in the given file -#appender.main.name = MainAppender -#appender.main.type = Console -#appender.main.layout.type = PatternLayout -#appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n -appender.main.name = MainAppender -appender.main.type = RollingFile -appender.main.append = true -appender.main.fileName = ${sys:log.file} -appender.main.filePattern = ${sys:log.file}.%i -appender.main.layout.type = PatternLayout -appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n -appender.main.policies.type = Policies -appender.main.policies.size.type = SizeBasedTriggeringPolicy -appender.main.policies.size.size = 100MB -appender.main.policies.startup.type = OnStartupTriggeringPolicy -appender.main.strategy.type = DefaultRolloverStrategy -appender.main.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10} - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline -logger.netty.level = OFF -logger.hadoop.configuration.name=org.apache.hadoop.conf.Configuration -logger.hadoop.configuration.level=ERROR diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnApplicationVO.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnApplicationVO.java index 98e13ee..a741d07 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnApplicationVO.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/YarnApplicationVO.java @@ -2,6 +2,7 @@ package com.lanyuanxiaoyao.service.web.entity; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.ReUtil; +import cn.hutool.core.util.StrUtil; import com.eshore.odcp.hudi.connector.utils.NameHelper; import com.fasterxml.jackson.annotation.JsonIgnore; import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication; @@ -17,7 +18,7 @@ import java.util.regex.Pattern; * @date 2023-04-21 */ public class YarnApplicationVO { - private static final Pattern TASK_REGEX = Pattern.compile("^Service_Task_(.+)_(.{8})$"); + private static final Pattern TASK_REGEX = Pattern.compile("^Service_Task(.*)#(.{8})$"); @JsonIgnore private final YarnApplication yarnApplication; @@ -48,8 +49,8 @@ public class YarnApplicationVO { Matcher matcher = TASK_REGEX.matcher(yarnApplication.getName()); isTaskApplication = matcher.matches(); if (isTaskApplication && matcher.groupCount() == 2) { - taskName = matcher.group(1); - taskId = matcher.group(2); + taskName = StrUtil.trim(matcher.group(1)); + taskId = StrUtil.format(matcher.group(2)); } long now = Instant.now().toEpochMilli(); diff --git a/service-web/src/main/resources/static/common/info.js b/service-web/src/main/resources/static/common/info.js index cc9ced5..46b629a 100644 --- a/service-web/src/main/resources/static/common/info.js +++ b/service-web/src/main/resources/static/common/info.js @@ -1,7 +1,7 @@ function commonInfo() { return { - baseUrl: 'http://132.126.207.131:35690/hudi_services/service_web', - // baseUrl: '/hudi_services/service_web', + // baseUrl: 'http://132.126.207.131:35690/hudi_services/service_web', + baseUrl: '/hudi_services/service_web', loki: { grafanaUrl: 'http://132.126.207.125:35700', hudi: { diff --git a/service-web/src/main/resources/static/components/task-tab.js b/service-web/src/main/resources/static/components/task-tab.js index e6313ae..b61ab75 100644 --- a/service-web/src/main/resources/static/components/task-tab.js +++ b/service-web/src/main/resources/static/components/task-tab.js @@ -139,6 +139,21 @@ function taskTab() { perPage: 10, headerToolbar: [ "reload", + { + label: "任务管理器日志", + type: "action", + tooltip: '打开Grafana日志', + onEvent: { + click: { + actions: [ + { + actionType: 'custom', + script: () => window.open(serviceLogByAppName("service-executor-manager"), '_blank'), + } + ] + } + } + }, paginationCommonOptions(), ], footerToolbar: [],