feat(executor): 完成任务提交执行的验证

2024-01-09 16:22:04 +08:00
parent a95aa5fc53
commit fe0d779afb
19 changed files with 536 additions and 63 deletions
--- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/ExecutorManagerApplication.java
+++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/ExecutorManagerApplication.java
@@ -1,7 +1,12 @@
 package com.lanyuanxiaoyao.service.executor.manager;

+import cn.hutool.core.util.IdUtil;
 import com.eshore.odcp.hudi.connector.utils.executor.Runner;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.lanyuanxiaoyao.service.executor.core.TaskConstants;
+import com.lanyuanxiaoyao.service.executor.core.TaskContext;
+import com.lanyuanxiaoyao.service.executor.manager.configuration.ExecutorConfiguration;
+import com.lanyuanxiaoyao.service.executor.manager.configuration.HadoopConfiguration;
 import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
 import java.time.Duration;
 import java.util.ArrayList;
@@ -11,6 +16,7 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions;
 import org.apache.flink.yarn.configuration.YarnDeploymentTarget;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.eclipse.collections.api.factory.Maps;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.boot.SpringApplication;
@@ -19,6 +25,7 @@ import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
+import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
 import org.springframework.retry.annotation.EnableRetry;

 import static com.eshore.odcp.hudi.connector.Constants.HALF_HOUR;
@@ -40,12 +47,24 @@ import static com.eshore.odcp.hudi.connector.Constants.MINUTE;
@EnableEncryptableProperties
@EnableRetry
 public class ExecutorManagerApplication implements ApplicationRunner {
+    private final HadoopConfiguration hadoopConfiguration;
+    private final ExecutorConfiguration executorConfiguration;
+    private final ObjectMapper mapper;
+
+    public ExecutorManagerApplication(HadoopConfiguration hadoopConfiguration, ExecutorConfiguration executorConfiguration, Jackson2ObjectMapperBuilder builder) {
+        this.hadoopConfiguration = hadoopConfiguration;
+        this.executorConfiguration = executorConfiguration;
+        this.mapper = builder.build();
+    }
+
    public static void main(String[] args) {
        SpringApplication.run(ExecutorManagerApplication.class, args);
    }

    @Override
    public void run(ApplicationArguments args) throws Exception {
+        String taskId = IdUtil.nanoId(8);
+
        Configuration configuration = new Configuration();
        configuration.setBoolean(JobManagerOptions.JVM_DIRECT_MEMORY_LIMIT_ENABLED, true);
        configuration.setString(AkkaOptions.ASK_TIMEOUT, "10 min");
@@ -54,15 +73,15 @@ public class ExecutorManagerApplication implements ApplicationRunner {
        configuration.set(ClientOptions.CLIENT_TIMEOUT, Duration.ofMinutes(30));
        // Kerberos认证
        configuration.setBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE, true);
-        configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, "/etc/security/keytabs/datalake.app.keytab");
-        configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, "datalake/b5s119.hdp.dc@ECLD.COM");
+        configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, hadoopConfiguration.getKerberosKeytabPath());
+        configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, hadoopConfiguration.getKerberosPrincipal());
        configuration.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, MINUTE);
        configuration.setLong(HeartbeatManagerOptions.HEARTBEAT_TIMEOUT, HALF_HOUR);
        configuration.setString(AkkaOptions.ASK_TIMEOUT, "1 min");
        configuration.setString(AkkaOptions.TCP_TIMEOUT, "2 min");
        configuration.setBoolean(CoreOptions.CHECK_LEAKED_CLASSLOADER, false);
        configuration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "4");
-        configuration.setString(YarnConfigOptions.STAGING_DIRECTORY, "hdfs://b2/apps/datalake/yarn");
+        configuration.setString(YarnConfigOptions.STAGING_DIRECTORY, executorConfiguration.getStagingDirectory());
        configuration.setString(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX + "MALLOC_ARENA_MAX", "1");
        configuration.setString(ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX + "MALLOC_ARENA_MAX", "1");
        configuration.setInteger(RestOptions.PORT, 8081);
@@ -74,22 +93,28 @@ public class ExecutorManagerApplication implements ApplicationRunner {
        configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("0m"));
        configuration.set(TaskManagerOptions.JVM_METASPACE, MemorySize.parse("128m"));
        configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 10);
-        configuration.setString(YarnConfigOptions.APPLICATION_NAME, "HudiService_faee2e95-660d-4b1c-9cec-13473b3cd5b7");
-        configuration.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, "hdfs://b2/apps/flink/completed-jobs/");
+        configuration.setString(YarnConfigOptions.APPLICATION_NAME, "Service_Task_" + taskId);
+        configuration.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, executorConfiguration.getHistoryServerArchiveDir());
        configuration.setLong(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_REFRESH_INTERVAL, 10000);

        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.setString(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, "/app-logs");

        configuration.set(PipelineOptions.JARS, new ArrayList<String>() {{
-            add("hdfs://b2/apps/datalake/jars/service/service-executor-task.jar");
+            add(executorConfiguration.getTaskJarPath());
        }});
        ApplicationId applicationId = Runner.run(
                configuration,
-                "com.lanyuanxiaoyao.service.executor.task.Hello",
+                "com.lanyuanxiaoyao.service.executor.task.AvroScanner",
                new String[]{
                        TaskConstants.TASK_CONTEXT_OPTION,
-                        "{\"metadata\": {\"id\":\"faee2e95-660d-4b1c-9cec-13473b3cd5b7\"}}"
+                        mapper.writeValueAsString(
+                                new TaskContext(
+                                        taskId,
+                                        executorConfiguration.getTaskResultPath(),
+                                        Maps.mutable.of("key", "123456", "hdfs", "hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr")
+                                )
+                        )
                }
        );
        System.out.println(applicationId);
--- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/configuration/ExecutorConfiguration.java
+++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/configuration/ExecutorConfiguration.java
@@ -0,0 +1,66 @@
+package com.lanyuanxiaoyao.service.executor.manager.configuration;
+
+import javax.annotation.PostConstruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * yarn 配置
+ *
+ * @author ZhangJiacheng
+ * @date 2022-03-30
+ */
+@ConfigurationProperties("executor")
+@Component
+public class ExecutorConfiguration {
+    private static final Logger logger = LoggerFactory.getLogger(ExecutorConfiguration.class);
+
+    private String stagingDirectory;
+    private String historyServerArchiveDir;
+    private String taskJarPath;
+    private String taskResultPath;
+
+    public String getStagingDirectory() {
+        return stagingDirectory;
+    }
+
+    public void setStagingDirectory(String stagingDirectory) {
+        this.stagingDirectory = stagingDirectory;
+    }
+
+    public String getHistoryServerArchiveDir() {
+        return historyServerArchiveDir;
+    }
+
+    public void setHistoryServerArchiveDir(String historyServerArchiveDir) {
+        this.historyServerArchiveDir = historyServerArchiveDir;
+    }
+
+    public String getTaskJarPath() {
+        return taskJarPath;
+    }
+
+    public void setTaskJarPath(String taskJarPath) {
+        this.taskJarPath = taskJarPath;
+    }
+
+    public String getTaskResultPath() {
+        return taskResultPath;
+    }
+
+    public void setTaskResultPath(String taskResultPath) {
+        this.taskResultPath = taskResultPath;
+    }
+
+    @Override
+    public String toString() {
+        return "ExecutorConfiguration{" +
+               "stagingDirectory='" + stagingDirectory + '\'' +
+               ", historyServerArchiveDir='" + historyServerArchiveDir + '\'' +
+               ", taskJarPath='" + taskJarPath + '\'' +
+               ", taskResultPath='" + taskResultPath + '\'' +
+               '}';
+    }
+}
--- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/configuration/HadoopConfiguration.java
+++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/configuration/HadoopConfiguration.java
@@ -0,0 +1,51 @@
+package com.lanyuanxiaoyao.service.executor.manager.configuration;
+
+import javax.annotation.PostConstruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * yarn 配置
+ *
+ * @author ZhangJiacheng
+ * @date 2022-03-30
+ */
+@ConfigurationProperties("connector.hadoop")
+@Component
+public class HadoopConfiguration {
+    private static final Logger logger = LoggerFactory.getLogger(HadoopConfiguration.class);
+
+    private String kerberosPrincipal;
+    private String kerberosKeytabPath;
+
+    @PostConstruct
+    private void init() {
+        logger.info("Configuration initial: {}", this);
+    }
+
+    public String getKerberosPrincipal() {
+        return kerberosPrincipal;
+    }
+
+    public void setKerberosPrincipal(String kerberosPrincipal) {
+        this.kerberosPrincipal = kerberosPrincipal;
+    }
+
+    public String getKerberosKeytabPath() {
+        return kerberosKeytabPath;
+    }
+
+    public void setKerberosKeytabPath(String kerberosKeytabPath) {
+        this.kerberosKeytabPath = kerberosKeytabPath;
+    }
+
+    @Override
+    public String toString() {
+        return "HadoopConfiguration{" +
+                "kerberosPrincipal='" + kerberosPrincipal + '\'' +
+                ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
+                '}';
+    }
+}
--- a/service-executor/service-executor-manager/src/main/resources/application.yml
+++ b/service-executor/service-executor-manager/src/main/resources/application.yml
@@ -2,4 +2,9 @@ spring:
  application:
    name: service-executor-manager
  profiles:
-    include: random-port,common,discovery,metrics
+    include: random-port,common,discovery,metrics
+executor:
+  staging-directory: hdfs://b2/apps/datalake/yarn
+  history-server-archive-dir: hdfs://b2/apps/flink/completed-jobs/
+  task-jar-path: hdfs://b2/apps/datalake/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar
+  task-result-path: hdfs://b2/apps/datalake/task-results