refactor(executor-manager): 使用sync同款jar更新方式

2024-03-01 15:18:30 +08:00
parent 36c31df2c1
commit 7dc38383c5
6 changed files with 111 additions and 31 deletions
--- a/bin/build-all.sh
+++ b/bin/build-all.sh
@@ -1,39 +1,41 @@
 #!/bin/bash
-
-current_path=$(dirname $(cd `dirname $0`;pwd))
-
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn deploy -N -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-common,service-dependencies,service-configuration,service-forest,service-cli,service-cli/service-cli-core,service-executor,service-executor/service-executor-core,utils/executor clean source:jar deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-api,service-check,service-cli/service-cli-runner,service-cloud-query,service-command,service-exporter,service-flink-query,service-gateway,service-info-query,service-loki-query,service-pulsar-query,service-queue,service-scheduler,service-web,service-yarn-query,service-zookeeper-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml -P b2b12
 mvn -pl service-executor/service-executor-manager,service-executor/service-executor-task,service-hudi-query,utils/sync clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12

 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2a4
-${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar

 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b1
-${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar

 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b5
-${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar

 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
-${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar

-${current_path}/bin/upload-rm.sh ${current_path}/service-api/target/service-api-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-check/target/service-check-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-command/target/service-command-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-web/target/service-web-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar
-${current_path}/bin/upload-rm.sh ${current_path}/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-api/target/service-api-1.0.0-SNAPSHOT.jar
+upload $root_path/service-check/target/service-check-1.0.0-SNAPSHOT.jar
+upload $root_path/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-command/target/service-command-1.0.0-SNAPSHOT.jar
+upload $root_path/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar
+upload $root_path/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar
+upload $root_path/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar
+upload $root_path/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar
+upload $root_path/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar
+upload $root_path/service-web/target/service-web-1.0.0-SNAPSHOT.jar
+upload $root_path/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar
+upload $root_path/utils/sync/target/sync-1.0.0-SNAPSHOT.jar
+
+$root_path/bin/upload-rm $root_path/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar
+$root_path/bin/upload-rm $root_path/service-uploader/target/service-uploader-1.0.0-SNAPSHOT.jar
--- a/bin/generate
+++ b/bin/generate
@@ -5,9 +5,10 @@ jdk_path=/opt/jdk1.8.0_162/bin/java

 arguments=$@
 curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-cli-runner-1.0.0-SNAPSHOT.jar -o ${jars_path}/service-cli-runner.jar
-${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true $arguments
+${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true --deploy.generate.update-jar=true $arguments

 chmod +x cloud/*.sh
 chmod +x command/cli*
 chmod +x command/*.sh
 chmod +x uploader/*.sh
+chmod +x *.sh
--- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java
+++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java
@@ -78,6 +78,7 @@ public class DeployInformationProperties {
        private Boolean cloud = true;
        private Boolean command = true;
        private Boolean uploader = true;
+        private Boolean updateJar = true;

        public Boolean getCloud() {
            return cloud;
@@ -103,12 +104,21 @@ public class DeployInformationProperties {
            this.uploader = uploader;
        }

+        public Boolean getUpdateJar() {
+            return updateJar;
+        }
+
+        public void setUpdateJar(Boolean updateJar) {
+            this.updateJar = updateJar;
+        }
+
        @Override
        public String toString() {
            return "Generate{" +
                   "cloud=" + cloud +
                   ", command=" + command +
                   ", uploader=" + uploader +
+                   ", updateJar=" + updateJar +
                   '}';
        }
    }
--- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
+++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
@@ -29,6 +29,7 @@ import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.api.map.MutableMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.boot.SpringApplication;
@@ -50,6 +51,8 @@ public class RunnerApplication implements ApplicationRunner {
    private final ImmutableList<ServiceInfoWrapper> serviceInfoList;
    private final ImmutableList<HostInfoWrapper> hostInfoList;
    private final ObjectMapper mapper;
+    @Value("${deploy.runtime.executor.task-jar-path}")
+    private String taskJarPath;

    public RunnerApplication(DeployInformationProperties deployInformationProperties) {
        this.deployInformationProperties = deployInformationProperties;
@@ -95,9 +98,10 @@ public class RunnerApplication implements ApplicationRunner {
            generateCloud(Paths.get("cloud"));
        if (deployInformationProperties.getGenerate().getCommand())
            generateCommand(Paths.get("command"));
-        if (deployInformationProperties.getGenerate().getUploader()) {
+        if (deployInformationProperties.getGenerate().getUploader())
            generateUploader(Paths.get("uploader"));
-        }
+        if (deployInformationProperties.getGenerate().getUpdateJar())
+            generateUpdateJar(Paths.get(""));
    }

    private void generateCloud(Path root) throws IOException {
@@ -302,4 +306,34 @@ public class RunnerApplication implements ApplicationRunner {
        stopScript = stopScript.replaceAll("\\$1", "service-uploader.jar");
        Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
    }
+
+    private void generateUpdateJar(Path root) throws IOException {
+        String absolutRootPath = root.toAbsolutePath().toString();
+        logger.info("Current path: {}", absolutRootPath);
+
+        TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
+        Template syncTemplate = engine.getTemplate("update-jar.ftl");
+        String syncScript = syncTemplate.render(MapUtil.builder()
+                .put("currentPath", absolutRootPath)
+                .put("runtime", runtimeInfo)
+                .put("jarPrefix", "sync")
+                .put("jarName", "sync-1.0.0-SNAPSHOT.jar")
+                .put("uploadPath", runtimeInfo.getHudi().getAppHdfsPath())
+                .build());
+        Path syncScriptFile = Paths.get(root.toString(), "update-sync.sh");
+        Files.deleteIfExists(syncScriptFile);
+        Files.write(syncScriptFile, syncScript.getBytes());
+
+        Template taskTemplate = engine.getTemplate("update-jar.ftl");
+        String taskScript = taskTemplate.render(MapUtil.builder()
+                .put("currentPath", absolutRootPath)
+                .put("runtime", runtimeInfo)
+                .put("jarPrefix", "task")
+                .put("jarName", "service-executor-task-1.0.0-SNAPSHOT.jar")
+                .put("uploadPath", taskJarPath)
+                .build());
+        Path taskScriptFile = Paths.get(root.toString(), "update-task.sh");
+        Files.deleteIfExists(taskScriptFile);
+        Files.write(taskScriptFile, taskScript.getBytes());
+    }
 }
--- a/service-cli/service-cli-runner/src/main/resources/template/update-jar.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/update-jar.ftl
@@ -0,0 +1,7 @@
+#!/bin/bash
+timestamp=`date +%s%N`
+jar_prefix=${jarPrefix}
+jar_name=${jarName}
+echo "Now version: <#noparse>${timestamp}</#noparse>"
+curl ${runtime.downloadUrl}/<#noparse>${jar_name}</#noparse> -o <#noparse>/tmp/${jar_prefix}-${timestamp}.jar</#noparse>
+hadoop fs -put <#noparse>/tmp/${jar_prefix}-${timestamp}.jar</#noparse> ${uploadPath}
--- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java
+++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java
@@ -1,9 +1,11 @@
 package com.lanyuanxiaoyao.service.executor.manager.service;

 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.lang.Tuple;
 import cn.hutool.core.map.MapBuilder;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.IdUtil;
+import cn.hutool.core.util.ReUtil;
 import cn.hutool.core.util.StrUtil;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.lanyuanxiaoyao.service.executor.Runner;
@@ -15,6 +17,7 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.flink.client.cli.ClientOptions;
@@ -26,7 +29,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.factory.Maps;
 import org.eclipse.collections.api.list.ImmutableList;
@@ -48,6 +50,7 @@ import static com.lanyuanxiaoyao.service.common.Constants.MINUTE;
@Service
 public class ExecutorTaskService {
    private static final Logger logger = LoggerFactory.getLogger(ExecutorTaskService.class);
+    private static final Pattern EXECUTOR_JAR_NAME = Pattern.compile(".+task-(\\d+)\\.jar");

    private final HadoopConfiguration hadoopConfiguration;
    private final ExecutorConfiguration executorConfiguration;
@@ -68,7 +71,27 @@ public class ExecutorTaskService {
        configuration.setString(ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX + key, value);
    }

-    private Configuration generateConfiguration(String taskId, String name) {
+    private String getLatestExecutorJarPath() throws IOException {
+        try (FileSystem fileSystem = FileSystem.get(new org.apache.hadoop.conf.Configuration())) {
+            Path root = new Path(executorConfiguration.getTaskJarPath());
+            return Lists.immutable.of(fileSystem.listStatus(root))
+                    .select(FileStatus::isFile)
+                    .collect(FileStatus::getPath)
+                    .collect(Path::toString)
+                    .select(path -> ReUtil.isMatch(EXECUTOR_JAR_NAME, path))
+                    .collect(path -> new Tuple(path, getLatestExecutorJarVersion(path)))
+                    .reject(tuple -> tuple.<Long>get(1) < 0)
+                    .maxByOptional(tuple -> tuple.<Long>get(1))
+                    .orElseThrow(() -> new RuntimeException("Latest jar path not found"))
+                    .get(0);
+        }
+    }
+
+    private Long getLatestExecutorJarVersion(String path) {
+        return Optional.ofNullable(ReUtil.get(EXECUTOR_JAR_NAME, path, 1)).map(Long::valueOf).orElse(-1L);
+    }
+
+    private Configuration generateConfiguration(String taskId, String name) throws IOException {
        Configuration configuration = new Configuration();
        configuration.setBoolean(JobManagerOptions.JVM_DIRECT_MEMORY_LIMIT_ENABLED, true);
        configuration.setString(AkkaOptions.ASK_TIMEOUT, "10 min");
@@ -100,8 +123,11 @@ public class ExecutorTaskService {

        setEnvironment(configuration, "task_id", taskId);

+        // 业务jar包
+        String executorJarPath = getLatestExecutorJarPath();
+        logger.info("Executor jar path: {}", executorJarPath);
        configuration.set(PipelineOptions.JARS, new ArrayList<String>() {{
-            add(executorConfiguration.getTaskJarPath());
+            add(executorJarPath);
        }});
        return configuration;
    }