diff --git a/bin/build-all.sh b/bin/build-all.sh index 9e454de..66a0f74 100755 --- a/bin/build-all.sh +++ b/bin/build-all.sh @@ -1,39 +1,41 @@ #!/bin/bash - -current_path=$(dirname $(cd `dirname $0`;pwd)) - +root_path=$(dirname $(cd $(dirname $0);pwd)) +source $root_path/bin/library.sh mvn deploy -N -D skipTests -P local -s ~/.m2/settings-development.xml mvn -pl service-common,service-dependencies,service-configuration,service-forest,service-cli,service-cli/service-cli-core,service-executor,service-executor/service-executor-core,utils/executor clean source:jar deploy -D skipTests -P local -s ~/.m2/settings-development.xml mvn -pl service-api,service-check,service-cli/service-cli-runner,service-cloud-query,service-command,service-exporter,service-flink-query,service-gateway,service-info-query,service-loki-query,service-pulsar-query,service-queue,service-scheduler,service-web,service-yarn-query,service-zookeeper-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml -P b2b12 mvn -pl service-executor/service-executor-manager,service-executor/service-executor-task,service-hudi-query,utils/sync clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2a4 -${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar +upload $root_path/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b1 -${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar +upload $root_path/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b5 -${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar +upload $root_path/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12 -${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar +upload $root_path/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-api/target/service-api-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-check/target/service-check-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-command/target/service-command-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-web/target/service-web-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar -${current_path}/bin/upload-rm.sh ${current_path}/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-api/target/service-api-1.0.0-SNAPSHOT.jar +upload $root_path/service-check/target/service-check-1.0.0-SNAPSHOT.jar +upload $root_path/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-command/target/service-command-1.0.0-SNAPSHOT.jar +upload $root_path/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar +upload $root_path/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar +upload $root_path/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar +upload $root_path/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar +upload $root_path/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar +upload $root_path/service-web/target/service-web-1.0.0-SNAPSHOT.jar +upload $root_path/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar +upload $root_path/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar +upload $root_path/utils/sync/target/sync-1.0.0-SNAPSHOT.jar + +$root_path/bin/upload-rm $root_path/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar +$root_path/bin/upload-rm $root_path/service-uploader/target/service-uploader-1.0.0-SNAPSHOT.jar diff --git a/bin/generate b/bin/generate index 25a5e60..a844e16 100644 --- a/bin/generate +++ b/bin/generate @@ -5,9 +5,10 @@ jdk_path=/opt/jdk1.8.0_162/bin/java arguments=$@ curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-cli-runner-1.0.0-SNAPSHOT.jar -o ${jars_path}/service-cli-runner.jar -${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true $arguments +${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true --deploy.generate.update-jar=true $arguments chmod +x cloud/*.sh chmod +x command/cli* chmod +x command/*.sh chmod +x uploader/*.sh +chmod +x *.sh diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java index ab9d986..95640a6 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java @@ -78,6 +78,7 @@ public class DeployInformationProperties { private Boolean cloud = true; private Boolean command = true; private Boolean uploader = true; + private Boolean updateJar = true; public Boolean getCloud() { return cloud; @@ -103,12 +104,21 @@ public class DeployInformationProperties { this.uploader = uploader; } + public Boolean getUpdateJar() { + return updateJar; + } + + public void setUpdateJar(Boolean updateJar) { + this.updateJar = updateJar; + } + @Override public String toString() { return "Generate{" + "cloud=" + cloud + ", command=" + command + ", uploader=" + uploader + + ", updateJar=" + updateJar + '}'; } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 6eba03c..67f3b3f 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -29,6 +29,7 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.map.MutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.SpringApplication; @@ -50,6 +51,8 @@ public class RunnerApplication implements ApplicationRunner { private final ImmutableList serviceInfoList; private final ImmutableList hostInfoList; private final ObjectMapper mapper; + @Value("${deploy.runtime.executor.task-jar-path}") + private String taskJarPath; public RunnerApplication(DeployInformationProperties deployInformationProperties) { this.deployInformationProperties = deployInformationProperties; @@ -95,9 +98,10 @@ public class RunnerApplication implements ApplicationRunner { generateCloud(Paths.get("cloud")); if (deployInformationProperties.getGenerate().getCommand()) generateCommand(Paths.get("command")); - if (deployInformationProperties.getGenerate().getUploader()) { + if (deployInformationProperties.getGenerate().getUploader()) generateUploader(Paths.get("uploader")); - } + if (deployInformationProperties.getGenerate().getUpdateJar()) + generateUpdateJar(Paths.get("")); } private void generateCloud(Path root) throws IOException { @@ -302,4 +306,34 @@ public class RunnerApplication implements ApplicationRunner { stopScript = stopScript.replaceAll("\\$1", "service-uploader.jar"); Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); } + + private void generateUpdateJar(Path root) throws IOException { + String absolutRootPath = root.toAbsolutePath().toString(); + logger.info("Current path: {}", absolutRootPath); + + TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH)); + Template syncTemplate = engine.getTemplate("update-jar.ftl"); + String syncScript = syncTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .put("jarPrefix", "sync") + .put("jarName", "sync-1.0.0-SNAPSHOT.jar") + .put("uploadPath", runtimeInfo.getHudi().getAppHdfsPath()) + .build()); + Path syncScriptFile = Paths.get(root.toString(), "update-sync.sh"); + Files.deleteIfExists(syncScriptFile); + Files.write(syncScriptFile, syncScript.getBytes()); + + Template taskTemplate = engine.getTemplate("update-jar.ftl"); + String taskScript = taskTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .put("jarPrefix", "task") + .put("jarName", "service-executor-task-1.0.0-SNAPSHOT.jar") + .put("uploadPath", taskJarPath) + .build()); + Path taskScriptFile = Paths.get(root.toString(), "update-task.sh"); + Files.deleteIfExists(taskScriptFile); + Files.write(taskScriptFile, taskScript.getBytes()); + } } diff --git a/service-cli/service-cli-runner/src/main/resources/template/update-jar.ftl b/service-cli/service-cli-runner/src/main/resources/template/update-jar.ftl new file mode 100644 index 0000000..da6a02b --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/update-jar.ftl @@ -0,0 +1,7 @@ +#!/bin/bash +timestamp=`date +%s%N` +jar_prefix=${jarPrefix} +jar_name=${jarName} +echo "Now version: <#noparse>${timestamp}" +curl ${runtime.downloadUrl}/<#noparse>${jar_name} -o <#noparse>/tmp/${jar_prefix}-${timestamp}.jar +hadoop fs -put <#noparse>/tmp/${jar_prefix}-${timestamp}.jar ${uploadPath} diff --git a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java index 4c30f4c..cc0a2fb 100644 --- a/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java +++ b/service-executor/service-executor-manager/src/main/java/com/lanyuanxiaoyao/service/executor/manager/service/ExecutorTaskService.java @@ -1,9 +1,11 @@ package com.lanyuanxiaoyao.service.executor.manager.service; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.lang.Tuple; import cn.hutool.core.map.MapBuilder; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.IdUtil; +import cn.hutool.core.util.ReUtil; import cn.hutool.core.util.StrUtil; import com.fasterxml.jackson.databind.ObjectMapper; import com.lanyuanxiaoyao.service.executor.Runner; @@ -15,6 +17,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.time.Duration; import java.util.ArrayList; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.flink.client.cli.ClientOptions; @@ -26,7 +29,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.list.ImmutableList; @@ -48,6 +50,7 @@ import static com.lanyuanxiaoyao.service.common.Constants.MINUTE; @Service public class ExecutorTaskService { private static final Logger logger = LoggerFactory.getLogger(ExecutorTaskService.class); + private static final Pattern EXECUTOR_JAR_NAME = Pattern.compile(".+task-(\\d+)\\.jar"); private final HadoopConfiguration hadoopConfiguration; private final ExecutorConfiguration executorConfiguration; @@ -68,7 +71,27 @@ public class ExecutorTaskService { configuration.setString(ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX + key, value); } - private Configuration generateConfiguration(String taskId, String name) { + private String getLatestExecutorJarPath() throws IOException { + try (FileSystem fileSystem = FileSystem.get(new org.apache.hadoop.conf.Configuration())) { + Path root = new Path(executorConfiguration.getTaskJarPath()); + return Lists.immutable.of(fileSystem.listStatus(root)) + .select(FileStatus::isFile) + .collect(FileStatus::getPath) + .collect(Path::toString) + .select(path -> ReUtil.isMatch(EXECUTOR_JAR_NAME, path)) + .collect(path -> new Tuple(path, getLatestExecutorJarVersion(path))) + .reject(tuple -> tuple.get(1) < 0) + .maxByOptional(tuple -> tuple.get(1)) + .orElseThrow(() -> new RuntimeException("Latest jar path not found")) + .get(0); + } + } + + private Long getLatestExecutorJarVersion(String path) { + return Optional.ofNullable(ReUtil.get(EXECUTOR_JAR_NAME, path, 1)).map(Long::valueOf).orElse(-1L); + } + + private Configuration generateConfiguration(String taskId, String name) throws IOException { Configuration configuration = new Configuration(); configuration.setBoolean(JobManagerOptions.JVM_DIRECT_MEMORY_LIMIT_ENABLED, true); configuration.setString(AkkaOptions.ASK_TIMEOUT, "10 min"); @@ -100,8 +123,11 @@ public class ExecutorTaskService { setEnvironment(configuration, "task_id", taskId); + // 业务jar包 + String executorJarPath = getLatestExecutorJarPath(); + logger.info("Executor jar path: {}", executorJarPath); configuration.set(PipelineOptions.JARS, new ArrayList() {{ - add(executorConfiguration.getTaskJarPath()); + add(executorJarPath); }}); return configuration; }