diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java index 82ab199..bb7cb80 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java @@ -1,6 +1,8 @@ package com.lanyuanxiaoyao.service.cli.core; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -11,6 +13,7 @@ import java.util.Map; */ public class ServiceInfo { private String name; + private List groups = new ArrayList<>(); private Integer replicas = 0; private String sourceJar; private Map environments = new HashMap<>(); @@ -24,6 +27,14 @@ public class ServiceInfo { this.name = name; } + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups; + } + public Integer getReplicas() { return replicas; } @@ -59,11 +70,12 @@ public class ServiceInfo { @Override public String toString() { return "ServiceInfo{" + - "name='" + name + '\'' + - ", replicas=" + replicas + - ", sourceJar='" + sourceJar + '\'' + - ", environments=" + environments + - ", arguments=" + arguments + - '}'; + "name='" + name + '\'' + + ", groups=" + groups + + ", replicas=" + replicas + + ", sourceJar='" + sourceJar + '\'' + + ", environments=" + environments + + ", arguments=" + arguments + + '}'; } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 5b42be0..0646912 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -17,10 +17,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,6 +91,10 @@ public class RunnerApplication implements ApplicationRunner { Path root = Paths.get(""); Files.createDirectories(root); + + String absolutRootPath = root.toAbsolutePath().toString(); + logger.info("Current path: {}", absolutRootPath); + TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH)); Template deployTemplate = engine.getTemplate("deploy.ftl"); for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) { @@ -112,6 +113,7 @@ public class RunnerApplication implements ApplicationRunner { deployPlans.put(serviceInfo.getName(), selectedHosts); } String deployScript = deployTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) .put("hosts", deployInformationProperties.getHosts() .stream() .map(HostInfo::getHostnameIp) @@ -132,6 +134,7 @@ public class RunnerApplication implements ApplicationRunner { Template stopTemplate = engine.getTemplate("stop.ftl"); String stopScript = stopTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) .put("hosts", deployInformationProperties.getHosts() .stream() .map(HostInfo::getIp) @@ -151,6 +154,7 @@ public class RunnerApplication implements ApplicationRunner { Template logTemplate = engine.getTemplate("log.ftl"); String logScript = logTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) .put("hosts", deployInformationProperties.getHosts() .stream() .map(HostInfo::getIp) @@ -212,6 +216,46 @@ public class RunnerApplication implements ApplicationRunner { "done"; Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); + Map> groups = new HashMap<>(); + for (ServiceInfo service : deployInformationProperties.getServices()) { + service.getGroups().add(0, "all"); + for (String group : service.getGroups()) { + if (!groups.containsKey(group)) { + groups.put(group, new ArrayList<>()); + } + List list = groups.get(group); + list.add(service); + } + } + for (Map.Entry> entry : groups.entrySet()) { + String group = entry.getKey(); + List infos = entry.getValue(); + + Template batchDeployTemplate = engine.getTemplate("batch-deploy.ftl"); + String batchDeployScript = batchDeployTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList())) + .build()); + Path batchDeployScriptFile = Paths.get( + root.toString(), + StrUtil.format("batch-deploy-{}.sh", group) + ); + Files.deleteIfExists(batchDeployScriptFile); + Files.write(batchDeployScriptFile, batchDeployScript.getBytes()); + + Template batchStopTemplate = engine.getTemplate("batch-stop.ftl"); + String batchStopScript = batchStopTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList())) + .build()); + Path batchStopScriptFile = Paths.get( + root.toString(), + StrUtil.format("batch-stop-{}.sh", group) + ); + Files.deleteIfExists(batchStopScriptFile); + Files.write(batchStopScriptFile, batchStopScript.getBytes()); + } + Files.deleteIfExists(planPath); Files.write(planPath, mapper.writeValueAsString(deployPlans).getBytes()); } diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index b10e9a7..04ba878 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -1,8 +1,5 @@ deploy: services: - - name: api - source-jar: api-1.0.0-SNAPSHOT.jar - replicas: 10 - name: service-gateway source-jar: service-gateway-1.0.0-SNAPSHOT.jar - name: service-queue @@ -11,10 +8,20 @@ deploy: arguments: data_save_enable: true data_save_location: ${deploy.runtime.data-path} + - name: api + source-jar: api-1.0.0-SNAPSHOT.jar + replicas: 10 - name: service-scheduler + groups: + - "service" + - "service-hudi" source-jar: service-scheduler-1.0.0-SNAPSHOT.jar replicas: 1 - name: service-launcher-runner-b1 + groups: + - "service" + - "service-hudi" + - "service-hudi-launcher" source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar replicas: 6 environments: @@ -31,6 +38,10 @@ deploy: connector_cluster_compaction-queue-name: compaction-queue-b1 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-b5 + groups: + - "service" + - "service-hudi" + - "service-hudi-launcher" source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar replicas: 6 environments: @@ -46,23 +57,11 @@ deploy: connector_cluster_sync-queue-name: sync-queue-b5 connector_cluster_compaction-queue-name: compaction-queue-b5 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-launcher-runner-b5-sync - source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar - replicas: 6 - environments: - connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM - connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} - connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path} - connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path} - connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url} - connector_hudi_loki-push-url: ${deploy.runtime.hudi.loki-push-url} - arguments: - spring_application_name: service-launcher-runner-b5-sync - connector_cluster_cluster: b5-sync - connector_cluster_sync-queue-name: sync-queue-b5-sync - connector_cluster_compaction-queue-name: compaction-queue-b5-sync - connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-a4 + groups: + - "service" + - "service-hudi" + - "service-hudi-launcher" source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar replicas: 6 environments: @@ -79,6 +78,10 @@ deploy: connector_cluster_compaction-queue-name: compaction-queue-a4 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-b12 + groups: + - "service" + - "service-hudi" + - "service-hudi-launcher" source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar replicas: 6 environments: @@ -95,9 +98,13 @@ deploy: connector_cluster_compaction-queue-name: compaction-queue-b12 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-info-query + groups: + - "service" source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 10 - name: service-yarn-query + groups: + - "service" source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 20 arguments: @@ -106,27 +113,36 @@ deploy: yarn_web-urls_b4: http://132.122.112.30:8088 yarn_web-urls_b5: http://132.122.116.12:8088 yarn_web-urls_b5-sync: http://132.122.116.143:8088 - - name: service-pulsar-query - source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar - replicas: 10 - name: service-zookeeper-query + groups: + - "service" source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar replicas: 10 arguments: connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-web - source-jar: service-web-1.0.0-SNAPSHOT.jar - replicas: 3 - name: service-hudi-query + groups: + - "service" source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar replicas: 20 + - name: service-pulsar-query + groups: + - "service" + source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar + replicas: 10 - name: service-flink-query + groups: + - "service" source-jar: service-flink-query-1.0.0-SNAPSHOT.jar replicas: 10 - name: service-cloud-query + groups: + - "service" source-jar: service-cloud-query-1.0.0-SNAPSHOT.jar replicas: 5 - name: service-executor-manager + groups: + - "service" source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar replicas: 5 environments: @@ -137,6 +153,13 @@ deploy: executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path} executor_task-jar-path: ${deploy.runtime.executor.task-jar-path} executor_task-result-path: ${deploy.runtime.executor.task-result-path} + - name: service-web + groups: + - "service" + source-jar: service-web-1.0.0-SNAPSHOT.jar + replicas: 3 - name: service-exporter + groups: + - "service" source-jar: service-exporter-1.0.0-SNAPSHOT.jar replicas: 3 diff --git a/service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl new file mode 100644 index 0000000..a0a09ce --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl @@ -0,0 +1,5 @@ +#!/bin/bash + +<#list services as service> +${currentPath}/deploy-${service}.sh + diff --git a/service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl b/service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl new file mode 100644 index 0000000..0c6a4d7 --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl @@ -0,0 +1,5 @@ +#!/bin/bash + +<#list services as service> +${currentPath}/stop-${service}.sh + diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index 31b5195..a4085b2 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -16,7 +16,7 @@ security_darkcode='${runtime.security.darkcode}' <#list hosts as host> host=${host.ip} echo "${info.name} ${host.ip} ${host.hostname} $datetime" -ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar +ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar <#if selectedHosts?seq_contains(host.ip)> hostname=`ssh $host 'hostname'` hostname_full=`ssh $host 'hostname -f'` diff --git a/service-cli/service-cli-runner/src/main/resources/template/log.ftl b/service-cli/service-cli-runner/src/main/resources/template/log.ftl index f1353d8..8a80471 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/log.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/log.ftl @@ -13,5 +13,5 @@ for host in <#noparse>${hosts[@]} do hostname=`ssh $host 'echo $HOSTNAME'` echo "$host $hostname" - ssh $host "cat ${runtime.logPath}/${info.name}.log" > logs/${info.name}/$hostname.log + ssh $host "cat ${runtime.logPath}/${info.name}.log" > ${currentPath}/logs/${info.name}/$hostname.log done diff --git a/service-cli/service-cli-runner/src/main/resources/template/stop.ftl b/service-cli/service-cli-runner/src/main/resources/template/stop.ftl index bab588c..62b6067 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/stop.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/stop.ftl @@ -10,5 +10,5 @@ for host in <#noparse>${hosts[@]} do hostname=`ssh $host 'echo $HOSTNAME'` echo "$host $hostname" - ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar + ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar done