feat(cli): 增加服务分组方便批量启停服务

服务可以分在多个分组里
2024-02-04 18:27:51 +08:00
parent b8607dbdc4
commit d155779952
8 changed files with 127 additions and 38 deletions
--- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java
+++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java
@@ -1,6 +1,8 @@
 package com.lanyuanxiaoyao.service.cli.core;

+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;

 /**
@@ -11,6 +13,7 @@ import java.util.Map;
 */
 public class ServiceInfo {
    private String name;
+    private List<String> groups = new ArrayList<>();
    private Integer replicas = 0;
    private String sourceJar;
    private Map<String, Object> environments = new HashMap<>();
@@ -24,6 +27,14 @@ public class ServiceInfo {
        this.name = name;
    }

+    public List<String> getGroups() {
+        return groups;
+    }
+
+    public void setGroups(List<String> groups) {
+        this.groups = groups;
+    }
+
    public Integer getReplicas() {
        return replicas;
    }
@@ -59,11 +70,12 @@ public class ServiceInfo {
    @Override
    public String toString() {
        return "ServiceInfo{" +
-                "name='" + name + '\'' +
-                ", replicas=" + replicas +
-                ", sourceJar='" + sourceJar + '\'' +
-                ", environments=" + environments +
-                ", arguments=" + arguments +
-                '}';
+               "name='" + name + '\'' +
+               ", groups=" + groups +
+               ", replicas=" + replicas +
+               ", sourceJar='" + sourceJar + '\'' +
+               ", environments=" + environments +
+               ", arguments=" + arguments +
+               '}';
    }
 }
--- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
+++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
@@ -17,10 +17,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -94,6 +91,10 @@ public class RunnerApplication implements ApplicationRunner {

        Path root = Paths.get("");
        Files.createDirectories(root);
+
+        String absolutRootPath = root.toAbsolutePath().toString();
+        logger.info("Current path: {}", absolutRootPath);
+
        TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
        Template deployTemplate = engine.getTemplate("deploy.ftl");
        for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) {
@@ -112,6 +113,7 @@ public class RunnerApplication implements ApplicationRunner {
                deployPlans.put(serviceInfo.getName(), selectedHosts);
            }
            String deployScript = deployTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                    .put("hosts", deployInformationProperties.getHosts()
                            .stream()
                            .map(HostInfo::getHostnameIp)
@@ -132,6 +134,7 @@ public class RunnerApplication implements ApplicationRunner {

            Template stopTemplate = engine.getTemplate("stop.ftl");
            String stopScript = stopTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                    .put("hosts", deployInformationProperties.getHosts()
                            .stream()
                            .map(HostInfo::getIp)
@@ -151,6 +154,7 @@ public class RunnerApplication implements ApplicationRunner {

            Template logTemplate = engine.getTemplate("log.ftl");
            String logScript = logTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                    .put("hosts", deployInformationProperties.getHosts()
                            .stream()
                            .map(HostInfo::getIp)
@@ -212,6 +216,46 @@ public class RunnerApplication implements ApplicationRunner {
                "done";
        Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());

+        Map<String, List<ServiceInfo>> groups = new HashMap<>();
+        for (ServiceInfo service : deployInformationProperties.getServices()) {
+            service.getGroups().add(0, "all");
+            for (String group : service.getGroups()) {
+                if (!groups.containsKey(group)) {
+                    groups.put(group, new ArrayList<>());
+                }
+                List<ServiceInfo> list = groups.get(group);
+                list.add(service);
+            }
+        }
+        for (Map.Entry<String, List<ServiceInfo>> entry : groups.entrySet()) {
+            String group = entry.getKey();
+            List<ServiceInfo> infos = entry.getValue();
+
+            Template batchDeployTemplate = engine.getTemplate("batch-deploy.ftl");
+            String batchDeployScript = batchDeployTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
+                    .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList()))
+                    .build());
+            Path batchDeployScriptFile = Paths.get(
+                    root.toString(),
+                    StrUtil.format("batch-deploy-{}.sh", group)
+            );
+            Files.deleteIfExists(batchDeployScriptFile);
+            Files.write(batchDeployScriptFile, batchDeployScript.getBytes());
+
+            Template batchStopTemplate = engine.getTemplate("batch-stop.ftl");
+            String batchStopScript = batchStopTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
+                    .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList()))
+                    .build());
+            Path batchStopScriptFile = Paths.get(
+                    root.toString(),
+                    StrUtil.format("batch-stop-{}.sh", group)
+            );
+            Files.deleteIfExists(batchStopScriptFile);
+            Files.write(batchStopScriptFile, batchStopScript.getBytes());
+        }
+
        Files.deleteIfExists(planPath);
        Files.write(planPath, mapper.writeValueAsString(deployPlans).getBytes());
    }
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -1,8 +1,5 @@
 deploy:
  services:
-    - name: api
-      source-jar: api-1.0.0-SNAPSHOT.jar
-      replicas: 10
    - name: service-gateway
      source-jar: service-gateway-1.0.0-SNAPSHOT.jar
    - name: service-queue
@@ -11,10 +8,20 @@ deploy:
      arguments:
        data_save_enable: true
        data_save_location: ${deploy.runtime.data-path}
+    - name: api
+      source-jar: api-1.0.0-SNAPSHOT.jar
+      replicas: 10
    - name: service-scheduler
+      groups:
+        - "service"
+        - "service-hudi"
      source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
      replicas: 1
    - name: service-launcher-runner-b1
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar
      replicas: 6
      environments:
@@ -31,6 +38,10 @@ deploy:
        connector_cluster_compaction-queue-name: compaction-queue-b1
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-b5
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
      replicas: 6
      environments:
@@ -46,23 +57,11 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-b5
        connector_cluster_compaction-queue-name: compaction-queue-b5
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-launcher-runner-b5-sync
-      source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar
-      replicas: 6
-      environments:
-        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
-        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
-        connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
-        connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
-        connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
-        connector_hudi_loki-push-url: ${deploy.runtime.hudi.loki-push-url}
-      arguments:
-        spring_application_name: service-launcher-runner-b5-sync
-        connector_cluster_cluster: b5-sync
-        connector_cluster_sync-queue-name: sync-queue-b5-sync
-        connector_cluster_compaction-queue-name: compaction-queue-b5-sync
-        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-a4
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
      replicas: 6
      environments:
@@ -79,6 +78,10 @@ deploy:
        connector_cluster_compaction-queue-name: compaction-queue-a4
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-b12
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar
      replicas: 6
      environments:
@@ -95,9 +98,13 @@ deploy:
        connector_cluster_compaction-queue-name: compaction-queue-b12
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-info-query
+      groups:
+        - "service"
      source-jar: service-info-query-1.0.0-SNAPSHOT.jar
      replicas: 10
    - name: service-yarn-query
+      groups:
+        - "service"
      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
      replicas: 20
      arguments:
@@ -106,27 +113,36 @@ deploy:
        yarn_web-urls_b4: http://132.122.112.30:8088
        yarn_web-urls_b5: http://132.122.116.12:8088
        yarn_web-urls_b5-sync: http://132.122.116.143:8088
-    - name: service-pulsar-query
-      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
-      replicas: 10
    - name: service-zookeeper-query
+      groups:
+        - "service"
      source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
      replicas: 10
      arguments:
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-web
-      source-jar: service-web-1.0.0-SNAPSHOT.jar
-      replicas: 3
    - name: service-hudi-query
+      groups:
+        - "service"
      source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
      replicas: 20
+    - name: service-pulsar-query
+      groups:
+        - "service"
+      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
+      replicas: 10
    - name: service-flink-query
+      groups:
+        - "service"
      source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
      replicas: 10
    - name: service-cloud-query
+      groups:
+        - "service"
      source-jar: service-cloud-query-1.0.0-SNAPSHOT.jar
      replicas: 5
    - name: service-executor-manager
+      groups:
+        - "service"
      source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
      replicas: 5
      environments:
@@ -137,6 +153,13 @@ deploy:
        executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path}
        executor_task-jar-path: ${deploy.runtime.executor.task-jar-path}
        executor_task-result-path: ${deploy.runtime.executor.task-result-path}
+    - name: service-web
+      groups:
+        - "service"
+      source-jar: service-web-1.0.0-SNAPSHOT.jar
+      replicas: 3
    - name: service-exporter
+      groups:
+        - "service"
      source-jar: service-exporter-1.0.0-SNAPSHOT.jar
      replicas: 3
--- a/service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+<#list services as service>
+${currentPath}/deploy-${service}.sh
+</#list>
--- a/service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+<#list services as service>
+${currentPath}/stop-${service}.sh
+</#list>
--- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
@@ -16,7 +16,7 @@ security_darkcode='${runtime.security.darkcode}'
 <#list hosts as host>
 host=${host.ip}
 echo "${info.name} ${host.ip} ${host.hostname} $datetime"
-ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar
 <#if selectedHosts?seq_contains(host.ip)>
 hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`
--- a/service-cli/service-cli-runner/src/main/resources/template/log.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/log.ftl
@@ -13,5 +13,5 @@ for host in <#noparse>${hosts[@]}</#noparse>
 do
  hostname=`ssh $host 'echo $HOSTNAME'`
  echo "$host $hostname"
-  ssh $host "cat ${runtime.logPath}/${info.name}.log" > logs/${info.name}/$hostname.log
+  ssh $host "cat ${runtime.logPath}/${info.name}.log" > ${currentPath}/logs/${info.name}/$hostname.log
 done
--- a/service-cli/service-cli-runner/src/main/resources/template/stop.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/stop.ftl
@@ -10,5 +10,5 @@ for host in <#noparse>${hosts[@]}</#noparse>
 do
  hostname=`ssh $host 'echo $HOSTNAME'`
  echo "$host $hostname"
-  ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+  ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar
 done