feat(cli): 增加服务分组方便批量启停服务

服务可以分在多个分组里

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java

@@ -17,10 +17,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -94,6 +91,10 @@ public class RunnerApplication implements ApplicationRunner {
 
         Path root = Paths.get("");
         Files.createDirectories(root);
+
+        String absolutRootPath = root.toAbsolutePath().toString();
+        logger.info("Current path: {}", absolutRootPath);
+
         TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
         Template deployTemplate = engine.getTemplate("deploy.ftl");
         for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) {
@@ -112,6 +113,7 @@ public class RunnerApplication implements ApplicationRunner {
                 deployPlans.put(serviceInfo.getName(), selectedHosts);
             }
             String deployScript = deployTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                     .put("hosts", deployInformationProperties.getHosts()
                             .stream()
                             .map(HostInfo::getHostnameIp)
@@ -132,6 +134,7 @@ public class RunnerApplication implements ApplicationRunner {
 
             Template stopTemplate = engine.getTemplate("stop.ftl");
             String stopScript = stopTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                     .put("hosts", deployInformationProperties.getHosts()
                             .stream()
                             .map(HostInfo::getIp)
@@ -151,6 +154,7 @@ public class RunnerApplication implements ApplicationRunner {
 
             Template logTemplate = engine.getTemplate("log.ftl");
             String logScript = logTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
                     .put("hosts", deployInformationProperties.getHosts()
                             .stream()
                             .map(HostInfo::getIp)
@@ -212,6 +216,46 @@ public class RunnerApplication implements ApplicationRunner {
                 "done";
         Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
 
+        Map<String, List<ServiceInfo>> groups = new HashMap<>();
+        for (ServiceInfo service : deployInformationProperties.getServices()) {
+            service.getGroups().add(0, "all");
+            for (String group : service.getGroups()) {
+                if (!groups.containsKey(group)) {
+                    groups.put(group, new ArrayList<>());
+                }
+                List<ServiceInfo> list = groups.get(group);
+                list.add(service);
+            }
+        }
+        for (Map.Entry<String, List<ServiceInfo>> entry : groups.entrySet()) {
+            String group = entry.getKey();
+            List<ServiceInfo> infos = entry.getValue();
+
+            Template batchDeployTemplate = engine.getTemplate("batch-deploy.ftl");
+            String batchDeployScript = batchDeployTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
+                    .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList()))
+                    .build());
+            Path batchDeployScriptFile = Paths.get(
+                    root.toString(),
+                    StrUtil.format("batch-deploy-{}.sh", group)
+            );
+            Files.deleteIfExists(batchDeployScriptFile);
+            Files.write(batchDeployScriptFile, batchDeployScript.getBytes());
+
+            Template batchStopTemplate = engine.getTemplate("batch-stop.ftl");
+            String batchStopScript = batchStopTemplate.render(MapUtil.builder()
+                    .put("currentPath", absolutRootPath)
+                    .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList()))
+                    .build());
+            Path batchStopScriptFile = Paths.get(
+                    root.toString(),
+                    StrUtil.format("batch-stop-{}.sh", group)
+            );
+            Files.deleteIfExists(batchStopScriptFile);
+            Files.write(batchStopScriptFile, batchStopScript.getBytes());
+        }
+
         Files.deleteIfExists(planPath);
         Files.write(planPath, mapper.writeValueAsString(deployPlans).getBytes());
     }

service-cli/service-cli-runner/src/main/resources/application.yml

@@ -1,8 +1,5 @@
 deploy:
   services:
-    - name: api
-      source-jar: api-1.0.0-SNAPSHOT.jar
-      replicas: 10
     - name: service-gateway
       source-jar: service-gateway-1.0.0-SNAPSHOT.jar
     - name: service-queue
@@ -11,10 +8,20 @@ deploy:
       arguments:
         data_save_enable: true
         data_save_location: ${deploy.runtime.data-path}
+    - name: api
+      source-jar: api-1.0.0-SNAPSHOT.jar
+      replicas: 10
     - name: service-scheduler
+      groups:
+        - "service"
+        - "service-hudi"
       source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
       replicas: 1
     - name: service-launcher-runner-b1
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
       source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar
       replicas: 6
       environments:
@@ -31,6 +38,10 @@ deploy:
         connector_cluster_compaction-queue-name: compaction-queue-b1
         connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
     - name: service-launcher-runner-b5
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
       source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
       replicas: 6
       environments:
@@ -46,23 +57,11 @@ deploy:
         connector_cluster_sync-queue-name: sync-queue-b5
         connector_cluster_compaction-queue-name: compaction-queue-b5
         connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-launcher-runner-b5-sync
-      source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar
-      replicas: 6
-      environments:
-        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
-        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
-        connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
-        connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
-        connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
-        connector_hudi_loki-push-url: ${deploy.runtime.hudi.loki-push-url}
-      arguments:
-        spring_application_name: service-launcher-runner-b5-sync
-        connector_cluster_cluster: b5-sync
-        connector_cluster_sync-queue-name: sync-queue-b5-sync
-        connector_cluster_compaction-queue-name: compaction-queue-b5-sync
-        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
     - name: service-launcher-runner-a4
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
       source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
       replicas: 6
       environments:
@@ -79,6 +78,10 @@ deploy:
         connector_cluster_compaction-queue-name: compaction-queue-a4
         connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
     - name: service-launcher-runner-b12
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
       source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar
       replicas: 6
       environments:
@@ -95,9 +98,13 @@ deploy:
         connector_cluster_compaction-queue-name: compaction-queue-b12
         connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
     - name: service-info-query
+      groups:
+        - "service"
       source-jar: service-info-query-1.0.0-SNAPSHOT.jar
       replicas: 10
     - name: service-yarn-query
+      groups:
+        - "service"
       source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
       replicas: 20
       arguments:
@@ -106,27 +113,36 @@ deploy:
         yarn_web-urls_b4: http://132.122.112.30:8088
         yarn_web-urls_b5: http://132.122.116.12:8088
         yarn_web-urls_b5-sync: http://132.122.116.143:8088
-    - name: service-pulsar-query
-      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
-      replicas: 10
     - name: service-zookeeper-query
+      groups:
+        - "service"
       source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
       replicas: 10
       arguments:
         connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-web
-      source-jar: service-web-1.0.0-SNAPSHOT.jar
-      replicas: 3
     - name: service-hudi-query
+      groups:
+        - "service"
       source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
       replicas: 20
+    - name: service-pulsar-query
+      groups:
+        - "service"
+      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
+      replicas: 10
     - name: service-flink-query
+      groups:
+        - "service"
       source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
       replicas: 10
     - name: service-cloud-query
+      groups:
+        - "service"
       source-jar: service-cloud-query-1.0.0-SNAPSHOT.jar
       replicas: 5
     - name: service-executor-manager
+      groups:
+        - "service"
       source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
       replicas: 5
       environments:
@@ -137,6 +153,13 @@ deploy:
         executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path}
         executor_task-jar-path: ${deploy.runtime.executor.task-jar-path}
         executor_task-result-path: ${deploy.runtime.executor.task-result-path}
+    - name: service-web
+      groups:
+        - "service"
+      source-jar: service-web-1.0.0-SNAPSHOT.jar
+      replicas: 3
     - name: service-exporter
+      groups:
+        - "service"
       source-jar: service-exporter-1.0.0-SNAPSHOT.jar
       replicas: 3

service-cli/service-cli-runner/src/main/resources/template/batch-deploy.ftl

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+<#list services as service>
+${currentPath}/deploy-${service}.sh
+</#list>

service-cli/service-cli-runner/src/main/resources/template/batch-stop.ftl

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+<#list services as service>
+${currentPath}/stop-${service}.sh
+</#list>

service-cli/service-cli-runner/src/main/resources/template/deploy.ftl

@@ -16,7 +16,7 @@ security_darkcode='${runtime.security.darkcode}'
 <#list hosts as host>
 host=${host.ip}
 echo "${info.name} ${host.ip} ${host.hostname} $datetime"
-ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar
 <#if selectedHosts?seq_contains(host.ip)>
 hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`

service-cli/service-cli-runner/src/main/resources/template/log.ftl

@@ -13,5 +13,5 @@ for host in <#noparse>${hosts[@]}</#noparse>
 do
   hostname=`ssh $host 'echo $HOSTNAME'`
   echo "$host $hostname"
-  ssh $host "cat ${runtime.logPath}/${info.name}.log" > logs/${info.name}/$hostname.log
+  ssh $host "cat ${runtime.logPath}/${info.name}.log" > ${currentPath}/logs/${info.name}/$hostname.log
 done

service-cli/service-cli-runner/src/main/resources/template/stop.ftl

@@ -10,5 +10,5 @@ for host in <#noparse>${hosts[@]}</#noparse>
 do
   hostname=`ssh $host 'echo $HOSTNAME'`
   echo "$host $hostname"
-  ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+  ssh $host 'bash -s' < ${currentPath}/stop.sh ${runtime.jarPath}/${info.name}.jar
 done