feature(cli): 增加 cli 模块

cli 模块批量生成服务启停脚本
2023-05-18 11:11:39 +08:00
parent 69314f62ec
commit 296c744b61
13 changed files with 861 additions and 0 deletions
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -0,0 +1,239 @@
+deploy:
+  runtime:
+    user: datalake
+    jar-path: /apps/datalake/hudi/jars
+    jdk-path: /opt/jdk1.8.0_162/bin/java
+    log-path: /apps/datalake/hudi/logs
+    data-path: /apps/datalake/hudi/data
+    kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
+    loki-url: http://132.122.116.142:33100/loki/api/v1/push
+    zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
+    eureka-url: http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.142:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.143:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.144:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.145:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.146:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.147:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.148:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.149:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.150:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.151:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.152:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.153:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.154:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.155:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.156:35670/eureka
+    hudi:
+      app-hdfs-path: hdfs://b2/apps/datalake/jars/app
+      archive-hdfs-path: hdfs://b2/apps/datalake/flink/completed-jobs-hudi
+      victoria-push-url: http://132.122.116.142:35710/api/v1/import/prometheus
+  hosts:
+    - host: b5s119
+      ip: 132.122.116.142
+    - host: b5s120
+      ip: 132.122.116.143
+    - host: b5s121
+      ip: 132.122.116.144
+    - host: b5s122
+      ip: 132.122.116.145
+    - host: b5s123
+      ip: 132.122.116.146
+    - host: b5s124
+      ip: 132.122.116.147
+    - host: b5s125
+      ip: 132.122.116.148
+    - host: b5s126
+      ip: 132.122.116.149
+    - host: b5s127
+      ip: 132.122.116.150
+    - host: b5s128
+      ip: 132.122.116.151
+    - host: b5s129
+      ip: 132.122.116.152
+    - host: b5s130
+      ip: 132.122.116.153
+    - host: b5s131
+      ip: 132.122.116.154
+    - host: b5s132
+      ip: 132.122.116.155
+    - host: b5s133
+      ip: 132.122.116.156
+  services:
+    - name: api
+      source-jar: api-1.0.0-SNAPSHOT.jar
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: gateway
+      source-jar: gateway-1.0.0.jar
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: gateway-new
+      source-jar: gateway-1.0.0.jar
+      arguments:
+        spring_application_name: gateway-new
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: registry
+      source-jar: registry-1.0.0.jar
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: queue
+      source-jar: queue-1.0.0.jar
+      replicas: 1
+      arguments:
+        data_save_enable: true
+        data_save_location: ${deploy.runtime.data-path}
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-scheduler
+      source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
+      replicas: 1
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-launcher-compaction-b1
+      source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
+      environments:
+        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
+        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
+        connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
+        connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
+        connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
+      arguments:
+        spring_application_name: service-launcher-compaction-b1
+        connector_cluster_cluster: ${hostname}
+        connector_cluster_queue-name: compaction-queue-b1
+        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-launcher-compaction-b5
+      source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
+      environments:
+        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
+        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
+        connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
+        connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
+        connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
+      arguments:
+        spring_application_name: service-launcher-compaction-b5
+        connector_cluster_cluster: ${hostname}
+        connector_cluster_queue-name: compaction-queue-b5
+        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-info-query
+      source-jar: service-info-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-yarn-query-b1
+      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        yarn_cluster: b1
+        yarn_web-url: http://132.122.98.13:8088
+        spring_application_name: service-yarn-query-b1
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-yarn-query-b4
+      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        yarn_cluster: b4
+        yarn_web-url: http://132.122.112.30:8088
+        spring_application_name: service-yarn-query-b4
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-yarn-query-b5
+      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        yarn_cluster: b5
+        yarn_web-url: http://132.122.116.12:8088
+        spring_application_name: service-yarn-query-b5
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-yarn-query-b5-sync
+      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        yarn_cluster: b5-sync
+        yarn_web-url: http://132.122.116.143:8088
+        spring_application_name: service-yarn-query-b5-sync
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-pulsar-query
+      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-zookeeper-query
+      source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
+      replicas: 2
+      arguments:
+        spring_cloud_connect-string: ${deploy.runtime.zk-url}
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-web
+      source-jar: service-web-1.0.0-SNAPSHOT.jar
+      replicas: 4
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-hudi-query
+      source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
+      replicas: 5
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
+    - name: service-flink-query
+      source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
+      replicas: 5
+      arguments:
+        logging_parent: ${deploy.runtime.log-path}
+        loki_url: ${deploy.runtime.loki-url}
+        datetime: ${datetime}
+        eureka_instance_hostname: ${hostname}
+        eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
--- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+hosts=(
+<#list hosts as host>
+  ${host}
+</#list>
+)
+datetime=`date +%Y%m%d%H%M%S`
+
+for host in <#noparse>${hosts[@]}</#noparse>
+do
+  hostname=`ssh $host 'echo $HOSTNAME'`
+  echo "$host $hostname"
+  ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+  ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
+  ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
+done
--- a/service-cli/service-cli-runner/src/main/resources/template/stop.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/stop.ftl
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+hosts=(
+<#list hosts as host>
+  ${host}
+</#list>
+)
+
+for host in <#noparse>${hosts[@]}</#noparse>
+do
+  hostname=`ssh $host 'echo $HOSTNAME'`
+  echo "$host $hostname"
+  ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
+done