feat(cli): 调整配置文件组成方便根据集群配置文件覆盖通用配置方案

2024-02-25 14:48:15 +08:00
parent 2817590617
commit 95bff3e48d
12 changed files with 374 additions and 187 deletions
--- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml
@@ -27,100 +27,123 @@ deploy:
      authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
      username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
      darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
+  services:
+    api:
+      replicas: 10
+    service-launcher-runner-b1:
+      replicas: 6
+    service-launcher-runner-b5:
+      replicas: 6
+    service-launcher-runner-a4:
+      replicas: 6
+    service-launcher-runner-b12:
+      replicas: 6
+    service-info-query:
+      replicas: 10
+    service-yarn-query:
+      replicas: 20
+    service-zookeeper-query:
+      replicas: 10
+    service-hudi-query:
+      replicas: 20
+    service-pulsar-query:
+      replicas: 10
+    service-flink-query:
+      replicas: 10
  hosts:
-    - host: b12s4.hdp.dc
+    b12s4_hdp_dc:
      ip: 132.126.207.130
-    - host: b12s5.hdp.dc
+    b12s5_hdp_dc:
      ip: 132.126.207.131
-    - host: b12s6.hdp.dc
+    b12s6_hdp_dc:
      ip: 132.126.207.132
-    - host: b12s7.hdp.dc
+    b12s7_hdp_dc:
      ip: 132.126.207.133
-    - host: b12s8.hdp.dc
+    b12s8_hdp_dc:
      ip: 132.126.207.134
-    - host: b12s9.hdp.dc
+    b12s9_hdp_dc:
      ip: 132.126.207.135
-    - host: b12s10.hdp.dc
+    b12s10_hdp_dc:
      ip: 132.126.207.136
-    - host: b12s11.hdp.dc
+    b12s11_hdp_dc:
      ip: 132.126.207.137
-    - host: b12s12.hdp.dc
+    b12s12_hdp_dc:
      ip: 132.126.207.138
-    - host: b12s13.hdp.dc
+    b12s13_hdp_dc:
      ip: 132.126.207.139
-    - host: b12s14.hdp.dc
+    b12s14_hdp_dc:
      ip: 132.126.207.140
-    - host: b12s15.hdp.dc
+    b12s15_hdp_dc:
      ip: 132.126.207.141
-    - host: b12s16.hdp.dc
+    b12s16_hdp_dc:
      ip: 132.126.207.142
-    - host: b12s17.hdp.dc
+    b12s17_hdp_dc:
      ip: 132.126.207.143
-    - host: b12s18.hdp.dc
+    b12s18_hdp_dc:
      ip: 132.126.207.144
-    - host: b12s19.hdp.dc
+    b12s19_hdp_dc:
      ip: 132.126.207.145
-    - host: b12s20.hdp.dc
+    b12s20_hdp_dc:
      ip: 132.126.207.146
-    - host: b12s21.hdp.dc
+    b12s21_hdp_dc:
      ip: 132.126.207.147
-    - host: b12s22.hdp.dc
+    b12s22_hdp_dc:
      ip: 132.126.207.148
-    - host: b12s23.hdp.dc
+    b12s23_hdp_dc:
      ip: 132.126.207.149
-    - host: b12s24.hdp.dc
+    b12s24_hdp_dc:
      ip: 132.126.207.150
-    - host: b12s25.hdp.dc
+    b12s25_hdp_dc:
      ip: 132.126.207.151
-    - host: b12s26.hdp.dc
+    b12s26_hdp_dc:
      ip: 132.126.207.152
-    - host: b12s27.hdp.dc
+    b12s27_hdp_dc:
      ip: 132.126.207.153
-    - host: b12s28.hdp.dc
+    b12s28_hdp_dc:
      ip: 132.126.207.154
-    - host: b12s29.hdp.dc
+    b12s29_hdp_dc:
      ip: 132.126.207.155
-    - host: b12s30.hdp.dc
+    b12s30_hdp_dc:
      ip: 132.126.207.156
-    - host: b12s31.hdp.dc
+    b12s31_hdp_dc:
      ip: 132.126.207.157
-    - host: b12s32.hdp.dc
+    b12s32_hdp_dc:
      ip: 132.126.207.158
-    - host: b12s33.hdp.dc
+    b12s33_hdp_dc:
      ip: 132.126.207.159
-    - host: b12s34.hdp.dc
+    b12s34_hdp_dc:
      ip: 132.126.207.160
-    - host: b12s35.hdp.dc
+    b12s35_hdp_dc:
      ip: 132.126.207.161
-    - host: b12s36.hdp.dc
+    b12s36_hdp_dc:
      ip: 132.126.207.162
-    - host: b12s37.hdp.dc
+    b12s37_hdp_dc:
      ip: 132.126.207.163
-    - host: b12s38.hdp.dc
+    b12s38_hdp_dc:
      ip: 132.126.207.164
-    - host: b12s39.hdp.dc
+    b12s39_hdp_dc:
      ip: 132.126.207.165
-    - host: b12s40.hdp.dc
+    b12s40_hdp_dc:
      ip: 132.126.207.166
-    - host: b12s41.hdp.dc
+    b12s41_hdp_dc:
      ip: 132.126.207.167
-    - host: b12s42.hdp.dc
+    b12s42_hdp_dc:
      ip: 132.126.207.168
-    - host: b12s43.hdp.dc
+    b12s43_hdp_dc:
      ip: 132.126.207.169
-    - host: b12s44.hdp.dc
+    b12s44_hdp_dc:
      ip: 132.126.207.170
-    - host: b12s45.hdp.dc
+    b12s45_hdp_dc:
      ip: 132.126.207.171
-    - host: b12s46.hdp.dc
+    b12s46_hdp_dc:
      ip: 132.126.207.172
-    - host: b12s47.hdp.dc
+    b12s47_hdp_dc:
      ip: 132.126.207.173
-    - host: b12s48.hdp.dc
+    b12s48_hdp_dc:
      ip: 132.126.207.174
-    - host: b12s49.hdp.dc
+    b12s49_hdp_dc:
      ip: 132.126.207.175
-    - host: b12s50.hdp.dc
+    b12s50_hdp_dc:
      ip: 132.126.207.176
-    - host: b12s51.hdp.dc
+    b12s51_hdp_dc:
      ip: 132.126.207.177
--- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml
@@ -28,33 +28,33 @@ deploy:
      username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
      darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
  hosts:
-    - host: b5s119
+    b5s119_hdp_dc:
      ip: 132.122.116.142
-    - host: b5s120
+    b5s120_hdp_dc:
      ip: 132.122.116.143
-    - host: b5s121
+    b5s121_hdp_dc:
      ip: 132.122.116.144
-    - host: b5s122
+    b5s122_hdp_dc:
      ip: 132.122.116.145
-    - host: b5s123
+    b5s123_hdp_dc:
      ip: 132.122.116.146
-    - host: b5s124
+    b5s124_hdp_dc:
      ip: 132.122.116.147
-    - host: b5s125
+    b5s125_hdp_dc:
      ip: 132.122.116.148
-    - host: b5s126
+    b5s126_hdp_dc:
      ip: 132.122.116.149
-    - host: b5s127
+    b5s127_hdp_dc:
      ip: 132.122.116.150
-    - host: b5s128
+    b5s128_hdp_dc:
      ip: 132.122.116.151
-    - host: b5s129
+    b5s129_hdp_dc:
      ip: 132.122.116.152
-    - host: b5s130
+    b5s130_hdp_dc:
      ip: 132.122.116.153
-    - host: b5s131
+    b5s131_hdp_dc:
      ip: 132.122.116.154
-    - host: b5s132
+    b5s132_hdp_dc:
      ip: 132.122.116.155
-    - host: b5s133
+    b5s133_hdp_dc:
      ip: 132.122.116.156
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -1,23 +1,27 @@
 deploy:
  services:
-    - name: service-gateway
+    service-gateway:
      source-jar: service-gateway-1.0.0-SNAPSHOT.jar
-    - name: service-queue
+    service-queue:
+      order: 1
      source-jar: service-queue-1.0.0-SNAPSHOT.jar
      replicas: 1
      arguments:
        data_save_enable: true
        data_save_location: ${deploy.runtime.data-path}
-    - name: api
+    api:
+      order: 1
      source-jar: api-1.0.0-SNAPSHOT.jar
      replicas: 10
-    - name: service-scheduler
+    service-scheduler:
+      order: 3
      groups:
        - "service"
        - "service-hudi"
      source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
      replicas: 1
-    - name: service-launcher-runner-b1
+    service-launcher-runner-b1:
+      order: 4
      groups:
        - "service"
        - "service-hudi"
@@ -37,7 +41,8 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-b1
        connector_cluster_compaction-queue-name: compaction-queue-b1
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-launcher-runner-b5
+    service-launcher-runner-b5:
+      order: 4
      groups:
        - "service"
        - "service-hudi"
@@ -57,7 +62,8 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-b5
        connector_cluster_compaction-queue-name: compaction-queue-b5
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-launcher-runner-a4
+    service-launcher-runner-a4:
+      order: 4
      groups:
        - "service"
        - "service-hudi"
@@ -77,7 +83,8 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-a4
        connector_cluster_compaction-queue-name: compaction-queue-a4
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-launcher-runner-b12
+    service-launcher-runner-b12:
+      order: 4
      groups:
        - "service"
        - "service-hudi"
@@ -97,12 +104,14 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-b12
        connector_cluster_compaction-queue-name: compaction-queue-b12
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-info-query
+    service-info-query:
+      order: 1
      groups:
        - "service"
      source-jar: service-info-query-1.0.0-SNAPSHOT.jar
      replicas: 10
-    - name: service-yarn-query
+    service-yarn-query:
+      order: 2
      groups:
        - "service"
      source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
@@ -113,34 +122,40 @@ deploy:
        yarn_web-urls_b4: http://132.122.112.30:8088
        yarn_web-urls_b5: http://132.122.116.12:8088
        yarn_web-urls_b5-sync: http://132.122.116.143:8088
-    - name: service-zookeeper-query
+    service-zookeeper-query:
+      order: 2
      groups:
        - "service"
      source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
      replicas: 10
      arguments:
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
-    - name: service-hudi-query
+    service-hudi-query:
+      order: 2
      groups:
        - "service"
      source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
      replicas: 20
-    - name: service-pulsar-query
+    service-pulsar-query:
+      order: 2
      groups:
        - "service"
      source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
      replicas: 10
-    - name: service-flink-query
+    service-flink-query:
+      order: 2
      groups:
        - "service"
      source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
      replicas: 10
-    - name: service-cloud-query
+    service-cloud-query:
+      order: 1
      groups:
        - "service"
      source-jar: service-cloud-query-1.0.0-SNAPSHOT.jar
      replicas: 5
-    - name: service-executor-manager
+    service-executor-manager:
+      order: 4
      groups:
        - "service"
      source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
@@ -153,12 +168,14 @@ deploy:
        executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path}
        executor_task-jar-path: ${deploy.runtime.executor.task-jar-path}
        executor_task-result-path: ${deploy.runtime.executor.task-result-path}
-    - name: service-web
+    service-web:
+      order: 5
      groups:
        - "service"
      source-jar: service-web-1.0.0-SNAPSHOT.jar
      replicas: 3
-    - name: service-exporter
+    service-exporter:
+      order: 5
      groups:
        - "service"
      source-jar: service-exporter-1.0.0-SNAPSHOT.jar
--- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
@@ -22,7 +22,7 @@ hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`
 ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
 start_time=`date +%Y%m%d%H%M%S`
-ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
+ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
 </#if>
 echo ''
 </#list>