feat(cli): 增加配置发布

将hadoop配置分发到各个主机上
2024-02-27 15:18:30 +08:00
parent 73f0c4c82c
commit 871b01522a
32 changed files with 380 additions and 8143 deletions
--- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml
@@ -4,6 +4,7 @@ deploy:
    jar-path: /data/datalake/jars
    jdk-path: /opt/jdk8u252-b09/bin/java
    log-path: /data/datalake/logs
+    conf-path: /data/datalake/conf
    data-path: /data/datalake/data
    kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
    loki:
--- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml
@@ -4,6 +4,7 @@ deploy:
    jar-path: /apps/datalake/hudi/jars
    jdk-path: /opt/jdk1.8.0_162/bin/java
    log-path: /apps/datalake/hudi/logs
+    conf-path: /apps/datalake/hudi/conf
    data-path: /apps/datalake/hudi/data
    kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
    loki:
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -28,6 +28,8 @@ deploy:
        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar
      replicas: 6
+      classpath:
+        - ${deploy.runtime.conf-path}/b2b1
      environments:
        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
@@ -49,6 +51,8 @@ deploy:
        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
      replicas: 6
+      classpath:
+        - ${deploy.runtime.conf-path}/b2b5
      environments:
        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
@@ -70,6 +74,8 @@ deploy:
        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
      replicas: 6
+      classpath:
+        - ${deploy.runtime.conf-path}/b2a4
      environments:
        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
@@ -91,6 +97,8 @@ deploy:
        - "service-hudi-launcher"
      source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar
      replicas: 6
+      classpath:
+        - ${deploy.runtime.conf-path}/b2b12
      environments:
        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
@@ -154,6 +162,8 @@ deploy:
        - "service"
      source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
      replicas: 5
+      classpath:
+        - ${deploy.runtime.conf-path}/b2b12
      environments:
        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
--- a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy-config.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy-config.ftl
@@ -0,0 +1,5 @@
+#!/bin/bash
+<#list hosts as host>
+echo ${host.ip} ${host.hostname}
+ssh ${host.ip} 'bash -s' < ${currentPath}/download-config.sh
+</#list>
--- a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl
@@ -24,7 +24,7 @@ hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`
 ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
 start_time=`date +%Y%m%d%H%M%S`
-ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
+ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -Xbootclasspath/a:${classpath} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
 </#if>
 echo ''
 </#list>
--- a/service-cli/service-cli-runner/src/main/resources/template/cloud/download-config.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/download-config.ftl
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+config_names=(
+<#list classpath as cp>
+  ${cp}
+</#list>
+)
+
+for config_name in <#noparse>${config_names[@]}</#noparse>
+do
+  conf_path=${runtime.confPath}/<#noparse>${config_name}</#noparse>
+  tar_path=<#noparse>${conf_path}</#noparse>.tar.gz
+  rm -rf <#noparse>${conf_path}</#noparse>
+  mkdir -p <#noparse>${conf_path}</#noparse>
+  curl -s ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/<#noparse>${config_name}</#noparse>.tar.gz -o <#noparse>${tar_path}</#noparse>
+  tar -zxf <#noparse>${tar_path}</#noparse> -C <#noparse>${conf_path}</#noparse>
+  rm -rf <#noparse>${tar_path}</#noparse>
+done