diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index e0139bb..ab7baad 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -16,6 +16,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Resource; import org.slf4j.Logger; @@ -42,53 +43,19 @@ public class RunnerApplication implements ApplicationRunner { @Resource private DeployInformationProperties deployInformationProperties; - private final String stopScript = "#!/bin/bash\n" + - "\n" + - "# 应用jar包文件名\n" + - "app_jar_name=\"$1\"\n" + - "\n" + - "# 获取应用进程ID\n" + - "get_pid() {\n" + - " STR=$1\n" + - " PID=$2\n" + - " if [[ ! -z \"$PID\" ]]; then\n" + - " ID=$(ps -ef | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{ print $2 }')\n" + - " if [[ -z \"$ID\" ]]; then\n" + - " ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" + - " if [[ -z \"$ID\" ]]; then\n" + - " ID=$(ps aux | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" + - " fi\n" + - " fi\n" + - " else\n" + - " ID=$(ps -ef | grep \"$app_jar_name\" | grep -v grep | awk '{ print $2 }')\n" + - " if [[ -z \"$ID\" ]]; then\n" + - " ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" + - " if [[ -z \"$ID\" ]]; then\n" + - " ID=$(ps aux | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" + - " fi\n" + - " fi\n" + - " fi\n" + - " echo ${ID}\n" + - "}\n" + - "\n" + - "pid=$(get_pid \"$app_jar_name\")\n" + - "if [[ -z \"$pid\" ]]; then\n" + - " echo \"Application is already stopped.\"\n" + - "else\n" + - " echo \"Stopping $pid...\"\n" + - " kill ${pid}\n" + - "fi\n" + - "\n" + - "LOOPS=0\n" + - "while (true); do\n" + - " gpid=$(get_pid \"$app_jar_name\" \"$pid\")\n" + - " if [[ \"$gpid\" == \"\" ]]; then\n" + - " echo \"Oook! cost:$LOOPS\"\n" + - " break\n" + - " fi\n" + - " let LOOPS=LOOPS+1\n" + - " sleep 1\n" + - "done\n"; + private Map escape(Map map) { + // https://github.com/spring-projects/spring-framework/issues/9628 + // spring boot 的配置文件没有办法转义 $,使用 $\{ -> ${ 来绕过一下 + return map.entrySet() + .stream() + .peek(entry -> { + if (entry.getValue() instanceof String) { + String value = (String) entry.getValue(); + entry.setValue(value.replace("$\\{", "${")); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } @Override public void run(ApplicationArguments args) throws IOException { @@ -98,7 +65,9 @@ public class RunnerApplication implements ApplicationRunner { Template deployTemplate = engine.getTemplate("deploy.ftl"); for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) { logger.info("Generate script for {}", serviceInfo.getName()); - List hosts = serviceInfo.getReplicas() == 0 + serviceInfo.setEnvironments(escape(serviceInfo.getEnvironments())); + serviceInfo.setArguments(escape(serviceInfo.getArguments())); + List selectedHosts = serviceInfo.getReplicas() == 0 ? deployInformationProperties.getHosts() .stream() .map(HostInfo::getIp) @@ -113,7 +82,12 @@ public class RunnerApplication implements ApplicationRunner { ), serviceInfo.getReplicas() ), Comparator.naturalOrder()); String deployScript = deployTemplate.render(MapUtil.builder() - .put("hosts", hosts) + .put("hosts", deployInformationProperties.getHosts() + .stream() + .map(HostInfo::getIp) + .sorted(Comparator.naturalOrder()) + .collect(Collectors.toList())) + .put("selectedHosts", selectedHosts) .put("runtime", deployInformationProperties.getRuntime()) .put("info", serviceInfo) .put("arguments", serviceInfo.getArguments()) @@ -145,6 +119,48 @@ public class RunnerApplication implements ApplicationRunner { Files.deleteIfExists(stopScriptFile); Files.write(stopScriptFile, stopScript.getBytes()); } + String stopScript = "original_command=\"$0\"\n" + + "application_name=\"$1\"\n" + + "function get_pid() {\n" + + " PID=$1\n" + + " if [[ ! -z \"$PID\" ]]; then\n" + + " ID=$(ps -ef | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{ print $2 }')\n" + + " if [[ -z \"$ID\" ]]; then\n" + + " ID=$(ps -C java -f --width 1000 | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" + + " if [[ -z \"$ID\" ]]; then\n" + + " ID=$(ps aux | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" + + " fi\n" + + " fi\n" + + " else\n" + + " ID=$(ps -ef | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{ print $2 }')\n" + + " if [[ -z \"$ID\" ]]; then\n" + + " ID=$(ps -C java -f --width 1000 | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" + + " if [[ -z \"$ID\" ]]; then\n" + + " ID=$(ps aux | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" + + " fi\n" + + " fi\n" + + " fi\n" + + " echo ${ID}\n" + + "}\n" + + "\n" + + "pid=$(get_pid)\n" + + "if [[ -z \"$pid\" ]]; then\n" + + " echo \"Application is already stopped.\"\n" + + "else\n" + + " echo \"Stopping $pid...\"\n" + + " kill ${pid}\n" + + "fi\n" + + "\n" + + "LOOPS=0\n" + + "while (true); do\n" + + " gpid=$(get_pid \"$pid\")\n" + + " if [[ \"$gpid\" == \"\" ]]; then\n" + + " echo \"Oook! cost:$LOOPS\"\n" + + " break\n" + + " fi\n" + + " let LOOPS=LOOPS+1\n" + + " sleep 1\n" + + "done\n"; Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); } } diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index c27316f..081f236 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -47,102 +47,52 @@ deploy: services: - name: api source-jar: api-1.0.0-SNAPSHOT.jar - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: gateway source-jar: gateway-1.0.0.jar - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: gateway-new source-jar: gateway-1.0.0.jar arguments: spring_application_name: gateway-new - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: registry source-jar: registry-1.0.0.jar - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: queue source-jar: queue-1.0.0.jar replicas: 1 arguments: data_save_enable: true data_save_location: ${deploy.runtime.data-path} - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-scheduler source-jar: service-scheduler-1.0.0-SNAPSHOT.jar replicas: 1 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-launcher-compaction-b1 source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar environments: - connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM + connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path} connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path} connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url} arguments: spring_application_name: service-launcher-compaction-b1 - connector_cluster_cluster: ${hostname} + connector_cluster_cluster: $\{hostname} connector_cluster_queue-name: compaction-queue-b1 connector_zookeeper_connect-url: ${deploy.runtime.zk-url} - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-launcher-compaction-b5 source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar environments: - connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM + connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path} connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path} connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path} connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url} arguments: spring_application_name: service-launcher-compaction-b5 - connector_cluster_cluster: ${hostname} + connector_cluster_cluster: $\{hostname} connector_cluster_queue-name: compaction-queue-b5 connector_zookeeper_connect-url: ${deploy.runtime.zk-url} - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-info-query source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 4 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-yarn-query-b1 source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 @@ -150,11 +100,6 @@ deploy: yarn_cluster: b1 yarn_web-url: http://132.122.98.13:8088 spring_application_name: service-yarn-query-b1 - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-yarn-query-b4 source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 @@ -162,11 +107,6 @@ deploy: yarn_cluster: b4 yarn_web-url: http://132.122.112.30:8088 spring_application_name: service-yarn-query-b4 - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-yarn-query-b5 source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 @@ -174,11 +114,6 @@ deploy: yarn_cluster: b5 yarn_web-url: http://132.122.116.12:8088 spring_application_name: service-yarn-query-b5 - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-yarn-query-b5-sync source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 @@ -186,54 +121,20 @@ deploy: yarn_cluster: b5-sync yarn_web-url: http://132.122.116.143:8088 spring_application_name: service-yarn-query-b5-sync - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-pulsar-query source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar replicas: 4 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-zookeeper-query source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar replicas: 2 arguments: spring_cloud_connect-string: ${deploy.runtime.zk-url} - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-web source-jar: service-web-1.0.0-SNAPSHOT.jar replicas: 4 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-hudi-query source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar replicas: 5 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} - name: service-flink-query source-jar: service-flink-query-1.0.0-SNAPSHOT.jar replicas: 5 - arguments: - logging_parent: ${deploy.runtime.log-path} - loki_url: ${deploy.runtime.loki-url} - datetime: ${datetime} - eureka_instance_hostname: ${hostname} - eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url} diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index 23356bf..68e2913 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -1,17 +1,31 @@ #!/bin/bash -hosts=( +<#--hosts=( <#list hosts as host> ${host} -) +)--> datetime=`date +%Y%m%d%H%M%S` - -for host in <#noparse>${hosts[@]} +log_path='${runtime.logPath}' +loki_url='${runtime.lokiUrl}' +eureka_url='${runtime.eurekaUrl}' +<#--for host in <#noparse>${hosts[@]} do hostname=`ssh $host 'echo $HOSTNAME'` echo "$host $hostname" ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" - ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" -done + ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar <#noparse>--datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone=${eureka_url} ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" +done--> +<#list hosts as host> + +host=${host} +hostname=`ssh $host 'echo $HOSTNAME'` +echo "$host $hostname $datetime" +ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar +<#if selectedHosts?seq_contains(host)> +ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" +ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone=${eureka_url} <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" + +echo '' +