lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(cli): 优化生成的脚本，转义配置文件的$符号

Browse Source 
直接循环生成启停语句，实现停止已有的，启动新增的「动态变化」功能
This commit is contained in:
兰缘小妖
2023-05-18 17:38:07 +08:00
parent 296c744b61
commit f2aac867b3
3 changed files with 89 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
View File

@@ -16,6 +16,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.slf4j.Logger;
@@ -42,53 +43,19 @@ public class RunnerApplication implements ApplicationRunner {
@Resource
private DeployInformationProperties deployInformationProperties;
private final String stopScript = "#!/bin/bash\n" +
"\n" +
"# 应用jar包文件名\n" +
"app_jar_name=\"$1\"\n" +
"\n" +
"# 获取应用进程ID\n" +
"get_pid() {\n" +
" STR=$1\n" +
" PID=$2\n" +
" if [[ ! -z \"$PID\" ]]; then\n" +
" ID=$(ps -ef | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" else\n" +
" ID=$(ps -ef | grep \"$app_jar_name\" | grep -v grep | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" fi\n" +
" echo ${ID}\n" +
"}\n" +
"\n" +
"pid=$(get_pid \"$app_jar_name\")\n" +
"if [[ -z \"$pid\" ]]; then\n" +
" echo \"Application is already stopped.\"\n" +
"else\n" +
" echo \"Stopping $pid...\"\n" +
" kill ${pid}\n" +
"fi\n" +
"\n" +
"LOOPS=0\n" +
"while (true); do\n" +
" gpid=$(get_pid \"$app_jar_name\" \"$pid\")\n" +
" if [[ \"$gpid\" == \"\" ]]; then\n" +
" echo \"Oook! cost:$LOOPS\"\n" +
" break\n" +
" fi\n" +
" let LOOPS=LOOPS+1\n" +
" sleep 1\n" +
"done\n";
private Map<String, Object> escape(Map<String, Object> map) {
// https://github.com/spring-projects/spring-framework/issues/9628
// spring boot 的配置文件没有办法转义 $，使用 $\{ -> ${ 来绕过一下
return map.entrySet()
.stream()
.peek(entry -> {
if (entry.getValue() instanceof String) {
String value = (String) entry.getValue();
entry.setValue(value.replace("$\\{", "${"));
}
})
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
@Override
public void run(ApplicationArguments args) throws IOException {
@@ -98,7 +65,9 @@ public class RunnerApplication implements ApplicationRunner {
Template deployTemplate = engine.getTemplate("deploy.ftl");
for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) {
logger.info("Generate script for {}", serviceInfo.getName());
List<String> hosts = serviceInfo.getReplicas() == 0
serviceInfo.setEnvironments(escape(serviceInfo.getEnvironments()));
serviceInfo.setArguments(escape(serviceInfo.getArguments()));
List<String> selectedHosts = serviceInfo.getReplicas() == 0
? deployInformationProperties.getHosts()
.stream()
.map(HostInfo::getIp)
@@ -113,7 +82,12 @@ public class RunnerApplication implements ApplicationRunner {
), serviceInfo.getReplicas()
), Comparator.naturalOrder());
String deployScript = deployTemplate.render(MapUtil.builder()
.put("hosts", hosts)
.put("hosts", deployInformationProperties.getHosts()
.stream()
.map(HostInfo::getIp)
.sorted(Comparator.naturalOrder())
.collect(Collectors.toList()))
.put("selectedHosts", selectedHosts)
.put("runtime", deployInformationProperties.getRuntime())
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
@@ -145,6 +119,48 @@ public class RunnerApplication implements ApplicationRunner {
Files.deleteIfExists(stopScriptFile);
Files.write(stopScriptFile, stopScript.getBytes());
}
String stopScript = "original_command=\"$0\"\n" +
"application_name=\"$1\"\n" +
"function get_pid() {\n" +
" PID=$1\n" +
" if [[ ! -z \"$PID\" ]]; then\n" +
" ID=$(ps -ef | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$application_name\" | grep \"$PID\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" else\n" +
" ID=$(ps -ef | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$application_name\" | grep -v grep | grep -v $original_command | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" fi\n" +
" echo ${ID}\n" +
"}\n" +
"\n" +
"pid=$(get_pid)\n" +
"if [[ -z \"$pid\" ]]; then\n" +
" echo \"Application is already stopped.\"\n" +
"else\n" +
" echo \"Stopping $pid...\"\n" +
" kill ${pid}\n" +
"fi\n" +
"\n" +
"LOOPS=0\n" +
"while (true); do\n" +
" gpid=$(get_pid \"$pid\")\n" +
" if [[ \"$gpid\" == \"\" ]]; then\n" +
" echo \"Oook! cost:$LOOPS\"\n" +
" break\n" +
" fi\n" +
" let LOOPS=LOOPS+1\n" +
" sleep 1\n" +
"done\n";
Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
}
}

107
service-cli/service-cli-runner/src/main/resources/application.yml
View File

@@ -47,102 +47,52 @@ deploy:
services:
- name: api
source-jar: api-1.0.0-SNAPSHOT.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: gateway
source-jar: gateway-1.0.0.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: gateway-new
source-jar: gateway-1.0.0.jar
arguments:
spring_application_name: gateway-new
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: registry
source-jar: registry-1.0.0.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: queue
source-jar: queue-1.0.0.jar
replicas: 1
arguments:
data_save_enable: true
data_save_location: ${deploy.runtime.data-path}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-scheduler
source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
replicas: 1
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-launcher-compaction-b1
source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
environments:
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
arguments:
spring_application_name: service-launcher-compaction-b1
connector_cluster_cluster: ${hostname}
connector_cluster_cluster: $\{hostname}
connector_cluster_queue-name: compaction-queue-b1
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-launcher-compaction-b5
source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
environments:
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
arguments:
spring_application_name: service-launcher-compaction-b5
connector_cluster_cluster: ${hostname}
connector_cluster_cluster: $\{hostname}
connector_cluster_queue-name: compaction-queue-b5
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-info-query
source-jar: service-info-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b1
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
@@ -150,11 +100,6 @@ deploy:
yarn_cluster: b1
yarn_web-url: http://132.122.98.13:8088
spring_application_name: service-yarn-query-b1
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b4
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
@@ -162,11 +107,6 @@ deploy:
yarn_cluster: b4
yarn_web-url: http://132.122.112.30:8088
spring_application_name: service-yarn-query-b4
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b5
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
@@ -174,11 +114,6 @@ deploy:
yarn_cluster: b5
yarn_web-url: http://132.122.116.12:8088
spring_application_name: service-yarn-query-b5
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b5-sync
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
@@ -186,54 +121,20 @@ deploy:
yarn_cluster: b5-sync
yarn_web-url: http://132.122.116.143:8088
spring_application_name: service-yarn-query-b5-sync
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-pulsar-query
source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-zookeeper-query
source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
replicas: 2
arguments:
spring_cloud_connect-string: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-web
source-jar: service-web-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-hudi-query
source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
replicas: 5
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-flink-query
source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
replicas: 5
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}

26
service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
View File

@@ -1,17 +1,31 @@
#!/bin/bash
hosts=(
<#--hosts=(
<#list hosts as host>
${host}
</#list>
)
)-->
datetime=`date +%Y%m%d%H%M%S`
for host in <#noparse>${hosts[@]}</#noparse>
log_path='${runtime.logPath}'
loki_url='${runtime.lokiUrl}'
eureka_url='${runtime.eurekaUrl}'
<#--for host in <#noparse>${hosts[@]}</#noparse>
do
hostname=`ssh $host 'echo $HOSTNAME'`
echo "$host $hostname"
ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
done
ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar <#noparse>--datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone=${eureka_url}</#noparse> ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
done-->
<#list hosts as host>
host=${host}
hostname=`ssh $host 'echo $HOSTNAME'`
echo "$host $hostname $datetime"
ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
<#if selectedHosts?seq_contains(host)>
ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone=${eureka_url}</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
</#if>
echo ''
</#list>