feature(cli): 增加 cli 模块

cli 模块批量生成服务启停脚本
This commit is contained in:
2023-05-18 11:11:39 +08:00
parent 69314f62ec
commit 296c744b61
13 changed files with 861 additions and 0 deletions

View File

@@ -0,0 +1,55 @@
package com.lanyuanxiaoyao.service.cli.runner;
import com.lanyuanxiaoyao.service.cli.core.HostInfo;
import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo;
import com.lanyuanxiaoyao.service.cli.core.ServiceInfo;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* 配置
*
* @author lanyuanxiaoyao
* @date 2023-05-17
*/
@ConfigurationProperties("deploy")
@Component
public class DeployInformationProperties {
private RuntimeInfo runtime;
private List<HostInfo> hosts;
private List<ServiceInfo> services;
public RuntimeInfo getRuntime() {
return runtime;
}
public void setRuntime(RuntimeInfo runtime) {
this.runtime = runtime;
}
public List<HostInfo> getHosts() {
return hosts;
}
public void setHosts(List<HostInfo> hosts) {
this.hosts = hosts;
}
public List<ServiceInfo> getServices() {
return services;
}
public void setServices(List<ServiceInfo> services) {
this.services = services;
}
@Override
public String toString() {
return "DeployInformationProperties{" +
"runtime=" + runtime +
", hosts=" + hosts +
", services=" + services +
'}';
}
}

View File

@@ -0,0 +1,150 @@
package com.lanyuanxiaoyao.service.cli.runner;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.extra.template.Template;
import cn.hutool.extra.template.TemplateConfig;
import cn.hutool.extra.template.TemplateEngine;
import cn.hutool.extra.template.TemplateUtil;
import com.lanyuanxiaoyao.service.cli.core.HostInfo;
import com.lanyuanxiaoyao.service.cli.core.ServiceInfo;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* 启动类
*
* @author lanyuanxiaoyao
* @date 2023-05-17
*/
@SpringBootApplication
public class RunnerApplication implements ApplicationRunner {
private static final Logger logger = LoggerFactory.getLogger(RunnerApplication.class);
public static void main(String[] args) {
SpringApplication.run(RunnerApplication.class, args);
}
@Resource
private DeployInformationProperties deployInformationProperties;
private final String stopScript = "#!/bin/bash\n" +
"\n" +
"# 应用jar包文件名\n" +
"app_jar_name=\"$1\"\n" +
"\n" +
"# 获取应用进程ID\n" +
"get_pid() {\n" +
" STR=$1\n" +
" PID=$2\n" +
" if [[ ! -z \"$PID\" ]]; then\n" +
" ID=$(ps -ef | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$app_jar_name\" | grep \"$PID\" | grep -v grep | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" else\n" +
" ID=$(ps -ef | grep \"$app_jar_name\" | grep -v grep | awk '{ print $2 }')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps -C java -f --width 1000 | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" +
" if [[ -z \"$ID\" ]]; then\n" +
" ID=$(ps aux | grep \"$app_jar_name\" | grep -v grep | awk '{print $2}')\n" +
" fi\n" +
" fi\n" +
" fi\n" +
" echo ${ID}\n" +
"}\n" +
"\n" +
"pid=$(get_pid \"$app_jar_name\")\n" +
"if [[ -z \"$pid\" ]]; then\n" +
" echo \"Application is already stopped.\"\n" +
"else\n" +
" echo \"Stopping $pid...\"\n" +
" kill ${pid}\n" +
"fi\n" +
"\n" +
"LOOPS=0\n" +
"while (true); do\n" +
" gpid=$(get_pid \"$app_jar_name\" \"$pid\")\n" +
" if [[ \"$gpid\" == \"\" ]]; then\n" +
" echo \"Oook! cost:$LOOPS\"\n" +
" break\n" +
" fi\n" +
" let LOOPS=LOOPS+1\n" +
" sleep 1\n" +
"done\n";
@Override
public void run(ApplicationArguments args) throws IOException {
Path root = Paths.get("");
Files.createDirectories(root);
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template deployTemplate = engine.getTemplate("deploy.ftl");
for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) {
logger.info("Generate script for {}", serviceInfo.getName());
List<String> hosts = serviceInfo.getReplicas() == 0
? deployInformationProperties.getHosts()
.stream()
.map(HostInfo::getIp)
.sorted(Comparator.naturalOrder())
.collect(Collectors.toList())
: ListUtil.sort(
RandomUtil.randomEleList(
deployInformationProperties.getHosts()
.stream()
.map(HostInfo::getIp)
.collect(Collectors.toList()
), serviceInfo.getReplicas()
), Comparator.naturalOrder());
String deployScript = deployTemplate.render(MapUtil.builder()
.put("hosts", hosts)
.put("runtime", deployInformationProperties.getRuntime())
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build());
Path deployScriptFile = Paths.get(
root.toString(),
StrUtil.format("deploy-{}.sh", serviceInfo.getName())
);
Files.deleteIfExists(deployScriptFile);
Files.write(deployScriptFile, deployScript.getBytes());
Template stopTemplate = engine.getTemplate("stop.ftl");
String stopScript = stopTemplate.render(MapUtil.builder()
.put("hosts", deployInformationProperties.getHosts()
.stream()
.map(HostInfo::getIp)
.sorted(Comparator.naturalOrder())
.collect(Collectors.toList()))
.put("runtime", deployInformationProperties.getRuntime())
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build());
Path stopScriptFile = Paths.get(
root.toString(),
StrUtil.format("stop-{}.sh", serviceInfo.getName())
);
Files.deleteIfExists(stopScriptFile);
Files.write(stopScriptFile, stopScript.getBytes());
}
Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
}
}

View File

@@ -0,0 +1,239 @@
deploy:
runtime:
user: datalake
jar-path: /apps/datalake/hudi/jars
jdk-path: /opt/jdk1.8.0_162/bin/java
log-path: /apps/datalake/hudi/logs
data-path: /apps/datalake/hudi/data
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
loki-url: http://132.122.116.142:33100/loki/api/v1/push
zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
eureka-url: http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.142:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.143:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.144:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.145:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.146:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.147:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.148:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.149:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.150:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.151:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.152:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.153:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.154:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.155:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.156:35670/eureka
hudi:
app-hdfs-path: hdfs://b2/apps/datalake/jars/app
archive-hdfs-path: hdfs://b2/apps/datalake/flink/completed-jobs-hudi
victoria-push-url: http://132.122.116.142:35710/api/v1/import/prometheus
hosts:
- host: b5s119
ip: 132.122.116.142
- host: b5s120
ip: 132.122.116.143
- host: b5s121
ip: 132.122.116.144
- host: b5s122
ip: 132.122.116.145
- host: b5s123
ip: 132.122.116.146
- host: b5s124
ip: 132.122.116.147
- host: b5s125
ip: 132.122.116.148
- host: b5s126
ip: 132.122.116.149
- host: b5s127
ip: 132.122.116.150
- host: b5s128
ip: 132.122.116.151
- host: b5s129
ip: 132.122.116.152
- host: b5s130
ip: 132.122.116.153
- host: b5s131
ip: 132.122.116.154
- host: b5s132
ip: 132.122.116.155
- host: b5s133
ip: 132.122.116.156
services:
- name: api
source-jar: api-1.0.0-SNAPSHOT.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: gateway
source-jar: gateway-1.0.0.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: gateway-new
source-jar: gateway-1.0.0.jar
arguments:
spring_application_name: gateway-new
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: registry
source-jar: registry-1.0.0.jar
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: queue
source-jar: queue-1.0.0.jar
replicas: 1
arguments:
data_save_enable: true
data_save_location: ${deploy.runtime.data-path}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-scheduler
source-jar: service-scheduler-1.0.0-SNAPSHOT.jar
replicas: 1
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-launcher-compaction-b1
source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
environments:
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
arguments:
spring_application_name: service-launcher-compaction-b1
connector_cluster_cluster: ${hostname}
connector_cluster_queue-name: compaction-queue-b1
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-launcher-compaction-b5
source-jar: service-launcher-compaction-1.0.0-SNAPSHOT.jar
environments:
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/${hostname}.hdp.dc@ECLD.COM
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
arguments:
spring_application_name: service-launcher-compaction-b5
connector_cluster_cluster: ${hostname}
connector_cluster_queue-name: compaction-queue-b5
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-info-query
source-jar: service-info-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b1
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
yarn_cluster: b1
yarn_web-url: http://132.122.98.13:8088
spring_application_name: service-yarn-query-b1
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b4
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
yarn_cluster: b4
yarn_web-url: http://132.122.112.30:8088
spring_application_name: service-yarn-query-b4
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b5
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
yarn_cluster: b5
yarn_web-url: http://132.122.116.12:8088
spring_application_name: service-yarn-query-b5
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-yarn-query-b5-sync
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
yarn_cluster: b5-sync
yarn_web-url: http://132.122.116.143:8088
spring_application_name: service-yarn-query-b5-sync
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-pulsar-query
source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-zookeeper-query
source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
replicas: 2
arguments:
spring_cloud_connect-string: ${deploy.runtime.zk-url}
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-web
source-jar: service-web-1.0.0-SNAPSHOT.jar
replicas: 4
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-hudi-query
source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar
replicas: 5
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}
- name: service-flink-query
source-jar: service-flink-query-1.0.0-SNAPSHOT.jar
replicas: 5
arguments:
logging_parent: ${deploy.runtime.log-path}
loki_url: ${deploy.runtime.loki-url}
datetime: ${datetime}
eureka_instance_hostname: ${hostname}
eureka_client_service-url_defaultZone: ${deploy.runtime.eureka-url}

View File

@@ -0,0 +1,17 @@
#!/bin/bash
hosts=(
<#list hosts as host>
${host}
</#list>
)
datetime=`date +%Y%m%d%H%M%S`
for host in <#noparse>${hosts[@]}</#noparse>
do
hostname=`ssh $host 'echo $HOSTNAME'`
echo "$host $hostname"
ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
ssh $host "curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
ssh $host "nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
done

View File

@@ -0,0 +1,14 @@
#!/bin/bash
hosts=(
<#list hosts as host>
${host}
</#list>
)
for host in <#noparse>${hosts[@]}</#noparse>
do
hostname=`ssh $host 'echo $HOSTNAME'`
echo "$host $hostname"
ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
done