diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/HostInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/HostInfo.java index 547cca4..0c556d1 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/HostInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/HostInfo.java @@ -7,20 +7,11 @@ package com.lanyuanxiaoyao.service.cli.core; * @date 2023-05-17 */ public class HostInfo { - private String host; private String ip; private Boolean useAuthority = false; private String username; private String password; - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - public String getIp() { return ip; } @@ -53,44 +44,13 @@ public class HostInfo { this.password = password; } - public HostnameIp getHostnameIp() { - return new HostnameIp(host, ip); - } - @Override public String toString() { return "HostInfo{" + - "host='" + host + '\'' + - ", ip='" + ip + '\'' + - ", useAuthority=" + useAuthority + - ", username='" + username + '\'' + - ", password='" + password + '\'' + - '}'; - } - - public static class HostnameIp { - private final String hostname; - private final String ip; - - private HostnameIp(String hostname, String ip) { - this.hostname = hostname; - this.ip = ip; - } - - public String getHostname() { - return hostname; - } - - public String getIp() { - return ip; - } - - @Override - public String toString() { - return "HostnameIp{" + - "hostname='" + hostname + '\'' + - ", ip='" + ip + '\'' + - '}'; - } + "ip='" + ip + '\'' + + ", useAuthority=" + useAuthority + + ", username='" + username + '\'' + + ", password='" + password + '\'' + + '}'; } } diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java index bb7cb80..38e82af 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/ServiceInfo.java @@ -12,19 +12,19 @@ import java.util.Map; * @date 2023-05-17 */ public class ServiceInfo { - private String name; + private Integer order = 0; private List groups = new ArrayList<>(); private Integer replicas = 0; private String sourceJar; private Map environments = new HashMap<>(); private Map arguments = new HashMap<>(); - public String getName() { - return name; + public Integer getOrder() { + return order; } - public void setName(String name) { - this.name = name; + public void setOrder(Integer order) { + this.order = order; } public List getGroups() { @@ -70,7 +70,7 @@ public class ServiceInfo { @Override public String toString() { return "ServiceInfo{" + - "name='" + name + '\'' + + "order=" + order + ", groups=" + groups + ", replicas=" + replicas + ", sourceJar='" + sourceJar + '\'' + diff --git a/service-cli/service-cli-runner/pom.xml b/service-cli/service-cli-runner/pom.xml index b9d7c78..77c59a3 100644 --- a/service-cli/service-cli-runner/pom.xml +++ b/service-cli/service-cli-runner/pom.xml @@ -26,6 +26,15 @@ spring-boot-configuration-processor true + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + runtime + org.springframework.boot spring-boot-starter-json diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java index 3188f10..1cc8a41 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java @@ -3,8 +3,9 @@ package com.lanyuanxiaoyao.service.cli.runner; import com.lanyuanxiaoyao.service.cli.core.HostInfo; import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo; import com.lanyuanxiaoyao.service.cli.core.ServiceInfo; -import java.util.List; +import java.util.Map; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; /** @@ -13,13 +14,13 @@ import org.springframework.stereotype.Component; * @author lanyuanxiaoyao * @date 2023-05-17 */ +@Configuration @ConfigurationProperties("deploy") -@Component public class DeployInformationProperties { private Boolean shuffler = false; private RuntimeInfo runtime; - private List hosts; - private List services; + private Map hosts; + private Map services; public Boolean getShuffler() { return shuffler; @@ -37,29 +38,29 @@ public class DeployInformationProperties { this.runtime = runtime; } - public List getHosts() { + public Map getHosts() { return hosts; } - public void setHosts(List hosts) { + public void setHosts(Map hosts) { this.hosts = hosts; } - public List getServices() { + public Map getServices() { return services; } - public void setServices(List services) { + public void setServices(Map services) { this.services = services; } @Override public String toString() { return "DeployInformationProperties{" + - "shuffler=" + shuffler + - ", runtime=" + runtime + - ", hosts=" + hosts + - ", services=" + services + - '}'; + "shuffler=" + shuffler + + ", runtime=" + runtime + + ", hosts=" + hosts + + ", services=" + services + + '}'; } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/HostInfoWrapper.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/HostInfoWrapper.java new file mode 100644 index 0000000..f06956f --- /dev/null +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/HostInfoWrapper.java @@ -0,0 +1,78 @@ +package com.lanyuanxiaoyao.service.cli.runner; + +import com.lanyuanxiaoyao.service.cli.core.HostInfo; + +/** + * 增补HostInfo属性 + * + * @author lanyuanxiaoyao + * @date 2024-02-25 + */ +public class HostInfoWrapper { + private final HostInfo hostInfo; + + private final String host; + + public HostInfoWrapper(String host, HostInfo hostInfo) { + this.host = host; + this.hostInfo = hostInfo; + } + + public String getHost() { + return host; + } + + public String getIp() { + return hostInfo.getIp(); + } + + public Boolean getUseAuthority() { + return hostInfo.getUseAuthority(); + } + + public String getUsername() { + return hostInfo.getUsername(); + } + + public String getPassword() { + return hostInfo.getPassword(); + } + + public HostnameIp getHostnameIp() { + return new HostnameIp(getHost(), getIp()); + } + + @Override + public String toString() { + return "HostInfoWrapper{" + + "hostInfo=" + hostInfo + + ", host='" + host + '\'' + + '}'; + } + + public static class HostnameIp { + private final String hostname; + private final String ip; + + private HostnameIp(String hostname, String ip) { + this.hostname = hostname; + this.ip = ip; + } + + public String getHostname() { + return hostname; + } + + public String getIp() { + return ip; + } + + @Override + public String toString() { + return "HostnameIp{" + + "hostname='" + hostname + '\'' + + ", ip='" + ip + '\'' + + '}'; + } + } +} diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 0646912..8157bdf 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -12,6 +12,7 @@ import cn.hutool.extra.template.TemplateUtil; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.lanyuanxiaoyao.service.cli.core.HostInfo; +import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo; import com.lanyuanxiaoyao.service.cli.core.ServiceInfo; import java.io.IOException; import java.nio.file.Files; @@ -19,6 +20,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -37,10 +40,22 @@ public class RunnerApplication implements ApplicationRunner { private static final Logger logger = LoggerFactory.getLogger(RunnerApplication.class); private final DeployInformationProperties deployInformationProperties; + private final RuntimeInfo runtimeInfo; + private final ImmutableList serviceInfoList; + private final ImmutableList hostInfoList; private final ObjectMapper mapper; public RunnerApplication(DeployInformationProperties deployInformationProperties) { this.deployInformationProperties = deployInformationProperties; + runtimeInfo = deployInformationProperties.getRuntime(); + serviceInfoList = Lists.immutable.ofAll(deployInformationProperties.getServices().entrySet()) + .collect(entry -> new ServiceInfoWrapper(entry.getKey(), entry.getValue())) + .toSortedListBy(ServiceInfoWrapper::getOrder) + .toImmutable(); + hostInfoList = Lists.immutable.ofAll(SpringPropertiesEscapeHelper.escapeMapKey(deployInformationProperties.getHosts()).entrySet()) + .collect(entry -> new HostInfoWrapper(entry.getKey(), entry.getValue())) + .toSortedListBy(HostInfoWrapper::getHost) + .toImmutable(); mapper = new ObjectMapper(); } @@ -48,32 +63,18 @@ public class RunnerApplication implements ApplicationRunner { SpringApplication.run(RunnerApplication.class, args); } - private Map escape(Map map) { - // https://github.com/spring-projects/spring-framework/issues/9628 - // spring boot 的配置文件没有办法转义 $,使用 $\{ -> ${ 来绕过一下 - return map.entrySet() - .stream() - .peek(entry -> { - if (entry.getValue() instanceof String) { - String value = (String) entry.getValue(); - entry.setValue(value.replace("$\\{", "${")); - } - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private List selectHosts(ServiceInfo serviceInfo) { + private List selectHosts(ServiceInfoWrapper serviceInfo) { return serviceInfo.getReplicas() == 0 - ? deployInformationProperties.getHosts() + ? hostInfoList .stream() - .map(HostInfo::getIp) + .map(HostInfoWrapper::getIp) .sorted(Comparator.naturalOrder()) .collect(Collectors.toList()) : ListUtil.sort( RandomUtil.randomEleList( - deployInformationProperties.getHosts() + hostInfoList .stream() - .map(HostInfo::getIp) + .map(HostInfoWrapper::getIp) .collect(Collectors.toList() ), serviceInfo.getReplicas() ), Comparator.naturalOrder()); @@ -81,12 +82,11 @@ public class RunnerApplication implements ApplicationRunner { @Override public void run(ApplicationArguments args) throws IOException { - logger.info("Loading runtime info: {}", mapper.writeValueAsString(deployInformationProperties.getRuntime())); + logger.info("Loading runtime info: {}", mapper.writeValueAsString(runtimeInfo)); Path planPath = Paths.get("deploy.plan"); Map> deployPlans = new HashMap<>(); if (Files.exists(planPath) && !deployInformationProperties.getShuffler()) { - deployPlans = mapper.readValue(new String(Files.readAllBytes(planPath)), new TypeReference>>() { - }); + deployPlans = mapper.readValue(new String(Files.readAllBytes(planPath)), new TypeReference>>() {}); } Path root = Paths.get(""); @@ -97,10 +97,8 @@ public class RunnerApplication implements ApplicationRunner { TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH)); Template deployTemplate = engine.getTemplate("deploy.ftl"); - for (ServiceInfo serviceInfo : deployInformationProperties.getServices()) { + for (ServiceInfoWrapper serviceInfo : serviceInfoList) { logger.info("Generate script for {}", serviceInfo.getName()); - serviceInfo.setEnvironments(escape(serviceInfo.getEnvironments())); - serviceInfo.setArguments(escape(serviceInfo.getArguments())); List selectedHosts; if (deployPlans.containsKey(serviceInfo.getName())) { selectedHosts = deployPlans.get(serviceInfo.getName()); @@ -114,13 +112,13 @@ public class RunnerApplication implements ApplicationRunner { } String deployScript = deployTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("hosts", deployInformationProperties.getHosts() + .put("hosts", hostInfoList .stream() - .map(HostInfo::getHostnameIp) + .map(HostInfoWrapper::getHostnameIp) .sorted((o1, o2) -> Comparator.naturalOrder().compare(o1.getIp(), o2.getIp())) .collect(Collectors.toList())) .put("selectedHosts", selectedHosts) - .put("runtime", deployInformationProperties.getRuntime()) + .put("runtime", runtimeInfo) .put("info", serviceInfo) .put("arguments", serviceInfo.getArguments()) .put("environments", serviceInfo.getEnvironments()) @@ -135,12 +133,12 @@ public class RunnerApplication implements ApplicationRunner { Template stopTemplate = engine.getTemplate("stop.ftl"); String stopScript = stopTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("hosts", deployInformationProperties.getHosts() + .put("hosts", hostInfoList .stream() - .map(HostInfo::getIp) + .map(HostInfoWrapper::getIp) .sorted(Comparator.naturalOrder()) .collect(Collectors.toList())) - .put("runtime", deployInformationProperties.getRuntime()) + .put("runtime", runtimeInfo) .put("info", serviceInfo) .put("arguments", serviceInfo.getArguments()) .put("environments", serviceInfo.getEnvironments()) @@ -155,13 +153,13 @@ public class RunnerApplication implements ApplicationRunner { Template logTemplate = engine.getTemplate("log.ftl"); String logScript = logTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("hosts", deployInformationProperties.getHosts() + .put("hosts", hostInfoList .stream() - .map(HostInfo::getIp) + .map(HostInfoWrapper::getIp) .sorted(Comparator.naturalOrder()) .collect(Collectors.toList())) .put("selectedHosts", selectedHosts) - .put("runtime", deployInformationProperties.getRuntime()) + .put("runtime", runtimeInfo) .put("info", serviceInfo) .put("arguments", serviceInfo.getArguments()) .put("environments", serviceInfo.getEnvironments()) @@ -216,25 +214,25 @@ public class RunnerApplication implements ApplicationRunner { "done"; Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); - Map> groups = new HashMap<>(); - for (ServiceInfo service : deployInformationProperties.getServices()) { + Map> groups = new HashMap<>(); + for (ServiceInfoWrapper service : serviceInfoList) { service.getGroups().add(0, "all"); for (String group : service.getGroups()) { if (!groups.containsKey(group)) { groups.put(group, new ArrayList<>()); } - List list = groups.get(group); + List list = groups.get(group); list.add(service); } } - for (Map.Entry> entry : groups.entrySet()) { + for (Map.Entry> entry : groups.entrySet()) { String group = entry.getKey(); - List infos = entry.getValue(); + List infos = entry.getValue(); Template batchDeployTemplate = engine.getTemplate("batch-deploy.ftl"); String batchDeployScript = batchDeployTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList())) + .put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList())) .build()); Path batchDeployScriptFile = Paths.get( root.toString(), @@ -246,7 +244,7 @@ public class RunnerApplication implements ApplicationRunner { Template batchStopTemplate = engine.getTemplate("batch-stop.ftl"); String batchStopScript = batchStopTemplate.render(MapUtil.builder() .put("currentPath", absolutRootPath) - .put("services", infos.stream().map(ServiceInfo::getName).collect(Collectors.toList())) + .put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList())) .build()); Path batchStopScriptFile = Paths.get( root.toString(), diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java new file mode 100644 index 0000000..a664d41 --- /dev/null +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/ServiceInfoWrapper.java @@ -0,0 +1,78 @@ +package com.lanyuanxiaoyao.service.cli.runner; + +import com.lanyuanxiaoyao.service.cli.core.ServiceInfo; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * ServiceInfo 增补属性 + * + * @author lanyuanxiaoyao + * @date 2024-02-25 + */ +public class ServiceInfoWrapper { + private final ServiceInfo serviceInfo; + + private final String name; + private final Map environments; + private final Map arguments; + + public ServiceInfoWrapper(String name, ServiceInfo serviceInfo) { + this.name = name; + this.serviceInfo = serviceInfo; + this.environments = SpringPropertiesEscapeHelper.escapeMapKey(serviceInfo.getEnvironments()); + this.arguments = SpringPropertiesEscapeHelper.escapeMapKey(serviceInfo.getArguments()); + } + + public String getName() { + return name; + } + + public Integer getOrder() { + return serviceInfo.getOrder(); + } + + public List getGroups() { + return serviceInfo.getGroups(); + } + + public Integer getReplicas() { + return serviceInfo.getReplicas(); + } + + public String getSourceJar() { + return serviceInfo.getSourceJar(); + } + + public Map getEnvironments() { + return environments; + } + + public Map getArguments() { + return arguments; + } + + @Override + public String toString() { + return "ServiceInfoWrapper{" + + "serviceInfo=" + serviceInfo + + ", name='" + name + '\'' + + '}'; + } + + private Map escape(Map map) { + // https://github.com/spring-projects/spring-framework/issues/9628 + // spring boot 的配置文件没有办法转义 $,使用 $\{ -> ${ 来绕过一下 + return map.entrySet() + .stream() + .peek(entry -> { + if (entry.getValue() instanceof String) { + String key = entry.getKey(); + String value = (String) entry.getValue(); + entry.setValue(value.replace("$\\{", "${")); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } +} diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/SpringPropertiesEscapeHelper.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/SpringPropertiesEscapeHelper.java new file mode 100644 index 0000000..0a25cda --- /dev/null +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/SpringPropertiesEscapeHelper.java @@ -0,0 +1,23 @@ +package com.lanyuanxiaoyao.service.cli.runner; + +import java.util.HashMap; +import java.util.Map; + +/** + * spring boot 配置文件转义 + * + * @author lanyuanxiaoyao + * @date 2024-02-25 + */ +public class SpringPropertiesEscapeHelper { + /** + * 使用 _ 代替 map key 的 . + */ + public static Map escapeMapKey(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey().replaceAll("_", "."), entry.getValue()); + } + return result; + } +} diff --git a/service-cli/service-cli-runner/src/main/resources/application-b12.yml b/service-cli/service-cli-runner/src/main/resources/application-b12.yml index 717f576..bdfb3d1 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml @@ -27,100 +27,123 @@ deploy: authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) + services: + api: + replicas: 10 + service-launcher-runner-b1: + replicas: 6 + service-launcher-runner-b5: + replicas: 6 + service-launcher-runner-a4: + replicas: 6 + service-launcher-runner-b12: + replicas: 6 + service-info-query: + replicas: 10 + service-yarn-query: + replicas: 20 + service-zookeeper-query: + replicas: 10 + service-hudi-query: + replicas: 20 + service-pulsar-query: + replicas: 10 + service-flink-query: + replicas: 10 hosts: - - host: b12s4.hdp.dc + b12s4_hdp_dc: ip: 132.126.207.130 - - host: b12s5.hdp.dc + b12s5_hdp_dc: ip: 132.126.207.131 - - host: b12s6.hdp.dc + b12s6_hdp_dc: ip: 132.126.207.132 - - host: b12s7.hdp.dc + b12s7_hdp_dc: ip: 132.126.207.133 - - host: b12s8.hdp.dc + b12s8_hdp_dc: ip: 132.126.207.134 - - host: b12s9.hdp.dc + b12s9_hdp_dc: ip: 132.126.207.135 - - host: b12s10.hdp.dc + b12s10_hdp_dc: ip: 132.126.207.136 - - host: b12s11.hdp.dc + b12s11_hdp_dc: ip: 132.126.207.137 - - host: b12s12.hdp.dc + b12s12_hdp_dc: ip: 132.126.207.138 - - host: b12s13.hdp.dc + b12s13_hdp_dc: ip: 132.126.207.139 - - host: b12s14.hdp.dc + b12s14_hdp_dc: ip: 132.126.207.140 - - host: b12s15.hdp.dc + b12s15_hdp_dc: ip: 132.126.207.141 - - host: b12s16.hdp.dc + b12s16_hdp_dc: ip: 132.126.207.142 - - host: b12s17.hdp.dc + b12s17_hdp_dc: ip: 132.126.207.143 - - host: b12s18.hdp.dc + b12s18_hdp_dc: ip: 132.126.207.144 - - host: b12s19.hdp.dc + b12s19_hdp_dc: ip: 132.126.207.145 - - host: b12s20.hdp.dc + b12s20_hdp_dc: ip: 132.126.207.146 - - host: b12s21.hdp.dc + b12s21_hdp_dc: ip: 132.126.207.147 - - host: b12s22.hdp.dc + b12s22_hdp_dc: ip: 132.126.207.148 - - host: b12s23.hdp.dc + b12s23_hdp_dc: ip: 132.126.207.149 - - host: b12s24.hdp.dc + b12s24_hdp_dc: ip: 132.126.207.150 - - host: b12s25.hdp.dc + b12s25_hdp_dc: ip: 132.126.207.151 - - host: b12s26.hdp.dc + b12s26_hdp_dc: ip: 132.126.207.152 - - host: b12s27.hdp.dc + b12s27_hdp_dc: ip: 132.126.207.153 - - host: b12s28.hdp.dc + b12s28_hdp_dc: ip: 132.126.207.154 - - host: b12s29.hdp.dc + b12s29_hdp_dc: ip: 132.126.207.155 - - host: b12s30.hdp.dc + b12s30_hdp_dc: ip: 132.126.207.156 - - host: b12s31.hdp.dc + b12s31_hdp_dc: ip: 132.126.207.157 - - host: b12s32.hdp.dc + b12s32_hdp_dc: ip: 132.126.207.158 - - host: b12s33.hdp.dc + b12s33_hdp_dc: ip: 132.126.207.159 - - host: b12s34.hdp.dc + b12s34_hdp_dc: ip: 132.126.207.160 - - host: b12s35.hdp.dc + b12s35_hdp_dc: ip: 132.126.207.161 - - host: b12s36.hdp.dc + b12s36_hdp_dc: ip: 132.126.207.162 - - host: b12s37.hdp.dc + b12s37_hdp_dc: ip: 132.126.207.163 - - host: b12s38.hdp.dc + b12s38_hdp_dc: ip: 132.126.207.164 - - host: b12s39.hdp.dc + b12s39_hdp_dc: ip: 132.126.207.165 - - host: b12s40.hdp.dc + b12s40_hdp_dc: ip: 132.126.207.166 - - host: b12s41.hdp.dc + b12s41_hdp_dc: ip: 132.126.207.167 - - host: b12s42.hdp.dc + b12s42_hdp_dc: ip: 132.126.207.168 - - host: b12s43.hdp.dc + b12s43_hdp_dc: ip: 132.126.207.169 - - host: b12s44.hdp.dc + b12s44_hdp_dc: ip: 132.126.207.170 - - host: b12s45.hdp.dc + b12s45_hdp_dc: ip: 132.126.207.171 - - host: b12s46.hdp.dc + b12s46_hdp_dc: ip: 132.126.207.172 - - host: b12s47.hdp.dc + b12s47_hdp_dc: ip: 132.126.207.173 - - host: b12s48.hdp.dc + b12s48_hdp_dc: ip: 132.126.207.174 - - host: b12s49.hdp.dc + b12s49_hdp_dc: ip: 132.126.207.175 - - host: b12s50.hdp.dc + b12s50_hdp_dc: ip: 132.126.207.176 - - host: b12s51.hdp.dc + b12s51_hdp_dc: ip: 132.126.207.177 \ No newline at end of file diff --git a/service-cli/service-cli-runner/src/main/resources/application-b5.yml b/service-cli/service-cli-runner/src/main/resources/application-b5.yml index 2cc625d..8e55442 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml @@ -28,33 +28,33 @@ deploy: username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) hosts: - - host: b5s119 + b5s119_hdp_dc: ip: 132.122.116.142 - - host: b5s120 + b5s120_hdp_dc: ip: 132.122.116.143 - - host: b5s121 + b5s121_hdp_dc: ip: 132.122.116.144 - - host: b5s122 + b5s122_hdp_dc: ip: 132.122.116.145 - - host: b5s123 + b5s123_hdp_dc: ip: 132.122.116.146 - - host: b5s124 + b5s124_hdp_dc: ip: 132.122.116.147 - - host: b5s125 + b5s125_hdp_dc: ip: 132.122.116.148 - - host: b5s126 + b5s126_hdp_dc: ip: 132.122.116.149 - - host: b5s127 + b5s127_hdp_dc: ip: 132.122.116.150 - - host: b5s128 + b5s128_hdp_dc: ip: 132.122.116.151 - - host: b5s129 + b5s129_hdp_dc: ip: 132.122.116.152 - - host: b5s130 + b5s130_hdp_dc: ip: 132.122.116.153 - - host: b5s131 + b5s131_hdp_dc: ip: 132.122.116.154 - - host: b5s132 + b5s132_hdp_dc: ip: 132.122.116.155 - - host: b5s133 + b5s133_hdp_dc: ip: 132.122.116.156 \ No newline at end of file diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 04ba878..893596b 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -1,23 +1,27 @@ deploy: services: - - name: service-gateway + service-gateway: source-jar: service-gateway-1.0.0-SNAPSHOT.jar - - name: service-queue + service-queue: + order: 1 source-jar: service-queue-1.0.0-SNAPSHOT.jar replicas: 1 arguments: data_save_enable: true data_save_location: ${deploy.runtime.data-path} - - name: api + api: + order: 1 source-jar: api-1.0.0-SNAPSHOT.jar replicas: 10 - - name: service-scheduler + service-scheduler: + order: 3 groups: - "service" - "service-hudi" source-jar: service-scheduler-1.0.0-SNAPSHOT.jar replicas: 1 - - name: service-launcher-runner-b1 + service-launcher-runner-b1: + order: 4 groups: - "service" - "service-hudi" @@ -37,7 +41,8 @@ deploy: connector_cluster_sync-queue-name: sync-queue-b1 connector_cluster_compaction-queue-name: compaction-queue-b1 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-launcher-runner-b5 + service-launcher-runner-b5: + order: 4 groups: - "service" - "service-hudi" @@ -57,7 +62,8 @@ deploy: connector_cluster_sync-queue-name: sync-queue-b5 connector_cluster_compaction-queue-name: compaction-queue-b5 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-launcher-runner-a4 + service-launcher-runner-a4: + order: 4 groups: - "service" - "service-hudi" @@ -77,7 +83,8 @@ deploy: connector_cluster_sync-queue-name: sync-queue-a4 connector_cluster_compaction-queue-name: compaction-queue-a4 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-launcher-runner-b12 + service-launcher-runner-b12: + order: 4 groups: - "service" - "service-hudi" @@ -97,12 +104,14 @@ deploy: connector_cluster_sync-queue-name: sync-queue-b12 connector_cluster_compaction-queue-name: compaction-queue-b12 connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-info-query + service-info-query: + order: 1 groups: - "service" source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 10 - - name: service-yarn-query + service-yarn-query: + order: 2 groups: - "service" source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar @@ -113,34 +122,40 @@ deploy: yarn_web-urls_b4: http://132.122.112.30:8088 yarn_web-urls_b5: http://132.122.116.12:8088 yarn_web-urls_b5-sync: http://132.122.116.143:8088 - - name: service-zookeeper-query + service-zookeeper-query: + order: 2 groups: - "service" source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar replicas: 10 arguments: connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - - name: service-hudi-query + service-hudi-query: + order: 2 groups: - "service" source-jar: service-hudi-query-1.0.0-SNAPSHOT.jar replicas: 20 - - name: service-pulsar-query + service-pulsar-query: + order: 2 groups: - "service" source-jar: service-pulsar-query-1.0.0-SNAPSHOT.jar replicas: 10 - - name: service-flink-query + service-flink-query: + order: 2 groups: - "service" source-jar: service-flink-query-1.0.0-SNAPSHOT.jar replicas: 10 - - name: service-cloud-query + service-cloud-query: + order: 1 groups: - "service" source-jar: service-cloud-query-1.0.0-SNAPSHOT.jar replicas: 5 - - name: service-executor-manager + service-executor-manager: + order: 4 groups: - "service" source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar @@ -153,12 +168,14 @@ deploy: executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path} executor_task-jar-path: ${deploy.runtime.executor.task-jar-path} executor_task-result-path: ${deploy.runtime.executor.task-result-path} - - name: service-web + service-web: + order: 5 groups: - "service" source-jar: service-web-1.0.0-SNAPSHOT.jar replicas: 3 - - name: service-exporter + service-exporter: + order: 5 groups: - "service" source-jar: service-exporter-1.0.0-SNAPSHOT.jar diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index a4085b2..6bbea5d 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -22,7 +22,7 @@ hostname=`ssh $host 'hostname'` hostname_full=`ssh $host 'hostname -f'` ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" start_time=`date +%Y%m%d%H%M%S` -ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" +ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' <#list arguments?keys as key>--${key}=${arguments[key]?string} > /dev/null 2>&1 &" echo ''