feat(cli): 增加配置发布
将hadoop配置分发到各个主机上
This commit is contained in:
@@ -17,10 +17,16 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.list.MutableList;
|
||||
import org.eclipse.collections.api.map.MutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
@@ -120,13 +126,12 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String deployScript = deployTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getHostnameIp)
|
||||
.sorted((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp()))
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getHostnameIp)
|
||||
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
|
||||
.put("selectedHosts", selectedHosts)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
.put("classpath", String.join(":", serviceInfo.getClasspath()))
|
||||
.put("arguments", serviceInfo.getArguments())
|
||||
.put("environments", serviceInfo.getEnvironments())
|
||||
.build());
|
||||
@@ -141,10 +146,8 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String stopScript = stopTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getIp)
|
||||
.sorted(Comparator.naturalOrder())
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getIp)
|
||||
.toSortedList(Comparator.naturalOrder()))
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
.put("arguments", serviceInfo.getArguments())
|
||||
@@ -161,10 +164,8 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
String logScript = logTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("hosts", hostInfoList
|
||||
.stream()
|
||||
.map(HostInfoWrapper::getIp)
|
||||
.sorted(Comparator.naturalOrder())
|
||||
.collect(Collectors.toList()))
|
||||
.collect(HostInfoWrapper::getIp)
|
||||
.toSortedList(Comparator.naturalOrder()))
|
||||
.put("selectedHosts", selectedHosts)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("info", serviceInfo)
|
||||
@@ -179,29 +180,56 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
Files.write(logScriptFile, logScript.getBytes());
|
||||
}
|
||||
|
||||
ImmutableList<String> clusters = Lists.mutable.of(runtimeInfo.getYarn().getSyncClusters().split(","))
|
||||
.withAll(Lists.mutable.of(runtimeInfo.getYarn().getCompactionClusters().split(",")))
|
||||
.collect(n -> runtimeInfo.getYarn().getStoreCluster() + n)
|
||||
.distinct()
|
||||
.toImmutable();
|
||||
Template downloadConfigTemplate = engine.getTemplate("cloud/download-config.ftl");
|
||||
String downloadConfigScript = downloadConfigTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("classpath", clusters)
|
||||
.build());
|
||||
Path downloadConfigScriptFile = Paths.get(root.toString(), "download-config.sh");
|
||||
Files.deleteIfExists(downloadConfigScriptFile);
|
||||
Files.write(downloadConfigScriptFile, downloadConfigScript.getBytes());
|
||||
|
||||
Template deployConfigTemplate = engine.getTemplate("cloud/deploy-config.ftl");
|
||||
String deployConfigScript = deployConfigTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("runtime", runtimeInfo)
|
||||
.put("hosts", hostInfoList
|
||||
.collect(HostInfoWrapper::getHostnameIp)
|
||||
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
|
||||
.build());
|
||||
Path deployConfigScriptFile = Paths.get(root.toString(), "deploy-config.sh");
|
||||
Files.deleteIfExists(deployConfigScriptFile);
|
||||
Files.write(deployConfigScriptFile, deployConfigScript.getBytes());
|
||||
|
||||
ClassPathResource stopFile = new ClassPathResource("script/stop.sh");
|
||||
String stopScript = new String(IoUtil.readBytes(stopFile.getInputStream()));
|
||||
Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
|
||||
|
||||
Map<String, List<ServiceInfoWrapper>> groups = new HashMap<>();
|
||||
MutableMap<String, MutableList<ServiceInfoWrapper>> groups = Maps.mutable.empty();
|
||||
for (ServiceInfoWrapper service : serviceInfoList) {
|
||||
service.getGroups().add(0, "all");
|
||||
for (String group : service.getGroups()) {
|
||||
if (!groups.containsKey(group)) {
|
||||
groups.put(group, new ArrayList<>());
|
||||
groups.put(group, Lists.mutable.empty());
|
||||
}
|
||||
List<ServiceInfoWrapper> list = groups.get(group);
|
||||
list.add(service);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, List<ServiceInfoWrapper>> entry : groups.entrySet()) {
|
||||
for (Map.Entry<String, MutableList<ServiceInfoWrapper>> entry : groups.entrySet()) {
|
||||
String group = entry.getKey();
|
||||
List<ServiceInfoWrapper> infos = entry.getValue();
|
||||
MutableList<ServiceInfoWrapper> infos = entry.getValue();
|
||||
|
||||
Template batchDeployTemplate = engine.getTemplate("cloud/batch-deploy.ftl");
|
||||
String batchDeployScript = batchDeployTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList()))
|
||||
.put("services", infos.collect(ServiceInfoWrapper::getName))
|
||||
.build());
|
||||
Path batchDeployScriptFile = Paths.get(
|
||||
root.toString(),
|
||||
@@ -213,7 +241,7 @@ public class RunnerApplication implements ApplicationRunner {
|
||||
Template batchStopTemplate = engine.getTemplate("cloud/batch-stop.ftl");
|
||||
String batchStopScript = batchStopTemplate.render(MapUtil.builder()
|
||||
.put("currentPath", absolutRootPath)
|
||||
.put("services", infos.stream().map(ServiceInfoWrapper::getName).collect(Collectors.toList()))
|
||||
.put("services", infos.collect(ServiceInfoWrapper::getName))
|
||||
.build());
|
||||
Path batchStopScriptFile = Paths.get(
|
||||
root.toString(),
|
||||
|
||||
@@ -49,6 +49,10 @@ public class ServiceInfoWrapper {
|
||||
return serviceInfo.getSourceJar();
|
||||
}
|
||||
|
||||
public List<String> getClasspath() {
|
||||
return serviceInfo.getClasspath();
|
||||
}
|
||||
|
||||
public Map<String, Object> getEnvironments() {
|
||||
return environments;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ deploy:
|
||||
jar-path: /data/datalake/jars
|
||||
jdk-path: /opt/jdk8u252-b09/bin/java
|
||||
log-path: /data/datalake/logs
|
||||
conf-path: /data/datalake/conf
|
||||
data-path: /data/datalake/data
|
||||
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
|
||||
loki:
|
||||
|
||||
@@ -4,6 +4,7 @@ deploy:
|
||||
jar-path: /apps/datalake/hudi/jars
|
||||
jdk-path: /opt/jdk1.8.0_162/bin/java
|
||||
log-path: /apps/datalake/hudi/logs
|
||||
conf-path: /apps/datalake/hudi/conf
|
||||
data-path: /apps/datalake/hudi/data
|
||||
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
|
||||
loki:
|
||||
|
||||
@@ -28,6 +28,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b1-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b1
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -49,6 +51,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b5
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -70,6 +74,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2a4
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -91,6 +97,8 @@ deploy:
|
||||
- "service-hudi-launcher"
|
||||
source-jar: service-launcher-runner-b2b12-1.0.0-SNAPSHOT.jar
|
||||
replicas: 6
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b12
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
@@ -154,6 +162,8 @@ deploy:
|
||||
- "service"
|
||||
source-jar: service-executor-manager-1.0.0-SNAPSHOT.jar
|
||||
replicas: 5
|
||||
classpath:
|
||||
- ${deploy.runtime.conf-path}/b2b12
|
||||
environments:
|
||||
connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
|
||||
connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
<#list hosts as host>
|
||||
echo ${host.ip} ${host.hostname}
|
||||
ssh ${host.ip} 'bash -s' < ${currentPath}/download-config.sh
|
||||
</#list>
|
||||
@@ -24,7 +24,7 @@ hostname=`ssh $host 'hostname'`
|
||||
hostname_full=`ssh $host 'hostname -f'`
|
||||
ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
|
||||
start_time=`date +%Y%m%d%H%M%S`
|
||||
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
|
||||
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -Xbootclasspath/a:${classpath} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
|
||||
</#if>
|
||||
echo ''
|
||||
</#list>
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
config_names=(
|
||||
<#list classpath as cp>
|
||||
${cp}
|
||||
</#list>
|
||||
)
|
||||
|
||||
for config_name in <#noparse>${config_names[@]}</#noparse>
|
||||
do
|
||||
conf_path=${runtime.confPath}/<#noparse>${config_name}</#noparse>
|
||||
tar_path=<#noparse>${conf_path}</#noparse>.tar.gz
|
||||
rm -rf <#noparse>${conf_path}</#noparse>
|
||||
mkdir -p <#noparse>${conf_path}</#noparse>
|
||||
curl -s ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/<#noparse>${config_name}</#noparse>.tar.gz -o <#noparse>${tar_path}</#noparse>
|
||||
tar -zxf <#noparse>${tar_path}</#noparse> -C <#noparse>${conf_path}</#noparse>
|
||||
rm -rf <#noparse>${tar_path}</#noparse>
|
||||
done
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.cli.runner;
|
||||
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-02-27
|
||||
*/
|
||||
public class UrlTest {
|
||||
public static void main(String[] args) {
|
||||
System.out.println(URLUtil.toURI("ftp://yyy:QeY!68)4nH1@132.121.122.15:2222/b9b9.tar.gz").getPath());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user