lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(configuration): 增加yarn集群的相关配置

Browse Source
This commit is contained in:
兰缘小妖
2024-02-26 18:31:47 +08:00
parent 8dfc797dfa
commit 834980f880
17 changed files with 254 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
View File

@@ -18,6 +18,7 @@ public class RuntimeInfo {
private String connectorZkUrl;
private HudiInfo hudi;
private SecurityInfo security;
private YarnInfo yarn;
public String getUser() {
return user;
@@ -107,6 +108,14 @@ public class RuntimeInfo {
this.security = security;
}
public YarnInfo getYarn() {
return yarn;
}
public void setYarn(YarnInfo yarn) {
this.yarn = yarn;
}
@Override
public String toString() {
return "RuntimeInfo{" +
@@ -299,4 +308,43 @@ public class RuntimeInfo {
'}';
}
}
public static final class YarnInfo {
private String storeCluster;
private String syncClusters;
private String compactionClusters;
public String getStoreCluster() {
return storeCluster;
}
public void setStoreCluster(String storeCluster) {
this.storeCluster = storeCluster;
}
public String getSyncClusters() {
return syncClusters;
}
public void setSyncClusters(String syncClusters) {
this.syncClusters = syncClusters;
}
public String getCompactionClusters() {
return compactionClusters;
}
public void setCompactionClusters(String compactionClusters) {
this.compactionClusters = compactionClusters;
}
@Override
public String toString() {
return "YarnInfo{" +
"storeCluster='" + storeCluster + '\'' +
", syncClusters='" + syncClusters + '\'' +
", compactionClusters='" + compactionClusters + '\'' +
'}';
}
}
}

2
service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
View File

@@ -11,9 +11,7 @@ import cn.hutool.extra.template.TemplateEngine;
import cn.hutool.extra.template.TemplateUtil;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.cli.core.HostInfo;
import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo;
import com.lanyuanxiaoyao.service.cli.core.ServiceInfo;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

4
service-cli/service-cli-runner/src/main/resources/application-b12.yml
View File

@@ -27,6 +27,10 @@ deploy:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
yarn:
store-cluster: b2
sync-clusters: b12
compaction-clusters: b12,b1,b5,a4
services:
api:
replicas: 10

4
service-cli/service-cli-runner/src/main/resources/application-b5.yml
View File

@@ -27,6 +27,10 @@ deploy:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
yarn:
store-cluster: b2
sync-clusters: b5-sync
compaction-clusters: b12,b1,b5,a4
hosts:
b5s119_hdp_dc:
ip: 132.122.116.142

4
service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
View File

@@ -12,6 +12,8 @@ zk_url='${runtime.zkUrl}'
security_authority='${runtime.security.authority}'
security_username='${runtime.security.username}'
security_darkcode='${runtime.security.darkcode}'
sync_clusters='${runtime.yarn.syncClusters}'
compaction_clusters='${runtime.yarn.compactionClusters}'
<#list hosts as host>
host=${host.ip}
@@ -22,7 +24,7 @@ hostname=`ssh $host 'hostname'`
hostname_full=`ssh $host 'hostname -f'`
ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
start_time=`date +%Y%m%d%H%M%S`
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
</#if>
echo ''
</#list>