lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(configuration): 增加yarn集群的相关配置

Browse Source
This commit is contained in:
兰缘小妖
2024-02-26 18:31:47 +08:00
parent 8dfc797dfa
commit 834980f880
17 changed files with 254 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bin/upload-config.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/bash
root_path=$(dirname $(cd $(dirname $0); pwd))
clusters=(
b2b12
b9b9
)
for cluster in ${clusters[*]};
do
cd ${root_path}/config/${cluster}
echo Upload config ${cluster}
tar_path=/tmp/${cluster}.tar.gz
tar -czvf ${tar_path} *
ytp-transfer ${tar_path}
rm -rf ${tar_path}
done

48
service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
View File

@@ -18,6 +18,7 @@ public class RuntimeInfo {
private String connectorZkUrl;
private HudiInfo hudi;
private SecurityInfo security;
private YarnInfo yarn;
public String getUser() {
return user;
@@ -107,6 +108,14 @@ public class RuntimeInfo {
this.security = security;
}
public YarnInfo getYarn() {
return yarn;
}
public void setYarn(YarnInfo yarn) {
this.yarn = yarn;
}
@Override
public String toString() {
return "RuntimeInfo{" +
@@ -299,4 +308,43 @@ public class RuntimeInfo {
'}';
}
}
public static final class YarnInfo {
private String storeCluster;
private String syncClusters;
private String compactionClusters;
public String getStoreCluster() {
return storeCluster;
}
public void setStoreCluster(String storeCluster) {
this.storeCluster = storeCluster;
}
public String getSyncClusters() {
return syncClusters;
}
public void setSyncClusters(String syncClusters) {
this.syncClusters = syncClusters;
}
public String getCompactionClusters() {
return compactionClusters;
}
public void setCompactionClusters(String compactionClusters) {
this.compactionClusters = compactionClusters;
}
@Override
public String toString() {
return "YarnInfo{" +
"storeCluster='" + storeCluster + '\'' +
", syncClusters='" + syncClusters + '\'' +
", compactionClusters='" + compactionClusters + '\'' +
'}';
}
}
}

2
service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
View File

@@ -11,9 +11,7 @@ import cn.hutool.extra.template.TemplateEngine;
import cn.hutool.extra.template.TemplateUtil;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.cli.core.HostInfo;
import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo;
import com.lanyuanxiaoyao.service.cli.core.ServiceInfo;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

4
service-cli/service-cli-runner/src/main/resources/application-b12.yml
View File

@@ -27,6 +27,10 @@ deploy:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
yarn:
store-cluster: b2
sync-clusters: b12
compaction-clusters: b12,b1,b5,a4
services:
api:
replicas: 10

4
service-cli/service-cli-runner/src/main/resources/application-b5.yml
View File

@@ -27,6 +27,10 @@ deploy:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
yarn:
store-cluster: b2
sync-clusters: b5-sync
compaction-clusters: b12,b1,b5,a4
hosts:
b5s119_hdp_dc:
ip: 132.122.116.142

4
service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
View File

@@ -12,6 +12,8 @@ zk_url='${runtime.zkUrl}'
security_authority='${runtime.security.authority}'
security_username='${runtime.security.username}'
security_darkcode='${runtime.security.darkcode}'
sync_clusters='${runtime.yarn.syncClusters}'
compaction_clusters='${runtime.yarn.compactionClusters}'
<#list hosts as host>
host=${host.ip}
@@ -22,7 +24,7 @@ hostname=`ssh $host 'hostname'`
hostname_full=`ssh $host 'hostname -f'`
ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
start_time=`date +%Y%m%d%H%M%S`
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
</#if>
echo ''
</#list>

35
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusterProperties.java Normal file
View File

@@ -0,0 +1,35 @@
package com.lanyuanxiaoyao.service.configuration.entity.yarn;
import java.util.List;
import java.util.Map;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Yarn 部署信息
*
* @author lanyuanxiaoyao
* @date 2024-02-26
*/
@ConfigurationProperties("yarn-cluster")
@Component
public class YarnClusterProperties {
private String syncClusters = "";
private String compactionClusters = "";
public String getSyncClusters() {
return syncClusters;
}
public void setSyncClusters(String syncClusters) {
this.syncClusters = syncClusters;
}
public String getCompactionClusters() {
return compactionClusters;
}
public void setCompactionClusters(String compactionClusters) {
this.compactionClusters = compactionClusters;
}
}

94
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java Normal file
View File

@@ -0,0 +1,94 @@
package com.lanyuanxiaoyao.service.configuration.entity.yarn;
import cn.hutool.core.map.MapUtil;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
/**
* Yarn 集群信息
*
* @author lanyuanxiaoyao
* @date 2024-02-26
*/
@Component
public class YarnClusters {
private static final Logger logger = LoggerFactory.getLogger(YarnClusters.class);
private final ImmutableMap<String, Cluster> clusters = Maps.immutable.ofAll(
MapUtil.<String, Cluster>builder()
.put("a4", new Cluster("http://132.121.107.91:8088"))
.put("b1", new Cluster("http://132.122.98.13:8088"))
.put("b4", new Cluster("http://132.122.112.30:8088"))
.put("b5", new Cluster("http://132.122.116.12:8088"))
.put("t5", new Cluster("http://132.121.126.84:8088"))
.put("b12", new Cluster("http://132.126.207.125:8088"))
.build()
);
private final ImmutableList<String> activeClusters;
private final String defaultSyncCluster;
private final ImmutableList<String> syncClusters;
private final String defaultCompactionCluster;
private final ImmutableList<String> compactionClusters;
public YarnClusters(YarnClusterProperties yarnClusterProperties) {
syncClusters = Lists.immutable.of(yarnClusterProperties.getSyncClusters().split(",")).select(clusters::containsKey);
defaultSyncCluster = syncClusters.getFirstOptional().orElse("");
compactionClusters = Lists.immutable.of(yarnClusterProperties.getCompactionClusters().split(",")).select(clusters::containsKey);
defaultCompactionCluster = compactionClusters.getFirstOptional().orElse("");
activeClusters = syncClusters.newWithAll(compactionClusters).distinct();
}
public ImmutableMap<String, Cluster> getClusters() {
return clusters;
}
public ImmutableList<String> getActiveClusters() {
return activeClusters;
}
public String getDefaultSyncCluster() {
return defaultSyncCluster;
}
public ImmutableList<String> getSyncClusters() {
return syncClusters;
}
public String getDefaultCompactionCluster() {
return defaultCompactionCluster;
}
public ImmutableList<String> getCompactionClusters() {
return compactionClusters;
}
public static final class Cluster {
private String webUrl;
public Cluster(String webUrl) {
this.webUrl = webUrl;
}
public String getWebUrl() {
return webUrl;
}
public void setWebUrl(String webUrl) {
this.webUrl = webUrl;
}
@Override
public String toString() {
return "Cluster{" +
"webUrl='" + webUrl + '\'' +
'}';
}
}
}

11
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/A4LauncherService.java
View File

@@ -1,11 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
import com.dtflys.forest.annotation.BaseRequest;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-a4")
public interface A4LauncherService extends LauncherService{
}

11
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B12LauncherService.java
View File

@@ -1,11 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
import com.dtflys.forest.annotation.BaseRequest;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b12")
public interface B12LauncherService extends LauncherService{
}

11
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5LauncherService.java
View File

@@ -1,11 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
import com.dtflys.forest.annotation.BaseRequest;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b5")
public interface B5LauncherService extends LauncherService{
}

11
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5SyncLauncherService.java
View File

@@ -1,11 +0,0 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
import com.dtflys.forest.annotation.BaseRequest;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b5-sync")
public interface B5SyncLauncherService extends LauncherService{
}

20
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java
View File

@@ -1,7 +1,7 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
import cn.hutool.core.util.StrUtil;
import com.eshore.odcp.hudi.connector.Constants;
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnClusters;
import java.util.HashMap;
import java.util.Map;
import org.eclipse.collections.api.factory.Maps;
@@ -9,6 +9,7 @@ import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Service;
/**
@@ -22,18 +23,15 @@ public class LaunchersService {
private final ImmutableMap<String, LauncherService> serviceMap;
public LaunchersService(
A4LauncherService a4LauncherService,
B1LauncherService b1LauncherService,
B5LauncherService b5LauncherService,
B5SyncLauncherService b5SyncLauncherService,
B12LauncherService b12LauncherService
YarnClusters yarnClusters,
ApplicationContext applicationContext
) {
Map<String, LauncherService> map = new HashMap<>(5);
map.put(Constants.CLUSTER_A4, a4LauncherService);
map.put(Constants.CLUSTER_B1, b1LauncherService);
map.put(Constants.CLUSTER_B5, b5LauncherService);
map.put(Constants.CLUSTER_B5_SYNC, b5SyncLauncherService);
map.put(Constants.CLUSTER_B12, b12LauncherService);
for (String cluster : yarnClusters.getActiveClusters()) {
logger.info("Load {} cluster service", cluster);
LauncherService service = applicationContext.getBean(StrUtil.format("{}LauncherService", StrUtil.toCamelCase(cluster)), LauncherService.class);
map.put(cluster, service);
}
serviceMap = Maps.immutable.ofAll(map);
}

12
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/A4LauncherService.java Normal file
View File

@@ -0,0 +1,12 @@
package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
import com.dtflys.forest.annotation.BaseRequest;
import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-a4")
public interface A4LauncherService extends LauncherService {
}

12
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B12LauncherService.java Normal file
View File

@@ -0,0 +1,12 @@
package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
import com.dtflys.forest.annotation.BaseRequest;
import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b12")
public interface B12LauncherService extends LauncherService {
}

5
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B1LauncherService.java → service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B1LauncherService.java
View File

@@ -1,13 +1,14 @@
package com.lanyuanxiaoyao.service.forest.service.launcher;
package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Query;
import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b1")
public interface B1LauncherService extends LauncherService{
public interface B1LauncherService extends LauncherService {
}

12
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B5LauncherService.java Normal file
View File

@@ -0,0 +1,12 @@
package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
import com.dtflys.forest.annotation.BaseRequest;
import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
/**
* @author lanyuanxiaoyao
* @date 2023-06-06
*/
@BaseRequest(baseURL = "http://service-launcher-runner-b5")
public interface B5LauncherService extends LauncherService {
}