diff --git a/bin/upload-config.sh b/bin/upload-config.sh new file mode 100755 index 0000000..f1bd3e3 --- /dev/null +++ b/bin/upload-config.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +root_path=$(dirname $(cd $(dirname $0); pwd)) + +clusters=( + b2b12 + b9b9 +) + +for cluster in ${clusters[*]}; +do + cd ${root_path}/config/${cluster} + echo Upload config ${cluster} + tar_path=/tmp/${cluster}.tar.gz + tar -czvf ${tar_path} * + ytp-transfer ${tar_path} + rm -rf ${tar_path} +done diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java index 495ebcd..f61ba5d 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java @@ -18,6 +18,7 @@ public class RuntimeInfo { private String connectorZkUrl; private HudiInfo hudi; private SecurityInfo security; + private YarnInfo yarn; public String getUser() { return user; @@ -107,6 +108,14 @@ public class RuntimeInfo { this.security = security; } + public YarnInfo getYarn() { + return yarn; + } + + public void setYarn(YarnInfo yarn) { + this.yarn = yarn; + } + @Override public String toString() { return "RuntimeInfo{" + @@ -299,4 +308,43 @@ public class RuntimeInfo { '}'; } } + + public static final class YarnInfo { + private String storeCluster; + private String syncClusters; + private String compactionClusters; + + public String getStoreCluster() { + return storeCluster; + } + + public void setStoreCluster(String storeCluster) { + this.storeCluster = storeCluster; + } + + public String getSyncClusters() { + return syncClusters; + } + + public void setSyncClusters(String syncClusters) { + this.syncClusters = syncClusters; + } + + public String getCompactionClusters() { + return compactionClusters; + } + + public void setCompactionClusters(String compactionClusters) { + this.compactionClusters = compactionClusters; + } + + @Override + public String toString() { + return "YarnInfo{" + + "storeCluster='" + storeCluster + '\'' + + ", syncClusters='" + syncClusters + '\'' + + ", compactionClusters='" + compactionClusters + '\'' + + '}'; + } + } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 0da906a..31a8ae6 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -11,9 +11,7 @@ import cn.hutool.extra.template.TemplateEngine; import cn.hutool.extra.template.TemplateUtil; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.lanyuanxiaoyao.service.cli.core.HostInfo; import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo; -import com.lanyuanxiaoyao.service.cli.core.ServiceInfo; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/service-cli/service-cli-runner/src/main/resources/application-b12.yml b/service-cli/service-cli-runner/src/main/resources/application-b12.yml index bdfb3d1..3d35ab7 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml @@ -27,6 +27,10 @@ deploy: authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) + yarn: + store-cluster: b2 + sync-clusters: b12 + compaction-clusters: b12,b1,b5,a4 services: api: replicas: 10 diff --git a/service-cli/service-cli-runner/src/main/resources/application-b5.yml b/service-cli/service-cli-runner/src/main/resources/application-b5.yml index 8e55442..27baf62 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml @@ -27,6 +27,10 @@ deploy: authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) + yarn: + store-cluster: b2 + sync-clusters: b5-sync + compaction-clusters: b12,b1,b5,a4 hosts: b5s119_hdp_dc: ip: 132.122.116.142 diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index 6bbea5d..eff7841 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -12,6 +12,8 @@ zk_url='${runtime.zkUrl}' security_authority='${runtime.security.authority}' security_username='${runtime.security.username}' security_darkcode='${runtime.security.darkcode}' +sync_clusters='${runtime.yarn.syncClusters}' +compaction_clusters='${runtime.yarn.compactionClusters}' <#list hosts as host> host=${host.ip} @@ -22,7 +24,7 @@ hostname=`ssh $host 'hostname'` hostname_full=`ssh $host 'hostname -f'` ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" start_time=`date +%Y%m%d%H%M%S` -ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' <#list arguments?keys as key>--${key}=${arguments[key]?string} > /dev/null 2>&1 &" +ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters} <#list arguments?keys as key>--${key}=${arguments[key]?string} > /dev/null 2>&1 &" echo '' diff --git a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusterProperties.java b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusterProperties.java new file mode 100644 index 0000000..1c49055 --- /dev/null +++ b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusterProperties.java @@ -0,0 +1,35 @@ +package com.lanyuanxiaoyao.service.configuration.entity.yarn; + +import java.util.List; +import java.util.Map; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Yarn 部署信息 + * + * @author lanyuanxiaoyao + * @date 2024-02-26 + */ +@ConfigurationProperties("yarn-cluster") +@Component +public class YarnClusterProperties { + private String syncClusters = ""; + private String compactionClusters = ""; + + public String getSyncClusters() { + return syncClusters; + } + + public void setSyncClusters(String syncClusters) { + this.syncClusters = syncClusters; + } + + public String getCompactionClusters() { + return compactionClusters; + } + + public void setCompactionClusters(String compactionClusters) { + this.compactionClusters = compactionClusters; + } +} diff --git a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java new file mode 100644 index 0000000..d4c62d3 --- /dev/null +++ b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java @@ -0,0 +1,94 @@ +package com.lanyuanxiaoyao.service.configuration.entity.yarn; + +import cn.hutool.core.map.MapUtil; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * Yarn 集群信息 + * + * @author lanyuanxiaoyao + * @date 2024-02-26 + */ +@Component +public class YarnClusters { + private static final Logger logger = LoggerFactory.getLogger(YarnClusters.class); + private final ImmutableMap clusters = Maps.immutable.ofAll( + MapUtil.builder() + .put("a4", new Cluster("http://132.121.107.91:8088")) + .put("b1", new Cluster("http://132.122.98.13:8088")) + .put("b4", new Cluster("http://132.122.112.30:8088")) + .put("b5", new Cluster("http://132.122.116.12:8088")) + .put("t5", new Cluster("http://132.121.126.84:8088")) + .put("b12", new Cluster("http://132.126.207.125:8088")) + .build() + ); + + private final ImmutableList activeClusters; + private final String defaultSyncCluster; + private final ImmutableList syncClusters; + private final String defaultCompactionCluster; + private final ImmutableList compactionClusters; + + public YarnClusters(YarnClusterProperties yarnClusterProperties) { + syncClusters = Lists.immutable.of(yarnClusterProperties.getSyncClusters().split(",")).select(clusters::containsKey); + defaultSyncCluster = syncClusters.getFirstOptional().orElse(""); + + compactionClusters = Lists.immutable.of(yarnClusterProperties.getCompactionClusters().split(",")).select(clusters::containsKey); + defaultCompactionCluster = compactionClusters.getFirstOptional().orElse(""); + + activeClusters = syncClusters.newWithAll(compactionClusters).distinct(); + } + + public ImmutableMap getClusters() { + return clusters; + } + + public ImmutableList getActiveClusters() { + return activeClusters; + } + + public String getDefaultSyncCluster() { + return defaultSyncCluster; + } + + public ImmutableList getSyncClusters() { + return syncClusters; + } + + public String getDefaultCompactionCluster() { + return defaultCompactionCluster; + } + + public ImmutableList getCompactionClusters() { + return compactionClusters; + } + + public static final class Cluster { + private String webUrl; + + public Cluster(String webUrl) { + this.webUrl = webUrl; + } + + public String getWebUrl() { + return webUrl; + } + + public void setWebUrl(String webUrl) { + this.webUrl = webUrl; + } + + @Override + public String toString() { + return "Cluster{" + + "webUrl='" + webUrl + '\'' + + '}'; + } + } +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/A4LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/A4LauncherService.java deleted file mode 100644 index 6f0b4bf..0000000 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/A4LauncherService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher; - -import com.dtflys.forest.annotation.BaseRequest; - -/** - * @author lanyuanxiaoyao - * @date 2023-06-06 - */ -@BaseRequest(baseURL = "http://service-launcher-runner-a4") -public interface A4LauncherService extends LauncherService{ -} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B12LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B12LauncherService.java deleted file mode 100644 index c48af18..0000000 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B12LauncherService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher; - -import com.dtflys.forest.annotation.BaseRequest; - -/** - * @author lanyuanxiaoyao - * @date 2023-06-06 - */ -@BaseRequest(baseURL = "http://service-launcher-runner-b12") -public interface B12LauncherService extends LauncherService{ -} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5LauncherService.java deleted file mode 100644 index e1ea99a..0000000 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5LauncherService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher; - -import com.dtflys.forest.annotation.BaseRequest; - -/** - * @author lanyuanxiaoyao - * @date 2023-06-06 - */ -@BaseRequest(baseURL = "http://service-launcher-runner-b5") -public interface B5LauncherService extends LauncherService{ -} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5SyncLauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5SyncLauncherService.java deleted file mode 100644 index d100717..0000000 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5SyncLauncherService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher; - -import com.dtflys.forest.annotation.BaseRequest; - -/** - * @author lanyuanxiaoyao - * @date 2023-06-06 - */ -@BaseRequest(baseURL = "http://service-launcher-runner-b5-sync") -public interface B5SyncLauncherService extends LauncherService{ -} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java index 6db5fe1..f3a6538 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java @@ -1,7 +1,7 @@ package com.lanyuanxiaoyao.service.forest.service.launcher; import cn.hutool.core.util.StrUtil; -import com.eshore.odcp.hudi.connector.Constants; +import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnClusters; import java.util.HashMap; import java.util.Map; import org.eclipse.collections.api.factory.Maps; @@ -9,6 +9,7 @@ import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.map.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; /** @@ -22,18 +23,15 @@ public class LaunchersService { private final ImmutableMap serviceMap; public LaunchersService( - A4LauncherService a4LauncherService, - B1LauncherService b1LauncherService, - B5LauncherService b5LauncherService, - B5SyncLauncherService b5SyncLauncherService, - B12LauncherService b12LauncherService + YarnClusters yarnClusters, + ApplicationContext applicationContext ) { Map map = new HashMap<>(5); - map.put(Constants.CLUSTER_A4, a4LauncherService); - map.put(Constants.CLUSTER_B1, b1LauncherService); - map.put(Constants.CLUSTER_B5, b5LauncherService); - map.put(Constants.CLUSTER_B5_SYNC, b5SyncLauncherService); - map.put(Constants.CLUSTER_B12, b12LauncherService); + for (String cluster : yarnClusters.getActiveClusters()) { + logger.info("Load {} cluster service", cluster); + LauncherService service = applicationContext.getBean(StrUtil.format("{}LauncherService", StrUtil.toCamelCase(cluster)), LauncherService.class); + map.put(cluster, service); + } serviceMap = Maps.immutable.ofAll(map); } diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/A4LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/A4LauncherService.java new file mode 100644 index 0000000..55fe4e6 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/A4LauncherService.java @@ -0,0 +1,12 @@ +package com.lanyuanxiaoyao.service.forest.service.launcher.impl; + +import com.dtflys.forest.annotation.BaseRequest; +import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService; + +/** + * @author lanyuanxiaoyao + * @date 2023-06-06 + */ +@BaseRequest(baseURL = "http://service-launcher-runner-a4") +public interface A4LauncherService extends LauncherService { +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B12LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B12LauncherService.java new file mode 100644 index 0000000..b4c3fb1 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B12LauncherService.java @@ -0,0 +1,12 @@ +package com.lanyuanxiaoyao.service.forest.service.launcher.impl; + +import com.dtflys.forest.annotation.BaseRequest; +import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService; + +/** + * @author lanyuanxiaoyao + * @date 2023-06-06 + */ +@BaseRequest(baseURL = "http://service-launcher-runner-b12") +public interface B12LauncherService extends LauncherService { +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B1LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B1LauncherService.java similarity index 55% rename from service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B1LauncherService.java rename to service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B1LauncherService.java index eefbee3..a674a63 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B1LauncherService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B1LauncherService.java @@ -1,13 +1,14 @@ -package com.lanyuanxiaoyao.service.forest.service.launcher; +package com.lanyuanxiaoyao.service.forest.service.launcher.impl; import com.dtflys.forest.annotation.BaseRequest; import com.dtflys.forest.annotation.Get; import com.dtflys.forest.annotation.Query; +import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService; /** * @author lanyuanxiaoyao * @date 2023-06-06 */ @BaseRequest(baseURL = "http://service-launcher-runner-b1") -public interface B1LauncherService extends LauncherService{ +public interface B1LauncherService extends LauncherService { } diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B5LauncherService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B5LauncherService.java new file mode 100644 index 0000000..a14d558 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B5LauncherService.java @@ -0,0 +1,12 @@ +package com.lanyuanxiaoyao.service.forest.service.launcher.impl; + +import com.dtflys.forest.annotation.BaseRequest; +import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService; + +/** + * @author lanyuanxiaoyao + * @date 2023-06-06 + */ +@BaseRequest(baseURL = "http://service-launcher-runner-b5") +public interface B5LauncherService extends LauncherService { +}