feat(configuration): 增加yarn集群的相关配置

bin/upload-config.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+root_path=$(dirname $(cd $(dirname $0); pwd))
+
+clusters=(
+  b2b12
+  b9b9
+)
+
+for cluster in ${clusters[*]};
+do
+  cd ${root_path}/config/${cluster}
+  echo Upload config ${cluster}
+  tar_path=/tmp/${cluster}.tar.gz
+  tar -czvf ${tar_path} *
+  ytp-transfer ${tar_path}
+  rm -rf ${tar_path}
+done

service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java

@@ -18,6 +18,7 @@ public class RuntimeInfo {
     private String connectorZkUrl;
     private HudiInfo hudi;
     private SecurityInfo security;
+    private YarnInfo yarn;
 
     public String getUser() {
         return user;
@@ -107,6 +108,14 @@ public class RuntimeInfo {
         this.security = security;
     }
 
+    public YarnInfo getYarn() {
+        return yarn;
+    }
+
+    public void setYarn(YarnInfo yarn) {
+        this.yarn = yarn;
+    }
+
     @Override
     public String toString() {
         return "RuntimeInfo{" +
@@ -299,4 +308,43 @@ public class RuntimeInfo {
                    '}';
         }
     }
+
+    public static final class YarnInfo {
+        private String storeCluster;
+        private String syncClusters;
+        private String compactionClusters;
+
+        public String getStoreCluster() {
+            return storeCluster;
+        }
+
+        public void setStoreCluster(String storeCluster) {
+            this.storeCluster = storeCluster;
+        }
+
+        public String getSyncClusters() {
+            return syncClusters;
+        }
+
+        public void setSyncClusters(String syncClusters) {
+            this.syncClusters = syncClusters;
+        }
+
+        public String getCompactionClusters() {
+            return compactionClusters;
+        }
+
+        public void setCompactionClusters(String compactionClusters) {
+            this.compactionClusters = compactionClusters;
+        }
+
+        @Override
+        public String toString() {
+            return "YarnInfo{" +
+                   "storeCluster='" + storeCluster + '\'' +
+                   ", syncClusters='" + syncClusters + '\'' +
+                   ", compactionClusters='" + compactionClusters + '\'' +
+                   '}';
+        }
+    }
 }

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java

@@ -11,9 +11,7 @@ import cn.hutool.extra.template.TemplateEngine;
 import cn.hutool.extra.template.TemplateUtil;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.lanyuanxiaoyao.service.cli.core.HostInfo;
 import com.lanyuanxiaoyao.service.cli.core.RuntimeInfo;
-import com.lanyuanxiaoyao.service.cli.core.ServiceInfo;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;

service-cli/service-cli-runner/src/main/resources/application-b12.yml

@@ -27,6 +27,10 @@ deploy:
       authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
       username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
       darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
+    yarn:
+      store-cluster: b2
+      sync-clusters: b12
+      compaction-clusters: b12,b1,b5,a4
   services:
     api:
       replicas: 10

service-cli/service-cli-runner/src/main/resources/application-b5.yml

@@ -27,6 +27,10 @@ deploy:
       authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
       username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
       darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
+    yarn:
+      store-cluster: b2
+      sync-clusters: b5-sync
+      compaction-clusters: b12,b1,b5,a4
   hosts:
     b5s119_hdp_dc:
       ip: 132.122.116.142

service-cli/service-cli-runner/src/main/resources/template/deploy.ftl

@@ -12,6 +12,8 @@ zk_url='${runtime.zkUrl}'
 security_authority='${runtime.security.authority}'
 security_username='${runtime.security.username}'
 security_darkcode='${runtime.security.darkcode}'
+sync_clusters='${runtime.yarn.syncClusters}'
+compaction_clusters='${runtime.yarn.compactionClusters}'
 
 <#list hosts as host>
 host=${host.ip}
@@ -22,7 +24,7 @@ hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`
 ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
 start_time=`date +%Y%m%d%H%M%S`
-ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
+ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' --yarn-cluster.sync-clusters=${sync_clusters} --yarn-cluster.compaction-clusters=${compaction_clusters}</#noparse> <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
 </#if>
 echo ''
 </#list>

service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusterProperties.java

@@ -0,0 +1,35 @@
+package com.lanyuanxiaoyao.service.configuration.entity.yarn;
+
+import java.util.List;
+import java.util.Map;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * Yarn 部署信息
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-02-26
+ */
+@ConfigurationProperties("yarn-cluster")
+@Component
+public class YarnClusterProperties {
+    private String syncClusters = "";
+    private String compactionClusters = "";
+
+    public String getSyncClusters() {
+        return syncClusters;
+    }
+
+    public void setSyncClusters(String syncClusters) {
+        this.syncClusters = syncClusters;
+    }
+
+    public String getCompactionClusters() {
+        return compactionClusters;
+    }
+
+    public void setCompactionClusters(String compactionClusters) {
+        this.compactionClusters = compactionClusters;
+    }
+}

service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/yarn/YarnClusters.java

@@ -0,0 +1,94 @@
+package com.lanyuanxiaoyao.service.configuration.entity.yarn;
+
+import cn.hutool.core.map.MapUtil;
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.factory.Maps;
+import org.eclipse.collections.api.list.ImmutableList;
+import org.eclipse.collections.api.map.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+/**
+ * Yarn 集群信息
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-02-26
+ */
+@Component
+public class YarnClusters {
+    private static final Logger logger = LoggerFactory.getLogger(YarnClusters.class);
+    private final ImmutableMap<String, Cluster> clusters = Maps.immutable.ofAll(
+            MapUtil.<String, Cluster>builder()
+                    .put("a4", new Cluster("http://132.121.107.91:8088"))
+                    .put("b1", new Cluster("http://132.122.98.13:8088"))
+                    .put("b4", new Cluster("http://132.122.112.30:8088"))
+                    .put("b5", new Cluster("http://132.122.116.12:8088"))
+                    .put("t5", new Cluster("http://132.121.126.84:8088"))
+                    .put("b12", new Cluster("http://132.126.207.125:8088"))
+                    .build()
+    );
+
+    private final ImmutableList<String> activeClusters;
+    private final String defaultSyncCluster;
+    private final ImmutableList<String> syncClusters;
+    private final String defaultCompactionCluster;
+    private final ImmutableList<String> compactionClusters;
+
+    public YarnClusters(YarnClusterProperties yarnClusterProperties) {
+        syncClusters = Lists.immutable.of(yarnClusterProperties.getSyncClusters().split(",")).select(clusters::containsKey);
+        defaultSyncCluster = syncClusters.getFirstOptional().orElse("");
+
+        compactionClusters = Lists.immutable.of(yarnClusterProperties.getCompactionClusters().split(",")).select(clusters::containsKey);
+        defaultCompactionCluster = compactionClusters.getFirstOptional().orElse("");
+
+        activeClusters = syncClusters.newWithAll(compactionClusters).distinct();
+    }
+
+    public ImmutableMap<String, Cluster> getClusters() {
+        return clusters;
+    }
+
+    public ImmutableList<String> getActiveClusters() {
+        return activeClusters;
+    }
+
+    public String getDefaultSyncCluster() {
+        return defaultSyncCluster;
+    }
+
+    public ImmutableList<String> getSyncClusters() {
+        return syncClusters;
+    }
+
+    public String getDefaultCompactionCluster() {
+        return defaultCompactionCluster;
+    }
+
+    public ImmutableList<String> getCompactionClusters() {
+        return compactionClusters;
+    }
+
+    public static final class Cluster {
+        private String webUrl;
+
+        public Cluster(String webUrl) {
+            this.webUrl = webUrl;
+        }
+
+        public String getWebUrl() {
+            return webUrl;
+        }
+
+        public void setWebUrl(String webUrl) {
+            this.webUrl = webUrl;
+        }
+
+        @Override
+        public String toString() {
+            return "Cluster{" +
+                   "webUrl='" + webUrl + '\'' +
+                   '}';
+        }
+    }
+}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/A4LauncherService.java

@@ -1,11 +0,0 @@
-package com.lanyuanxiaoyao.service.forest.service.launcher;
-
-import com.dtflys.forest.annotation.BaseRequest;
-
-/**
- * @author lanyuanxiaoyao
- * @date 2023-06-06
- */
-@BaseRequest(baseURL = "http://service-launcher-runner-a4")
-public interface A4LauncherService extends LauncherService{
-}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B12LauncherService.java

@@ -1,11 +0,0 @@
-package com.lanyuanxiaoyao.service.forest.service.launcher;
-
-import com.dtflys.forest.annotation.BaseRequest;
-
-/**
- * @author lanyuanxiaoyao
- * @date 2023-06-06
- */
-@BaseRequest(baseURL = "http://service-launcher-runner-b12")
-public interface B12LauncherService extends LauncherService{
-}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5LauncherService.java

@@ -1,11 +0,0 @@
-package com.lanyuanxiaoyao.service.forest.service.launcher;
-
-import com.dtflys.forest.annotation.BaseRequest;
-
-/**
- * @author lanyuanxiaoyao
- * @date 2023-06-06
- */
-@BaseRequest(baseURL = "http://service-launcher-runner-b5")
-public interface B5LauncherService extends LauncherService{
-}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B5SyncLauncherService.java

@@ -1,11 +0,0 @@
-package com.lanyuanxiaoyao.service.forest.service.launcher;
-
-import com.dtflys.forest.annotation.BaseRequest;
-
-/**
- * @author lanyuanxiaoyao
- * @date 2023-06-06
- */
-@BaseRequest(baseURL = "http://service-launcher-runner-b5-sync")
-public interface B5SyncLauncherService extends LauncherService{
-}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/LaunchersService.java

@@ -1,7 +1,7 @@
 package com.lanyuanxiaoyao.service.forest.service.launcher;
 
 import cn.hutool.core.util.StrUtil;
-import com.eshore.odcp.hudi.connector.Constants;
+import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnClusters;
 import java.util.HashMap;
 import java.util.Map;
 import org.eclipse.collections.api.factory.Maps;
@@ -9,6 +9,7 @@ import org.eclipse.collections.api.list.ImmutableList;
 import org.eclipse.collections.api.map.ImmutableMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
 import org.springframework.stereotype.Service;
 
 /**
@@ -22,18 +23,15 @@ public class LaunchersService {
     private final ImmutableMap<String, LauncherService> serviceMap;
 
     public LaunchersService(
-            A4LauncherService a4LauncherService,
+            YarnClusters yarnClusters,
-            B1LauncherService b1LauncherService,
+            ApplicationContext applicationContext
-            B5LauncherService b5LauncherService,
-            B5SyncLauncherService b5SyncLauncherService,
-            B12LauncherService b12LauncherService
     ) {
         Map<String, LauncherService> map = new HashMap<>(5);
-        map.put(Constants.CLUSTER_A4, a4LauncherService);
+        for (String cluster : yarnClusters.getActiveClusters()) {
-        map.put(Constants.CLUSTER_B1, b1LauncherService);
+            logger.info("Load {} cluster service", cluster);
-        map.put(Constants.CLUSTER_B5, b5LauncherService);
+            LauncherService service = applicationContext.getBean(StrUtil.format("{}LauncherService", StrUtil.toCamelCase(cluster)), LauncherService.class);
-        map.put(Constants.CLUSTER_B5_SYNC, b5SyncLauncherService);
+            map.put(cluster, service);
-        map.put(Constants.CLUSTER_B12, b12LauncherService);
+        }
         serviceMap = Maps.immutable.ofAll(map);
     }
 

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/A4LauncherService.java

@@ -0,0 +1,12 @@
+package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
+
+import com.dtflys.forest.annotation.BaseRequest;
+import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-06-06
+ */
+@BaseRequest(baseURL = "http://service-launcher-runner-a4")
+public interface A4LauncherService extends LauncherService {
+}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B12LauncherService.java

@@ -0,0 +1,12 @@
+package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
+
+import com.dtflys.forest.annotation.BaseRequest;
+import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-06-06
+ */
+@BaseRequest(baseURL = "http://service-launcher-runner-b12")
+public interface B12LauncherService extends LauncherService {
+}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B1LauncherService.java

@@ -1,13 +1,14 @@
-package com.lanyuanxiaoyao.service.forest.service.launcher;
+package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
 
 import com.dtflys.forest.annotation.BaseRequest;
 import com.dtflys.forest.annotation.Get;
 import com.dtflys.forest.annotation.Query;
+import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
 
 /**
  * @author lanyuanxiaoyao
  * @date 2023-06-06
  */
 @BaseRequest(baseURL = "http://service-launcher-runner-b1")
-public interface B1LauncherService extends LauncherService{
+public interface B1LauncherService extends LauncherService {
 }

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/impl/B5LauncherService.java

@@ -0,0 +1,12 @@
+package com.lanyuanxiaoyao.service.forest.service.launcher.impl;
+
+import com.dtflys.forest.annotation.BaseRequest;
+import com.lanyuanxiaoyao.service.forest.service.launcher.LauncherService;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-06-06
+ */
+@BaseRequest(baseURL = "http://service-launcher-runner-b5")
+public interface B5LauncherService extends LauncherService {
+}