feature(all): 增加zookeeper作为注册中心

2023-07-12 10:26:23 +08:00
parent 2cfc009c51
commit 72bdda5546
17 changed files with 95 additions and 30 deletions
--- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml
@@ -9,6 +9,7 @@ deploy:
    loki-base-url: http://132.122.116.142:33100
    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
    zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
+    connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
    eureka-url: ENC(1gFdUy0/FYxDMK910ZS65qNqdULnnSOPrR8EkqrZ3zYXsTP9kk8y55C7WbcxM5vaa1KIDNEqilEIA2sOLX3920gu3Woq805pf8dzxl9/velQWdaTFUxn9XqypuP+p/bSyY3upCYehCZFWgPPSn8ag0m3JKSofkk9V4mO+WswxFDlWiM1P5Om7Ev7WsWSIYMqhN2viRjCscctrIkxh3trIgsb9qfw5snNYZvh+gHRQJ1GA0VYlLmqJLvasKcHSzKGabu8iRAnXMHaAv7jwgdMpHBz7uBeeQh+BIFqurSEotjRG+TlmidLA8X0BWiRTGiGp9X9mV/5Zm5QkBp1em8L1D9+wz8R9rn8eFPXMSHhlqocxKl7ZJgnzSfu9CS67y5k7PzSIjkViN01HTPzg36zyv9xzqVt5W+7rXbQJpmXLXfUAPsjRP/ns0a6ly0jkHoIOHqapZcJ8gVCVH46D0e5C59h57Q6iX/BW0f8VIsH4HfbSGi2doRqNVqbIRwdVnkzUMDihhKNvLFTh70H20v1uw6wNZclYF1wh9Lkn28TKqz/y45FbxhRwjBCvWu55R8/qSkfR2K/QyLw4WvE3raG23fssN2BYlNA/x9OaGBWPXVUZcphGl6rPUZeAqXVKms/qcRuLOzZrZMv42OeKFk/NzR8BrwiXN7NafhXrvH1CNN0b69t7jGqS5+sIwkBjNtD+J17G9G8qUB9ETH6C+uRO4MWJ+KesERdI3E0PIqzimTQiV60lzVL4kYB5KJuDnw0LAsxpu3a5VgKWIZzF/zWXZcBQTeea0IKRBEeujwTCecvaNTjd+x53jwTSobCyL4ADt2hcGHOGMcynuryZYh++XTMEjx0P/cMxXqHDXqb21liJftWmMzqyQ5HHQZCY6Mg1Lh8FsLaV3vyl142rbXan0NwolIGJ96QZ7PoNa9zeQOgcwXvGjHnpzZWsOwq/90Bnl1JDcm0MJpjAq2LnZN5r/7aa74TvC54Pu4jMMzZMdyrUS0bWAxF9NS2erfP2BPywI2AwSGgvl7qh2Sx4d/nh1984TTwtK5jDd+LIdml8SfVIcBfNG3dPvlrJN6TmvbBxtRRFo1lXX47axfr6kL6DNDUPTFbnZRjK0RLRuNdIueLhcUCLAJfVrP0jz4axcg12ZdRU4oPUTQg7m6blN5IjJu90GcHXia2kPXudSQidQLBIrE6RrZxUYO3DPwUXzVvOwYmuGzu2TJWS4vLYR00jDMCEv/Gosy0zurzs/zLCA/awGlR4c3InQo8IahyDh2faSNljCnfUPMMgfl/vE8BBODCKX18m2a+xvyHn+out4sB6MwBqRwINn5/QWKw8qqMOPS5rmLHvPUi8BdZZnetFk2najdRib7o1SdUG1Sn3pDOYRHXVXxoHxebSyAl5ybzis9PAlc9MTJnxM1qVEEdzwUBPElDuPOE8Zr2lYhz4/RKWnwd7BG2WUP3YYVyQmDL/H00xQCagUBoQ6CD3sKmTZSFkJUAKX3r1t+K1LqqjkyYP6XSvgp7ha8CsW4ssPhCFKgieUMR5T8cFVV2tSxLEJTlm15GLVFgiWtmTsHlZ32CU388BUYqVvtv4gNvgxOxfr9suFNa4V/xRTxlpsLjQ+GSSBMHznVykxf9Buk05CuuqfW8U4Ulgc+9LxXpKFcKAHjXe3Hj5RghtuOz+Nb+Iyi3sZibeLjqYoS0pWOE5g44azVPyeSunOWCNmlQI6lwEm/st6FoupwkI70X+V5Mthd90lnFpUtp49GlC81jJZ2zSiKV7ZFGsZfLEDQRU12S)
    hudi:
      app-hdfs-path: hdfs://b2/apps/datalake/jars/app
--- a/service-cli/service-cli-runner/src/main/resources/application-t5.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-t5.yml
@@ -9,6 +9,7 @@ deploy:
    loki-base-url: http://132.121.126.84:33100
    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
    zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
+    connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
    eureka-url: ENC(qoyFTyMe+qtjLai596ZScZJ4crAPgIzivL/kiUPJ3rIZwTfNTb+JaMj4ZCakcSVtVj5CBtXXfsEY/VIrwWiociuBjlyxOULyRsgZC37G9q1Qvgh7YxocKe7POn/M7unzPQE5qLJpmqRHjQFZtqUK3bUdqVtS0S4+Kkd1qYNUo0s1+rkVeigfYG15ntkiVvTav63ZxJMVgEnLZTdG8iOW/f1xgNZNYU4yolCAwhxi1nap5blWYdISF0JUwt5W8sPvPoaQ+3tGQMrIw4E7hSs5lNqsF4efz6KbJ7H4og4NHAM8Kq51k4sJZkrani1H7BxVjy721j4LgpiKoia9blIa1Nj9n2uFni7hSrsrwd0z5YP9FCLerC8YSx9WXLZFhRlXgo6LE4p7OLpYsTy5w2eRLVaonS8kXCLLo21OG/scbGa1ieFcQnt4L9BkBu1FhcpAjPBdu2P0UZJA/R2K0Y+ju/pP3oQJG8Pa+asNF8CcVJovuym+Y5FC3WoVkvRwHDYx74GXe2i2sUbLurQlYXXnnepdfDyL8g8khRD1mri9lM4O52H7PZ36jyhFnx4YlmlNjBpxcf/ppSBZ5ASFpEKy0RDcPzzEpLP+8BL8LF94VhQ=)
    hudi:
      app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -33,7 +33,7 @@ deploy:
        connector_cluster_cluster: b1
        connector_cluster_sync-queue-name: sync-queue-b1
        connector_cluster_compaction-queue-name: compaction-queue-b1
-        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-b5
      source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
      replicas: 6
@@ -48,7 +48,7 @@ deploy:
        connector_cluster_cluster: b5
        connector_cluster_sync-queue-name: sync-queue-b5
        connector_cluster_compaction-queue-name: compaction-queue-b5
-        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-b5-sync
      source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar
      replicas: 6
@@ -63,7 +63,7 @@ deploy:
        connector_cluster_cluster: b5-sync
        connector_cluster_sync-queue-name: sync-queue-b5-sync
        connector_cluster_compaction-queue-name: compaction-queue-b5-sync
-        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-launcher-runner-a4
      source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
      replicas: 6
@@ -78,7 +78,7 @@ deploy:
        connector_cluster_cluster: a4
        connector_cluster_sync-queue-name: sync-queue-a4
        connector_cluster_compaction-queue-name: compaction-queue-a4
-        connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-info-query
      source-jar: service-info-query-1.0.0-SNAPSHOT.jar
      replicas: 4
@@ -103,7 +103,7 @@ deploy:
      source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
      replicas: 6
      arguments:
-        spring_cloud_connect-string: ${deploy.runtime.zk-url}
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    - name: service-web
      source-jar: service-web-1.0.0-SNAPSHOT.jar
      replicas: 4
--- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
@@ -9,6 +9,7 @@ datetime=`date +%Y%m%d%H%M%S`
 log_path='${runtime.logPath}'
 loki_url='${runtime.lokiUrl}'
 eureka_url='${runtime.eurekaUrl}'
+zk_url='${runtime.zkUrl}'
 security_authority='${runtime.security.authority}'
 security_username='${runtime.security.username}'
 security_darkcode='${runtime.security.darkcode}'
@@ -20,7 +21,7 @@ echo "${info.name} $host $hostname $datetime"
 ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
 <#if selectedHosts?seq_contains(host)>
 ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
-ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
+ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --spring.cloud.zookeeper.discovery.instance-host=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
 </#if>
 echo ''
 </#list>
--- a/service-configuration/src/main/resources/application-discovery.yml
+++ b/service-configuration/src/main/resources/application-discovery.yml
@@ -1,9 +1,19 @@
+spring:
+  cloud:
+    zookeeper:
+      discovery:
+        enabled: false
+        root: /hudi-services
+        instance-host: localhost
+        instance-id: ${spring.application.name}-${spring.cloud.zookeeper.discovery.instance-host}-${random.uuid}-${datetime:19700101}
+      connect-string: 127.0.0.1:2181
 eureka:
  instance:
    hostname: localhost
    prefer-ip-address: true
    instance-id: ${spring.application.name}-${eureka.instance.hostname}-${random.uuid}-${datetime}
  client:
+    enabled: true
    service-url:
      defaultZone: http://localhost:35670/eureka/
    registry-fetch-interval-seconds: 10
--- a/service-dependencies/pom.xml
+++ b/service-dependencies/pom.xml
@@ -21,6 +21,10 @@
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-zookeeper-discovery</artifactId>
+        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-security</artifactId>
--- a/service-flink-query/src/main/resources/application.yml
+++ b/service-flink-query/src/main/resources/application.yml
@@ -2,4 +2,4 @@ spring:
  application:
    name: service-flink-query
  profiles:
-    include: random-port,common,eureka,metrics
+    include: random-port,common,discovery,metrics
--- a/service-hudi-query/src/main/resources/application.yml
+++ b/service-hudi-query/src/main/resources/application.yml
@@ -2,4 +2,4 @@ spring:
  application:
    name: service-hudi-query
  profiles:
-    include: random-port,common,eureka,metrics,forest
+    include: random-port,common,discovery,metrics,forest
--- a/service-info-query/src/main/resources/application.yml
+++ b/service-info-query/src/main/resources/application.yml
@@ -2,7 +2,7 @@ spring:
  application:
    name: service-info-query
  profiles:
-    include: random-port,common,eureka,metrics
+    include: random-port,common,discovery,metrics
  datasource:
    url: ENC(tDeB9gYs1IHN90VV+KhNQAVEKSqeInEmFhgma7A0g6surB5pOyJC3lJx0QrvQo4zYz0WOhSgqjglHDSX7nh7k9ak3OQMgLYkHQGVawAZfcsGT/1m0csyjQzCxTCOZr5r)
    username: ENC(29BKiU1cMKlA61gszFeYfWvoDVtLCCQAtfraxMq+f6Gm2LFu+67lkkBhoWgWIJga)
--- a/service-loki-query/src/main/resources/application.yml
+++ b/service-loki-query/src/main/resources/application.yml
@@ -2,6 +2,6 @@ spring:
  application:
    name: service-loki-query
  profiles:
-    include: random-port,common,eureka,metrics
+    include: random-port,common,discovery,metrics
 loki:
  host: http://132.122.116.142:33100
--- a/service-pulsar-query/src/main/resources/application.yml
+++ b/service-pulsar-query/src/main/resources/application.yml
@@ -2,4 +2,4 @@ spring:
  application:
    name: service-pulsar-query
  profiles:
-    include: random-port,common,eureka,metrics
+    include: random-port,common,discovery,metrics
--- a/service-web/src/main/resources/application.yml
+++ b/service-web/src/main/resources/application.yml
@@ -4,6 +4,6 @@ spring:
  application:
    name: service-web
  profiles:
-    include: common,eureka,metrics,forest
+    include: common,discovery,metrics,forest
  jackson:
    default-property-inclusion: non_null
--- a/service-yarn-query/src/main/resources/application.yml
+++ b/service-yarn-query/src/main/resources/application.yml
@@ -2,7 +2,7 @@ spring:
  application:
    name: service-yarn-query
  profiles:
-    include: random-port,common,eureka,metrics
+    include: random-port,common,discovery,metrics
 yarn:
  web-urls:
    a4: http://132.121.107.91:8088
--- a/service-zookeeper-query/pom.xml
+++ b/service-zookeeper-query/pom.xml
@@ -22,15 +22,6 @@
            <artifactId>service-configuration</artifactId>
            <version>1.0.0-SNAPSHOT</version>
        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-zookeeper-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.curator</groupId>
-            <artifactId>curator-recipes</artifactId>
-            <version>5.1.0</version>
-        </dependency>
    </dependencies>

    <build>
--- a/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java
+++ b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java
@@ -0,0 +1,41 @@
+package com.lanyuanxiaoyao.service.zookeeper;
+
+import javax.annotation.PostConstruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * Zookeeper 配置
+ *
+ * @author ZhangJiacheng
+ * @date 2023-05-04
+ */
+@ConfigurationProperties("connector.zookeeper")
+@Component
+public class ZookeeperConfiguration {
+    private static final Logger logger = LoggerFactory.getLogger(ZookeeperConfiguration.class);
+
+    private String connectUrl;
+
+    @PostConstruct
+    private void init() {
+        logger.info("Configuration initial: {}", this);
+    }
+
+    public String getConnectUrl() {
+        return connectUrl;
+    }
+
+    public void setConnectUrl(String connectUrl) {
+        this.connectUrl = connectUrl;
+    }
+
+    @Override
+    public String toString() {
+        return "ZookeeperConfiguration{" +
+                "connectUrl='" + connectUrl + '\'' +
+                '}';
+    }
+}
--- a/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java
+++ b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java
@@ -1,10 +1,15 @@
 package com.lanyuanxiaoyao.service.zookeeper;

 import cn.hutool.core.util.ObjectUtil;
+import com.eshore.odcp.hudi.connector.Constants;
 import com.lanyuanxiaoyao.service.configuration.entity.zookeeper.ZookeeperNode;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.PreDestroy;
 import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.recipes.locks.InterProcessLock;
 import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.zookeeper.data.Stat;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.ImmutableList;
@@ -16,8 +21,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;

-import java.util.concurrent.TimeUnit;
-
 /**
 * Zookeeper 查询
 *
@@ -32,8 +35,21 @@ public class ZookeeperController {

    private final CuratorFramework client;

-    public ZookeeperController(CuratorFramework client) {
-        this.client = client;
+    public ZookeeperController(ZookeeperConfiguration zookeeperConfiguration) {
+        this.client = CuratorFrameworkFactory.builder()
+                .connectString(zookeeperConfiguration.getConnectUrl())
+                .retryPolicy(new ExponentialBackoffRetry((int) Constants.SECOND, 3))
+                .sessionTimeoutMs((int) Constants.SECOND)
+                .connectionTimeoutMs((int) (30 * Constants.SECOND))
+                .build();
+        client.start();
+    }
+
+    @PreDestroy
+    public void destroy() {
+        if (ObjectUtil.isNotNull(client)) {
+            client.close();
+        }
    }

    @GetMapping("exists_path")
--- a/service-zookeeper-query/src/main/resources/application.yml
+++ b/service-zookeeper-query/src/main/resources/application.yml
@@ -2,7 +2,7 @@ spring:
  application:
    name: service-zookeeper-query
  profiles:
-    include: random-port,common,eureka,metrics
-  cloud:
+    include: random-port,common,discovery,metrics
+connector:
  zookeeper:
    connect-string: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181