diff --git a/service-cli/service-cli-runner/src/main/resources/application-b5.yml b/service-cli/service-cli-runner/src/main/resources/application-b5.yml index 13e8a03..fea219d 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml @@ -9,6 +9,7 @@ deploy: loki-base-url: http://132.122.116.142:33100 loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 + connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 eureka-url: ENC(1gFdUy0/FYxDMK910ZS65qNqdULnnSOPrR8EkqrZ3zYXsTP9kk8y55C7WbcxM5vaa1KIDNEqilEIA2sOLX3920gu3Woq805pf8dzxl9/velQWdaTFUxn9XqypuP+p/bSyY3upCYehCZFWgPPSn8ag0m3JKSofkk9V4mO+WswxFDlWiM1P5Om7Ev7WsWSIYMqhN2viRjCscctrIkxh3trIgsb9qfw5snNYZvh+gHRQJ1GA0VYlLmqJLvasKcHSzKGabu8iRAnXMHaAv7jwgdMpHBz7uBeeQh+BIFqurSEotjRG+TlmidLA8X0BWiRTGiGp9X9mV/5Zm5QkBp1em8L1D9+wz8R9rn8eFPXMSHhlqocxKl7ZJgnzSfu9CS67y5k7PzSIjkViN01HTPzg36zyv9xzqVt5W+7rXbQJpmXLXfUAPsjRP/ns0a6ly0jkHoIOHqapZcJ8gVCVH46D0e5C59h57Q6iX/BW0f8VIsH4HfbSGi2doRqNVqbIRwdVnkzUMDihhKNvLFTh70H20v1uw6wNZclYF1wh9Lkn28TKqz/y45FbxhRwjBCvWu55R8/qSkfR2K/QyLw4WvE3raG23fssN2BYlNA/x9OaGBWPXVUZcphGl6rPUZeAqXVKms/qcRuLOzZrZMv42OeKFk/NzR8BrwiXN7NafhXrvH1CNN0b69t7jGqS5+sIwkBjNtD+J17G9G8qUB9ETH6C+uRO4MWJ+KesERdI3E0PIqzimTQiV60lzVL4kYB5KJuDnw0LAsxpu3a5VgKWIZzF/zWXZcBQTeea0IKRBEeujwTCecvaNTjd+x53jwTSobCyL4ADt2hcGHOGMcynuryZYh++XTMEjx0P/cMxXqHDXqb21liJftWmMzqyQ5HHQZCY6Mg1Lh8FsLaV3vyl142rbXan0NwolIGJ96QZ7PoNa9zeQOgcwXvGjHnpzZWsOwq/90Bnl1JDcm0MJpjAq2LnZN5r/7aa74TvC54Pu4jMMzZMdyrUS0bWAxF9NS2erfP2BPywI2AwSGgvl7qh2Sx4d/nh1984TTwtK5jDd+LIdml8SfVIcBfNG3dPvlrJN6TmvbBxtRRFo1lXX47axfr6kL6DNDUPTFbnZRjK0RLRuNdIueLhcUCLAJfVrP0jz4axcg12ZdRU4oPUTQg7m6blN5IjJu90GcHXia2kPXudSQidQLBIrE6RrZxUYO3DPwUXzVvOwYmuGzu2TJWS4vLYR00jDMCEv/Gosy0zurzs/zLCA/awGlR4c3InQo8IahyDh2faSNljCnfUPMMgfl/vE8BBODCKX18m2a+xvyHn+out4sB6MwBqRwINn5/QWKw8qqMOPS5rmLHvPUi8BdZZnetFk2najdRib7o1SdUG1Sn3pDOYRHXVXxoHxebSyAl5ybzis9PAlc9MTJnxM1qVEEdzwUBPElDuPOE8Zr2lYhz4/RKWnwd7BG2WUP3YYVyQmDL/H00xQCagUBoQ6CD3sKmTZSFkJUAKX3r1t+K1LqqjkyYP6XSvgp7ha8CsW4ssPhCFKgieUMR5T8cFVV2tSxLEJTlm15GLVFgiWtmTsHlZ32CU388BUYqVvtv4gNvgxOxfr9suFNa4V/xRTxlpsLjQ+GSSBMHznVykxf9Buk05CuuqfW8U4Ulgc+9LxXpKFcKAHjXe3Hj5RghtuOz+Nb+Iyi3sZibeLjqYoS0pWOE5g44azVPyeSunOWCNmlQI6lwEm/st6FoupwkI70X+V5Mthd90lnFpUtp49GlC81jJZ2zSiKV7ZFGsZfLEDQRU12S) hudi: app-hdfs-path: hdfs://b2/apps/datalake/jars/app diff --git a/service-cli/service-cli-runner/src/main/resources/application-t5.yml b/service-cli/service-cli-runner/src/main/resources/application-t5.yml index 3183109..1473a3c 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-t5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-t5.yml @@ -9,6 +9,7 @@ deploy: loki-base-url: http://132.121.126.84:33100 loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181 + connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181 eureka-url: ENC(qoyFTyMe+qtjLai596ZScZJ4crAPgIzivL/kiUPJ3rIZwTfNTb+JaMj4ZCakcSVtVj5CBtXXfsEY/VIrwWiociuBjlyxOULyRsgZC37G9q1Qvgh7YxocKe7POn/M7unzPQE5qLJpmqRHjQFZtqUK3bUdqVtS0S4+Kkd1qYNUo0s1+rkVeigfYG15ntkiVvTav63ZxJMVgEnLZTdG8iOW/f1xgNZNYU4yolCAwhxi1nap5blWYdISF0JUwt5W8sPvPoaQ+3tGQMrIw4E7hSs5lNqsF4efz6KbJ7H4og4NHAM8Kq51k4sJZkrani1H7BxVjy721j4LgpiKoia9blIa1Nj9n2uFni7hSrsrwd0z5YP9FCLerC8YSx9WXLZFhRlXgo6LE4p7OLpYsTy5w2eRLVaonS8kXCLLo21OG/scbGa1ieFcQnt4L9BkBu1FhcpAjPBdu2P0UZJA/R2K0Y+ju/pP3oQJG8Pa+asNF8CcVJovuym+Y5FC3WoVkvRwHDYx74GXe2i2sUbLurQlYXXnnepdfDyL8g8khRD1mri9lM4O52H7PZ36jyhFnx4YlmlNjBpxcf/ppSBZ5ASFpEKy0RDcPzzEpLP+8BL8LF94VhQ=) hudi: app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 85af98b..79d86f7 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -33,7 +33,7 @@ deploy: connector_cluster_cluster: b1 connector_cluster_sync-queue-name: sync-queue-b1 connector_cluster_compaction-queue-name: compaction-queue-b1 - connector_zookeeper_connect-url: ${deploy.runtime.zk-url} + connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-b5 source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar replicas: 6 @@ -48,7 +48,7 @@ deploy: connector_cluster_cluster: b5 connector_cluster_sync-queue-name: sync-queue-b5 connector_cluster_compaction-queue-name: compaction-queue-b5 - connector_zookeeper_connect-url: ${deploy.runtime.zk-url} + connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-b5-sync source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar replicas: 6 @@ -63,7 +63,7 @@ deploy: connector_cluster_cluster: b5-sync connector_cluster_sync-queue-name: sync-queue-b5-sync connector_cluster_compaction-queue-name: compaction-queue-b5-sync - connector_zookeeper_connect-url: ${deploy.runtime.zk-url} + connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-launcher-runner-a4 source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar replicas: 6 @@ -78,7 +78,7 @@ deploy: connector_cluster_cluster: a4 connector_cluster_sync-queue-name: sync-queue-a4 connector_cluster_compaction-queue-name: compaction-queue-a4 - connector_zookeeper_connect-url: ${deploy.runtime.zk-url} + connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-info-query source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 4 @@ -103,7 +103,7 @@ deploy: source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar replicas: 6 arguments: - spring_cloud_connect-string: ${deploy.runtime.zk-url} + connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url} - name: service-web source-jar: service-web-1.0.0-SNAPSHOT.jar replicas: 4 diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index a631897..cda1742 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -9,6 +9,7 @@ datetime=`date +%Y%m%d%H%M%S` log_path='${runtime.logPath}' loki_url='${runtime.lokiUrl}' eureka_url='${runtime.eurekaUrl}' +zk_url='${runtime.zkUrl}' security_authority='${runtime.security.authority}' security_username='${runtime.security.username}' security_darkcode='${runtime.security.darkcode}' @@ -20,7 +21,7 @@ echo "${info.name} $host $hostname $datetime" ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar <#if selectedHosts?seq_contains(host)> ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar" -ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" +ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} -jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --spring.cloud.zookeeper.discovery.instance-host=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}' <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} > /dev/null 2>&1 &" echo '' diff --git a/service-configuration/src/main/resources/application-eureka.yml b/service-configuration/src/main/resources/application-discovery.yml similarity index 55% rename from service-configuration/src/main/resources/application-eureka.yml rename to service-configuration/src/main/resources/application-discovery.yml index 3daf96c..67d011a 100644 --- a/service-configuration/src/main/resources/application-eureka.yml +++ b/service-configuration/src/main/resources/application-discovery.yml @@ -1,9 +1,19 @@ +spring: + cloud: + zookeeper: + discovery: + enabled: false + root: /hudi-services + instance-host: localhost + instance-id: ${spring.application.name}-${spring.cloud.zookeeper.discovery.instance-host}-${random.uuid}-${datetime:19700101} + connect-string: 127.0.0.1:2181 eureka: instance: hostname: localhost prefer-ip-address: true instance-id: ${spring.application.name}-${eureka.instance.hostname}-${random.uuid}-${datetime} client: + enabled: true service-url: defaultZone: http://localhost:35670/eureka/ registry-fetch-interval-seconds: 10 diff --git a/service-dependencies/pom.xml b/service-dependencies/pom.xml index 51a8ad1..9d0be09 100644 --- a/service-dependencies/pom.xml +++ b/service-dependencies/pom.xml @@ -21,6 +21,10 @@ org.springframework.cloud spring-cloud-starter-netflix-eureka-client + + org.springframework.cloud + spring-cloud-starter-zookeeper-discovery + org.springframework.boot spring-boot-starter-security diff --git a/service-flink-query/src/main/resources/application.yml b/service-flink-query/src/main/resources/application.yml index cd366e0..273959d 100644 --- a/service-flink-query/src/main/resources/application.yml +++ b/service-flink-query/src/main/resources/application.yml @@ -2,4 +2,4 @@ spring: application: name: service-flink-query profiles: - include: random-port,common,eureka,metrics \ No newline at end of file + include: random-port,common,discovery,metrics \ No newline at end of file diff --git a/service-hudi-query/src/main/resources/application.yml b/service-hudi-query/src/main/resources/application.yml index cc236dd..3288138 100644 --- a/service-hudi-query/src/main/resources/application.yml +++ b/service-hudi-query/src/main/resources/application.yml @@ -2,4 +2,4 @@ spring: application: name: service-hudi-query profiles: - include: random-port,common,eureka,metrics,forest \ No newline at end of file + include: random-port,common,discovery,metrics,forest \ No newline at end of file diff --git a/service-info-query/src/main/resources/application.yml b/service-info-query/src/main/resources/application.yml index 600d837..0b817a2 100644 --- a/service-info-query/src/main/resources/application.yml +++ b/service-info-query/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: application: name: service-info-query profiles: - include: random-port,common,eureka,metrics + include: random-port,common,discovery,metrics datasource: url: ENC(tDeB9gYs1IHN90VV+KhNQAVEKSqeInEmFhgma7A0g6surB5pOyJC3lJx0QrvQo4zYz0WOhSgqjglHDSX7nh7k9ak3OQMgLYkHQGVawAZfcsGT/1m0csyjQzCxTCOZr5r) username: ENC(29BKiU1cMKlA61gszFeYfWvoDVtLCCQAtfraxMq+f6Gm2LFu+67lkkBhoWgWIJga) diff --git a/service-loki-query/src/main/resources/application.yml b/service-loki-query/src/main/resources/application.yml index e7aae58..e8a836e 100644 --- a/service-loki-query/src/main/resources/application.yml +++ b/service-loki-query/src/main/resources/application.yml @@ -2,6 +2,6 @@ spring: application: name: service-loki-query profiles: - include: random-port,common,eureka,metrics + include: random-port,common,discovery,metrics loki: host: http://132.122.116.142:33100 \ No newline at end of file diff --git a/service-pulsar-query/src/main/resources/application.yml b/service-pulsar-query/src/main/resources/application.yml index d24cecc..da3fe1a 100644 --- a/service-pulsar-query/src/main/resources/application.yml +++ b/service-pulsar-query/src/main/resources/application.yml @@ -2,4 +2,4 @@ spring: application: name: service-pulsar-query profiles: - include: random-port,common,eureka,metrics + include: random-port,common,discovery,metrics diff --git a/service-web/src/main/resources/application.yml b/service-web/src/main/resources/application.yml index 40b865f..c1935a0 100644 --- a/service-web/src/main/resources/application.yml +++ b/service-web/src/main/resources/application.yml @@ -4,6 +4,6 @@ spring: application: name: service-web profiles: - include: common,eureka,metrics,forest + include: common,discovery,metrics,forest jackson: default-property-inclusion: non_null \ No newline at end of file diff --git a/service-yarn-query/src/main/resources/application.yml b/service-yarn-query/src/main/resources/application.yml index 6e0331d..1279d7d 100644 --- a/service-yarn-query/src/main/resources/application.yml +++ b/service-yarn-query/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: application: name: service-yarn-query profiles: - include: random-port,common,eureka,metrics + include: random-port,common,discovery,metrics yarn: web-urls: a4: http://132.121.107.91:8088 diff --git a/service-zookeeper-query/pom.xml b/service-zookeeper-query/pom.xml index 8e064b6..e591389 100644 --- a/service-zookeeper-query/pom.xml +++ b/service-zookeeper-query/pom.xml @@ -22,15 +22,6 @@ service-configuration 1.0.0-SNAPSHOT - - org.springframework.cloud - spring-cloud-zookeeper-core - - - org.apache.curator - curator-recipes - 5.1.0 - diff --git a/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java new file mode 100644 index 0000000..8e2cc0c --- /dev/null +++ b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java @@ -0,0 +1,41 @@ +package com.lanyuanxiaoyao.service.zookeeper; + +import javax.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Zookeeper 配置 + * + * @author ZhangJiacheng + * @date 2023-05-04 + */ +@ConfigurationProperties("connector.zookeeper") +@Component +public class ZookeeperConfiguration { + private static final Logger logger = LoggerFactory.getLogger(ZookeeperConfiguration.class); + + private String connectUrl; + + @PostConstruct + private void init() { + logger.info("Configuration initial: {}", this); + } + + public String getConnectUrl() { + return connectUrl; + } + + public void setConnectUrl(String connectUrl) { + this.connectUrl = connectUrl; + } + + @Override + public String toString() { + return "ZookeeperConfiguration{" + + "connectUrl='" + connectUrl + '\'' + + '}'; + } +} diff --git a/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java index 438b469..473b3df 100644 --- a/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java +++ b/service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java @@ -1,10 +1,15 @@ package com.lanyuanxiaoyao.service.zookeeper; import cn.hutool.core.util.ObjectUtil; +import com.eshore.odcp.hudi.connector.Constants; import com.lanyuanxiaoyao.service.configuration.entity.zookeeper.ZookeeperNode; +import java.util.concurrent.TimeUnit; +import javax.annotation.PreDestroy; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.recipes.locks.InterProcessLock; import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.zookeeper.data.Stat; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; @@ -16,8 +21,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.concurrent.TimeUnit; - /** * Zookeeper 查询 * @@ -32,8 +35,21 @@ public class ZookeeperController { private final CuratorFramework client; - public ZookeeperController(CuratorFramework client) { - this.client = client; + public ZookeeperController(ZookeeperConfiguration zookeeperConfiguration) { + this.client = CuratorFrameworkFactory.builder() + .connectString(zookeeperConfiguration.getConnectUrl()) + .retryPolicy(new ExponentialBackoffRetry((int) Constants.SECOND, 3)) + .sessionTimeoutMs((int) Constants.SECOND) + .connectionTimeoutMs((int) (30 * Constants.SECOND)) + .build(); + client.start(); + } + + @PreDestroy + public void destroy() { + if (ObjectUtil.isNotNull(client)) { + client.close(); + } } @GetMapping("exists_path") diff --git a/service-zookeeper-query/src/main/resources/application.yml b/service-zookeeper-query/src/main/resources/application.yml index e7be5b1..e776b6f 100644 --- a/service-zookeeper-query/src/main/resources/application.yml +++ b/service-zookeeper-query/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: application: name: service-zookeeper-query profiles: - include: random-port,common,eureka,metrics - cloud: - zookeeper: - connect-string: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 \ No newline at end of file + include: random-port,common,discovery,metrics +connector: + zookeeper: + connect-string: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 \ No newline at end of file