lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature(all): 增加zookeeper作为注册中心

Browse Source
This commit is contained in:
兰缘小妖
2023-07-12 10:26:23 +08:00
parent 2cfc009c51
commit 72bdda5546
17 changed files with 95 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
service-cli/service-cli-runner/src/main/resources/application-b5.yml
View File

@@ -9,6 +9,7 @@ deploy:
loki-base-url: http://132.122.116.142:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
eureka-url: ENC(1gFdUy0/FYxDMK910ZS65qNqdULnnSOPrR8EkqrZ3zYXsTP9kk8y55C7WbcxM5vaa1KIDNEqilEIA2sOLX3920gu3Woq805pf8dzxl9/velQWdaTFUxn9XqypuP+p/bSyY3upCYehCZFWgPPSn8ag0m3JKSofkk9V4mO+WswxFDlWiM1P5Om7Ev7WsWSIYMqhN2viRjCscctrIkxh3trIgsb9qfw5snNYZvh+gHRQJ1GA0VYlLmqJLvasKcHSzKGabu8iRAnXMHaAv7jwgdMpHBz7uBeeQh+BIFqurSEotjRG+TlmidLA8X0BWiRTGiGp9X9mV/5Zm5QkBp1em8L1D9+wz8R9rn8eFPXMSHhlqocxKl7ZJgnzSfu9CS67y5k7PzSIjkViN01HTPzg36zyv9xzqVt5W+7rXbQJpmXLXfUAPsjRP/ns0a6ly0jkHoIOHqapZcJ8gVCVH46D0e5C59h57Q6iX/BW0f8VIsH4HfbSGi2doRqNVqbIRwdVnkzUMDihhKNvLFTh70H20v1uw6wNZclYF1wh9Lkn28TKqz/y45FbxhRwjBCvWu55R8/qSkfR2K/QyLw4WvE3raG23fssN2BYlNA/x9OaGBWPXVUZcphGl6rPUZeAqXVKms/qcRuLOzZrZMv42OeKFk/NzR8BrwiXN7NafhXrvH1CNN0b69t7jGqS5+sIwkBjNtD+J17G9G8qUB9ETH6C+uRO4MWJ+KesERdI3E0PIqzimTQiV60lzVL4kYB5KJuDnw0LAsxpu3a5VgKWIZzF/zWXZcBQTeea0IKRBEeujwTCecvaNTjd+x53jwTSobCyL4ADt2hcGHOGMcynuryZYh++XTMEjx0P/cMxXqHDXqb21liJftWmMzqyQ5HHQZCY6Mg1Lh8FsLaV3vyl142rbXan0NwolIGJ96QZ7PoNa9zeQOgcwXvGjHnpzZWsOwq/90Bnl1JDcm0MJpjAq2LnZN5r/7aa74TvC54Pu4jMMzZMdyrUS0bWAxF9NS2erfP2BPywI2AwSGgvl7qh2Sx4d/nh1984TTwtK5jDd+LIdml8SfVIcBfNG3dPvlrJN6TmvbBxtRRFo1lXX47axfr6kL6DNDUPTFbnZRjK0RLRuNdIueLhcUCLAJfVrP0jz4axcg12ZdRU4oPUTQg7m6blN5IjJu90GcHXia2kPXudSQidQLBIrE6RrZxUYO3DPwUXzVvOwYmuGzu2TJWS4vLYR00jDMCEv/Gosy0zurzs/zLCA/awGlR4c3InQo8IahyDh2faSNljCnfUPMMgfl/vE8BBODCKX18m2a+xvyHn+out4sB6MwBqRwINn5/QWKw8qqMOPS5rmLHvPUi8BdZZnetFk2najdRib7o1SdUG1Sn3pDOYRHXVXxoHxebSyAl5ybzis9PAlc9MTJnxM1qVEEdzwUBPElDuPOE8Zr2lYhz4/RKWnwd7BG2WUP3YYVyQmDL/H00xQCagUBoQ6CD3sKmTZSFkJUAKX3r1t+K1LqqjkyYP6XSvgp7ha8CsW4ssPhCFKgieUMR5T8cFVV2tSxLEJTlm15GLVFgiWtmTsHlZ32CU388BUYqVvtv4gNvgxOxfr9suFNa4V/xRTxlpsLjQ+GSSBMHznVykxf9Buk05CuuqfW8U4Ulgc+9LxXpKFcKAHjXe3Hj5RghtuOz+Nb+Iyi3sZibeLjqYoS0pWOE5g44azVPyeSunOWCNmlQI6lwEm/st6FoupwkI70X+V5Mthd90lnFpUtp49GlC81jJZ2zSiKV7ZFGsZfLEDQRU12S)
hudi:
app-hdfs-path: hdfs://b2/apps/datalake/jars/app

1
service-cli/service-cli-runner/src/main/resources/application-t5.yml
View File

@@ -9,6 +9,7 @@ deploy:
loki-base-url: http://132.121.126.84:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
eureka-url: ENC(qoyFTyMe+qtjLai596ZScZJ4crAPgIzivL/kiUPJ3rIZwTfNTb+JaMj4ZCakcSVtVj5CBtXXfsEY/VIrwWiociuBjlyxOULyRsgZC37G9q1Qvgh7YxocKe7POn/M7unzPQE5qLJpmqRHjQFZtqUK3bUdqVtS0S4+Kkd1qYNUo0s1+rkVeigfYG15ntkiVvTav63ZxJMVgEnLZTdG8iOW/f1xgNZNYU4yolCAwhxi1nap5blWYdISF0JUwt5W8sPvPoaQ+3tGQMrIw4E7hSs5lNqsF4efz6KbJ7H4og4NHAM8Kq51k4sJZkrani1H7BxVjy721j4LgpiKoia9blIa1Nj9n2uFni7hSrsrwd0z5YP9FCLerC8YSx9WXLZFhRlXgo6LE4p7OLpYsTy5w2eRLVaonS8kXCLLo21OG/scbGa1ieFcQnt4L9BkBu1FhcpAjPBdu2P0UZJA/R2K0Y+ju/pP3oQJG8Pa+asNF8CcVJovuym+Y5FC3WoVkvRwHDYx74GXe2i2sUbLurQlYXXnnepdfDyL8g8khRD1mri9lM4O52H7PZ36jyhFnx4YlmlNjBpxcf/ppSBZ5ASFpEKy0RDcPzzEpLP+8BL8LF94VhQ=)
hudi:
app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars

10
service-cli/service-cli-runner/src/main/resources/application.yml
View File

@@ -33,7 +33,7 @@ deploy:
connector_cluster_cluster: b1
connector_cluster_sync-queue-name: sync-queue-b1
connector_cluster_compaction-queue-name: compaction-queue-b1
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
- name: service-launcher-runner-b5
source-jar: service-launcher-runner-b2b5-1.0.0-SNAPSHOT.jar
replicas: 6
@@ -48,7 +48,7 @@ deploy:
connector_cluster_cluster: b5
connector_cluster_sync-queue-name: sync-queue-b5
connector_cluster_compaction-queue-name: compaction-queue-b5
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
- name: service-launcher-runner-b5-sync
source-jar: service-launcher-runner-b2b5sync-1.0.0-SNAPSHOT.jar
replicas: 6
@@ -63,7 +63,7 @@ deploy:
connector_cluster_cluster: b5-sync
connector_cluster_sync-queue-name: sync-queue-b5-sync
connector_cluster_compaction-queue-name: compaction-queue-b5-sync
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
- name: service-launcher-runner-a4
source-jar: service-launcher-runner-b2a4-1.0.0-SNAPSHOT.jar
replicas: 6
@@ -78,7 +78,7 @@ deploy:
connector_cluster_cluster: a4
connector_cluster_sync-queue-name: sync-queue-a4
connector_cluster_compaction-queue-name: compaction-queue-a4
connector_zookeeper_connect-url: ${deploy.runtime.zk-url}
connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
- name: service-info-query
source-jar: service-info-query-1.0.0-SNAPSHOT.jar
replicas: 4
@@ -103,7 +103,7 @@ deploy:
source-jar: service-zookeeper-query-1.0.0-SNAPSHOT.jar
replicas: 6
arguments:
spring_cloud_connect-string: ${deploy.runtime.zk-url}
connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
- name: service-web
source-jar: service-web-1.0.0-SNAPSHOT.jar
replicas: 4

3
service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
View File

@@ -9,6 +9,7 @@ datetime=`date +%Y%m%d%H%M%S`
log_path='${runtime.logPath}'
loki_url='${runtime.lokiUrl}'
eureka_url='${runtime.eurekaUrl}'
zk_url='${runtime.zkUrl}'
security_authority='${runtime.security.authority}'
security_username='${runtime.security.username}'
security_darkcode='${runtime.security.darkcode}'
@@ -20,7 +21,7 @@ echo "${info.name} $host $hostname $datetime"
ssh $host 'bash -s' < stop.sh ${runtime.jarPath}/${info.name}.jar
<#if selectedHosts?seq_contains(host)>
ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key?replace("_", ".")}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar<#noparse> --datetime=${datetime} --hostname=${hostname} --spring.cloud.zookeeper.discovery.instance-host=${hostname} --eureka.instance.hostname=${hostname} --logging.parent=${log_path} --loki.url=${loki_url} --spring.cloud.zookeeper.connect-string=${zk_url} --eureka.client.service-url.defaultZone='${eureka_url}' --spring.security.meta.authority='${security_authority}' --spring.security.meta.username='${security_username}' --spring.security.meta.darkcode='${security_darkcode}'</#noparse> <#list arguments?keys as key>--${key?replace("_", ".")}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
</#if>
echo ''
</#list>

10
service-configuration/src/main/resources/application-eureka.yml → service-configuration/src/main/resources/application-discovery.yml
View File

@@ -1,9 +1,19 @@
spring:
cloud:
zookeeper:
discovery:
enabled: false
root: /hudi-services
instance-host: localhost
instance-id: ${spring.application.name}-${spring.cloud.zookeeper.discovery.instance-host}-${random.uuid}-${datetime:19700101}
connect-string: 127.0.0.1:2181
eureka:
instance:
hostname: localhost
prefer-ip-address: true
instance-id: ${spring.application.name}-${eureka.instance.hostname}-${random.uuid}-${datetime}
client:
enabled: true
service-url:
defaultZone: http://localhost:35670/eureka/
registry-fetch-interval-seconds: 10

4
service-dependencies/pom.xml
View File

@@ -21,6 +21,10 @@
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-zookeeper-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>

2
service-flink-query/src/main/resources/application.yml
View File

@@ -2,4 +2,4 @@ spring:
application:
name: service-flink-query
profiles:
include: random-port,common,eureka,metrics
include: random-port,common,discovery,metrics

2
service-hudi-query/src/main/resources/application.yml
View File

@@ -2,4 +2,4 @@ spring:
application:
name: service-hudi-query
profiles:
include: random-port,common,eureka,metrics,forest
include: random-port,common,discovery,metrics,forest

2
service-info-query/src/main/resources/application.yml
View File

@@ -2,7 +2,7 @@ spring:
application:
name: service-info-query
profiles:
include: random-port,common,eureka,metrics
include: random-port,common,discovery,metrics
datasource:
url: ENC(tDeB9gYs1IHN90VV+KhNQAVEKSqeInEmFhgma7A0g6surB5pOyJC3lJx0QrvQo4zYz0WOhSgqjglHDSX7nh7k9ak3OQMgLYkHQGVawAZfcsGT/1m0csyjQzCxTCOZr5r)
username: ENC(29BKiU1cMKlA61gszFeYfWvoDVtLCCQAtfraxMq+f6Gm2LFu+67lkkBhoWgWIJga)

2
service-loki-query/src/main/resources/application.yml
View File

@@ -2,6 +2,6 @@ spring:
application:
name: service-loki-query
profiles:
include: random-port,common,eureka,metrics
include: random-port,common,discovery,metrics
loki:
host: http://132.122.116.142:33100

2
service-pulsar-query/src/main/resources/application.yml
View File

@@ -2,4 +2,4 @@ spring:
application:
name: service-pulsar-query
profiles:
include: random-port,common,eureka,metrics
include: random-port,common,discovery,metrics

2
service-web/src/main/resources/application.yml
View File

@@ -4,6 +4,6 @@ spring:
application:
name: service-web
profiles:
include: common,eureka,metrics,forest
include: common,discovery,metrics,forest
jackson:
default-property-inclusion: non_null

2
service-yarn-query/src/main/resources/application.yml
View File

@@ -2,7 +2,7 @@ spring:
application:
name: service-yarn-query
profiles:
include: random-port,common,eureka,metrics
include: random-port,common,discovery,metrics
yarn:
web-urls:
a4: http://132.121.107.91:8088

9
service-zookeeper-query/pom.xml
View File

@@ -22,15 +22,6 @@
<artifactId>service-configuration</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-zookeeper-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>5.1.0</version>
</dependency>
</dependencies>
<build>

41
service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperConfiguration.java Normal file
View File

@@ -0,0 +1,41 @@
package com.lanyuanxiaoyao.service.zookeeper;
import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* Zookeeper 配置
*
* @author ZhangJiacheng
* @date 2023-05-04
*/
@ConfigurationProperties("connector.zookeeper")
@Component
public class ZookeeperConfiguration {
private static final Logger logger = LoggerFactory.getLogger(ZookeeperConfiguration.class);
private String connectUrl;
@PostConstruct
private void init() {
logger.info("Configuration initial: {}", this);
}
public String getConnectUrl() {
return connectUrl;
}
public void setConnectUrl(String connectUrl) {
this.connectUrl = connectUrl;
}
@Override
public String toString() {
return "ZookeeperConfiguration{" +
"connectUrl='" + connectUrl + '\'' +
'}';
}
}

24
service-zookeeper-query/src/main/java/com/lanyuanxiaoyao/service/zookeeper/ZookeeperController.java
View File

@@ -1,10 +1,15 @@
package com.lanyuanxiaoyao.service.zookeeper;
import cn.hutool.core.util.ObjectUtil;
import com.eshore.odcp.hudi.connector.Constants;
import com.lanyuanxiaoyao.service.configuration.entity.zookeeper.ZookeeperNode;
import java.util.concurrent.TimeUnit;
import javax.annotation.PreDestroy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.locks.InterProcessLock;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.data.Stat;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
@@ -16,8 +21,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.concurrent.TimeUnit;
/**
* Zookeeper 查询
*
@@ -32,8 +35,21 @@ public class ZookeeperController {
private final CuratorFramework client;
public ZookeeperController(CuratorFramework client) {
this.client = client;
public ZookeeperController(ZookeeperConfiguration zookeeperConfiguration) {
this.client = CuratorFrameworkFactory.builder()
.connectString(zookeeperConfiguration.getConnectUrl())
.retryPolicy(new ExponentialBackoffRetry((int) Constants.SECOND, 3))
.sessionTimeoutMs((int) Constants.SECOND)
.connectionTimeoutMs((int) (30 * Constants.SECOND))
.build();
client.start();
}
@PreDestroy
public void destroy() {
if (ObjectUtil.isNotNull(client)) {
client.close();
}
}
@GetMapping("exists_path")

8
service-zookeeper-query/src/main/resources/application.yml
View File

@@ -2,7 +2,7 @@ spring:
application:
name: service-zookeeper-query
profiles:
include: random-port,common,eureka,metrics
cloud:
zookeeper:
connect-string: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
include: random-port,common,discovery,metrics
connector:
zookeeper:
connect-string: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181