lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(check): 增加hdfs/juicefs的访问校验

Browse Source
This commit is contained in:
兰缘小妖
2024-02-23 14:13:22 +08:00
parent 3ac4f80ab7
commit ecb1773238
14 changed files with 2499 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/HdfsChecker.java Normal file
View File

@@ -0,0 +1,44 @@
package com.lanyuanxiaoyao.service.check.actions;
import com.lanyuanxiaoyao.service.check.configuration.HdfsConfigurationProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Component
@ConditionalOnBean(HdfsConfigurationProperties.class)
public class HdfsChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(HdfsChecker.class);
private final HdfsConfigurationProperties hdfsConfigurationProperties;
public HdfsChecker(HdfsConfigurationProperties hdfsConfigurationProperties) {this.hdfsConfigurationProperties = hdfsConfigurationProperties;}
@Override
public void check() throws Exception {
logger.info("Check hdfs path");
FileSystem fileSystem = FileSystem.get(new Configuration());
for (String path : hdfsConfigurationProperties.getExistsPaths()) {
if (!fileSystem.exists(new Path(path))) {
logger.warn("Path not exists {}", path);
}
for (FileStatus fileStatus : fileSystem.listStatus(new Path(path))) {
logger.info("{}", fileStatus.getPath().toString());
}
}
}
@Override
public String description() {
return "Check HDFS";
}
}

2
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/MysqlChecker.java
View File

@@ -7,6 +7,7 @@ import java.sql.ResultSet;
import java.sql.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
@@ -14,6 +15,7 @@ import org.springframework.stereotype.Component;
* @date 2024-02-22
*/
@Component
@ConditionalOnBean(MysqlConfigurationProperties.class)
public class MysqlChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(MysqlChecker.class);

9
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/PulsarChecker.java
View File

@@ -1,11 +1,13 @@
package com.lanyuanxiaoyao.service.check.actions;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.check.configuration.PulsarConfigurationProperties;
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarInfo;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.PulsarClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
@@ -15,15 +17,10 @@ import org.springframework.stereotype.Component;
* @date 2024-02-22
*/
@Component
@ConditionalOnBean(PulsarConfigurationProperties.class)
public class PulsarChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(PulsarChecker.class);
private void checkConnect(String url) {
if (!ping(url)) {
logger.warn("{} ping failure", url);
}
}
private String adminUrl(PulsarInfo info) {
return StrUtil.format("http://{}/admin/v2", info.getAdmin());
}

42
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/HdfsConfigurationProperties.java Normal file
View File

@@ -0,0 +1,42 @@
package com.lanyuanxiaoyao.service.check.configuration;
import java.util.List;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Configuration
@ConfigurationProperties("checker.hdfs")
@ConditionalOnProperty(value = "checker.hdfs.enabled", matchIfMissing = true)
public class HdfsConfigurationProperties {
private Boolean enabled = true;
private List<String> existsPaths;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public List<String> getExistsPaths() {
return existsPaths;
}
public void setExistsPaths(List<String> existsPaths) {
this.existsPaths = existsPaths;
}
@Override
public String toString() {
return "HdfsConfigurationProperties{" +
"enabled=" + enabled +
", existsPaths=" + existsPaths +
'}';
}
}

16
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/MysqlConfigurationProperties.java
View File

@@ -1,6 +1,7 @@
package com.lanyuanxiaoyao.service.check.configuration;
import java.util.List;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@@ -9,10 +10,20 @@ import org.springframework.context.annotation.Configuration;
* @date 2024-02-22
*/
@Configuration
@ConfigurationProperties(prefix = "checker.mysql")
@ConfigurationProperties("checker.mysql")
@ConditionalOnProperty(value = "checker.mysql.enabled", matchIfMissing = true)
public class MysqlConfigurationProperties {
private Boolean enabled = true;
private List<MysqlInfo> targets;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public List<MysqlInfo> getTargets() {
return targets;
}
@@ -24,7 +35,8 @@ public class MysqlConfigurationProperties {
@Override
public String toString() {
return "MysqlConfigurationProperties{" +
"targets=" + targets +
"enabled=" + enabled +
", targets=" + targets +
'}';
}

31
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/PulsarConfigurationProperties.java Normal file
View File

@@ -0,0 +1,31 @@
package com.lanyuanxiaoyao.service.check.configuration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Configuration
@ConfigurationProperties("checker.pulsar")
@ConditionalOnProperty(value = "checker.pulsar.enabled", matchIfMissing = true)
public class PulsarConfigurationProperties {
private Boolean enabled = true;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
@Override
public String toString() {
return "PulsarConfigurationProperties{" +
"enabled=" + enabled +
'}';
}
}

9
service-check/src/main/resources/application.yml
View File

@@ -4,8 +4,15 @@ spring:
profiles:
include: common
checker:
pulsar:
enabled: false
mysql:
enabled: false
targets:
- url: jdbc:mysql://132.121.204.217:17906/hudi_collect_build_2?useSSL=false
username: odcp
password: wFg_fR492#&
password: wFg_fR492#&
hdfs:
exists-paths:
- "/"
- "jfs://tdsc/user"