lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(check): 增加hdfs/juicefs的访问校验

Browse Source
This commit is contained in:
兰缘小妖
2024-02-23 14:13:22 +08:00
parent 3ac4f80ab7
commit ecb1773238
14 changed files with 2499 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
service-check/pom.xml
View File

@@ -50,6 +50,21 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.juicefs</groupId>
<artifactId>juicefs-hadoop</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-configuration</artifactId>
@@ -65,13 +80,92 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>copy-config-file</id>
<phase>validate</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
<resources>
<resource>
<directory>${project.parent.basedir}/config/${build-tag}</directory>
<includes>
<include>*.xml</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer
implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
<resource>META-INF/spring.factories</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.lanyuanxiaoyao.service.check.CheckApplication</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>log4j-surefire*.properties</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>

44
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/HdfsChecker.java Normal file
View File

@@ -0,0 +1,44 @@
package com.lanyuanxiaoyao.service.check.actions;
import com.lanyuanxiaoyao.service.check.configuration.HdfsConfigurationProperties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Component
@ConditionalOnBean(HdfsConfigurationProperties.class)
public class HdfsChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(HdfsChecker.class);
private final HdfsConfigurationProperties hdfsConfigurationProperties;
public HdfsChecker(HdfsConfigurationProperties hdfsConfigurationProperties) {this.hdfsConfigurationProperties = hdfsConfigurationProperties;}
@Override
public void check() throws Exception {
logger.info("Check hdfs path");
FileSystem fileSystem = FileSystem.get(new Configuration());
for (String path : hdfsConfigurationProperties.getExistsPaths()) {
if (!fileSystem.exists(new Path(path))) {
logger.warn("Path not exists {}", path);
}
for (FileStatus fileStatus : fileSystem.listStatus(new Path(path))) {
logger.info("{}", fileStatus.getPath().toString());
}
}
}
@Override
public String description() {
return "Check HDFS";
}
}

2
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/MysqlChecker.java
View File

@@ -7,6 +7,7 @@ import java.sql.ResultSet;
import java.sql.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
@@ -14,6 +15,7 @@ import org.springframework.stereotype.Component;
* @date 2024-02-22
*/
@Component
@ConditionalOnBean(MysqlConfigurationProperties.class)
public class MysqlChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(MysqlChecker.class);

9
service-check/src/main/java/com/lanyuanxiaoyao/service/check/actions/PulsarChecker.java
View File

@@ -1,11 +1,13 @@
package com.lanyuanxiaoyao.service.check.actions;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.check.configuration.PulsarConfigurationProperties;
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarInfo;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.PulsarClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.stereotype.Component;
/**
@@ -15,15 +17,10 @@ import org.springframework.stereotype.Component;
* @date 2024-02-22
*/
@Component
@ConditionalOnBean(PulsarConfigurationProperties.class)
public class PulsarChecker extends Checker {
private static final Logger logger = LoggerFactory.getLogger(PulsarChecker.class);
private void checkConnect(String url) {
if (!ping(url)) {
logger.warn("{} ping failure", url);
}
}
private String adminUrl(PulsarInfo info) {
return StrUtil.format("http://{}/admin/v2", info.getAdmin());
}

42
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/HdfsConfigurationProperties.java Normal file
View File

@@ -0,0 +1,42 @@
package com.lanyuanxiaoyao.service.check.configuration;
import java.util.List;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Configuration
@ConfigurationProperties("checker.hdfs")
@ConditionalOnProperty(value = "checker.hdfs.enabled", matchIfMissing = true)
public class HdfsConfigurationProperties {
private Boolean enabled = true;
private List<String> existsPaths;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public List<String> getExistsPaths() {
return existsPaths;
}
public void setExistsPaths(List<String> existsPaths) {
this.existsPaths = existsPaths;
}
@Override
public String toString() {
return "HdfsConfigurationProperties{" +
"enabled=" + enabled +
", existsPaths=" + existsPaths +
'}';
}
}

16
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/MysqlConfigurationProperties.java
View File

@@ -1,6 +1,7 @@
package com.lanyuanxiaoyao.service.check.configuration;
import java.util.List;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@@ -9,10 +10,20 @@ import org.springframework.context.annotation.Configuration;
* @date 2024-02-22
*/
@Configuration
@ConfigurationProperties(prefix = "checker.mysql")
@ConfigurationProperties("checker.mysql")
@ConditionalOnProperty(value = "checker.mysql.enabled", matchIfMissing = true)
public class MysqlConfigurationProperties {
private Boolean enabled = true;
private List<MysqlInfo> targets;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public List<MysqlInfo> getTargets() {
return targets;
}
@@ -24,7 +35,8 @@ public class MysqlConfigurationProperties {
@Override
public String toString() {
return "MysqlConfigurationProperties{" +
"targets=" + targets +
"enabled=" + enabled +
", targets=" + targets +
'}';
}

31
service-check/src/main/java/com/lanyuanxiaoyao/service/check/configuration/PulsarConfigurationProperties.java Normal file
View File

@@ -0,0 +1,31 @@
package com.lanyuanxiaoyao.service.check.configuration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @date 2024-02-23
*/
@Configuration
@ConfigurationProperties("checker.pulsar")
@ConditionalOnProperty(value = "checker.pulsar.enabled", matchIfMissing = true)
public class PulsarConfigurationProperties {
private Boolean enabled = true;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
@Override
public String toString() {
return "PulsarConfigurationProperties{" +
"enabled=" + enabled +
'}';
}
}

9
service-check/src/main/resources/application.yml
View File

@@ -4,8 +4,15 @@ spring:
profiles:
include: common
checker:
pulsar:
enabled: false
mysql:
enabled: false
targets:
- url: jdbc:mysql://132.121.204.217:17906/hudi_collect_build_2?useSSL=false
username: odcp
password: wFg_fR492#&
password: wFg_fR492#&
hdfs:
exists-paths:
- "/"
- "jfs://tdsc/user"