feat(uploader): 增加一个关于文件上传下载的项目，后续方便jar包传递
This commit is contained in:
1
pom.xml
1
pom.xml
@@ -24,6 +24,7 @@
|
||||
<module>service-cli</module>
|
||||
<module>service-loki-query</module>
|
||||
<module>service-test-query</module>
|
||||
<module>service-uploader</module>
|
||||
<module>service-executor</module>
|
||||
<module>service-cloud-query</module>
|
||||
</modules>
|
||||
|
||||
49
service-uploader/pom.xml
Normal file
49
service-uploader/pom.xml
Normal file
@@ -0,0 +1,49 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>hudi-service</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-uploader</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-dependencies</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-configuration</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.1.2</version>
|
||||
</dependency>
|
||||
<!-- <dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
<version>3.1.2</version>
|
||||
</dependency> -->
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.lanyuanxiaoyao.service.uploader;
|
||||
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-01-24
|
||||
*/
|
||||
@EnableDiscoveryClient
|
||||
@SpringBootApplication(scanBasePackages = {"com.lanyuanxiaoyao.service"})
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
public class UploaderApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(UploaderApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lanyuanxiaoyao.service.uploader.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-01-24
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties("uploader")
|
||||
public class UploaderConfiguration {
|
||||
private String tmpDir;
|
||||
|
||||
public String getTmpDir() {
|
||||
return tmpDir;
|
||||
}
|
||||
|
||||
public void setTmpDir(String tmpDir) {
|
||||
this.tmpDir = tmpDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "UploaderConfiguration{" +
|
||||
"tmpDir='" + tmpDir + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.lanyuanxiaoyao.service.uploader.service.filesystem;
|
||||
|
||||
import cn.hutool.core.io.file.FileNameUtil;
|
||||
import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 处理上传事宜
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-01-24
|
||||
*/
|
||||
@Service
|
||||
public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
|
||||
private final UploaderConfiguration uploaderConfiguration;
|
||||
private final Configuration configuration;
|
||||
|
||||
public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
|
||||
this.uploaderConfiguration = uploaderConfiguration;
|
||||
configuration = new Configuration();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void download(String path) throws Exception {
|
||||
String filename = FileNameUtil.getName(path);
|
||||
try (FileSystem fileSystem = FileSystem.get(configuration)) {
|
||||
fileSystem.copyToLocalFile(new Path(path), new Path(uploaderConfiguration.getTmpDir(), filename));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(String filename, String targetPath) throws Exception {
|
||||
try (FileSystem fileSystem = FileSystem.get(configuration)) {
|
||||
fileSystem.copyFromLocalFile(new Path(uploaderConfiguration.getTmpDir(), filename), new Path(targetPath, filename));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.lanyuanxiaoyao.service.uploader.service.filesystem;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* 上传和下载
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-01-24
|
||||
*/
|
||||
public interface UploadAndDownloadService {
|
||||
void download(String path) throws Exception;
|
||||
|
||||
void upload(String filename, String targetPath) throws Exception;
|
||||
}
|
||||
5
service-uploader/src/main/resources/application.yml
Normal file
5
service-uploader/src/main/resources/application.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-uploader
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics
|
||||
51
service-uploader/src/main/resources/logback-spring.xml
Normal file
51
service-uploader/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,51 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Loki" class="com.github.loki4j.logback.Loki4jAppender">
|
||||
<metricsEnabled>true</metricsEnabled>
|
||||
<http class="com.github.loki4j.logback.ApacheHttpSender">
|
||||
<url>${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push}</url>
|
||||
</http>
|
||||
<format>
|
||||
<label>
|
||||
<pattern>app=${APP_NAME:- },host=${HOSTNAME},level=%level</pattern>
|
||||
</label>
|
||||
<message>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</message>
|
||||
<sortByTime>true</sortByTime>
|
||||
</format>
|
||||
</appender>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Loki"/>
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.lanyuanxiaoyao.service.uploader;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.io.file.FileNameUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-01-24
|
||||
*/
|
||||
public class PathTest {
|
||||
public static void main(String[] args) {
|
||||
System.out.println(new Path("/apps/", "hello.txt"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user