lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(uploader): 增加文件上传下载模块

Browse Source
This commit is contained in:
兰缘小妖
2024-03-01 12:28:58 +08:00
parent c8126684a1
commit f3c1d12c60
14 changed files with 256 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
service-uploader/pom.xml
View File

@@ -20,6 +20,14 @@
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-configuration</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -30,11 +38,87 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-config-file</id>
<phase>validate</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
<resources>
<resource>
<directory>${project.parent.basedir}/config/${build-tag}</directory>
<includes>
<include>*.xml</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer
implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
<resource>META-INF/spring.factories</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.lanyuanxiaoyao.service.uploader.UploaderApplication</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>core-default.xml</exclude>
<exclude>hdfs-default.xml</exclude>
<exclude>yarn-default.xml</exclude>
<exclude>log4j-surefire*.properties</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>

11
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java
View File

@@ -1,21 +1,18 @@
package com.lanyuanxiaoyao.service.uploader;
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.retry.annotation.EnableRetry;
/**
* @author lanyuanxiaoyao
* @date 2024-01-24
*/
@EnableDiscoveryClient
@SpringBootApplication(scanBasePackages = {"com.lanyuanxiaoyao.service"})
// @EnableDiscoveryClient
@SpringBootApplication/* (scanBasePackages = {"com.lanyuanxiaoyao.service"}) */
@EnableConfigurationProperties
@EnableEncryptableProperties
@EnableRetry
// @EnableEncryptableProperties
// @EnableRetry
public class UploaderApplication {
public static void main(String[] args) {
SpringApplication.run(UploaderApplication.class, args);

11
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java
View File

@@ -10,7 +10,16 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties("uploader")
public class UploaderConfiguration {
private String tmpDir;
private String backend = "local";
private String tmpDir = "./";
public String getBackend() {
return backend;
}
public void setBackend(String backend) {
this.backend = backend;
}
public String getTmpDir() {
return tmpDir;

42
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java Normal file
View File

@@ -0,0 +1,42 @@
package com.lanyuanxiaoyao.service.uploader.controller;
import cn.hutool.core.io.IoUtil;
import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
import com.lanyuanxiaoyao.service.uploader.service.filesystem.UploadAndDownloadService;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Enumeration;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author lanyuanxiaoyao
* @date 2024-03-01
*/
@RestController
@RequestMapping("file")
public class FileController {
private static final Logger logger = LoggerFactory.getLogger(FileController.class);
private final UploadAndDownloadService service;
public FileController(UploaderConfiguration uploaderConfiguration, ApplicationContext applicationContext) {
this.service = applicationContext.getBean(uploaderConfiguration.getBackend(), UploadAndDownloadService.class);
}
@RequestMapping("upload/{filename}")
public void upload(@PathVariable("filename") String filename, HttpServletRequest request) throws Exception {
service.upload(filename, request.getInputStream());
}
@RequestMapping("download/{filename}")
public void download(@PathVariable("filename") String filename, HttpServletResponse response) throws Exception {
service.download(filename, response.getOutputStream());
}
}

21
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
View File

@@ -1,8 +1,12 @@
package com.lanyuanxiaoyao.service.uploader.service.filesystem;
import cn.hutool.core.io.file.FileNameUtil;
import cn.hutool.core.io.IoUtil;
import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
@@ -15,7 +19,7 @@ import org.springframework.stereotype.Service;
* @author lanyuanxiaoyao
* @date 2024-01-24
*/
@Service
@Service("hdfs")
public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration;
@@ -23,21 +27,22 @@ public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
this.uploaderConfiguration = uploaderConfiguration;
configuration = new Configuration();
this.configuration = new Configuration();
}
@Override
public void download(String path) throws Exception {
String filename = FileNameUtil.getName(path);
public void upload(String filename, InputStream inputStream) throws Exception {
try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.copyToLocalFile(new Path(path), new Path(uploaderConfiguration.getTmpDir(), filename));
FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true);
IoUtil.copy(inputStream, outputStream);
}
}
@Override
public void upload(String filename, String targetPath) throws Exception {
public void download(String filename, OutputStream outputStream) throws Exception {
try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.copyFromLocalFile(new Path(uploaderConfiguration.getTmpDir(), filename), new Path(targetPath, filename));
FSDataInputStream inputStream = fileSystem.open(new Path(uploaderConfiguration.getTmpDir(), filename));
IoUtil.copy(inputStream, outputStream);
}
}
}

37
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java Normal file
View File

@@ -0,0 +1,37 @@
package com.lanyuanxiaoyao.service.uploader.service.filesystem;
import cn.hutool.core.io.IoUtil;
import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
/**
* 处理上传事宜
*
* @author lanyuanxiaoyao
* @date 2024-01-24
*/
@Service("local")
public class LocalUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration;
public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
this.uploaderConfiguration = uploaderConfiguration;
}
@Override
public void upload(String filename, InputStream inputStream) throws Exception {
IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)));
}
@Override
public void download(String filename, OutputStream outputStream) throws Exception {
IoUtil.copy(Files.newInputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)), outputStream);
}
}

7
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java
View File

@@ -1,5 +1,8 @@
package com.lanyuanxiaoyao.service.uploader.service.filesystem;
import java.io.InputStream;
import java.io.OutputStream;
/**
* 上传和下载
*
@@ -7,7 +10,7 @@ package com.lanyuanxiaoyao.service.uploader.service.filesystem;
* @date 2024-01-24
*/
public interface UploadAndDownloadService {
void download(String path) throws Exception;
void upload(String filename, InputStream inputStream) throws Exception;
void upload(String filename, String targetPath) throws Exception;
void download(String filename, OutputStream outputStream) throws Exception;
}

2
service-uploader/src/main/resources/application.yml
View File

@@ -3,3 +3,5 @@ spring:
name: service-uploader
profiles:
include: random-port,common,discovery,metrics
uploader:
tmp-dir: