lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(uploader): 修复hdfs默认构造导致问题

Browse Source
This commit is contained in:
兰缘小妖
2024-03-01 18:06:32 +08:00
parent 4a0db374fe
commit c1d423bc32
4 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/library.sh
View File

@@ -3,7 +3,7 @@
encrypt_username=AxhEbscwsJDbYMH2
encrypt_password=cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
upload_url=http://$encrypt_username:$encrypt_password@132.126.207.34:36800
upload_url=http://$encrypt_username:$encrypt_password@132.126.207.124:36800
root_path=$(dirname $(cd $(dirname $0);pwd))

5
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java
View File

@@ -26,8 +26,9 @@ public class FileController {
private static final Logger logger = LoggerFactory.getLogger(FileController.class);
private final UploadAndDownloadService service;
public FileController(UploaderConfiguration uploaderConfiguration, ApplicationContext applicationContext) {
this.service = applicationContext.getBean(uploaderConfiguration.getBackend(), UploadAndDownloadService.class);
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
public FileController(UploadAndDownloadService service) {
this.service = service;
}
@RequestMapping("upload/{filename}")

10
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
View File

@@ -12,6 +12,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
/**
@@ -20,7 +21,8 @@ import org.springframework.stereotype.Service;
* @author lanyuanxiaoyao
* @date 2024-01-24
*/
@Service("hdfs")
@Service
@ConditionalOnProperty(value = "uploader.backend", havingValue = "hdfs")
public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration;
@@ -29,16 +31,12 @@ public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException {
this.uploaderConfiguration = uploaderConfiguration;
this.configuration = new Configuration();
logger.info("Create tmp dir: {}", uploaderConfiguration.getTmpDir());
try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.mkdirs(new Path(uploaderConfiguration.getTmpDir()));
}
}
@Override
public void upload(String filename, InputStream inputStream) throws Exception {
try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.mkdirs(new Path(uploaderConfiguration.getTmpDir()));
FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true);
IoUtil.copy(inputStream, outputStream);
}

7
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java
View File

@@ -9,6 +9,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
/**
@@ -17,19 +18,19 @@ import org.springframework.stereotype.Service;
* @author lanyuanxiaoyao
* @date 2024-01-24
*/
@Service("local")
@Service
@ConditionalOnProperty(value = "uploader.backend", havingValue = "local")
public class LocalUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration;
public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException {
this.uploaderConfiguration = uploaderConfiguration;
logger.info("Create tmp dir: {}", uploaderConfiguration.getTmpDir());
Files.createDirectories(Paths.get(uploaderConfiguration.getTmpDir()));
}
@Override
public void upload(String filename, InputStream inputStream) throws Exception {
Files.createDirectories(Paths.get(uploaderConfiguration.getTmpDir()));
IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)));
}