lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(uploader): 修复hdfs默认构造导致问题

Browse Source
This commit is contained in:
兰缘小妖
2024-03-01 18:06:32 +08:00
parent 4a0db374fe
commit c1d423bc32
4 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/library.sh
View File

@@ -3,7 +3,7 @@
encrypt_username=AxhEbscwsJDbYMH2 encrypt_username=AxhEbscwsJDbYMH2
encrypt_password=cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4 encrypt_password=cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
upload_url=http://$encrypt_username:$encrypt_password@132.126.207.34:36800 upload_url=http://$encrypt_username:$encrypt_password@132.126.207.124:36800
root_path=$(dirname $(cd $(dirname $0);pwd)) root_path=$(dirname $(cd $(dirname $0);pwd))

5
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java
View File

@@ -26,8 +26,9 @@ public class FileController {
private static final Logger logger = LoggerFactory.getLogger(FileController.class); private static final Logger logger = LoggerFactory.getLogger(FileController.class);
private final UploadAndDownloadService service; private final UploadAndDownloadService service;
public FileController(UploaderConfiguration uploaderConfiguration, ApplicationContext applicationContext) { @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
this.service = applicationContext.getBean(uploaderConfiguration.getBackend(), UploadAndDownloadService.class); public FileController(UploadAndDownloadService service) {
this.service = service;
} }
@RequestMapping("upload/{filename}") @RequestMapping("upload/{filename}")

10
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
View File

@@ -12,6 +12,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
/** /**
@@ -20,7 +21,8 @@ import org.springframework.stereotype.Service;
* @author lanyuanxiaoyao * @author lanyuanxiaoyao
* @date 2024-01-24 * @date 2024-01-24
*/ */
@Service("hdfs") @Service
@ConditionalOnProperty(value = "uploader.backend", havingValue = "hdfs")
public class HdfsUploadAndDownloadService implements UploadAndDownloadService { public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class); private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration; private final UploaderConfiguration uploaderConfiguration;
@@ -29,16 +31,12 @@ public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException { public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException {
this.uploaderConfiguration = uploaderConfiguration; this.uploaderConfiguration = uploaderConfiguration;
this.configuration = new Configuration(); this.configuration = new Configuration();
logger.info("Create tmp dir: {}", uploaderConfiguration.getTmpDir());
try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.mkdirs(new Path(uploaderConfiguration.getTmpDir()));
}
} }
@Override @Override
public void upload(String filename, InputStream inputStream) throws Exception { public void upload(String filename, InputStream inputStream) throws Exception {
try (FileSystem fileSystem = FileSystem.get(configuration)) { try (FileSystem fileSystem = FileSystem.get(configuration)) {
fileSystem.mkdirs(new Path(uploaderConfiguration.getTmpDir()));
FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true); FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true);
IoUtil.copy(inputStream, outputStream); IoUtil.copy(inputStream, outputStream);
} }

7
service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java
View File

@@ -9,6 +9,7 @@ import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
/** /**
@@ -17,19 +18,19 @@ import org.springframework.stereotype.Service;
* @author lanyuanxiaoyao * @author lanyuanxiaoyao
* @date 2024-01-24 * @date 2024-01-24
*/ */
@Service("local") @Service
@ConditionalOnProperty(value = "uploader.backend", havingValue = "local")
public class LocalUploadAndDownloadService implements UploadAndDownloadService { public class LocalUploadAndDownloadService implements UploadAndDownloadService {
private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class); private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class);
private final UploaderConfiguration uploaderConfiguration; private final UploaderConfiguration uploaderConfiguration;
public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException { public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) throws IOException {
this.uploaderConfiguration = uploaderConfiguration; this.uploaderConfiguration = uploaderConfiguration;
logger.info("Create tmp dir: {}", uploaderConfiguration.getTmpDir());
Files.createDirectories(Paths.get(uploaderConfiguration.getTmpDir()));
} }
@Override @Override
public void upload(String filename, InputStream inputStream) throws Exception { public void upload(String filename, InputStream inputStream) throws Exception {
Files.createDirectories(Paths.get(uploaderConfiguration.getTmpDir()));
IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename))); IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)));
} }