From d68f8a27ee6c66ca58649d88fc3957ba158601d7 Mon Sep 17 00:00:00 2001 From: lanyuanxiaoyao Date: Thu, 21 Nov 2024 12:25:19 +0800 Subject: [PATCH] =?UTF-8?q?feat(web):=20=E5=A2=9E=E5=8A=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=8A=E4=BC=A0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gringotts-frontend/components/constants.js | 8 + .../components/resource/dialog-resource.js | 10 +- gringotts-frontend/pages/index/tab-data.js | 16 ++ .../configuration/UploadConfiguration.java | 18 ++ .../web/domain/entity/DatetimeOnlyEntity.java | 29 ++++ .../upload/controller/UploadController.java | 162 ++++++++++++++++++ .../web/domain/upload/entity/DataFile.java | 26 +++ .../upload/repository/DataFileRepository.java | 13 ++ .../domain/upload/service/UploadService.java | 53 ++++++ .../src/main/resources/application.yml | 9 +- 10 files changed, 338 insertions(+), 6 deletions(-) create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/configuration/UploadConfiguration.java create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/DatetimeOnlyEntity.java create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/controller/UploadController.java create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/entity/DataFile.java create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/repository/DataFileRepository.java create mode 100644 gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/service/UploadService.java diff --git a/gringotts-frontend/components/constants.js b/gringotts-frontend/components/constants.js index 535e90b..0b49e51 100644 --- a/gringotts-frontend/components/constants.js +++ b/gringotts-frontend/components/constants.js @@ -83,6 +83,14 @@ export function paginationTemplate(perPage = 20, maxButtons = 5) { } } +export function inputFileFormItemCommonOptions(accept = '*', maxSize = 5242880) { + return { + accept: accept, + maxSize: maxSize, + autoUpload: false, + } +} + export function copyField(field, tips = '复制', ignoreLength = 0, extra = undefined) { let tpl = ignoreLength === 0 ? `\${${field}}` : `\${TRUNCATE(${field}, ${ignoreLength})}` let content = extra diff --git a/gringotts-frontend/components/resource/dialog-resource.js b/gringotts-frontend/components/resource/dialog-resource.js index 0cb660b..020ae55 100644 --- a/gringotts-frontend/components/resource/dialog-resource.js +++ b/gringotts-frontend/components/resource/dialog-resource.js @@ -1,5 +1,5 @@ import './dialog-resource.css' -import {apiPost, horizontalFormOptions} from "../constants.js"; +import {apiPost, horizontalFormOptions, inputFileFormItemCommonOptions} from "../constants.js"; const clearable = { clearable: true, @@ -99,8 +99,8 @@ export function resourceAddDialog() { { type: 'input-file', name: 'filePath', - accept: '.zip', description: '只适合小于2M的资源文件使用，大文件请使用其他资源类型', + ...inputFileFormItemCommonOptions('.zip', 2097152), }, ] }, @@ -148,13 +148,13 @@ export function resourceAddDialog() { type: 'input-file', description: 'core-site.xml', name: 'coreSiteFile', - accept: '.xml', + ...inputFileFormItemCommonOptions('.xml', 1048576), }, { type: 'input-file', description: 'hdfs-site.xml', name: 'hdfsSiteFile', - accept: '.xml', + ...inputFileFormItemCommonOptions('.xml', 1048576), }, ] }, @@ -257,7 +257,7 @@ export function resourceAddDialog() { label: '资源示例', description: '可以上传用于作为格式示范的样例数据', name: 'example', - accept: '*', + ...inputFileFormItemCommonOptions(), }, ] } diff --git a/gringotts-frontend/pages/index/tab-data.js b/gringotts-frontend/pages/index/tab-data.js index 9f9fd1f..fc2c3a3 100644 --- a/gringotts-frontend/pages/index/tab-data.js +++ b/gringotts-frontend/pages/index/tab-data.js @@ -1,4 +1,5 @@ import {resourceAddDialog} from "../../components/resource/dialog-resource.js"; +import {apiPost, inputFileFormItemCommonOptions} from "../../components/constants.js"; export function tabData() { return { @@ -10,6 +11,21 @@ export function tabData() { label: '', icon: 'fa fa-plus', ...resourceAddDialog() + }, + { + type: 'form', + body: [ + { + type: 'input-file', + name: 'file', + label: '测试文件上传', + ...inputFileFormItemCommonOptions(undefined, 1073741824), + useChunk: true, + startChunkApi: apiPost('${base}/upload/start'), + chunkApi: apiPost('${base}/upload/slice'), + finishChunkApi: apiPost('${base}/upload/finish') + } + ] } ] } diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/configuration/UploadConfiguration.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/configuration/UploadConfiguration.java new file mode 100644 index 0000000..8656036 --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/configuration/UploadConfiguration.java @@ -0,0 +1,18 @@ +package com.eshore.gringotts.web.configuration; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * 上传文件相关配置 + * + * @author lanyuanxiaoyao + * @date 2024-11-21 + */ +@Data +@ConfigurationProperties(prefix = "gringotts.upload") +@Configuration +public class UploadConfiguration { + private String uploadPath = "./upload"; +} diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/DatetimeOnlyEntity.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/DatetimeOnlyEntity.java new file mode 100644 index 0000000..c6be58a --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/DatetimeOnlyEntity.java @@ -0,0 +1,29 @@ +package com.eshore.gringotts.web.domain.entity; + +import java.time.LocalDateTime; +import javax.persistence.EntityListeners; +import javax.persistence.MappedSuperclass; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +/** + * 实体类公共字段 + * + * @author lanyuanxiaoyao + * @date 2024-11-20 + */ +@Getter +@Setter +@ToString +@MappedSuperclass +@EntityListeners(AuditingEntityListener.class) +public class DatetimeOnlyEntity extends IdOnlyEntity { + @CreatedDate + private LocalDateTime createdTime; + @LastModifiedDate + private LocalDateTime modifiedTime; +} diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/controller/UploadController.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/controller/UploadController.java new file mode 100644 index 0000000..df12dbd --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/controller/UploadController.java @@ -0,0 +1,162 @@ +package com.eshore.gringotts.web.domain.upload.controller; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.SecureUtil; +import com.eshore.gringotts.web.configuration.UploadConfiguration; +import com.eshore.gringotts.web.configuration.amis.AmisResponse; +import com.eshore.gringotts.web.domain.upload.service.UploadService; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.eclipse.collections.api.list.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +/** + * 文件上传接口 + * + * @author lanyuanxiaoyao + * @date 2024-11-21 + */ +@RestController +@RequestMapping("/upload") +public class UploadController { + private static final Logger logger = LoggerFactory.getLogger(UploadController.class); + + private final UploadService uploadService; + private final String uploadFolderPath; + private final String cacheFolderPath; + private final String sliceFolderPath; + + public UploadController(UploadConfiguration uploadConfiguration, UploadService uploadService) { + this.uploadService = uploadService; + + this.uploadFolderPath = uploadConfiguration.getUploadPath(); + this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath); + this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath); + } + + @PostMapping("/start") + public AmisResponse start(@RequestBody StartRequest request) { + logger.info("Request: {}", request); + Long id = uploadService.initialDataFile(request.filename); + return AmisResponse.responseSuccess(new StartResponse(id.toString())); + } + + @PostMapping("/slice") + public AmisResponse slice( + @RequestParam("uploadId") + Long uploadId, + @RequestParam("partNumber") + Integer sequence, + @RequestParam("partSize") + Long size, + @RequestParam("file") + MultipartFile file + ) throws IOException { + logger.info("UploadId: {}, sequence: {}, size: {}, file: {}", uploadId, sequence, size, file.getName()); + byte[] bytes = file.getBytes(); + String md5 = SecureUtil.md5(new ByteArrayInputStream(bytes)); + String targetFilename = StrUtil.format("{}-{}", sequence, md5); + String targetFilePath = sliceFilePath(uploadId, targetFilename); + FileUtil.mkParentDirs(targetFilePath); + FileUtil.writeBytes(bytes, targetFilePath); + return AmisResponse.responseSuccess(new SliceResponse(targetFilename)); + } + + private String sliceFilePath(Long uploadId, String sliceFilename) { + return StrUtil.format("{}/{}/{}", sliceFolderPath, uploadId, sliceFilename); + } + + @PostMapping("finish") + public AmisResponse finish(@RequestBody FinishRequest request) { + logger.info("Request: {}", request); + if (request.partList.anySatisfy(part -> !FileUtil.exist(sliceFilePath(request.uploadId, part.eTag)))) { + throw new RuntimeException("文件校验失败，请重新上传"); + } + try { + File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId)); + FileUtil.mkParentDirs(cacheFile); + if (cacheFile.createNewFile()) { + try (FileOutputStream fos = new FileOutputStream(cacheFile)) { + try (FileChannel fosChannel = fos.getChannel()) { + for (FinishRequest.Part part : request.partList) { + File sliceFile = new File(sliceFilePath(request.uploadId, part.eTag)); + try (FileInputStream fis = new FileInputStream(sliceFile)) { + try (FileChannel fisChannel = fis.getChannel()) { + fisChannel.transferTo(0, fisChannel.size(), fosChannel); + } + } + } + } + } + File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, request.uploadId)); + FileUtil.move(cacheFile, targetFile, true); + uploadService.updateDataFile(request.uploadId, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), SecureUtil.md5(targetFile)); + return AmisResponse.responseSuccess(new FinishResponse(request.uploadId.toString())); + } else { + throw new RuntimeException("合并文件失败"); + } + } catch (Throwable throwable) { + throw new RuntimeException(throwable); + } finally { + FileUtil.del(StrUtil.format("{}/{}", sliceFolderPath, request.uploadId)); + } + } + + @Data + public static final class StartRequest { + private String name; + private String filename; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static final class StartResponse { + private String uploadId; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static final class SliceResponse { + @JsonProperty("eTag") + private String eTag; + } + + @Data + public static final class FinishRequest { + private String filename; + private Long uploadId; + private ImmutableList partList; + + @Data + public static final class Part { + private Integer partNumber; + @JsonProperty("eTag") + private String eTag; + } + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static final class FinishResponse { + private String value; + } +} diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/entity/DataFile.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/entity/DataFile.java new file mode 100644 index 0000000..7ef02e7 --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/entity/DataFile.java @@ -0,0 +1,26 @@ +package com.eshore.gringotts.web.domain.upload.entity; + +import com.eshore.gringotts.web.domain.entity.SimpleEntity; +import javax.persistence.Entity; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.hibernate.annotations.DynamicUpdate; + +/** + * 上传文件 + * + * @author lanyuanxiaoyao + * @date 2024-11-21 + */ +@Getter +@Setter +@ToString +@Entity +@DynamicUpdate +public class DataFile extends SimpleEntity { + private String filename; + private Long size; + private String md5; + private String path; +} diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/repository/DataFileRepository.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/repository/DataFileRepository.java new file mode 100644 index 0000000..165f3a5 --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/repository/DataFileRepository.java @@ -0,0 +1,13 @@ +package com.eshore.gringotts.web.domain.upload.repository; + +import com.blinkfox.fenix.jpa.FenixJpaRepository; +import com.blinkfox.fenix.specification.FenixJpaSpecificationExecutor; +import com.eshore.gringotts.web.domain.upload.entity.DataFile; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +@Repository +public interface DataFileRepository extends FenixJpaRepository, FenixJpaSpecificationExecutor { +} \ No newline at end of file diff --git a/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/service/UploadService.java b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/service/UploadService.java new file mode 100644 index 0000000..9dc78bc --- /dev/null +++ b/gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/service/UploadService.java @@ -0,0 +1,53 @@ +package com.eshore.gringotts.web.domain.upload.service; + +import com.eshore.gringotts.web.domain.upload.entity.DataFile; +import com.eshore.gringotts.web.domain.upload.repository.DataFileRepository; +import com.eshore.gringotts.web.domain.user.entity.User; +import com.eshore.gringotts.web.domain.user.service.UserService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +/** + * 上传服务 + * + * @author lanyuanxiaoyao + * @date 2024-11-21 + */ +@Service +public class UploadService { + private static final Logger logger = LoggerFactory.getLogger(UploadService.class); + + private final DataFileRepository dataFileRepository; + private final UserService userService; + + public UploadService(DataFileRepository dataFileRepository, UserService userService) { + this.dataFileRepository = dataFileRepository; + this.userService = userService; + } + + public Long initialDataFile(String filename) { + DataFile dataFile = new DataFile(); + dataFile.setFilename(filename); + User loginUser = userService.currentLoginUser(); + dataFile.setCreatedUser(loginUser); + dataFile.setModifiedUser(loginUser); + return dataFileRepository.save(dataFile).getId(); + } + + public void updateDataFile(Long id, String path, Long size, String md5) { + DataFile dataFile = dataFileRepository.findById(id).orElseThrow(UpdateDataFileFailedException::new); + dataFile.setSize(size); + dataFile.setMd5(md5); + dataFile.setPath(path); + User loginUser = userService.currentLoginUser(); + dataFile.setModifiedUser(loginUser); + dataFileRepository.save(dataFile); + } + + public static final class UpdateDataFileFailedException extends RuntimeException { + public UpdateDataFileFailedException() { + super("更新文件信息失败，请重新上传"); + } + } +} diff --git a/gringotts-web/src/main/resources/application.yml b/gringotts-web/src/main/resources/application.yml index 70c1596..1e6aef9 100644 --- a/gringotts-web/src/main/resources/application.yml +++ b/gringotts-web/src/main/resources/application.yml @@ -13,6 +13,10 @@ spring: jpa: generate-ddl: true show-sql: true + servlet: + multipart: + max-file-size: 10MB + max-request-size: 20MB fenix: print-banner: false print-sql: false @@ -27,4 +31,7 @@ logging: bcos: sdk: eventsub: - EventSubscribeImp: error \ No newline at end of file + EventSubscribeImp: error +gringotts: + upload: + upload-path: /Users/lanyuanxiaoyao/Project/IdeaProjects/gringotts/gringotts-web/target/upload \ No newline at end of file