feat(web): 增加文件上传接口

gringotts-frontend/components/constants.js

@@ -83,6 +83,14 @@ export function paginationTemplate(perPage = 20, maxButtons = 5) {
     }
 }
 
+export function inputFileFormItemCommonOptions(accept = '*', maxSize = 5242880) {
+    return {
+        accept: accept,
+        maxSize: maxSize,
+        autoUpload: false,
+    }
+}
+
 export function copyField(field, tips = '复制', ignoreLength = 0, extra = undefined) {
     let tpl = ignoreLength === 0 ? `\${${field}}` : `\${TRUNCATE(${field}, ${ignoreLength})}`
     let content = extra

gringotts-frontend/components/resource/dialog-resource.js

@@ -1,5 +1,5 @@
 import './dialog-resource.css'
-import {apiPost, horizontalFormOptions} from "../constants.js";
+import {apiPost, horizontalFormOptions, inputFileFormItemCommonOptions} from "../constants.js";
 
 const clearable = {
     clearable: true,
@@ -99,8 +99,8 @@ export function resourceAddDialog() {
                                     {
                                         type: 'input-file',
                                         name: 'filePath',
-                                        accept: '.zip',
                                         description: '只适合小于2M的资源文件使用，大文件请使用其他资源类型',
+                                        ...inputFileFormItemCommonOptions('.zip', 2097152),
                                     },
                                 ]
                             },
@@ -148,13 +148,13 @@ export function resourceAddDialog() {
                                         type: 'input-file',
                                         description: 'core-site.xml',
                                         name: 'coreSiteFile',
-                                        accept: '.xml',
+                                        ...inputFileFormItemCommonOptions('.xml', 1048576),
                                     },
                                     {
                                         type: 'input-file',
                                         description: 'hdfs-site.xml',
                                         name: 'hdfsSiteFile',
-                                        accept: '.xml',
+                                        ...inputFileFormItemCommonOptions('.xml', 1048576),
                                     },
                                 ]
                             },
@@ -257,7 +257,7 @@ export function resourceAddDialog() {
                                 label: '资源示例',
                                 description: '可以上传用于作为格式示范的样例数据',
                                 name: 'example',
-                                accept: '*',
+                                ...inputFileFormItemCommonOptions(),
                             },
                         ]
                     }

gringotts-frontend/pages/index/tab-data.js

@@ -1,4 +1,5 @@
 import {resourceAddDialog} from "../../components/resource/dialog-resource.js";
+import {apiPost, inputFileFormItemCommonOptions} from "../../components/constants.js";
 
 export function tabData() {
     return {
@@ -10,6 +11,21 @@ export function tabData() {
                 label: '',
                 icon: 'fa fa-plus',
                 ...resourceAddDialog()
+            },
+            {
+                type: 'form',
+                body: [
+                    {
+                        type: 'input-file',
+                        name: 'file',
+                        label: '测试文件上传',
+                        ...inputFileFormItemCommonOptions(undefined, 1073741824),
+                        useChunk: true,
+                        startChunkApi: apiPost('${base}/upload/start'),
+                        chunkApi: apiPost('${base}/upload/slice'),
+                        finishChunkApi: apiPost('${base}/upload/finish')
+                    }
+                ]
             }
         ]
     }

gringotts-web/src/main/java/com/eshore/gringotts/web/configuration/UploadConfiguration.java

@@ -0,0 +1,18 @@
+package com.eshore.gringotts.web.configuration;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * 上传文件相关配置
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-11-21
+ */
+@Data
+@ConfigurationProperties(prefix = "gringotts.upload")
+@Configuration
+public class UploadConfiguration {
+    private String uploadPath = "./upload";
+}

gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/DatetimeOnlyEntity.java

@@ -0,0 +1,29 @@
+package com.eshore.gringotts.web.domain.entity;
+
+import java.time.LocalDateTime;
+import javax.persistence.EntityListeners;
+import javax.persistence.MappedSuperclass;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+/**
+ * 实体类公共字段
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-11-20
+ */
+@Getter
+@Setter
+@ToString
+@MappedSuperclass
+@EntityListeners(AuditingEntityListener.class)
+public class DatetimeOnlyEntity extends IdOnlyEntity {
+    @CreatedDate
+    private LocalDateTime createdTime;
+    @LastModifiedDate
+    private LocalDateTime modifiedTime;
+}

gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/controller/UploadController.java

@@ -0,0 +1,162 @@
+package com.eshore.gringotts.web.domain.upload.controller;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.crypto.SecureUtil;
+import com.eshore.gringotts.web.configuration.UploadConfiguration;
+import com.eshore.gringotts.web.configuration.amis.AmisResponse;
+import com.eshore.gringotts.web.domain.upload.service.UploadService;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.eclipse.collections.api.list.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * 文件上传接口
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-11-21
+ */
+@RestController
+@RequestMapping("/upload")
+public class UploadController {
+    private static final Logger logger = LoggerFactory.getLogger(UploadController.class);
+
+    private final UploadService uploadService;
+    private final String uploadFolderPath;
+    private final String cacheFolderPath;
+    private final String sliceFolderPath;
+
+    public UploadController(UploadConfiguration uploadConfiguration, UploadService uploadService) {
+        this.uploadService = uploadService;
+
+        this.uploadFolderPath = uploadConfiguration.getUploadPath();
+        this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath);
+        this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath);
+    }
+
+    @PostMapping("/start")
+    public AmisResponse<StartResponse> start(@RequestBody StartRequest request) {
+        logger.info("Request: {}", request);
+        Long id = uploadService.initialDataFile(request.filename);
+        return AmisResponse.responseSuccess(new StartResponse(id.toString()));
+    }
+
+    @PostMapping("/slice")
+    public AmisResponse<SliceResponse> slice(
+            @RequestParam("uploadId")
+            Long uploadId,
+            @RequestParam("partNumber")
+            Integer sequence,
+            @RequestParam("partSize")
+            Long size,
+            @RequestParam("file")
+            MultipartFile file
+    ) throws IOException {
+        logger.info("UploadId: {}, sequence: {}, size: {}, file: {}", uploadId, sequence, size, file.getName());
+        byte[] bytes = file.getBytes();
+        String md5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
+        String targetFilename = StrUtil.format("{}-{}", sequence, md5);
+        String targetFilePath = sliceFilePath(uploadId, targetFilename);
+        FileUtil.mkParentDirs(targetFilePath);
+        FileUtil.writeBytes(bytes, targetFilePath);
+        return AmisResponse.responseSuccess(new SliceResponse(targetFilename));
+    }
+
+    private String sliceFilePath(Long uploadId, String sliceFilename) {
+        return StrUtil.format("{}/{}/{}", sliceFolderPath, uploadId, sliceFilename);
+    }
+
+    @PostMapping("finish")
+    public AmisResponse<FinishResponse> finish(@RequestBody FinishRequest request) {
+        logger.info("Request: {}", request);
+        if (request.partList.anySatisfy(part -> !FileUtil.exist(sliceFilePath(request.uploadId, part.eTag)))) {
+            throw new RuntimeException("文件校验失败，请重新上传");
+        }
+        try {
+            File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId));
+            FileUtil.mkParentDirs(cacheFile);
+            if (cacheFile.createNewFile()) {
+                try (FileOutputStream fos = new FileOutputStream(cacheFile)) {
+                    try (FileChannel fosChannel = fos.getChannel()) {
+                        for (FinishRequest.Part part : request.partList) {
+                            File sliceFile = new File(sliceFilePath(request.uploadId, part.eTag));
+                            try (FileInputStream fis = new FileInputStream(sliceFile)) {
+                                try (FileChannel fisChannel = fis.getChannel()) {
+                                    fisChannel.transferTo(0, fisChannel.size(), fosChannel);
+                                }
+                            }
+                        }
+                    }
+                }
+                File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, request.uploadId));
+                FileUtil.move(cacheFile, targetFile, true);
+                uploadService.updateDataFile(request.uploadId, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), SecureUtil.md5(targetFile));
+                return AmisResponse.responseSuccess(new FinishResponse(request.uploadId.toString()));
+            } else {
+                throw new RuntimeException("合并文件失败");
+            }
+        } catch (Throwable throwable) {
+            throw new RuntimeException(throwable);
+        } finally {
+            FileUtil.del(StrUtil.format("{}/{}", sliceFolderPath, request.uploadId));
+        }
+    }
+
+    @Data
+    public static final class StartRequest {
+        private String name;
+        private String filename;
+    }
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static final class StartResponse {
+        private String uploadId;
+    }
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static final class SliceResponse {
+        @JsonProperty("eTag")
+        private String eTag;
+    }
+
+    @Data
+    public static final class FinishRequest {
+        private String filename;
+        private Long uploadId;
+        private ImmutableList<Part> partList;
+
+        @Data
+        public static final class Part {
+            private Integer partNumber;
+            @JsonProperty("eTag")
+            private String eTag;
+        }
+    }
+
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    public static final class FinishResponse {
+        private String value;
+    }
+}

gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/entity/DataFile.java

@@ -0,0 +1,26 @@
+package com.eshore.gringotts.web.domain.upload.entity;
+
+import com.eshore.gringotts.web.domain.entity.SimpleEntity;
+import javax.persistence.Entity;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.hibernate.annotations.DynamicUpdate;
+
+/**
+ * 上传文件
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-11-21
+ */
+@Getter
+@Setter
+@ToString
+@Entity
+@DynamicUpdate
+public class DataFile extends SimpleEntity {
+    private String filename;
+    private Long size;
+    private String md5;
+    private String path;
+}

gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/repository/DataFileRepository.java

@@ -0,0 +1,13 @@
+package com.eshore.gringotts.web.domain.upload.repository;
+
+import com.blinkfox.fenix.jpa.FenixJpaRepository;
+import com.blinkfox.fenix.specification.FenixJpaSpecificationExecutor;
+import com.eshore.gringotts.web.domain.upload.entity.DataFile;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+@Repository
+public interface DataFileRepository extends FenixJpaRepository<DataFile, Long>, FenixJpaSpecificationExecutor<DataFile> {
+}

gringotts-web/src/main/java/com/eshore/gringotts/web/domain/upload/service/UploadService.java

@@ -0,0 +1,53 @@
+package com.eshore.gringotts.web.domain.upload.service;
+
+import com.eshore.gringotts.web.domain.upload.entity.DataFile;
+import com.eshore.gringotts.web.domain.upload.repository.DataFileRepository;
+import com.eshore.gringotts.web.domain.user.entity.User;
+import com.eshore.gringotts.web.domain.user.service.UserService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * 上传服务
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-11-21
+ */
+@Service
+public class UploadService {
+    private static final Logger logger = LoggerFactory.getLogger(UploadService.class);
+
+    private final DataFileRepository dataFileRepository;
+    private final UserService userService;
+
+    public UploadService(DataFileRepository dataFileRepository, UserService userService) {
+        this.dataFileRepository = dataFileRepository;
+        this.userService = userService;
+    }
+
+    public Long initialDataFile(String filename) {
+        DataFile dataFile = new DataFile();
+        dataFile.setFilename(filename);
+        User loginUser = userService.currentLoginUser();
+        dataFile.setCreatedUser(loginUser);
+        dataFile.setModifiedUser(loginUser);
+        return dataFileRepository.save(dataFile).getId();
+    }
+
+    public void updateDataFile(Long id, String path, Long size, String md5) {
+        DataFile dataFile = dataFileRepository.findById(id).orElseThrow(UpdateDataFileFailedException::new);
+        dataFile.setSize(size);
+        dataFile.setMd5(md5);
+        dataFile.setPath(path);
+        User loginUser = userService.currentLoginUser();
+        dataFile.setModifiedUser(loginUser);
+        dataFileRepository.save(dataFile);
+    }
+
+    public static final class UpdateDataFileFailedException extends RuntimeException {
+        public UpdateDataFileFailedException() {
+            super("更新文件信息失败，请重新上传");
+        }
+    }
+}

gringotts-web/src/main/resources/application.yml

@@ -13,6 +13,10 @@ spring:
   jpa:
     generate-ddl: true
     show-sql: true
+  servlet:
+    multipart:
+      max-file-size: 10MB
+      max-request-size: 20MB
 fenix:
   print-banner: false
   print-sql: false
@@ -28,3 +32,6 @@ logging:
           sdk:
             eventsub:
               EventSubscribeImp: error
+gringotts:
+  upload:
+    upload-path: /Users/lanyuanxiaoyao/Project/IdeaProjects/gringotts/gringotts-web/target/upload