feat(uploader): 增加文件上传下载模块

bin/build-uploader.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
+mvn -pl service-uploader clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
+ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-uploader/target/service-uploader-1.0.0-SNAPSHOT.jar

bin/generate

@@ -5,8 +5,9 @@ jdk_path=/opt/jdk1.8.0_162/bin/java
 
 arguments=$@
 curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-cli-runner-1.0.0-SNAPSHOT.jar -o ${jars_path}/service-cli-runner.jar
-${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true $arguments
+${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true $arguments
 
 chmod +x cloud/*.sh
 chmod +x command/cli*
 chmod +x command/*.sh
+chmod +x uploader/*.sh

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java

@@ -77,6 +77,7 @@ public class DeployInformationProperties {
     public static final class Generate {
         private Boolean cloud = true;
         private Boolean command = true;
+        private Boolean uploader = true;
 
         public Boolean getCloud() {
             return cloud;
@@ -94,11 +95,20 @@ public class DeployInformationProperties {
             this.command = command;
         }
 
+        public Boolean getUploader() {
+            return uploader;
+        }
+
+        public void setUploader(Boolean uploader) {
+            this.uploader = uploader;
+        }
+
         @Override
         public String toString() {
             return "Generate{" +
                    "cloud=" + cloud +
                    ", command=" + command +
+                   ", uploader=" + uploader +
                    '}';
         }
     }

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java

@@ -95,6 +95,9 @@ public class RunnerApplication implements ApplicationRunner {
             generateCloud(Paths.get("cloud"));
         if (deployInformationProperties.getGenerate().getCommand())
             generateCommand(Paths.get("command"));
+        if (deployInformationProperties.getGenerate().getUploader()) {
+            generateUploader(Paths.get("uploader"));
+        }
     }
 
     private void generateCloud(Path root) throws IOException {
@@ -269,4 +272,34 @@ public class RunnerApplication implements ApplicationRunner {
         Files.deleteIfExists(updateScriptFile);
         Files.write(updateScriptFile, updateScript.getBytes());
     }
+
+    private void generateUploader(Path root) throws IOException {
+        String absolutRootPath = root.toAbsolutePath().toString();
+        logger.info("Current path: {}", absolutRootPath);
+
+        TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
+        Template startTemplate = engine.getTemplate("uploader/start.ftl");
+        String startScript = startTemplate.render(MapUtil.builder()
+                .put("currentPath", absolutRootPath)
+                .put("runtime", runtimeInfo)
+                .build());
+        Path startScriptFile = Paths.get(root.toString(), "start.sh");
+        Files.deleteIfExists(startScriptFile);
+        Files.write(startScriptFile, startScript.getBytes());
+
+        Template updateTemplate = engine.getTemplate("uploader/update.ftl");
+        String updateScript = updateTemplate.render(MapUtil.builder()
+                .put("currentPath", absolutRootPath)
+                .put("runtime", runtimeInfo)
+                .build());
+        Path updateScriptFile = Paths.get(root.toString(), "update.sh");
+        Files.deleteIfExists(updateScriptFile);
+        Files.write(updateScriptFile, updateScript.getBytes());
+
+        ClassPathResource stopFile = new ClassPathResource("script/stop.sh");
+        String stopScript = new String(IoUtil.readBytes(stopFile.getInputStream()));
+        // 一个魔法操作
+        stopScript = stopScript.replaceAll("\\$1", "service-uploader.jar");
+        Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes());
+    }
 }

service-cli/service-cli-runner/src/main/resources/template/uploader/start.ftl

@@ -0,0 +1,4 @@
+#!/bin/bash
+mkdir -p ${runtime.jarPath}
+export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
+${runtime.jdkPath} <#noparse>-Ddeploy.datetime=$(date +%Y%m%d%H%M%S) -Ddeploy.hostname=$(hostname) -Ddeploy.hostname-full=$(hostname -f) -Ddeploy.start-time=$(date +%Y%m%d%H%M%S)</#noparse> -Dlogging.parent=${runtime.logPath} -Dloki.url=${runtime.loki.servicePushUrl} -Dspring.cloud.zookeeper.connect-string=${runtime.zkUrl} -Duploader.backend=local -Duploader.tmp-dir=${runtime.dataPath}/uploader -jar ${runtime.jarPath}/service-uploader.jar

service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-uploader-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-uploader.jar

service-uploader/pom.xml

@@ -20,6 +20,14 @@
             <groupId>com.lanyuanxiaoyao</groupId>
             <artifactId>service-configuration</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
@@ -30,11 +38,87 @@
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-config-file</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${project.parent.basedir}/config/${build-tag}</directory>
+                                    <includes>
+                                        <include>*.xml</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
             <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <configuration>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+                    <transformers>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>META-INF/spring.handlers</resource>
+                        </transformer>
+                        <transformer
+                                implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
+                            <resource>META-INF/spring.factories</resource>
+                        </transformer>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>META-INF/spring.schemas</resource>
+                        </transformer>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <mainClass>com.lanyuanxiaoyao.service.uploader.UploaderApplication</mainClass>
+                        </transformer>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>reference.conf</resource>
+                        </transformer>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                    </transformers>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                                <exclude>core-default.xml</exclude>
+                                <exclude>hdfs-default.xml</exclude>
+                                <exclude>yarn-default.xml</exclude>
+                                <exclude>log4j-surefire*.properties</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
                         <groupId>org.springframework.boot</groupId>
                         <artifactId>spring-boot-maven-plugin</artifactId>
+                        <version>${spring-boot.version}</version>
+                    </dependency>
+                </dependencies>
             </plugin>
         </plugins>
     </build>

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java

@@ -1,21 +1,18 @@
 package com.lanyuanxiaoyao.service.uploader;
 
-import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
-import org.springframework.retry.annotation.EnableRetry;
 
 /**
  * @author lanyuanxiaoyao
  * @date 2024-01-24
  */
-@EnableDiscoveryClient
-@SpringBootApplication(scanBasePackages = {"com.lanyuanxiaoyao.service"})
+// @EnableDiscoveryClient
+@SpringBootApplication/* (scanBasePackages = {"com.lanyuanxiaoyao.service"}) */
 @EnableConfigurationProperties
-@EnableEncryptableProperties
-@EnableRetry
+// @EnableEncryptableProperties
+// @EnableRetry
 public class UploaderApplication {
     public static void main(String[] args) {
         SpringApplication.run(UploaderApplication.class, args);

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java

@@ -10,7 +10,16 @@ import org.springframework.context.annotation.Configuration;
 @Configuration
 @ConfigurationProperties("uploader")
 public class UploaderConfiguration {
-    private String tmpDir;
+    private String backend = "local";
+    private String tmpDir = "./";
+
+    public String getBackend() {
+        return backend;
+    }
+
+    public void setBackend(String backend) {
+        this.backend = backend;
+    }
 
     public String getTmpDir() {
         return tmpDir;

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java

@@ -0,0 +1,42 @@
+package com.lanyuanxiaoyao.service.uploader.controller;
+
+import cn.hutool.core.io.IoUtil;
+import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
+import com.lanyuanxiaoyao.service.uploader.service.filesystem.UploadAndDownloadService;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Enumeration;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2024-03-01
+ */
+@RestController
+@RequestMapping("file")
+public class FileController {
+    private static final Logger logger = LoggerFactory.getLogger(FileController.class);
+    private final UploadAndDownloadService service;
+
+    public FileController(UploaderConfiguration uploaderConfiguration, ApplicationContext applicationContext) {
+        this.service = applicationContext.getBean(uploaderConfiguration.getBackend(), UploadAndDownloadService.class);
+    }
+
+    @RequestMapping("upload/{filename}")
+    public void upload(@PathVariable("filename") String filename, HttpServletRequest request) throws Exception {
+        service.upload(filename, request.getInputStream());
+    }
+
+    @RequestMapping("download/{filename}")
+    public void download(@PathVariable("filename") String filename, HttpServletResponse response) throws Exception {
+        service.download(filename, response.getOutputStream());
+    }
+}

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java

@@ -1,8 +1,12 @@
 package com.lanyuanxiaoyao.service.uploader.service.filesystem;
 
-import cn.hutool.core.io.file.FileNameUtil;
+import cn.hutool.core.io.IoUtil;
 import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
+import java.io.InputStream;
+import java.io.OutputStream;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
@@ -15,7 +19,7 @@ import org.springframework.stereotype.Service;
  * @author lanyuanxiaoyao
  * @date 2024-01-24
  */
-@Service
+@Service("hdfs")
 public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
     private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
     private final UploaderConfiguration uploaderConfiguration;
@@ -23,21 +27,22 @@ public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
 
     public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
         this.uploaderConfiguration = uploaderConfiguration;
-        configuration = new Configuration();
+        this.configuration = new Configuration();
     }
 
     @Override
-    public void download(String path) throws Exception {
-        String filename = FileNameUtil.getName(path);
+    public void upload(String filename, InputStream inputStream) throws Exception {
         try (FileSystem fileSystem = FileSystem.get(configuration)) {
-            fileSystem.copyToLocalFile(new Path(path), new Path(uploaderConfiguration.getTmpDir(), filename));
+            FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true);
+            IoUtil.copy(inputStream, outputStream);
         }
     }
 
     @Override
-    public void upload(String filename, String targetPath) throws Exception {
+    public void download(String filename, OutputStream outputStream) throws Exception {
         try (FileSystem fileSystem = FileSystem.get(configuration)) {
-            fileSystem.copyFromLocalFile(new Path(uploaderConfiguration.getTmpDir(), filename), new Path(targetPath, filename));
+            FSDataInputStream inputStream = fileSystem.open(new Path(uploaderConfiguration.getTmpDir(), filename));
+            IoUtil.copy(inputStream, outputStream);
         }
     }
 }

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java

@@ -0,0 +1,37 @@
+package com.lanyuanxiaoyao.service.uploader.service.filesystem;
+
+import cn.hutool.core.io.IoUtil;
+import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * 处理上传事宜
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+@Service("local")
+public class LocalUploadAndDownloadService implements UploadAndDownloadService {
+    private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class);
+    private final UploaderConfiguration uploaderConfiguration;
+
+    public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
+        this.uploaderConfiguration = uploaderConfiguration;
+    }
+
+    @Override
+    public void upload(String filename, InputStream inputStream) throws Exception {
+        IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)));
+    }
+
+    @Override
+    public void download(String filename, OutputStream outputStream) throws Exception {
+        IoUtil.copy(Files.newInputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)), outputStream);
+    }
+}

service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java

@@ -1,5 +1,8 @@
 package com.lanyuanxiaoyao.service.uploader.service.filesystem;
 
+import java.io.InputStream;
+import java.io.OutputStream;
+
 /**
  * 上传和下载
  *
@@ -7,7 +10,7 @@ package com.lanyuanxiaoyao.service.uploader.service.filesystem;
  * @date 2024-01-24
  */
 public interface UploadAndDownloadService {
-    void download(String path) throws Exception;
+    void upload(String filename, InputStream inputStream) throws Exception;
 
-    void upload(String filename, String targetPath) throws Exception;
+    void download(String filename, OutputStream outputStream) throws Exception;
 }

service-uploader/src/main/resources/application.yml

@@ -3,3 +3,5 @@ spring:
     name: service-uploader
   profiles:
     include: random-port,common,discovery,metrics
+uploader:
+  tmp-dir: