diff --git a/bin/build-uploader.sh b/bin/build-uploader.sh new file mode 100755 index 0000000..442f57e --- /dev/null +++ b/bin/build-uploader.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml +mvn -pl service-uploader clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12 +ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-uploader/target/service-uploader-1.0.0-SNAPSHOT.jar \ No newline at end of file diff --git a/bin/generate b/bin/generate index 2eabd06..25a5e60 100644 --- a/bin/generate +++ b/bin/generate @@ -5,8 +5,9 @@ jdk_path=/opt/jdk1.8.0_162/bin/java arguments=$@ curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-cli-runner-1.0.0-SNAPSHOT.jar -o ${jars_path}/service-cli-runner.jar -${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true $arguments +${jdk_path} -jar ${jars_path}/service-cli-runner.jar --spring.profiles.active=b12 --deploy.generate.command=true --deploy.generate.uploader=true $arguments chmod +x cloud/*.sh chmod +x command/cli* chmod +x command/*.sh +chmod +x uploader/*.sh diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java index a71be2f..ab9d986 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/DeployInformationProperties.java @@ -77,6 +77,7 @@ public class DeployInformationProperties { public static final class Generate { private Boolean cloud = true; private Boolean command = true; + private Boolean uploader = true; public Boolean getCloud() { return cloud; @@ -94,11 +95,20 @@ public class DeployInformationProperties { this.command = command; } + public Boolean getUploader() { + return uploader; + } + + public void setUploader(Boolean uploader) { + this.uploader = uploader; + } + @Override public String toString() { return "Generate{" + "cloud=" + cloud + ", command=" + command + + ", uploader=" + uploader + '}'; } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 6ce96c6..6eba03c 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -95,6 +95,9 @@ public class RunnerApplication implements ApplicationRunner { generateCloud(Paths.get("cloud")); if (deployInformationProperties.getGenerate().getCommand()) generateCommand(Paths.get("command")); + if (deployInformationProperties.getGenerate().getUploader()) { + generateUploader(Paths.get("uploader")); + } } private void generateCloud(Path root) throws IOException { @@ -269,4 +272,34 @@ public class RunnerApplication implements ApplicationRunner { Files.deleteIfExists(updateScriptFile); Files.write(updateScriptFile, updateScript.getBytes()); } + + private void generateUploader(Path root) throws IOException { + String absolutRootPath = root.toAbsolutePath().toString(); + logger.info("Current path: {}", absolutRootPath); + + TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH)); + Template startTemplate = engine.getTemplate("uploader/start.ftl"); + String startScript = startTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .build()); + Path startScriptFile = Paths.get(root.toString(), "start.sh"); + Files.deleteIfExists(startScriptFile); + Files.write(startScriptFile, startScript.getBytes()); + + Template updateTemplate = engine.getTemplate("uploader/update.ftl"); + String updateScript = updateTemplate.render(MapUtil.builder() + .put("currentPath", absolutRootPath) + .put("runtime", runtimeInfo) + .build()); + Path updateScriptFile = Paths.get(root.toString(), "update.sh"); + Files.deleteIfExists(updateScriptFile); + Files.write(updateScriptFile, updateScript.getBytes()); + + ClassPathResource stopFile = new ClassPathResource("script/stop.sh"); + String stopScript = new String(IoUtil.readBytes(stopFile.getInputStream())); + // 一个魔法操作 + stopScript = stopScript.replaceAll("\\$1", "service-uploader.jar"); + Files.write(Paths.get(root.toString(), "stop.sh"), stopScript.getBytes()); + } } diff --git a/service-cli/service-cli-runner/src/main/resources/template/uploader/start.ftl b/service-cli/service-cli-runner/src/main/resources/template/uploader/start.ftl new file mode 100644 index 0000000..5982de6 --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/uploader/start.ftl @@ -0,0 +1,4 @@ +#!/bin/bash +mkdir -p ${runtime.jarPath} +export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.' +${runtime.jdkPath} <#noparse>-Ddeploy.datetime=$(date +%Y%m%d%H%M%S) -Ddeploy.hostname=$(hostname) -Ddeploy.hostname-full=$(hostname -f) -Ddeploy.start-time=$(date +%Y%m%d%H%M%S) -Dlogging.parent=${runtime.logPath} -Dloki.url=${runtime.loki.servicePushUrl} -Dspring.cloud.zookeeper.connect-string=${runtime.zkUrl} -Duploader.backend=local -Duploader.tmp-dir=${runtime.dataPath}/uploader -jar ${runtime.jarPath}/service-uploader.jar diff --git a/service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl b/service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl new file mode 100644 index 0000000..6a9f18e --- /dev/null +++ b/service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl @@ -0,0 +1,3 @@ +#!/bin/bash + +curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-uploader-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-uploader.jar \ No newline at end of file diff --git a/service-uploader/pom.xml b/service-uploader/pom.xml index 44d15f2..1222721 100644 --- a/service-uploader/pom.xml +++ b/service-uploader/pom.xml @@ -20,6 +20,14 @@ com.lanyuanxiaoyao service-configuration + + org.springframework.boot + spring-boot-starter-web + + + cn.hutool + hutool-all + org.apache.hadoop hadoop-client @@ -30,11 +38,87 @@ org.apache.maven.plugins - maven-source-plugin + maven-resources-plugin + + + copy-config-file + validate + + copy-resources + + + ${project.build.directory}/classes + + + ${project.parent.basedir}/config/${build-tag} + + *.xml + + + + + + - org.springframework.boot - spring-boot-maven-plugin + org.apache.maven.plugins + maven-shade-plugin + + false + true + + + META-INF/spring.handlers + + + META-INF/spring.factories + + + META-INF/spring.schemas + + + com.lanyuanxiaoyao.service.uploader.UploaderApplication + + + reference.conf + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + core-default.xml + hdfs-default.xml + yarn-default.xml + log4j-surefire*.properties + + + + + + + package + + shade + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java index e0daf45..41d4778 100644 --- a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java @@ -1,21 +1,18 @@ package com.lanyuanxiaoyao.service.uploader; -import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; -import org.springframework.retry.annotation.EnableRetry; /** * @author lanyuanxiaoyao * @date 2024-01-24 */ -@EnableDiscoveryClient -@SpringBootApplication(scanBasePackages = {"com.lanyuanxiaoyao.service"}) +// @EnableDiscoveryClient +@SpringBootApplication/* (scanBasePackages = {"com.lanyuanxiaoyao.service"}) */ @EnableConfigurationProperties -@EnableEncryptableProperties -@EnableRetry +// @EnableEncryptableProperties +// @EnableRetry public class UploaderApplication { public static void main(String[] args) { SpringApplication.run(UploaderApplication.class, args); diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java index 7131b6d..15792ae 100644 --- a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java @@ -10,7 +10,16 @@ import org.springframework.context.annotation.Configuration; @Configuration @ConfigurationProperties("uploader") public class UploaderConfiguration { - private String tmpDir; + private String backend = "local"; + private String tmpDir = "./"; + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } public String getTmpDir() { return tmpDir; diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java new file mode 100644 index 0000000..73f5652 --- /dev/null +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/controller/FileController.java @@ -0,0 +1,42 @@ +package com.lanyuanxiaoyao.service.uploader.controller; + +import cn.hutool.core.io.IoUtil; +import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration; +import com.lanyuanxiaoyao.service.uploader.service.filesystem.UploadAndDownloadService; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Enumeration; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @author lanyuanxiaoyao + * @date 2024-03-01 + */ +@RestController +@RequestMapping("file") +public class FileController { + private static final Logger logger = LoggerFactory.getLogger(FileController.class); + private final UploadAndDownloadService service; + + public FileController(UploaderConfiguration uploaderConfiguration, ApplicationContext applicationContext) { + this.service = applicationContext.getBean(uploaderConfiguration.getBackend(), UploadAndDownloadService.class); + } + + @RequestMapping("upload/{filename}") + public void upload(@PathVariable("filename") String filename, HttpServletRequest request) throws Exception { + service.upload(filename, request.getInputStream()); + } + + @RequestMapping("download/{filename}") + public void download(@PathVariable("filename") String filename, HttpServletResponse response) throws Exception { + service.download(filename, response.getOutputStream()); + } +} diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java index cd333f2..81338ed 100644 --- a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java @@ -1,8 +1,12 @@ package com.lanyuanxiaoyao.service.uploader.service.filesystem; -import cn.hutool.core.io.file.FileNameUtil; +import cn.hutool.core.io.IoUtil; import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration; +import java.io.InputStream; +import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -15,7 +19,7 @@ import org.springframework.stereotype.Service; * @author lanyuanxiaoyao * @date 2024-01-24 */ -@Service +@Service("hdfs") public class HdfsUploadAndDownloadService implements UploadAndDownloadService { private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class); private final UploaderConfiguration uploaderConfiguration; @@ -23,21 +27,22 @@ public class HdfsUploadAndDownloadService implements UploadAndDownloadService { public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) { this.uploaderConfiguration = uploaderConfiguration; - configuration = new Configuration(); + this.configuration = new Configuration(); } @Override - public void download(String path) throws Exception { - String filename = FileNameUtil.getName(path); + public void upload(String filename, InputStream inputStream) throws Exception { try (FileSystem fileSystem = FileSystem.get(configuration)) { - fileSystem.copyToLocalFile(new Path(path), new Path(uploaderConfiguration.getTmpDir(), filename)); + FSDataOutputStream outputStream = fileSystem.create(new Path(uploaderConfiguration.getTmpDir(), filename), true); + IoUtil.copy(inputStream, outputStream); } } @Override - public void upload(String filename, String targetPath) throws Exception { + public void download(String filename, OutputStream outputStream) throws Exception { try (FileSystem fileSystem = FileSystem.get(configuration)) { - fileSystem.copyFromLocalFile(new Path(uploaderConfiguration.getTmpDir(), filename), new Path(targetPath, filename)); + FSDataInputStream inputStream = fileSystem.open(new Path(uploaderConfiguration.getTmpDir(), filename)); + IoUtil.copy(inputStream, outputStream); } } } diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java new file mode 100644 index 0000000..fb53627 --- /dev/null +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/LocalUploadAndDownloadService.java @@ -0,0 +1,37 @@ +package com.lanyuanxiaoyao.service.uploader.service.filesystem; + +import cn.hutool.core.io.IoUtil; +import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +/** + * 处理上传事宜 + * + * @author lanyuanxiaoyao + * @date 2024-01-24 + */ +@Service("local") +public class LocalUploadAndDownloadService implements UploadAndDownloadService { + private static final Logger logger = LoggerFactory.getLogger(LocalUploadAndDownloadService.class); + private final UploaderConfiguration uploaderConfiguration; + + public LocalUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) { + this.uploaderConfiguration = uploaderConfiguration; + } + + @Override + public void upload(String filename, InputStream inputStream) throws Exception { + IoUtil.copy(inputStream, Files.newOutputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename))); + } + + @Override + public void download(String filename, OutputStream outputStream) throws Exception { + IoUtil.copy(Files.newInputStream(Paths.get(uploaderConfiguration.getTmpDir(), filename)), outputStream); + } +} diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java index aad6936..59a63d8 100644 --- a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java +++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java @@ -1,5 +1,8 @@ package com.lanyuanxiaoyao.service.uploader.service.filesystem; +import java.io.InputStream; +import java.io.OutputStream; + /** * 上传和下载 * @@ -7,7 +10,7 @@ package com.lanyuanxiaoyao.service.uploader.service.filesystem; * @date 2024-01-24 */ public interface UploadAndDownloadService { - void download(String path) throws Exception; + void upload(String filename, InputStream inputStream) throws Exception; - void upload(String filename, String targetPath) throws Exception; + void download(String filename, OutputStream outputStream) throws Exception; } diff --git a/service-uploader/src/main/resources/application.yml b/service-uploader/src/main/resources/application.yml index 1c1ac8f..3af3d69 100644 --- a/service-uploader/src/main/resources/application.yml +++ b/service-uploader/src/main/resources/application.yml @@ -3,3 +3,5 @@ spring: name: service-uploader profiles: include: random-port,common,discovery,metrics +uploader: + tmp-dir: