From a7203977d5a6dac9f5272ca1240e9eaa40cd09ff Mon Sep 17 00:00:00 2001
From: lanyuanxiaoyao <lanyuanxiaoyao@gmail.com>
Date: Wed, 24 Jan 2024 10:12:26 +0800
Subject: [PATCH] =?UTF-8?q?feat(uploader):=20=E5=A2=9E=E5=8A=A0=E4=B8=80?=
 =?UTF-8?q?=E4=B8=AA=E5=85=B3=E4=BA=8E=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0?=
 =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E7=9A=84=E9=A1=B9=E7=9B=AE=EF=BC=8C=E5=90=8E?=
 =?UTF-8?q?=E7=BB=AD=E6=96=B9=E4=BE=BFjar=E5=8C=85=E4=BC=A0=E9=80=92?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pom.xml                                       |  1 +
 service-uploader/pom.xml                      | 49 ++++++++++++++++++
 .../service/uploader/UploaderApplication.java | 23 +++++++++
 .../configuration/UploaderConfiguration.java  | 29 +++++++++++
 .../HdfsUploadAndDownloadService.java         | 43 ++++++++++++++++
 .../filesystem/UploadAndDownloadService.java  | 15 ++++++
 .../src/main/resources/application.yml        |  5 ++
 .../src/main/resources/logback-spring.xml     | 51 +++++++++++++++++++
 .../service/uploader/PathTest.java            | 15 ++++++
 9 files changed, 231 insertions(+)
 create mode 100644 service-uploader/pom.xml
 create mode 100644 service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java
 create mode 100644 service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java
 create mode 100644 service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
 create mode 100644 service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java
 create mode 100644 service-uploader/src/main/resources/application.yml
 create mode 100644 service-uploader/src/main/resources/logback-spring.xml
 create mode 100644 service-uploader/src/test/java/com/lanyuanxiaoyao/service/uploader/PathTest.java

diff --git a/pom.xml b/pom.xml
index 6682932..9362862 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,6 +24,7 @@
         <module>service-cli</module>
         <module>service-loki-query</module>
         <module>service-test-query</module>
+        <module>service-uploader</module>
         <module>service-executor</module>
         <module>service-cloud-query</module>
     </modules>
diff --git a/service-uploader/pom.xml b/service-uploader/pom.xml
new file mode 100644
index 0000000..0e55bdf
--- /dev/null
+++ b/service-uploader/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.lanyuanxiaoyao</groupId>
+        <artifactId>hudi-service</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>service-uploader</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.lanyuanxiaoyao</groupId>
+            <artifactId>service-dependencies</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.lanyuanxiaoyao</groupId>
+            <artifactId>service-configuration</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>3.1.2</version>
+        </dependency>
+        <!-- <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>3.1.2</version>
+        </dependency> -->
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file
diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java
new file mode 100644
index 0000000..e0daf45
--- /dev/null
+++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/UploaderApplication.java
@@ -0,0 +1,23 @@
+package com.lanyuanxiaoyao.service.uploader;
+
+import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
+import org.springframework.retry.annotation.EnableRetry;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+@EnableDiscoveryClient
+@SpringBootApplication(scanBasePackages = {"com.lanyuanxiaoyao.service"})
+@EnableConfigurationProperties
+@EnableEncryptableProperties
+@EnableRetry
+public class UploaderApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(UploaderApplication.class, args);
+    }
+}
diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java
new file mode 100644
index 0000000..7131b6d
--- /dev/null
+++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/configuration/UploaderConfiguration.java
@@ -0,0 +1,29 @@
+package com.lanyuanxiaoyao.service.uploader.configuration;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+@Configuration
+@ConfigurationProperties("uploader")
+public class UploaderConfiguration {
+    private String tmpDir;
+
+    public String getTmpDir() {
+        return tmpDir;
+    }
+
+    public void setTmpDir(String tmpDir) {
+        this.tmpDir = tmpDir;
+    }
+
+    @Override
+    public String toString() {
+        return "UploaderConfiguration{" +
+               "tmpDir='" + tmpDir + '\'' +
+               '}';
+    }
+}
diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
new file mode 100644
index 0000000..cd333f2
--- /dev/null
+++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/HdfsUploadAndDownloadService.java
@@ -0,0 +1,43 @@
+package com.lanyuanxiaoyao.service.uploader.service.filesystem;
+
+import cn.hutool.core.io.file.FileNameUtil;
+import com.lanyuanxiaoyao.service.uploader.configuration.UploaderConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * 处理上传事宜
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+@Service
+public class HdfsUploadAndDownloadService implements UploadAndDownloadService {
+    private static final Logger logger = LoggerFactory.getLogger(HdfsUploadAndDownloadService.class);
+    private final UploaderConfiguration uploaderConfiguration;
+    private final Configuration configuration;
+
+    public HdfsUploadAndDownloadService(UploaderConfiguration uploaderConfiguration) {
+        this.uploaderConfiguration = uploaderConfiguration;
+        configuration = new Configuration();
+    }
+
+    @Override
+    public void download(String path) throws Exception {
+        String filename = FileNameUtil.getName(path);
+        try (FileSystem fileSystem = FileSystem.get(configuration)) {
+            fileSystem.copyToLocalFile(new Path(path), new Path(uploaderConfiguration.getTmpDir(), filename));
+        }
+    }
+
+    @Override
+    public void upload(String filename, String targetPath) throws Exception {
+        try (FileSystem fileSystem = FileSystem.get(configuration)) {
+            fileSystem.copyFromLocalFile(new Path(uploaderConfiguration.getTmpDir(), filename), new Path(targetPath, filename));
+        }
+    }
+}
diff --git a/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java
new file mode 100644
index 0000000..8685c10
--- /dev/null
+++ b/service-uploader/src/main/java/com/lanyuanxiaoyao/service/uploader/service/filesystem/UploadAndDownloadService.java
@@ -0,0 +1,15 @@
+package com.lanyuanxiaoyao.service.uploader.service.filesystem;
+
+import java.io.IOException;
+
+/**
+ * 上传和下载
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+public interface UploadAndDownloadService {
+    void download(String path) throws Exception;
+
+    void upload(String filename, String targetPath) throws Exception;
+}
diff --git a/service-uploader/src/main/resources/application.yml b/service-uploader/src/main/resources/application.yml
new file mode 100644
index 0000000..1c1ac8f
--- /dev/null
+++ b/service-uploader/src/main/resources/application.yml
@@ -0,0 +1,5 @@
+spring:
+  application:
+    name: service-uploader
+  profiles:
+    include: random-port,common,discovery,metrics
diff --git a/service-uploader/src/main/resources/logback-spring.xml b/service-uploader/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000..f955921
--- /dev/null
+++ b/service-uploader/src/main/resources/logback-spring.xml
@@ -0,0 +1,51 @@
+<configuration>
+    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
+    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
+    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
+
+    <springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
+    <springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
+    <springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
+
+    <appender name="Loki" class="com.github.loki4j.logback.Loki4jAppender">
+        <metricsEnabled>true</metricsEnabled>
+        <http class="com.github.loki4j.logback.ApacheHttpSender">
+            <url>${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push}</url>
+        </http>
+        <format>
+            <label>
+                <pattern>app=${APP_NAME:- },host=${HOSTNAME},level=%level</pattern>
+            </label>
+            <message>
+                <pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } -&#45;&#45; [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
+            </message>
+            <sortByTime>true</sortByTime>
+        </format>
+    </appender>
+
+    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
+            <MaxHistory>7</MaxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } -&#45;&#45; [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="com.zaxxer.hikari" level="ERROR"/>
+    <logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
+
+    <root level="INFO">
+        <appender-ref ref="Loki"/>
+        <appender-ref ref="Console"/>
+        <appender-ref ref="RollingFile"/>
+    </root>
+</configuration>
\ No newline at end of file
diff --git a/service-uploader/src/test/java/com/lanyuanxiaoyao/service/uploader/PathTest.java b/service-uploader/src/test/java/com/lanyuanxiaoyao/service/uploader/PathTest.java
new file mode 100644
index 0000000..aa5f6ec
--- /dev/null
+++ b/service-uploader/src/test/java/com/lanyuanxiaoyao/service/uploader/PathTest.java
@@ -0,0 +1,15 @@
+package com.lanyuanxiaoyao.service.uploader;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.io.file.FileNameUtil;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2024-01-24
+ */
+public class PathTest {
+    public static void main(String[] args) {
+        System.out.println(new Path("/apps/", "hello.txt"));
+    }
+}