feat(cli): 适配使用uploader上传jar包

2024-03-01 14:32:02 +08:00
parent 826f7bc780
commit 9a6eb1e0a4
26 changed files with 97 additions and 32 deletions
--- a/bin/build-api.sh
+++ b/bin/build-api.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-api clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-file_path=/Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-api/target/service-api-1.0.0-SNAPSHOT.jar
-ytp-transfer2 $file_path
-rm $file_path
+upload $root_path/service-api/target/service-api-1.0.0-SNAPSHOT.jar
--- a/bin/build-check.sh
+++ b/bin/build-check.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-check clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml -P b2b12
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-check/target/service-check-1.0.0-SNAPSHOT.jar
+upload $root_path/service-check/target/service-check-1.0.0-SNAPSHOT.jar
--- a/bin/build-cloud-query.sh
+++ b/bin/build-cloud-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-cloud-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-cloud-query/target/service-cloud-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-command.sh
+++ b/bin/build-command.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-command clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-command/target/service-command-1.0.0-SNAPSHOT.jar
+upload $root_path/service-command/target/service-command-1.0.0-SNAPSHOT.jar
--- a/bin/build-executor-manager.sh
+++ b/bin/build-executor-manager.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest,service-executor,service-executor/service-executor-core clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-executor/service-executor-manager clean package -D skipTests -s ~/.m2/settings-development.xml  -P b2b12
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar
+upload $root_path/service-executor/service-executor-manager/target/service-executor-manager-1.0.0-SNAPSHOT.jar
--- a/bin/build-executor-task.sh
+++ b/bin/build-executor-task.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest,service-executor,service-executor/service-executor-core clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-executor/service-executor-task clean package -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar
+upload $root_path/service-executor/service-executor-task/target/service-executor-task-1.0.0-SNAPSHOT.jar
--- a/bin/build-exporter.sh
+++ b/bin/build-exporter.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-exporter clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-exporter/target/service-exporter-1.0.0-SNAPSHOT.jar
+upload $root_path/service-exporter/target/service-exporter-1.0.0-SNAPSHOT.jar
--- a/bin/build-flink-query.sh
+++ b/bin/build-flink-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-flink-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-gateway.sh
+++ b/bin/build-gateway.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-gateway clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar
+upload $root_path/service-gateway/target/service-gateway-1.0.0-SNAPSHOT.jar
--- a/bin/build-hudi-query.sh
+++ b/bin/build-hudi-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-hudi-query clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-info-query.sh
+++ b/bin/build-info-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-info-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-launcher.sh
+++ b/bin/build-launcher.sh
@@ -1,22 +1,20 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml

 echo 'Build b2a4'
 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2a4
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar
-rm /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2a4-1.0.0-SNAPSHOT.jar

 echo 'Build b2b1'
 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b1
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar
-rm /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b1-1.0.0-SNAPSHOT.jar

 echo 'Build b2b5'
 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b5
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar
-rm /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b5-1.0.0-SNAPSHOT.jar

 echo 'Build b2b12'
 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar
-rm /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar
+upload $root_path/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar
--- a/bin/build-loki-query.sh
+++ b/bin/build-loki-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-loki-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-pulsar-query.sh
+++ b/bin/build-pulsar-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-pulsar-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-queue.sh
+++ b/bin/build-queue.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-queue clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar
+upload $root_path/service-queue/target/service-queue-1.0.0-SNAPSHOT.jar
--- a/bin/build-scheduler.sh
+++ b/bin/build-scheduler.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-scheduler clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar
+upload $root_path/service-scheduler/target/service-scheduler-1.0.0-SNAPSHOT.jar
--- a/bin/build-sync.sh
+++ b/bin/build-sync.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl utils/sync clean package -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/utils/sync/target/sync-1.0.0-SNAPSHOT.jar
+upload $root_path/utils/sync/target/sync-1.0.0-SNAPSHOT.jar
--- a/bin/build-web.sh
+++ b/bin/build-web.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-web/target/service-web-1.0.0-SNAPSHOT.jar
+upload $root_path/service-web/target/service-web-1.0.0-SNAPSHOT.jar
--- a/bin/build-yarn-query.sh
+++ b/bin/build-yarn-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-yarn-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar
--- a/bin/build-zookeeper-query.sh
+++ b/bin/build-zookeeper-query.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+root_path=$(dirname $(cd $(dirname $0);pwd))
+source $root_path/bin/library.sh
 mvn -pl service-common,service-dependencies,service-configuration clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
 mvn -pl service-zookeeper-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
-ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar
+upload $root_path/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar
--- a/bin/library.sh
+++ b/bin/library.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+encrypt_username=AxhEbscwsJDbYMH2
+encrypt_password=cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+
+upload_url=http://$encrypt_username:$encrypt_password@132.126.207.34:36800
+
+root_path=$(dirname $(cd $(dirname $0);pwd))
+
+function upload() {
+  source_file_path=$(realpath $1)
+  file_name=$(basename $source_file_path)
+  echo "↪ Source md5: $(md5sum $source_file_path | awk '{print $1}')"
+  echo "↪ Uploading $source_file_path"
+  curl $upload_url/file/upload/$file_name -T $source_file_path
+  echo "↪ Upload ytp success"
+  echo "↪ Download: curl $upload_url/file/download/$file_name -o $file_name"
+  echo "↪ Delete source"
+  rm $source_file_path
+}
--- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
+++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
@@ -13,6 +13,7 @@ public class RuntimeInfo {
    private String logPath;
    private String confPath;
    private String dataPath;
+    private String downloadUrl;
    private String kerberosKeytabPath;
    private LokiInfo loki;
    private String zkUrl;
@@ -69,6 +70,14 @@ public class RuntimeInfo {
        this.dataPath = dataPath;
    }

+    public String getDownloadUrl() {
+        return downloadUrl;
+    }
+
+    public void setDownloadUrl(String downloadUrl) {
+        this.downloadUrl = downloadUrl;
+    }
+
    public String getKerberosKeytabPath() {
        return kerberosKeytabPath;
    }
@@ -134,6 +143,7 @@ public class RuntimeInfo {
               ", logPath='" + logPath + '\'' +
               ", confPath='" + confPath + '\'' +
               ", dataPath='" + dataPath + '\'' +
+               ", downloadUrl='" + downloadUrl + '\'' +
               ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
               ", loki=" + loki +
               ", zkUrl='" + zkUrl + '\'' +
--- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml
@@ -6,6 +6,7 @@ deploy:
    log-path: /data/datalake/logs
    conf-path: /data/datalake/conf
    data-path: /data/datalake/data
+    download-url: http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.34:36800
    kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
    loki:
      service-url: http://132.126.207.125:33100
--- a/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/cloud/deploy.ftl
@@ -12,7 +12,7 @@ hostname=`ssh $host 'hostname'`
 hostname_full=`ssh $host 'hostname -f'`
 <#-- 获取当前时间 -->
 start_time=`date +%Y%m%d%H%M%S`
-ssh $host "mkdir -p ${runtime.jarPath};curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
+ssh $host "mkdir -p ${runtime.jarPath};curl ${runtime.downloadUrl}/${info.sourceJar} -o ${runtime.jarPath}/${info.name}.jar"
 ssh $host "export JASYPT_ENCRYPTOR_PASSWORD='r#(R,P\"Dp^A47>WSn:Wn].gs/+\"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.';nohup ${runtime.jdkPath} <#list environments?keys as key>-D${key}=${environments[key]?string} </#list>-jar ${runtime.jarPath}/${info.name}.jar <#noparse>--deploy.datetime=${datetime} --deploy.ip=${host} --deploy.hostname=${hostname} --deploy.hostname-full=${hostname_full} --deploy.start-time=${start_time}</#noparse> --logging.parent=${runtime.logPath} --loki.url=${runtime.loki.servicePushUrl} --spring.cloud.zookeeper.connect-string=${runtime.zkUrl} --spring.security.meta.authority='${runtime.security.authority}' --spring.security.meta.username='${runtime.security.username}' --spring.security.meta.darkcode='${runtime.security.darkcode}' --yarn-cluster.sync-clusters=${runtime.yarn.syncClusters} --yarn-cluster.compaction-clusters=${runtime.yarn.compactionClusters} <#list arguments?keys as key>--${key}=${arguments[key]?string} </#list>> /dev/null 2>&1 &"
 </#if>
 echo ''
--- a/service-cli/service-cli-runner/src/main/resources/template/command/update.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/command/update.ftl
@@ -1,3 +1,3 @@
 #!/bin/bash

-curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-command-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-command.jar
+curl ${runtime.downloadUrl}/service-command-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-command.jar
--- a/service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl
+++ b/service-cli/service-cli-runner/src/main/resources/template/uploader/update.ftl
@@ -1,3 +1,3 @@
 #!/bin/bash

-curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222/service-uploader-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-uploader.jar
+curl ${runtime.downloadUrl}/service-uploader-1.0.0-SNAPSHOT.jar -o ${runtime.jarPath}/service-uploader.jar