feat(loki-query): 拆分服务和hudi使用的loki，查询服务分开部署

service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java

@@ -13,9 +13,9 @@ public class RuntimeInfo {
     private String logPath;
     private String dataPath;
     private String kerberosKeytabPath;
-    private String lokiUrl;
+    private LokiInfo loki;
     private String zkUrl;
-    private String eurekaUrl;
+    private String connectorZkUrl;
     private HudiInfo hudi;
     private SecurityInfo security;
 
@@ -67,12 +67,12 @@ public class RuntimeInfo {
         this.kerberosKeytabPath = kerberosKeytabPath;
     }
 
-    public String getLokiUrl() {
+    public LokiInfo getLoki() {
-        return lokiUrl;
+        return loki;
     }
 
-    public void setLokiUrl(String lokiUrl) {
+    public void setLoki(LokiInfo loki) {
-        this.lokiUrl = lokiUrl;
+        this.loki = loki;
     }
 
     public String getZkUrl() {
@@ -83,12 +83,12 @@ public class RuntimeInfo {
         this.zkUrl = zkUrl;
     }
 
-    public String getEurekaUrl() {
+    public String getConnectorZkUrl() {
-        return eurekaUrl;
+        return connectorZkUrl;
     }
 
-    public void setEurekaUrl(String eurekaUrl) {
+    public void setConnectorZkUrl(String connectorZkUrl) {
-        this.eurekaUrl = eurekaUrl;
+        this.connectorZkUrl = connectorZkUrl;
     }
 
     public HudiInfo getHudi() {
@@ -110,18 +110,116 @@ public class RuntimeInfo {
     @Override
     public String toString() {
         return "RuntimeInfo{" +
-                "user='" + user + '\'' +
+               "user='" + user + '\'' +
-                ", jarPath='" + jarPath + '\'' +
+               ", jarPath='" + jarPath + '\'' +
-                ", jdkPath='" + jdkPath + '\'' +
+               ", jdkPath='" + jdkPath + '\'' +
-                ", logPath='" + logPath + '\'' +
+               ", logPath='" + logPath + '\'' +
-                ", dataPath='" + dataPath + '\'' +
+               ", dataPath='" + dataPath + '\'' +
-                ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
+               ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
-                ", lokiUrl='" + lokiUrl + '\'' +
+               ", loki=" + loki +
-                ", zkUrl='" + zkUrl + '\'' +
+               ", zkUrl='" + zkUrl + '\'' +
-                ", eurekaUrl='" + eurekaUrl + '\'' +
+               ", connectorZkUrl='" + connectorZkUrl + '\'' +
-                ", hudi=" + hudi +
+               ", hudi=" + hudi +
-                ", security=" + security +
+               ", security=" + security +
-                '}';
+               '}';
+    }
+
+    public static final class LokiInfo {
+        private String serviceUrl;
+        private String servicePushUrl;
+        private String hudiUrl;
+        private String hudiPushUrl;
+
+        public String getServiceUrl() {
+            return serviceUrl;
+        }
+
+        public void setServiceUrl(String serviceUrl) {
+            this.serviceUrl = serviceUrl;
+        }
+
+        public String getServicePushUrl() {
+            return servicePushUrl;
+        }
+
+        public void setServicePushUrl(String servicePushUrl) {
+            this.servicePushUrl = servicePushUrl;
+        }
+
+        public String getHudiUrl() {
+            return hudiUrl;
+        }
+
+        public void setHudiUrl(String hudiUrl) {
+            this.hudiUrl = hudiUrl;
+        }
+
+        public String getHudiPushUrl() {
+            return hudiPushUrl;
+        }
+
+        public void setHudiPushUrl(String hudiPushUrl) {
+            this.hudiPushUrl = hudiPushUrl;
+        }
+
+        @Override
+        public String toString() {
+            return "LokiInfo{" +
+                   "serviceUrl='" + serviceUrl + '\'' +
+                   ", servicePushUrl='" + servicePushUrl + '\'' +
+                   ", hudiUrl='" + hudiUrl + '\'' +
+                   ", hudiPushUrl='" + hudiPushUrl + '\'' +
+                   '}';
+        }
+    }
+
+    public static final class ExecutorInfo {
+        private String stagingPath;
+        private String archiveHdfsPath;
+        private String taskJarPath;
+        private String taskResultPath;
+
+        public String getStagingPath() {
+            return stagingPath;
+        }
+
+        public void setStagingPath(String stagingPath) {
+            this.stagingPath = stagingPath;
+        }
+
+        public String getArchiveHdfsPath() {
+            return archiveHdfsPath;
+        }
+
+        public void setArchiveHdfsPath(String archiveHdfsPath) {
+            this.archiveHdfsPath = archiveHdfsPath;
+        }
+
+        public String getTaskJarPath() {
+            return taskJarPath;
+        }
+
+        public void setTaskJarPath(String taskJarPath) {
+            this.taskJarPath = taskJarPath;
+        }
+
+        public String getTaskResultPath() {
+            return taskResultPath;
+        }
+
+        public void setTaskResultPath(String taskResultPath) {
+            this.taskResultPath = taskResultPath;
+        }
+
+        @Override
+        public String toString() {
+            return "ExecutorInfo{" +
+                   "stagingPath='" + stagingPath + '\'' +
+                   ", archiveHdfsPath='" + archiveHdfsPath + '\'' +
+                   ", taskJarPath='" + taskJarPath + '\'' +
+                   ", taskResultPath='" + taskResultPath + '\'' +
+                   '}';
+        }
     }
 
     public static final class HudiInfo {
@@ -156,10 +254,10 @@ public class RuntimeInfo {
         @Override
         public String toString() {
             return "HudiInfo{" +
-                    "appHdfsPath='" + appHdfsPath + '\'' +
+                   "appHdfsPath='" + appHdfsPath + '\'' +
-                    ", archiveHdfsPath='" + archiveHdfsPath + '\'' +
+                   ", archiveHdfsPath='" + archiveHdfsPath + '\'' +
-                    ", victoriaPushUrl='" + victoriaPushUrl + '\'' +
+                   ", victoriaPushUrl='" + victoriaPushUrl + '\'' +
-                    '}';
+                   '}';
         }
     }
 
@@ -195,10 +293,10 @@ public class RuntimeInfo {
         @Override
         public String toString() {
             return "SecurityInfo{" +
-                    "authority='" + authority + '\'' +
+                   "authority='" + authority + '\'' +
-                    ", username='" + username + '\'' +
+                   ", username='" + username + '\'' +
-                    ", darkcode='" + darkcode + '\'' +
+                   ", darkcode='" + darkcode + '\'' +
-                    '}';
+                   '}';
         }
     }
 }

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java

@@ -85,6 +85,7 @@ public class RunnerApplication implements ApplicationRunner {
 
     @Override
     public void run(ApplicationArguments args) throws IOException {
+        logger.info("Loading runtime info: {}", mapper.writeValueAsString(deployInformationProperties.getRuntime()));
         Path planPath = Paths.get("deploy.plan");
         Map<String, List<String>> deployPlans = new HashMap<>();
         if (Files.exists(planPath) && !deployInformationProperties.getShuffler()) {

service-cli/service-cli-runner/src/main/resources/application-b12.yml

@@ -6,8 +6,11 @@ deploy:
     log-path: /data/datalake/logs
     data-path: /data/datalake/data
     kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
-    loki-base-url: http://132.126.207.125:33100
+    loki:
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
+      service-url: http://132.126.207.125:33100
+      service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
+      hudi-url: http://132.126.207.126:33100
+      hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
     zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
     connector-zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
     hudi:

service-cli/service-cli-runner/src/main/resources/application-b5.yml

@@ -6,8 +6,11 @@ deploy:
     log-path: /apps/datalake/hudi/logs
     data-path: /apps/datalake/hudi/data
     kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
-    loki-base-url: http://132.122.116.142:33100
+    loki:
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
+      service-url: http://132.122.116.142:33100
+      service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
+      hudi-url: http://132.122.116.142:33100
+      hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
     zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
     connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
     hudi:

service-cli/service-cli-runner/src/main/resources/application-t5.yml

@@ -1,36 +0,0 @@
-deploy:
-  runtime:
-    user: datalake
-    jar-path: /home/datalake/hudi/jars
-    jdk-path: /opt/jdk1.8.0_162/bin/java
-    log-path: /home/datalake/hudi/logs
-    data-path: /home/datalake/hudi/data
-    kerberos-keytab-path: /opt/TDP/keytab/datalake.keytab
-    loki-base-url: http://132.121.126.84:33100
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
-    zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
-    connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
-    hudi:
-      app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars
-      archive-hdfs-path: jfs://ns1/apps/datalake/hudi/archive
-      victoria-push-url: http://132.121.126.84:35710/api/v1/import/prometheus
-    executor:
-      staging-path: hdfs://b2/apps/datalake/yarn
-      archive-hdfs-path: hdfs://b2/apps/flink/completed-jobs/
-      task-jar-path: hdfs://b2/apps/datalake/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar
-      task-result-path: hdfs://b2/apps/datalake/task-results
-    security:
-      authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
-      username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
-      darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
-  hosts:
-    - host: t5s1
-      ip: 132.121.126.83
-    - host: t5s2
-      ip: 132.121.126.84
-    - host: t5s3
-      ip: 132.121.126.86
-    - host: t5s4
-      ip: 132.121.126.89
-    - host: t5s5
-      ip: 132.121.126.90

service-cli/service-cli-runner/src/main/resources/application.yml

@@ -99,10 +99,17 @@ deploy:
       source-jar: service-info-query-1.0.0-SNAPSHOT.jar
       replicas: 10
     - name: service-loki-query
+      source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
+      replicas: 5
+      arguments:
+        spring_application_name: service-loki-query
+        loki_host: ${deploy.runtime.loki.service-url}
+    - name: service-loki-hudi-query
       source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
       replicas: 10
       arguments:
-        loki_host: ${deploy.runtime.loki-base-url}
+        spring_application_name: service-loki-hudi-query
+        loki_host: ${deploy.runtime.loki.hudi-url}
     - name: service-yarn-query
       source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
       replicas: 20

service-cli/service-cli-runner/src/main/resources/template/deploy.ftl

@@ -7,7 +7,7 @@
 )-->
 datetime=`date +%Y%m%d%H%M%S`
 log_path='${runtime.logPath}'
-loki_url='${runtime.lokiUrl}'
+loki_url='${runtime.loki.servicePushUrl}'
 zk_url='${runtime.zkUrl}'
 security_authority='${runtime.security.authority}'
 security_username='${runtime.security.username}'

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java

@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.forest.service.loki;
+
+import com.dtflys.forest.annotation.BaseRequest;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-05-25
+ */
+@BaseRequest(baseURL = "http://service-loki-hudi-query")
+public interface LokiHudiService extends LokiService {
+}

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiService.java

@@ -1,13 +1,11 @@
-package com.lanyuanxiaoyao.service.forest.service;
+package com.lanyuanxiaoyao.service.forest.service.loki;
 
 import com.dtflys.forest.annotation.BaseRequest;
 import com.dtflys.forest.annotation.Get;
 import com.dtflys.forest.annotation.Query;
 import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
-import org.eclipse.collections.api.list.ImmutableList;
-import org.springframework.web.bind.annotation.RequestParam;
-
 import java.util.Map;
+import org.eclipse.collections.api.list.ImmutableList;
 
 /**
  * @author lanyuanxiaoyao

service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java

@@ -0,0 +1,16 @@
+package com.lanyuanxiaoyao.service.forest.service.loki;
+
+import com.dtflys.forest.annotation.BaseRequest;
+import com.dtflys.forest.annotation.Get;
+import com.dtflys.forest.annotation.Query;
+import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
+import java.util.Map;
+import org.eclipse.collections.api.list.ImmutableList;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-05-25
+ */
+@BaseRequest(baseURL = "http://service-loki-query")
+public interface LokiServiceService extends LokiService {
+}

service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java

@@ -3,7 +3,7 @@ package com.lanyuanxiaoyao.service.web.controller;
 import cn.hutool.core.map.MapUtil;
 import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
 import com.lanyuanxiaoyao.service.forest.service.InfoService;
-import com.lanyuanxiaoyao.service.forest.service.LokiService;
+import com.lanyuanxiaoyao.service.forest.service.loki.LokiHudiService;
 import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse;
 import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse;
 import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
@@ -26,12 +26,12 @@ import org.springframework.web.bind.annotation.RestController;
 public class LogController extends BaseController {
     private static final Logger logger = LoggerFactory.getLogger(LogController.class);
 
-    private final LokiService lokiService;
+    private final LokiHudiService lokiHudiService;
     private final InfoService infoService;
 
     @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
-    public LogController(LokiService lokiService, InfoService infoService) {
+    public LogController(LokiHudiService lokiHudiService, InfoService infoService) {
-        this.lokiService = lokiService;
+        this.lokiHudiService = lokiHudiService;
         this.infoService = infoService;
     }
 
@@ -43,7 +43,7 @@ public class LogController extends BaseController {
 
     @GetMapping("query_sync_log")
     public AmisDetailResponse querySyncLog(@RequestParam("flink_job_id") Long flinkJobId) {
-        return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
+        return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
                         .put("app", "hudi-sync")
                         .put("run_type", "sync")
                         .put("flink_job_id", flinkJobId.toString())
@@ -56,7 +56,7 @@ public class LogController extends BaseController {
 
     @GetMapping("query_compaction_log")
     public AmisDetailResponse queryCompactionLog(@RequestParam("flink_job_id") Long flinkJobId, @RequestParam("alias") String alias) {
-        return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
+        return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
                         .put("app", "hudi-sync")
                         .put("run_type", "compaction")
                         .put("flink_job_id", flinkJobId.toString())
@@ -70,7 +70,7 @@ public class LogController extends BaseController {
 
     @GetMapping("query_application_log")
     public AmisDetailResponse queryApplicationLog(@RequestParam("application_id") String applicationId) {
-        return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
+        return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
                         .put("app", "hudi-sync")
                         .put("app_id", applicationId)
                         .build())