feat(loki-query): 拆分服务和hudi使用的loki，查询服务分开部署

service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java

@@ -85,6 +85,7 @@ public class RunnerApplication implements ApplicationRunner {
 
     @Override
     public void run(ApplicationArguments args) throws IOException {
+        logger.info("Loading runtime info: {}", mapper.writeValueAsString(deployInformationProperties.getRuntime()));
         Path planPath = Paths.get("deploy.plan");
         Map<String, List<String>> deployPlans = new HashMap<>();
         if (Files.exists(planPath) && !deployInformationProperties.getShuffler()) {

service-cli/service-cli-runner/src/main/resources/application-b12.yml

@@ -6,8 +6,11 @@ deploy:
     log-path: /data/datalake/logs
     data-path: /data/datalake/data
     kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
-    loki-base-url: http://132.126.207.125:33100
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
+    loki:
+      service-url: http://132.126.207.125:33100
+      service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
+      hudi-url: http://132.126.207.126:33100
+      hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
     zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
     connector-zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
     hudi:

service-cli/service-cli-runner/src/main/resources/application-b5.yml

@@ -6,8 +6,11 @@ deploy:
     log-path: /apps/datalake/hudi/logs
     data-path: /apps/datalake/hudi/data
     kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
-    loki-base-url: http://132.122.116.142:33100
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
+    loki:
+      service-url: http://132.122.116.142:33100
+      service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
+      hudi-url: http://132.122.116.142:33100
+      hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
     zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
     connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
     hudi:

service-cli/service-cli-runner/src/main/resources/application-t5.yml

@@ -1,36 +0,0 @@
-deploy:
-  runtime:
-    user: datalake
-    jar-path: /home/datalake/hudi/jars
-    jdk-path: /opt/jdk1.8.0_162/bin/java
-    log-path: /home/datalake/hudi/logs
-    data-path: /home/datalake/hudi/data
-    kerberos-keytab-path: /opt/TDP/keytab/datalake.keytab
-    loki-base-url: http://132.121.126.84:33100
-    loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
-    zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
-    connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
-    hudi:
-      app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars
-      archive-hdfs-path: jfs://ns1/apps/datalake/hudi/archive
-      victoria-push-url: http://132.121.126.84:35710/api/v1/import/prometheus
-    executor:
-      staging-path: hdfs://b2/apps/datalake/yarn
-      archive-hdfs-path: hdfs://b2/apps/flink/completed-jobs/
-      task-jar-path: hdfs://b2/apps/datalake/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar
-      task-result-path: hdfs://b2/apps/datalake/task-results
-    security:
-      authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
-      username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
-      darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
-  hosts:
-    - host: t5s1
-      ip: 132.121.126.83
-    - host: t5s2
-      ip: 132.121.126.84
-    - host: t5s3
-      ip: 132.121.126.86
-    - host: t5s4
-      ip: 132.121.126.89
-    - host: t5s5
-      ip: 132.121.126.90

service-cli/service-cli-runner/src/main/resources/application.yml

@@ -99,10 +99,17 @@ deploy:
       source-jar: service-info-query-1.0.0-SNAPSHOT.jar
       replicas: 10
     - name: service-loki-query
+      source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
+      replicas: 5
+      arguments:
+        spring_application_name: service-loki-query
+        loki_host: ${deploy.runtime.loki.service-url}
+    - name: service-loki-hudi-query
       source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
       replicas: 10
       arguments:
-        loki_host: ${deploy.runtime.loki-base-url}
+        spring_application_name: service-loki-hudi-query
+        loki_host: ${deploy.runtime.loki.hudi-url}
     - name: service-yarn-query
       source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
       replicas: 20

service-cli/service-cli-runner/src/main/resources/template/deploy.ftl

@@ -7,7 +7,7 @@
 )-->
 datetime=`date +%Y%m%d%H%M%S`
 log_path='${runtime.logPath}'
-loki_url='${runtime.lokiUrl}'
+loki_url='${runtime.loki.servicePushUrl}'
 zk_url='${runtime.zkUrl}'
 security_authority='${runtime.security.authority}'
 security_username='${runtime.security.username}'