From 86ad6df8995355d1fc94917aa9b15015164dc0a1 Mon Sep 17 00:00:00 2001 From: lanyuanxiaoyao Date: Wed, 10 Jan 2024 09:49:11 +0800 Subject: [PATCH] =?UTF-8?q?feat(loki-query):=20=E6=8B=86=E5=88=86=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=92=8Chudi=E4=BD=BF=E7=94=A8=E7=9A=84loki=EF=BC=8C?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=9C=8D=E5=8A=A1=E5=88=86=E5=BC=80=E9=83=A8?= =?UTF-8?q?=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/cli/core/RuntimeInfo.java | 158 ++++++++++++++---- .../service/cli/runner/RunnerApplication.java | 1 + .../src/main/resources/application-b12.yml | 7 +- .../src/main/resources/application-b5.yml | 7 +- .../src/main/resources/application-t5.yml | 36 ---- .../src/main/resources/application.yml | 9 +- .../src/main/resources/template/deploy.ftl | 2 +- .../forest/service/loki/LokiHudiService.java | 11 ++ .../service/{ => loki}/LokiService.java | 6 +- .../service/loki/LokiServiceService.java | 16 ++ .../service/web/controller/LogController.java | 14 +- 11 files changed, 184 insertions(+), 83 deletions(-) delete mode 100644 service-cli/service-cli-runner/src/main/resources/application-t5.yml create mode 100644 service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java rename service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/{ => loki}/LokiService.java (82%) create mode 100644 service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java diff --git a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java index 6ebca98..495ebcd 100644 --- a/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java +++ b/service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java @@ -13,9 +13,9 @@ public class RuntimeInfo { private String logPath; private String dataPath; private String kerberosKeytabPath; - private String lokiUrl; + private LokiInfo loki; private String zkUrl; - private String eurekaUrl; + private String connectorZkUrl; private HudiInfo hudi; private SecurityInfo security; @@ -67,12 +67,12 @@ public class RuntimeInfo { this.kerberosKeytabPath = kerberosKeytabPath; } - public String getLokiUrl() { - return lokiUrl; + public LokiInfo getLoki() { + return loki; } - public void setLokiUrl(String lokiUrl) { - this.lokiUrl = lokiUrl; + public void setLoki(LokiInfo loki) { + this.loki = loki; } public String getZkUrl() { @@ -83,12 +83,12 @@ public class RuntimeInfo { this.zkUrl = zkUrl; } - public String getEurekaUrl() { - return eurekaUrl; + public String getConnectorZkUrl() { + return connectorZkUrl; } - public void setEurekaUrl(String eurekaUrl) { - this.eurekaUrl = eurekaUrl; + public void setConnectorZkUrl(String connectorZkUrl) { + this.connectorZkUrl = connectorZkUrl; } public HudiInfo getHudi() { @@ -110,18 +110,116 @@ public class RuntimeInfo { @Override public String toString() { return "RuntimeInfo{" + - "user='" + user + '\'' + - ", jarPath='" + jarPath + '\'' + - ", jdkPath='" + jdkPath + '\'' + - ", logPath='" + logPath + '\'' + - ", dataPath='" + dataPath + '\'' + - ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' + - ", lokiUrl='" + lokiUrl + '\'' + - ", zkUrl='" + zkUrl + '\'' + - ", eurekaUrl='" + eurekaUrl + '\'' + - ", hudi=" + hudi + - ", security=" + security + - '}'; + "user='" + user + '\'' + + ", jarPath='" + jarPath + '\'' + + ", jdkPath='" + jdkPath + '\'' + + ", logPath='" + logPath + '\'' + + ", dataPath='" + dataPath + '\'' + + ", kerberosKeytabPath='" + kerberosKeytabPath + '\'' + + ", loki=" + loki + + ", zkUrl='" + zkUrl + '\'' + + ", connectorZkUrl='" + connectorZkUrl + '\'' + + ", hudi=" + hudi + + ", security=" + security + + '}'; + } + + public static final class LokiInfo { + private String serviceUrl; + private String servicePushUrl; + private String hudiUrl; + private String hudiPushUrl; + + public String getServiceUrl() { + return serviceUrl; + } + + public void setServiceUrl(String serviceUrl) { + this.serviceUrl = serviceUrl; + } + + public String getServicePushUrl() { + return servicePushUrl; + } + + public void setServicePushUrl(String servicePushUrl) { + this.servicePushUrl = servicePushUrl; + } + + public String getHudiUrl() { + return hudiUrl; + } + + public void setHudiUrl(String hudiUrl) { + this.hudiUrl = hudiUrl; + } + + public String getHudiPushUrl() { + return hudiPushUrl; + } + + public void setHudiPushUrl(String hudiPushUrl) { + this.hudiPushUrl = hudiPushUrl; + } + + @Override + public String toString() { + return "LokiInfo{" + + "serviceUrl='" + serviceUrl + '\'' + + ", servicePushUrl='" + servicePushUrl + '\'' + + ", hudiUrl='" + hudiUrl + '\'' + + ", hudiPushUrl='" + hudiPushUrl + '\'' + + '}'; + } + } + + public static final class ExecutorInfo { + private String stagingPath; + private String archiveHdfsPath; + private String taskJarPath; + private String taskResultPath; + + public String getStagingPath() { + return stagingPath; + } + + public void setStagingPath(String stagingPath) { + this.stagingPath = stagingPath; + } + + public String getArchiveHdfsPath() { + return archiveHdfsPath; + } + + public void setArchiveHdfsPath(String archiveHdfsPath) { + this.archiveHdfsPath = archiveHdfsPath; + } + + public String getTaskJarPath() { + return taskJarPath; + } + + public void setTaskJarPath(String taskJarPath) { + this.taskJarPath = taskJarPath; + } + + public String getTaskResultPath() { + return taskResultPath; + } + + public void setTaskResultPath(String taskResultPath) { + this.taskResultPath = taskResultPath; + } + + @Override + public String toString() { + return "ExecutorInfo{" + + "stagingPath='" + stagingPath + '\'' + + ", archiveHdfsPath='" + archiveHdfsPath + '\'' + + ", taskJarPath='" + taskJarPath + '\'' + + ", taskResultPath='" + taskResultPath + '\'' + + '}'; + } } public static final class HudiInfo { @@ -156,10 +254,10 @@ public class RuntimeInfo { @Override public String toString() { return "HudiInfo{" + - "appHdfsPath='" + appHdfsPath + '\'' + - ", archiveHdfsPath='" + archiveHdfsPath + '\'' + - ", victoriaPushUrl='" + victoriaPushUrl + '\'' + - '}'; + "appHdfsPath='" + appHdfsPath + '\'' + + ", archiveHdfsPath='" + archiveHdfsPath + '\'' + + ", victoriaPushUrl='" + victoriaPushUrl + '\'' + + '}'; } } @@ -195,10 +293,10 @@ public class RuntimeInfo { @Override public String toString() { return "SecurityInfo{" + - "authority='" + authority + '\'' + - ", username='" + username + '\'' + - ", darkcode='" + darkcode + '\'' + - '}'; + "authority='" + authority + '\'' + + ", username='" + username + '\'' + + ", darkcode='" + darkcode + '\'' + + '}'; } } } diff --git a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java index 0f87903..9d72147 100644 --- a/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java +++ b/service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java @@ -85,6 +85,7 @@ public class RunnerApplication implements ApplicationRunner { @Override public void run(ApplicationArguments args) throws IOException { + logger.info("Loading runtime info: {}", mapper.writeValueAsString(deployInformationProperties.getRuntime())); Path planPath = Paths.get("deploy.plan"); Map> deployPlans = new HashMap<>(); if (Files.exists(planPath) && !deployInformationProperties.getShuffler()) { diff --git a/service-cli/service-cli-runner/src/main/resources/application-b12.yml b/service-cli/service-cli-runner/src/main/resources/application-b12.yml index acc5f16..667a3af 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml @@ -6,8 +6,11 @@ deploy: log-path: /data/datalake/logs data-path: /data/datalake/data kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab - loki-base-url: http://132.126.207.125:33100 - loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push + loki: + service-url: http://132.126.207.125:33100 + service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push + hudi-url: http://132.126.207.126:33100 + hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 connector-zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181 hudi: diff --git a/service-cli/service-cli-runner/src/main/resources/application-b5.yml b/service-cli/service-cli-runner/src/main/resources/application-b5.yml index 7fa1060..3b70f29 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b5.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b5.yml @@ -6,8 +6,11 @@ deploy: log-path: /apps/datalake/hudi/logs data-path: /apps/datalake/hudi/data kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab - loki-base-url: http://132.122.116.142:33100 - loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push + loki: + service-url: http://132.122.116.142:33100 + service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push + hudi-url: http://132.122.116.142:33100 + hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 hudi: diff --git a/service-cli/service-cli-runner/src/main/resources/application-t5.yml b/service-cli/service-cli-runner/src/main/resources/application-t5.yml deleted file mode 100644 index d3312e9..0000000 --- a/service-cli/service-cli-runner/src/main/resources/application-t5.yml +++ /dev/null @@ -1,36 +0,0 @@ -deploy: - runtime: - user: datalake - jar-path: /home/datalake/hudi/jars - jdk-path: /opt/jdk1.8.0_162/bin/java - log-path: /home/datalake/hudi/logs - data-path: /home/datalake/hudi/data - kerberos-keytab-path: /opt/TDP/keytab/datalake.keytab - loki-base-url: http://132.121.126.84:33100 - loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push - zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181 - connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181 - hudi: - app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars - archive-hdfs-path: jfs://ns1/apps/datalake/hudi/archive - victoria-push-url: http://132.121.126.84:35710/api/v1/import/prometheus - executor: - staging-path: hdfs://b2/apps/datalake/yarn - archive-hdfs-path: hdfs://b2/apps/flink/completed-jobs/ - task-jar-path: hdfs://b2/apps/datalake/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar - task-result-path: hdfs://b2/apps/datalake/task-results - security: - authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) - username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) - darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) - hosts: - - host: t5s1 - ip: 132.121.126.83 - - host: t5s2 - ip: 132.121.126.84 - - host: t5s3 - ip: 132.121.126.86 - - host: t5s4 - ip: 132.121.126.89 - - host: t5s5 - ip: 132.121.126.90 \ No newline at end of file diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index a98a303..4bc511a 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -99,10 +99,17 @@ deploy: source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 10 - name: service-loki-query + source-jar: service-loki-query-1.0.0-SNAPSHOT.jar + replicas: 5 + arguments: + spring_application_name: service-loki-query + loki_host: ${deploy.runtime.loki.service-url} + - name: service-loki-hudi-query source-jar: service-loki-query-1.0.0-SNAPSHOT.jar replicas: 10 arguments: - loki_host: ${deploy.runtime.loki-base-url} + spring_application_name: service-loki-hudi-query + loki_host: ${deploy.runtime.loki.hudi-url} - name: service-yarn-query source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 20 diff --git a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl index e22a3a0..5a4d61f 100644 --- a/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl +++ b/service-cli/service-cli-runner/src/main/resources/template/deploy.ftl @@ -7,7 +7,7 @@ )--> datetime=`date +%Y%m%d%H%M%S` log_path='${runtime.logPath}' -loki_url='${runtime.lokiUrl}' +loki_url='${runtime.loki.servicePushUrl}' zk_url='${runtime.zkUrl}' security_authority='${runtime.security.authority}' security_username='${runtime.security.username}' diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java new file mode 100644 index 0000000..1f6de8f --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java @@ -0,0 +1,11 @@ +package com.lanyuanxiaoyao.service.forest.service.loki; + +import com.dtflys.forest.annotation.BaseRequest; + +/** + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@BaseRequest(baseURL = "http://service-loki-hudi-query") +public interface LokiHudiService extends LokiService { +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiService.java similarity index 82% rename from service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java rename to service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiService.java index 4f742d3..075ddce 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiService.java @@ -1,13 +1,11 @@ -package com.lanyuanxiaoyao.service.forest.service; +package com.lanyuanxiaoyao.service.forest.service.loki; import com.dtflys.forest.annotation.BaseRequest; import com.dtflys.forest.annotation.Get; import com.dtflys.forest.annotation.Query; import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; -import org.eclipse.collections.api.list.ImmutableList; -import org.springframework.web.bind.annotation.RequestParam; - import java.util.Map; +import org.eclipse.collections.api.list.ImmutableList; /** * @author lanyuanxiaoyao diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java new file mode 100644 index 0000000..b88f201 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java @@ -0,0 +1,16 @@ +package com.lanyuanxiaoyao.service.forest.service.loki; + +import com.dtflys.forest.annotation.BaseRequest; +import com.dtflys.forest.annotation.Get; +import com.dtflys.forest.annotation.Query; +import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; +import java.util.Map; +import org.eclipse.collections.api.list.ImmutableList; + +/** + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@BaseRequest(baseURL = "http://service-loki-query") +public interface LokiServiceService extends LokiService { +} diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java index c78c318..327ee57 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java @@ -3,7 +3,7 @@ package com.lanyuanxiaoyao.service.web.controller; import cn.hutool.core.map.MapUtil; import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; import com.lanyuanxiaoyao.service.forest.service.InfoService; -import com.lanyuanxiaoyao.service.forest.service.LokiService; +import com.lanyuanxiaoyao.service.forest.service.loki.LokiHudiService; import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse; import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse; import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse; @@ -26,12 +26,12 @@ import org.springframework.web.bind.annotation.RestController; public class LogController extends BaseController { private static final Logger logger = LoggerFactory.getLogger(LogController.class); - private final LokiService lokiService; + private final LokiHudiService lokiHudiService; private final InfoService infoService; @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") - public LogController(LokiService lokiService, InfoService infoService) { - this.lokiService = lokiService; + public LogController(LokiHudiService lokiHudiService, InfoService infoService) { + this.lokiHudiService = lokiHudiService; this.infoService = infoService; } @@ -43,7 +43,7 @@ public class LogController extends BaseController { @GetMapping("query_sync_log") public AmisDetailResponse querySyncLog(@RequestParam("flink_job_id") Long flinkJobId) { - return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.builder() + return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.builder() .put("app", "hudi-sync") .put("run_type", "sync") .put("flink_job_id", flinkJobId.toString()) @@ -56,7 +56,7 @@ public class LogController extends BaseController { @GetMapping("query_compaction_log") public AmisDetailResponse queryCompactionLog(@RequestParam("flink_job_id") Long flinkJobId, @RequestParam("alias") String alias) { - return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.builder() + return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.builder() .put("app", "hudi-sync") .put("run_type", "compaction") .put("flink_job_id", flinkJobId.toString()) @@ -70,7 +70,7 @@ public class LogController extends BaseController { @GetMapping("query_application_log") public AmisDetailResponse queryApplicationLog(@RequestParam("application_id") String applicationId) { - return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.builder() + return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.builder() .put("app", "hudi-sync") .put("app_id", applicationId) .build())