diff --git a/bin/build-all.sh b/bin/build-all.sh index dd4bee3..f8e359e 100644 --- a/bin/build-all.sh +++ b/bin/build-all.sh @@ -2,11 +2,12 @@ mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml -mvn -pl service-flink-query,service-hudi-query,service-info-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml +mvn -pl service-flink-query,service-hudi-query,service-info-query,service-loki-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao +sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao @@ -15,6 +16,7 @@ sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/i sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-flink-query-1.0.0-SNAPSHOT.jar' sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-hudi-query-1.0.0-SNAPSHOT.jar' sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-info-query-1.0.0-SNAPSHOT.jar' +sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-loki-query-1.0.0-SNAPSHOT.jar' sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-pulsar-query-1.0.0-SNAPSHOT.jar' sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-yarn-query-1.0.0-SNAPSHOT.jar' sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-zookeeper-query-1.0.0-SNAPSHOT.jar' diff --git a/bin/build-loki-query.sh b/bin/build-loki-query.sh new file mode 100755 index 0000000..112f926 --- /dev/null +++ b/bin/build-loki-query.sh @@ -0,0 +1,5 @@ +#!/bin/bash +mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml +mvn -pl service-loki-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml +sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao +sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-loki-query-1.0.0-SNAPSHOT.jar' \ No newline at end of file diff --git a/pom.xml b/pom.xml index cbbeea8..9562ad0 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,7 @@ service-zookeeper-query service-dependencies service-cli + service-loki-query diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 3e0222c..ca4e331 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -6,7 +6,8 @@ deploy: log-path: /apps/datalake/hudi/logs data-path: /apps/datalake/hudi/data kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab - loki-url: http://132.122.116.142:33100/loki/api/v1/push + loki-base-url: http://132.122.116.142:33100 + loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181 eureka-url: http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.142:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.143:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.144:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.145:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.146:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.147:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.148:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.149:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.150:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.151:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.152:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.153:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.154:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.155:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.156:35670/eureka hudi: @@ -95,6 +96,11 @@ deploy: - name: service-info-query source-jar: service-info-query-1.0.0-SNAPSHOT.jar replicas: 4 + - name: service-loki-query + source-jar: service-loki-query-1.0.0-SNAPSHOT.jar + replicas: 3 + arguments: + loki_host: ${deploy.runtime.loki-base-url} - name: service-yarn-query-b1 source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar replicas: 4 diff --git a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/loki/LokiLogLine.java b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/loki/LokiLogLine.java new file mode 100644 index 0000000..592f23e --- /dev/null +++ b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/loki/LokiLogLine.java @@ -0,0 +1,45 @@ +package com.lanyuanxiaoyao.service.configuration.entity.loki; + +import org.eclipse.collections.api.map.ImmutableMap; + +/** + * 一行日志 + * + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +public class LokiLogLine { + private ImmutableMap tags; + private Long timestamp; + private String data; + + public LokiLogLine() { + } + + public LokiLogLine(ImmutableMap tags, Long timestamp, String data) { + this.tags = tags; + this.timestamp = timestamp; + this.data = data; + } + + public ImmutableMap getTags() { + return tags; + } + + public Long getTimestamp() { + return timestamp; + } + + public String getData() { + return data; + } + + @Override + public String toString() { + return "LokiLogLine{" + + "tags=" + tags + + ", timestamp=" + timestamp + + ", data='" + data + '\'' + + '}'; + } +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java new file mode 100644 index 0000000..4f742d3 --- /dev/null +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java @@ -0,0 +1,20 @@ +package com.lanyuanxiaoyao.service.forest.service; + +import com.dtflys.forest.annotation.BaseRequest; +import com.dtflys.forest.annotation.Get; +import com.dtflys.forest.annotation.Query; +import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; +import org.eclipse.collections.api.list.ImmutableList; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.Map; + +/** + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@BaseRequest(baseURL = "http://service-loki-query") +public interface LokiService { + @Get("/loki/query_range") + ImmutableList queryRange(@Query Map queryMap); +} diff --git a/service-loki-query/pom.xml b/service-loki-query/pom.xml new file mode 100644 index 0000000..3613665 --- /dev/null +++ b/service-loki-query/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + com.lanyuanxiaoyao + hudi-service + 1.0.0-SNAPSHOT + + + service-loki-query + + + + com.lanyuanxiaoyao + service-dependencies + 1.0.0-SNAPSHOT + + + com.lanyuanxiaoyao + service-configuration + 1.0.0-SNAPSHOT + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + \ No newline at end of file diff --git a/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/LokiQueryApplication.java b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/LokiQueryApplication.java new file mode 100644 index 0000000..79aef6c --- /dev/null +++ b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/LokiQueryApplication.java @@ -0,0 +1,54 @@ +package com.lanyuanxiaoyao.service.loki; + +import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; +import com.lanyuanxiaoyao.service.loki.service.LokiQueryService; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.retry.annotation.EnableRetry; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * 启动类 + * + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@EnableDiscoveryClient +@SpringBootApplication( + scanBasePackages = {"com.lanyuanxiaoyao.service"}, + exclude = {GsonAutoConfiguration.class} +) +@EnableConfigurationProperties +@EnableRetry +@RequestMapping("loki") +@RestController +public class LokiQueryApplication { + private static final Logger logger = LoggerFactory.getLogger(LokiQueryApplication.class); + + public LokiQueryApplication(LokiQueryService lokiQueryService) { + this.lokiQueryService = lokiQueryService; + } + + public static void main(String[] args) { + SpringApplication.run(LokiQueryApplication.class, args); + } + + private final LokiQueryService lokiQueryService; + + @GetMapping("query_range") + public ImmutableList queryRange(@RequestParam Map queryMap) throws Exception { + return lokiQueryService.queryRange(Maps.immutable.ofAll(queryMap)); + } +} diff --git a/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/configuration/LokiConfiguration.java b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/configuration/LokiConfiguration.java new file mode 100644 index 0000000..bc5684a --- /dev/null +++ b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/configuration/LokiConfiguration.java @@ -0,0 +1,31 @@ +package com.lanyuanxiaoyao.service.loki.configuration; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 配置 + * + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@Component +@ConfigurationProperties("loki") +public class LokiConfiguration { + private String host; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + @Override + public String toString() { + return "LokiConfiguration{" + + "url='" + host + '\'' + + '}'; + } +} diff --git a/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/entity/LokiQueryRangeResponse.java b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/entity/LokiQueryRangeResponse.java new file mode 100644 index 0000000..a032f05 --- /dev/null +++ b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/entity/LokiQueryRangeResponse.java @@ -0,0 +1,77 @@ +package com.lanyuanxiaoyao.service.loki.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; + +import java.util.Map; + +/** + * Loki 查询接收 + * + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +public final class LokiQueryRangeResponse { + private String status; + private Result data; + + public String getStatus() { + return status; + } + + public Result getData() { + return data; + } + + @Override + public String toString() { + return "LokiQueryRangeResponse{" + + "status='" + status + '\'' + + ", data=" + data + + '}'; + } + + public static final class Result { + @JsonProperty("resultType") + private String type; + private ImmutableList result; + + public String getType() { + return type; + } + + public ImmutableList getResult() { + return result; + } + + @Override + public String toString() { + return "Result{" + + "type='" + type + '\'' + + ", result=" + result + + '}'; + } + + public static final class Line { + private ImmutableMap stream; + private ImmutableList> values; + + public ImmutableMap getStream() { + return stream; + } + + public ImmutableList> getValues() { + return values; + } + + @Override + public String toString() { + return "Line{" + + "stream=" + stream + + ", values=" + values + + '}'; + } + } + } +} diff --git a/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/service/LokiQueryService.java b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/service/LokiQueryService.java new file mode 100644 index 0000000..3b0d6f8 --- /dev/null +++ b/service-loki-query/src/main/java/com/lanyuanxiaoyao/service/loki/service/LokiQueryService.java @@ -0,0 +1,70 @@ +package com.lanyuanxiaoyao.service.loki.service; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.core.util.URLUtil; +import cn.hutool.http.HttpResponse; +import cn.hutool.http.HttpUtil; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine; +import com.lanyuanxiaoyao.service.loki.configuration.LokiConfiguration; +import com.lanyuanxiaoyao.service.loki.entity.LokiQueryRangeResponse; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; + +/** + * Loki 查询服务 + * + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +@Service +public class LokiQueryService { + private static final Logger logger = LoggerFactory.getLogger(LokiQueryService.class); + + private final LokiConfiguration lokiConfiguration; + private final ObjectMapper mapper; + + public LokiQueryService(LokiConfiguration lokiConfiguration, Jackson2ObjectMapperBuilder builder) { + this.lokiConfiguration = lokiConfiguration; + this.mapper = builder.build(); + } + + private String buildLokiQuery(ImmutableMap queryMap) { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + String query = queryMap + .collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value)) + .valuesView() + .makeString(","); + builder.append(query); + builder.append("}"); + return builder.toString(); + } + + @Retryable(Throwable.class) + public ImmutableList queryRange(ImmutableMap queryMap) throws Exception { + String query = URLUtil.encodeQuery(buildLokiQuery(queryMap)); + String queryUrl = StrUtil.format("{}/loki/api/v1/query_range?query={}&limit=200", lokiConfiguration.getHost(), query); + HttpResponse response = HttpUtil.createGet(queryUrl).execute(); + if (response.isOk()) { + LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class); + if (StrUtil.equals("success", queryRangeResponse.getStatus())) { + ImmutableList results = queryRangeResponse.getData().getResult(); + return results.collect(result -> result.getValues().collect(value -> { + Long timestamp = Long.valueOf(value.get(0)); + String data = value.get(1); + return new LokiLogLine(result.getStream(), timestamp, data); + })).flatCollect(list -> list); + } else { + throw new Exception("Loki receive not success: " + queryRangeResponse.getStatus()); + } + } else { + throw new Exception("Http query error: " + response.getStatus() + " for " + queryUrl); + } + } +} diff --git a/service-loki-query/src/main/resources/application.yml b/service-loki-query/src/main/resources/application.yml new file mode 100644 index 0000000..e7aae58 --- /dev/null +++ b/service-loki-query/src/main/resources/application.yml @@ -0,0 +1,7 @@ +spring: + application: + name: service-loki-query + profiles: + include: random-port,common,eureka,metrics +loki: + host: http://132.122.116.142:33100 \ No newline at end of file diff --git a/service-loki-query/src/main/resources/logback-spring.xml b/service-loki-query/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..f955921 --- /dev/null +++ b/service-loki-query/src/main/resources/logback-spring.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + true + + ${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push} + + + + + ${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}} + + true + + + + + + ${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}} + + + + + ${LOGGING_PARENT:-.}/${APP_NAME:-run}.log + + ${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz + 7 + + + ${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}} + + + + + + + + + + + + \ No newline at end of file diff --git a/service-loki-query/src/test/java/com/lanyuanxiaoyao/service/loki/LokiQueryParse.java b/service-loki-query/src/test/java/com/lanyuanxiaoyao/service/loki/LokiQueryParse.java new file mode 100644 index 0000000..beaf738 --- /dev/null +++ b/service-loki-query/src/test/java/com/lanyuanxiaoyao/service/loki/LokiQueryParse.java @@ -0,0 +1,41 @@ +package com.lanyuanxiaoyao.service.loki; + +import cn.hutool.core.util.StrUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.map.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author lanyuanxiaoyao + * @date 2023-05-25 + */ +public class LokiQueryParse { + private static final Logger logger = LoggerFactory.getLogger(LokiQueryParse.class); + + private static String buildLokiQuery(ImmutableMap queryMap) { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + String query = queryMap + .collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value)) + .valuesView() + .makeString(","); + builder.append(query); + builder.append("}"); + return builder.toString(); + } + + public static void main(String[] args) throws JsonProcessingException { +// String query = URLUtil.encodeQuery("{app=\"hudi-sync\",run_type=\"compaction\",level=~\"WARN|ERROR\"}"); +// HttpResponse response = HttpUtil.createGet(StrUtil.format("http://132.122.116.142:33100/loki/api/v1/query_range?query={}&limit=10", query)).execute(); +// // logger.info("{}\n{}\n{}", query, response.getStatus(), response.body()); +// ObjectMapper mapper = new ObjectMapper(); +// mapper.registerModule(new EclipseCollectionsModule()); +// mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); +// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); +// LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class); +// logger.info("{}", queryRangeResponse); + logger.info("{}", buildLokiQuery(Maps.immutable.of("app", "sync", "run_type", "compact"))); + } +}