feature(loki-query): 新增 loki 日志查询
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
package com.lanyuanxiaoyao.service.loki;
|
||||
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
|
||||
import com.lanyuanxiaoyao.service.loki.service.LokiQueryService;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 启动类
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@EnableDiscoveryClient
|
||||
@SpringBootApplication(
|
||||
scanBasePackages = {"com.lanyuanxiaoyao.service"},
|
||||
exclude = {GsonAutoConfiguration.class}
|
||||
)
|
||||
@EnableConfigurationProperties
|
||||
@EnableRetry
|
||||
@RequestMapping("loki")
|
||||
@RestController
|
||||
public class LokiQueryApplication {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryApplication.class);
|
||||
|
||||
public LokiQueryApplication(LokiQueryService lokiQueryService) {
|
||||
this.lokiQueryService = lokiQueryService;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(LokiQueryApplication.class, args);
|
||||
}
|
||||
|
||||
private final LokiQueryService lokiQueryService;
|
||||
|
||||
@GetMapping("query_range")
|
||||
public ImmutableList<LokiLogLine> queryRange(@RequestParam Map<String, String> queryMap) throws Exception {
|
||||
return lokiQueryService.queryRange(Maps.immutable.ofAll(queryMap));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.lanyuanxiaoyao.service.loki.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 配置
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@Component
|
||||
@ConfigurationProperties("loki")
|
||||
public class LokiConfiguration {
|
||||
private String host;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LokiConfiguration{" +
|
||||
"url='" + host + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package com.lanyuanxiaoyao.service.loki.entity;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Loki 查询接收
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
public final class LokiQueryRangeResponse {
|
||||
private String status;
|
||||
private Result data;
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public Result getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LokiQueryRangeResponse{" +
|
||||
"status='" + status + '\'' +
|
||||
", data=" + data +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Result {
|
||||
@JsonProperty("resultType")
|
||||
private String type;
|
||||
private ImmutableList<Line> result;
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public ImmutableList<Line> getResult() {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Result{" +
|
||||
"type='" + type + '\'' +
|
||||
", result=" + result +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Line {
|
||||
private ImmutableMap<String, String> stream;
|
||||
private ImmutableList<ImmutableList<String>> values;
|
||||
|
||||
public ImmutableMap<String, String> getStream() {
|
||||
return stream;
|
||||
}
|
||||
|
||||
public ImmutableList<ImmutableList<String>> getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Line{" +
|
||||
"stream=" + stream +
|
||||
", values=" + values +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.lanyuanxiaoyao.service.loki.service;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
|
||||
import com.lanyuanxiaoyao.service.loki.configuration.LokiConfiguration;
|
||||
import com.lanyuanxiaoyao.service.loki.entity.LokiQueryRangeResponse;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
|
||||
import org.springframework.retry.annotation.Retryable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Loki 查询服务
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@Service
|
||||
public class LokiQueryService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryService.class);
|
||||
|
||||
private final LokiConfiguration lokiConfiguration;
|
||||
private final ObjectMapper mapper;
|
||||
|
||||
public LokiQueryService(LokiConfiguration lokiConfiguration, Jackson2ObjectMapperBuilder builder) {
|
||||
this.lokiConfiguration = lokiConfiguration;
|
||||
this.mapper = builder.build();
|
||||
}
|
||||
|
||||
private String buildLokiQuery(ImmutableMap<String, String> queryMap) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
String query = queryMap
|
||||
.collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value))
|
||||
.valuesView()
|
||||
.makeString(",");
|
||||
builder.append(query);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Retryable(Throwable.class)
|
||||
public ImmutableList<LokiLogLine> queryRange(ImmutableMap<String, String> queryMap) throws Exception {
|
||||
String query = URLUtil.encodeQuery(buildLokiQuery(queryMap));
|
||||
String queryUrl = StrUtil.format("{}/loki/api/v1/query_range?query={}&limit=200", lokiConfiguration.getHost(), query);
|
||||
HttpResponse response = HttpUtil.createGet(queryUrl).execute();
|
||||
if (response.isOk()) {
|
||||
LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class);
|
||||
if (StrUtil.equals("success", queryRangeResponse.getStatus())) {
|
||||
ImmutableList<LokiQueryRangeResponse.Result.Line> results = queryRangeResponse.getData().getResult();
|
||||
return results.collect(result -> result.getValues().collect(value -> {
|
||||
Long timestamp = Long.valueOf(value.get(0));
|
||||
String data = value.get(1);
|
||||
return new LokiLogLine(result.getStream(), timestamp, data);
|
||||
})).flatCollect(list -> list);
|
||||
} else {
|
||||
throw new Exception("Loki receive not success: " + queryRangeResponse.getStatus());
|
||||
}
|
||||
} else {
|
||||
throw new Exception("Http query error: " + response.getStatus() + " for " + queryUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
7
service-loki-query/src/main/resources/application.yml
Normal file
7
service-loki-query/src/main/resources/application.yml
Normal file
@@ -0,0 +1,7 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-loki-query
|
||||
profiles:
|
||||
include: random-port,common,eureka,metrics
|
||||
loki:
|
||||
host: http://132.122.116.142:33100
|
||||
51
service-loki-query/src/main/resources/logback-spring.xml
Normal file
51
service-loki-query/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,51 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Loki" class="com.github.loki4j.logback.Loki4jAppender">
|
||||
<metricsEnabled>true</metricsEnabled>
|
||||
<http class="com.github.loki4j.logback.ApacheHttpSender">
|
||||
<url>${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push}</url>
|
||||
</http>
|
||||
<format>
|
||||
<label>
|
||||
<pattern>app=${APP_NAME:- },host=${HOSTNAME},level=%level</pattern>
|
||||
</label>
|
||||
<message>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</message>
|
||||
<sortByTime>true</sortByTime>
|
||||
</format>
|
||||
</appender>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Loki"/>
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.lanyuanxiaoyao.service.loki;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
public class LokiQueryParse {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryParse.class);
|
||||
|
||||
private static String buildLokiQuery(ImmutableMap<String, String> queryMap) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
String query = queryMap
|
||||
.collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value))
|
||||
.valuesView()
|
||||
.makeString(",");
|
||||
builder.append(query);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws JsonProcessingException {
|
||||
// String query = URLUtil.encodeQuery("{app=\"hudi-sync\",run_type=\"compaction\",level=~\"WARN|ERROR\"}");
|
||||
// HttpResponse response = HttpUtil.createGet(StrUtil.format("http://132.122.116.142:33100/loki/api/v1/query_range?query={}&limit=10", query)).execute();
|
||||
// // logger.info("{}\n{}\n{}", query, response.getStatus(), response.body());
|
||||
// ObjectMapper mapper = new ObjectMapper();
|
||||
// mapper.registerModule(new EclipseCollectionsModule());
|
||||
// mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
|
||||
// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
// LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class);
|
||||
// logger.info("{}", queryRangeResponse);
|
||||
logger.info("{}", buildLokiQuery(Maps.immutable.of("app", "sync", "run_type", "compact")));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user