feature(loki-query): 新增 loki 日志查询
This commit is contained in:
@@ -2,11 +2,12 @@
|
||||
|
||||
mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
|
||||
|
||||
mvn -pl service-flink-query,service-hudi-query,service-info-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
|
||||
mvn -pl service-flink-query,service-hudi-query,service-info-query,service-loki-query,service-pulsar-query,service-yarn-query,service-zookeeper-query,service-web clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
|
||||
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-flink-query/target/service-flink-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-hudi-query/target/service-hudi-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-info-query/target/service-info-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-pulsar-query/target/service-pulsar-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-yarn-query/target/service-yarn-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-zookeeper-query/target/service-zookeeper-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
@@ -15,6 +16,7 @@ sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/i
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-flink-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-hudi-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-info-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-loki-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-pulsar-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-yarn-query-1.0.0-SNAPSHOT.jar'
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-zookeeper-query-1.0.0-SNAPSHOT.jar'
|
||||
|
||||
5
bin/build-loki-query.sh
Executable file
5
bin/build-loki-query.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
|
||||
mvn -pl service-loki-query clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) scp /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-loki-query/target/service-loki-query-1.0.0-SNAPSHOT.jar iap@132.122.1.162:/apps/iap/tmp/lanyuanxiaoyao
|
||||
sshpass -p $(/Users/lanyuanxiaoyao/Project/Work/Host/keepassxc-password.sh SSH/iap/132.122.1.162) ssh -o 'StrictHostKeyChecking no' iap@132.122.1.162 'curl ftp://yyy:QeY\!68\)4nH1@132.121.122.15:2222 -T /apps/iap/tmp/lanyuanxiaoyao/service-loki-query-1.0.0-SNAPSHOT.jar'
|
||||
1
pom.xml
1
pom.xml
@@ -20,6 +20,7 @@
|
||||
<module>service-zookeeper-query</module>
|
||||
<module>service-dependencies</module>
|
||||
<module>service-cli</module>
|
||||
<module>service-loki-query</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
|
||||
@@ -6,7 +6,8 @@ deploy:
|
||||
log-path: /apps/datalake/hudi/logs
|
||||
data-path: /apps/datalake/hudi/data
|
||||
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
|
||||
loki-url: http://132.122.116.142:33100/loki/api/v1/push
|
||||
loki-base-url: http://132.122.116.142:33100
|
||||
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
|
||||
zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
|
||||
eureka-url: http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.142:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.143:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.144:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.145:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.146:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.147:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.148:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.149:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.150:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.151:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.152:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.153:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.154:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.155:35670/eureka/,http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.122.116.156:35670/eureka
|
||||
hudi:
|
||||
@@ -95,6 +96,11 @@ deploy:
|
||||
- name: service-info-query
|
||||
source-jar: service-info-query-1.0.0-SNAPSHOT.jar
|
||||
replicas: 4
|
||||
- name: service-loki-query
|
||||
source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
|
||||
replicas: 3
|
||||
arguments:
|
||||
loki_host: ${deploy.runtime.loki-base-url}
|
||||
- name: service-yarn-query-b1
|
||||
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
|
||||
replicas: 4
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.lanyuanxiaoyao.service.configuration.entity.loki;
|
||||
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
|
||||
/**
|
||||
* 一行日志
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
public class LokiLogLine {
|
||||
private ImmutableMap<String, String> tags;
|
||||
private Long timestamp;
|
||||
private String data;
|
||||
|
||||
public LokiLogLine() {
|
||||
}
|
||||
|
||||
public LokiLogLine(ImmutableMap<String, String> tags, Long timestamp, String data) {
|
||||
this.tags = tags;
|
||||
this.timestamp = timestamp;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public ImmutableMap<String, String> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public Long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public String getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LokiLogLine{" +
|
||||
"tags=" + tags +
|
||||
", timestamp=" + timestamp +
|
||||
", data='" + data + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.lanyuanxiaoyao.service.forest.service;
|
||||
|
||||
import com.dtflys.forest.annotation.BaseRequest;
|
||||
import com.dtflys.forest.annotation.Get;
|
||||
import com.dtflys.forest.annotation.Query;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@BaseRequest(baseURL = "http://service-loki-query")
|
||||
public interface LokiService {
|
||||
@Get("/loki/query_range")
|
||||
ImmutableList<LokiLogLine> queryRange(@Query Map<String, String> queryMap);
|
||||
}
|
||||
36
service-loki-query/pom.xml
Normal file
36
service-loki-query/pom.xml
Normal file
@@ -0,0 +1,36 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>hudi-service</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-loki-query</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-dependencies</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-configuration</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.lanyuanxiaoyao.service.loki;
|
||||
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
|
||||
import com.lanyuanxiaoyao.service.loki.service.LokiQueryService;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 启动类
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@EnableDiscoveryClient
|
||||
@SpringBootApplication(
|
||||
scanBasePackages = {"com.lanyuanxiaoyao.service"},
|
||||
exclude = {GsonAutoConfiguration.class}
|
||||
)
|
||||
@EnableConfigurationProperties
|
||||
@EnableRetry
|
||||
@RequestMapping("loki")
|
||||
@RestController
|
||||
public class LokiQueryApplication {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryApplication.class);
|
||||
|
||||
public LokiQueryApplication(LokiQueryService lokiQueryService) {
|
||||
this.lokiQueryService = lokiQueryService;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(LokiQueryApplication.class, args);
|
||||
}
|
||||
|
||||
private final LokiQueryService lokiQueryService;
|
||||
|
||||
@GetMapping("query_range")
|
||||
public ImmutableList<LokiLogLine> queryRange(@RequestParam Map<String, String> queryMap) throws Exception {
|
||||
return lokiQueryService.queryRange(Maps.immutable.ofAll(queryMap));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.lanyuanxiaoyao.service.loki.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 配置
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@Component
|
||||
@ConfigurationProperties("loki")
|
||||
public class LokiConfiguration {
|
||||
private String host;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LokiConfiguration{" +
|
||||
"url='" + host + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package com.lanyuanxiaoyao.service.loki.entity;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Loki 查询接收
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
public final class LokiQueryRangeResponse {
|
||||
private String status;
|
||||
private Result data;
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public Result getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LokiQueryRangeResponse{" +
|
||||
"status='" + status + '\'' +
|
||||
", data=" + data +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Result {
|
||||
@JsonProperty("resultType")
|
||||
private String type;
|
||||
private ImmutableList<Line> result;
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public ImmutableList<Line> getResult() {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Result{" +
|
||||
"type='" + type + '\'' +
|
||||
", result=" + result +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Line {
|
||||
private ImmutableMap<String, String> stream;
|
||||
private ImmutableList<ImmutableList<String>> values;
|
||||
|
||||
public ImmutableMap<String, String> getStream() {
|
||||
return stream;
|
||||
}
|
||||
|
||||
public ImmutableList<ImmutableList<String>> getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Line{" +
|
||||
"stream=" + stream +
|
||||
", values=" + values +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.lanyuanxiaoyao.service.loki.service;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
|
||||
import com.lanyuanxiaoyao.service.loki.configuration.LokiConfiguration;
|
||||
import com.lanyuanxiaoyao.service.loki.entity.LokiQueryRangeResponse;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
|
||||
import org.springframework.retry.annotation.Retryable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Loki 查询服务
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
@Service
|
||||
public class LokiQueryService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryService.class);
|
||||
|
||||
private final LokiConfiguration lokiConfiguration;
|
||||
private final ObjectMapper mapper;
|
||||
|
||||
public LokiQueryService(LokiConfiguration lokiConfiguration, Jackson2ObjectMapperBuilder builder) {
|
||||
this.lokiConfiguration = lokiConfiguration;
|
||||
this.mapper = builder.build();
|
||||
}
|
||||
|
||||
private String buildLokiQuery(ImmutableMap<String, String> queryMap) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
String query = queryMap
|
||||
.collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value))
|
||||
.valuesView()
|
||||
.makeString(",");
|
||||
builder.append(query);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
@Retryable(Throwable.class)
|
||||
public ImmutableList<LokiLogLine> queryRange(ImmutableMap<String, String> queryMap) throws Exception {
|
||||
String query = URLUtil.encodeQuery(buildLokiQuery(queryMap));
|
||||
String queryUrl = StrUtil.format("{}/loki/api/v1/query_range?query={}&limit=200", lokiConfiguration.getHost(), query);
|
||||
HttpResponse response = HttpUtil.createGet(queryUrl).execute();
|
||||
if (response.isOk()) {
|
||||
LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class);
|
||||
if (StrUtil.equals("success", queryRangeResponse.getStatus())) {
|
||||
ImmutableList<LokiQueryRangeResponse.Result.Line> results = queryRangeResponse.getData().getResult();
|
||||
return results.collect(result -> result.getValues().collect(value -> {
|
||||
Long timestamp = Long.valueOf(value.get(0));
|
||||
String data = value.get(1);
|
||||
return new LokiLogLine(result.getStream(), timestamp, data);
|
||||
})).flatCollect(list -> list);
|
||||
} else {
|
||||
throw new Exception("Loki receive not success: " + queryRangeResponse.getStatus());
|
||||
}
|
||||
} else {
|
||||
throw new Exception("Http query error: " + response.getStatus() + " for " + queryUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
7
service-loki-query/src/main/resources/application.yml
Normal file
7
service-loki-query/src/main/resources/application.yml
Normal file
@@ -0,0 +1,7 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-loki-query
|
||||
profiles:
|
||||
include: random-port,common,eureka,metrics
|
||||
loki:
|
||||
host: http://132.122.116.142:33100
|
||||
51
service-loki-query/src/main/resources/logback-spring.xml
Normal file
51
service-loki-query/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,51 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Loki" class="com.github.loki4j.logback.Loki4jAppender">
|
||||
<metricsEnabled>true</metricsEnabled>
|
||||
<http class="com.github.loki4j.logback.ApacheHttpSender">
|
||||
<url>${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push}</url>
|
||||
</http>
|
||||
<format>
|
||||
<label>
|
||||
<pattern>app=${APP_NAME:- },host=${HOSTNAME},level=%level</pattern>
|
||||
</label>
|
||||
<message>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</message>
|
||||
<sortByTime>true</sortByTime>
|
||||
</format>
|
||||
</appender>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} #@# : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Loki"/>
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.lanyuanxiaoyao.service.loki;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-05-25
|
||||
*/
|
||||
public class LokiQueryParse {
|
||||
private static final Logger logger = LoggerFactory.getLogger(LokiQueryParse.class);
|
||||
|
||||
private static String buildLokiQuery(ImmutableMap<String, String> queryMap) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("{");
|
||||
String query = queryMap
|
||||
.collectValues((key, value) -> StrUtil.format("{}=\"{}\"", key, value))
|
||||
.valuesView()
|
||||
.makeString(",");
|
||||
builder.append(query);
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws JsonProcessingException {
|
||||
// String query = URLUtil.encodeQuery("{app=\"hudi-sync\",run_type=\"compaction\",level=~\"WARN|ERROR\"}");
|
||||
// HttpResponse response = HttpUtil.createGet(StrUtil.format("http://132.122.116.142:33100/loki/api/v1/query_range?query={}&limit=10", query)).execute();
|
||||
// // logger.info("{}\n{}\n{}", query, response.getStatus(), response.body());
|
||||
// ObjectMapper mapper = new ObjectMapper();
|
||||
// mapper.registerModule(new EclipseCollectionsModule());
|
||||
// mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
|
||||
// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
// LokiQueryRangeResponse queryRangeResponse = mapper.readValue(response.body(), LokiQueryRangeResponse.class);
|
||||
// logger.info("{}", queryRangeResponse);
|
||||
logger.info("{}", buildLokiQuery(Maps.immutable.of("app", "sync", "run_type", "compact")));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user