完成全部功能,优化图表和对话

This commit is contained in:
2025-03-01 18:44:27 +08:00
parent bfffbc9868
commit d90029e7c2
15 changed files with 9338 additions and 75 deletions

37
.idea/csv-editor.xml generated Normal file
View File

@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CsvFileAttributes">
<option name="attributeMap">
<map>
<entry key="/dumpDataPreview">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
<entry key="/knowledge/data.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
<entry key="/knowledge/data2.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
<entry key="/knowledge/host_assets.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
</map>
</option>
</component>
</project>

View File

@@ -6,6 +6,16 @@
<option name="name" value="Central Repository" />
<option name="url" value="https://repo.maven.apache.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="local-releases" />
<option name="name" value="Local Releases" />
<option name="url" value="http://localhost:3105/threepartrepo" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Central Repository" />
<option name="url" value="http://localhost:3105/threepartrepo" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Maven Central repository" />

1
.idea/sqldialects.xml generated
View File

@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/knowledge/host_assets.sql" dialect="MySQL" />
<file url="PROJECT" dialect="MySQL" />
</component>
</project>

View File

@@ -6,7 +6,7 @@ import * as echarts from 'echarts'
import {CategoryScale, Chart, LinearScale, LineController, LineElement, PointElement, Title,} from 'chart.js'
import 'chartjs-adapter-moment'
import mermaid from 'mermaid'
import {each, isEqual, isJson, nextTick} from 'licia'
import {each, isEqual, isNull, nextTick} from 'licia'
Chart.register(
LineController,
@@ -35,14 +35,19 @@ md.use(function (md) {
const content = tokens[idx].content
// 此处判断是否为 echarts 代码块
if (tokens[idx].info === 'echarts') {
if (isJson(content)) {
chartsMap.value.echarts[idx] = JSON.parse(content) //此处表示将内容存起来,存到当前页面的变量去
try {
chartsMap.value.echarts[idx] = JSON.parse(content)
console.log('ok', content)
} catch (e) {
console.error(e, content)
}
return `<div id="echarts-${idx}" style="width: 600px;height:400px;"></div>`
} else if (tokens[idx].info === 'chartjs') {
// 此处判断是否为 chartjs 代码块
if (isJson(content)) {
try {
chartsMap.value.chartjs[idx] = JSON.parse(content)
} catch (e) {
console.error(e, content)
}
return `<canvas id="chartjs-${idx}"></canvas>`
} else if (tokens[idx].info === 'mermaid') {
@@ -97,6 +102,7 @@ const handleSubmit = async () => {
mermaid: {},
}
let eventSource
try {
const url = 'http://localhost:7891/chat/stream'
const formData = new FormData()
@@ -112,7 +118,7 @@ const handleSubmit = async () => {
formData.append('database_name', databaseName.value)
}
const eventSource = createEventSource({
eventSource = createEventSource({
url: url,
method: 'POST',
body: formData,
@@ -131,6 +137,9 @@ const handleSubmit = async () => {
answer.value = '抱歉,请求出现错误'
renderedAnswer.value = answer.value
showToastMessage('请求失败,请检查服务器连接', 'error')
if (!isNull(eventSource)) {
eventSource.close()
}
} finally {
loading.value = false
}
@@ -224,7 +233,7 @@ onMounted(() => {
<template>
<div class="chat-container">
<div class="chat-header">
<h1>数据库助手</h1>
<h1>知识库助手</h1>
</div>
<div class="messages-container">

4469
knowledge/host_assets.csv Normal file

File diff suppressed because it is too large Load Diff

30
knowledge/host_assets.sql Normal file
View File

@@ -0,0 +1,30 @@
CREATE TABLE `host_assets`
(
`id` bigint NOT NULL AUTO_INCREMENT COMMENT 'ID',
`host_ip` varchar(100) DEFAULT NULL COMMENT '业务IP地址该地址表示对外暴露业务的主机IP通常业务由1个或多个主机共同提供服务其他不对外暴露的主机的IP记录在all_ip字段中',
`all_ip` varchar(500) DEFAULT NULL COMMENT '提供业务支撑的所有主机的IP',
`system_name` varchar(255) DEFAULT NULL COMMENT '所属业务系统名称',
`location` varchar(255) DEFAULT NULL COMMENT '主机所在机房具体地址',
`manager_ip` varchar(100) DEFAULT NULL COMMENT '如果业务提供对外的管理控制台则控制台的IP地址记录在这里如果没有则记录为空',
`host_name` varchar(100) DEFAULT NULL COMMENT '主机host名称',
`host_os` varchar(100) DEFAULT NULL COMMENT '主机操作系统',
`manufacturer` varchar(255) DEFAULT NULL COMMENT '主机设备制造厂商',
`host_model` varchar(255) DEFAULT NULL COMMENT '主机设备型号',
`host_sn` varchar(100) DEFAULT NULL COMMENT '主机序列号',
`cpu_model` varchar(100) DEFAULT NULL COMMENT 'cpu型号',
`cpu_num` varchar(100) DEFAULT NULL COMMENT 'cpu数量',
`memory` varchar(100) DEFAULT NULL COMMENT '内存大小除非该字段记录标明单位否则默认单位为GB',
`system_disk` varchar(100) DEFAULT NULL COMMENT '系统盘容量除非该字段记录标明单位否则默认单位为GB',
`data_disk` varchar(100) DEFAULT NULL COMMENT '数据盘容量除非该字段记录标明单位否则默认单位为GB',
`start_date` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL COMMENT '设备生产日期',
`monitor` varchar(100) DEFAULT NULL COMMENT '是否使用基础监控',
`component` varchar(100) DEFAULT NULL COMMENT '主机上部署了哪些应用组件',
`host_state` varchar(100) DEFAULT NULL COMMENT '主机状态',
`state_remark` varchar(255) DEFAULT NULL COMMENT '主机状态详细说明',
`responsible_person` varchar(100) DEFAULT NULL COMMENT '主机责任人',
`manager` varchar(100) DEFAULT NULL COMMENT '主机所属业务的业务经理',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
AUTO_INCREMENT = 8532
DEFAULT CHARSET = utf8mb4
COLLATE = utf8mb4_0900_ai_ci COMMENT ='主机资产表,该表记录了主机的详细信息,包括配置、部署组件、业务等'

File diff suppressed because it is too large Load Diff

40
pom.xml
View File

@@ -47,6 +47,46 @@
<artifactId>hutool-all</artifactId>
<version>5.8.32</version>
</dependency>
<dependency>
<groupId>net.datafaker</groupId>
<artifactId>datafaker</artifactId>
<version>2.4.2</version>
</dependency>
<!--<dependency>
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty-http</artifactId>
<version>1.2.3</version>
</dependency>-->
<!--<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-reactive-httpclient</artifactId>
<version>4.0.8</version>
</dependency>-->
<!--<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>5.4.2</version>
</dependency>-->
<!--<dependency>
<groupId>org.apache.httpcomponents.core5</groupId>
<artifactId>httpcore5-reactive</artifactId>
<version>5.3.3</version>
</dependency>-->
<!--<dependency>
<groupId>com.openai</groupId>
<artifactId>openai-java</artifactId>
<version>0.30.0</version>
</dependency>-->
<!--<dependency>
<groupId>com.agentsflex</groupId>
<artifactId>agents-flex-bom</artifactId>
<version>1.0.0-rc.6</version>
</dependency>
<dependency>
<groupId>com.agentsflex</groupId>
<artifactId>agents-flex-spring-boot-starter</artifactId>
<version>1.0.0-rc.6</version>
</dependency>-->
<!-- <dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-qdrant-store-spring-boot-starter</artifactId>

View File

@@ -1,11 +1,35 @@
package com.lanyuanxiaoyao.ai.analysis;
import java.net.http.HttpClient;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Primary;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.web.client.RestClient;
@Slf4j
@SpringBootApplication
public class AnalysisApplication {
public class AnalysisApplication implements ApplicationRunner {
public static void main(String[] args) {
SpringApplication.run(AnalysisApplication.class, args);
}
@Override
public void run(ApplicationArguments args) {
}
@Bean
@Primary
public RestClient.Builder clientBuilder() {
return RestClient.builder()
.requestFactory(new JdkClientHttpRequestFactory(
HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build()
));
}
}

View File

@@ -1,10 +1,8 @@
package com.lanyuanxiaoyao.ai.analysis.controller;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.ai.analysis.tools.DatabaseTools;
import com.lanyuanxiaoyao.ai.analysis.tools.DatetimeTools;
import com.lanyuanxiaoyao.ai.analysis.tools.PromptTools;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -16,7 +14,6 @@ import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
@Slf4j
@@ -30,25 +27,6 @@ public class ChatController {
this.client = builder.build();
}
@ResponseBody
@PostMapping(value = "", produces = MediaType.TEXT_PLAIN_VALUE)
public String chat(
@RequestParam("prompt") String prompt,
@RequestParam(value = "use_tw", defaultValue = "false") Boolean useTw,
@RequestParam(value = "use_database", defaultValue = "false") Boolean useDatabase,
@RequestParam(value = "database_url", required = false) String databaseUrl,
@RequestParam(value = "database_username", required = false) String databaseUsername,
@RequestParam(value = "database_password", required = false) String databasePassword,
@RequestParam(value = "database_name", required = false) String databaseName
) {
logParameters(prompt, useTw, useDatabase, databaseUrl, databaseUsername, databasePassword, databaseName);
return client.prompt()
.system("始终在中文语境下回答")
.user(prompt)
.call()
.content();
}
@PostMapping(value = "stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter chatStream(
@RequestParam("prompt") String prompt,
@@ -58,29 +36,161 @@ public class ChatController {
@RequestParam(value = "database_username", required = false) String databaseUsername,
@RequestParam(value = "database_password", required = false) String databasePassword,
@RequestParam(value = "database_name", required = false) String databaseName
) {
log.info("chatStream");
logParameters(prompt, useTw, useDatabase, databaseUrl, databaseUsername, databasePassword, databaseName);
) throws IOException {
log.info("prompt: {}", prompt);
log.info("use_tw: {}", useTw);
log.info("use_database: {}", useDatabase);
if (useDatabase) {
log.info("database_url: {}", databaseUrl);
log.info("database_username: {}", databaseUsername);
log.info("database_password: {}", databasePassword);
log.info("database_name: {}", databaseName);
}
SseEmitter emitter = new SseEmitter();
StringBuilder systemPrompt = new StringBuilder("始终在中文语境下回答\n\n");
systemPrompt
.append("当遇到数据对比、占比、趋势等场景相关的提问优先考虑使用mermaid绘制图表图文结合来回答")
.append(PromptTools.MERMAID_PROMPT)
.append("\n\n");
List<Object> tools = new ArrayList<>();
tools.add(new DatetimeTools());
List<String> prompts = new ArrayList<>();
// language=TEXT
prompts.add("""
你是一名服务器主机运维管理员,对于用户提出的问题,你会尝试从多个方面思考分析,通过多种角度进行研究,使用尽可能多的数据对比对用户的问题进行回答
你会积极使用表格、echarts图表等形式对数据进行分类、统计力求能直观表达数据之间的关系和趋势尤其是用户问题中出现“趋势”、“占比”、“对比”等字眼的时候
你始终会在中文语境下回答对于用户的每个问题先提出4个和用户提问最为相关的问题针对以上5个问题分别进行回答
对于echarts图表严格按照如下格式进行绘制这部份代码必须新起一段
```echarts
(这里是echarts的json配置)
```
你能够生成echarts json配置以在回答中可视化数据。当您认为图表能更好地说明信息或增强回答的可理解性时应创建相应的echarts json配置。
请遵循以下步骤生成配置:
1. 确定是否需要图表:评估用户的问题或任务是否涉及数据可视化,例如比较数据、展示趋势或比例。
2. 选择合适的图表类型:根据数据特性和表达目的选择图表类型。例如:
- 柱状图bar适合比较分类数据如不同国家的销量。
- 折线图line适合展示时间序列或趋势如温度变化。
- 饼图pie适合展示比例或百分比如市场份额。
3. 准备数据将数据组织成echarts可理解的格式。例如
- 柱状图/折线图x轴为类别或时间点y轴为数值。
- 饼图包含名称name和值value的数组。
4. 生成json配置基于echarts的配置规则创建有效的json对象包括标题title、轴xAxis/yAxis如适用、系列series等必要组件。
5. 正确格式化输出确保json配置是有效的字符串使用双引号包裹字符串值数值不加引号。
以下是不同图表类型的示例,帮助您理解如何构建配置:
示例1柱状图
场景展示2023年五个城市的销售额。
数据北京5000、上海4500、广州3000、深圳2800、杭州2000。
输出:
```echarts
{
"title": {
"text": "2023 年城市销售额"
},
"xAxis": {
"type": "category",
"data": ["北京", "上海", "广州", "深圳", "杭州"]
},
"yAxis": {
"type": "value",
"axisLabel": {
"formatter": "{value} 万"
}
},
"series": [
{
"data": [5000, 4500, 3000, 2800, 2000],
"type": "bar"
}
]
}
```
示例2折线图
场景展示某地区过去6个月的平均温度。
数据1月 5°C、2月 8°C、3月 12°C、4月 18°C、5月 22°C、6月 25°C。
输出:
```echarts
{
"title": {
"text": "过去 6 个月平均温度"
},
"xAxis": {
"type": "category",
"data": ["1月", "2月", "3月", "4月", "5月", "6月"]
},
"yAxis": {
"type": "value",
"axisLabel": {
"formatter": "{value} °C"
}
},
"series": [
{
"data": [5, 8, 12, 18, 22, 25],
"type": "line"
}
]
}
```
示例3饼图
场景:展示三种水果的市场份额。
数据苹果40%、橙子35%、香蕉25%。
输出:
```echarts
{
"title": {
"text": "水果市场份额"
},
"series": [
{
"type": "pie",
"radius": "50%",
"data": [
{ "name": "苹果", "value": 40 },
{ "name": "橙子", "value": 35 },
{ "name": "香蕉", "value": 25 }
],
"label": {
"formatter": "{b}: {c}%"
}
}
]
}
```
注意事项
json格式字符串值使用双引号"text"数值不加引号如5000不能使用任何注释。
有效性确保json配置语法正确可直接用于echarts。
相关性:仅在图表能为回答增添价值时生成配置,避免不必要的可视化。""");
if (useDatabase) {
log.info("use database");
DatabaseTools databaseTools = new DatabaseTools(databaseUrl, databaseUsername, databasePassword);
systemPrompt.append("以下是Hudi数据库中已有表的详细信息包含表名、表描述、字段名和字段描述任何与这些表数据相关的问题都优先查询数据库获取答案\n");
systemPrompt.append(databaseTools.getTableInformation(databaseName)).append("\n");
systemPrompt.append("任何使用Hudi数据库的数据的回答都必须从实际的数据中获取,严禁虚构任何下列信息中不存在的表或表字段,没有描述的表或字段可以从表名或字段名中推测,如果找不到答案,就回复数据库中没有找到相关数据\n");
// language=TEXT
prompts.add("""
你将所管理的主机信息记录在据数据库中schema名为main、表名为host_assets的MySQL表中任何关于主机的数据的回答都必须从实际的数据中获取,严禁虚构任何不存在的表或表字段,没有描述的表或字段可以从表名或字段名中推测,如果找不到答案,就回复数据库中没有找到相关数据
任何时候你都只能查询数据库不会对数据库进行任何修改操作也不会查询除host_assets外的任何表数据库查询结果中出现null表示没有数据
用户问题中提到的关键词通常不是精准的字段名或字段内容,你需要结合精确匹配或模糊匹配等等多种方式综合查询数据,必要的时候可以多次查询数据库综合查询数据
以下是host_assets表的详细信息包含表名、表描述、字段名和字段描述""");
prompts.add(databaseTools.getTableInformation(databaseName));
prompts.add("");
tools.add(databaseTools);
}
client.prompt()
.system(systemPrompt.toString())
String content = client.prompt()
.system(String.join("\n", prompts))
.user(prompt)
.tools(tools.toArray(new Object[]{}))
.call()
.content();
log.info("{}", content);
for (String line : content.split("\n")) {
emitter.send(StrUtil.format("#/#{}#/#", line));
emitter.send(StrUtil.format("#/#{}#/#", "\n"));
}
emitter.send("#CLOSE#");
emitter.complete();
/* client.prompt()
.system(String.join("\n", prompts))
.user(prompt)
.tools(tools.toArray(new Object[]{}))
.stream()
@@ -94,20 +204,15 @@ public class ChatController {
}
},
emitter::completeWithError,
emitter::complete
);
() -> {
try {
emitter.send("#CLOSE#");
emitter.complete();
} catch (IOException e) {
emitter.completeWithError(e);
}
}
); */
return emitter;
}
private void logParameters(String prompt, Boolean useTw, Boolean useDatabase, String databaseUrl, String databaseUsername, String databasePassword, String databaseName) {
log.info("prompt: {}", prompt);
log.info("use_tw: {}", useTw);
log.info("use_database: {}", useDatabase);
if (useDatabase) {
log.info("database_url: {}", databaseUrl);
log.info("database_username: {}", databaseUsername);
log.info("database_password: {}", databasePassword);
log.info("database_name: {}", databaseName);
}
}
}

View File

@@ -52,6 +52,7 @@ public class DatabaseTools {
}
builder.append("\n");
}
log.info("\n{}", builder);
return builder.toString();
} catch (SQLException e) {
return StrUtil.format("查询失败,失败提示:{}", e.getMessage());

View File

@@ -3,7 +3,6 @@ package com.lanyuanxiaoyao.ai.analysis.tools;
import java.time.LocalDateTime;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.function.FunctionToolCallback;
/**
* @author lanyuanxiaoyao
@@ -11,14 +10,8 @@ import org.springframework.ai.tool.function.FunctionToolCallback;
*/
@Slf4j
public class DatetimeTools {
public static final FunctionToolCallback<Void, LocalDateTime> call = FunctionToolCallback.builder("current_time", () -> LocalDateTime.now())
.description("返回当前的日期时间")
.inputType(Void.class)
.build();
@Tool(description = "返回当前的日期时间")
public String currentTime() {
return LocalDateTime.now().toString();
}
}

View File

@@ -4,12 +4,32 @@ spring:
application:
name: ai-analysis
ai:
# openai:
# base-url: https://api.deepseek.com
# api-key: "sk-c338699912bb4ad891874a9ab713e735"
# chat:
# options:
# model: "deepseek-chat"
# ai:
# openai:
# base-url: http://132.121.116.79:31348
# api-key: "nopassword"
# chat:
# options:
# model: "deepseek-coder"
# openai:
# base-url: http://132.121.116.70:31409
# api-key: "nopassword"
# chat:
# options:
# model: "telechat2-7B"
openai:
base-url: https://api.siliconflow.cn
api-key: "sk-xrguybusoqndpqvgzgvllddzgjamksuecyqdaygdwnrnqfwo"
base-url: https://open.bigmodel.cn/api/paas/v4
api-key: "d1e97306540d12bb2f834be961fcacb1.SNBShlCxWYJCx0qZ"
chat:
completions-path: /chat/completions
options:
model: "Qwen/Qwen2.5-32B-Instruct"
model: "glm-4-flash"
ollama:
chat:
model: "qwen2.5:7b"
@@ -20,12 +40,10 @@ spring:
chat:
options:
model: "glm-4-flash"
# openai:
# base-url: http://173.0.59.240:8093
# api-key: test
mvc:
async:
request-timeout: -1
logging:
level:
org.springframework.ai: trace
# root: debug
org.springframework.ai: debug

View File

@@ -0,0 +1,51 @@
package com.lanyuanxiaoyao.ai.analysis;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.db.DbUtil;
import cn.hutool.db.Entity;
import cn.hutool.db.Session;
import cn.hutool.db.ds.simple.SimpleDataSource;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import lombok.extern.slf4j.Slf4j;
import net.datafaker.Faker;
/**
* @author lanyuanxiaoyao
* @version 20250301
*/
@Slf4j
public class FakeData {
public static void main(String[] args) {
Faker faker = new Faker(new Locale("zh-CN"));
try (Session session = DbUtil.newSession(new SimpleDataSource("jdbc:mysql://localhost:3307/main", "test", "test"))) {
String fieldName = "system_name";
List<String> items = new ArrayList<>();
List<Entity> entities = session.query(StrUtil.format("select distinct {} as name from host_assets where {} is not null", fieldName, fieldName));
for (Entity entity : entities) {
/* Set<String> fields = entity.getFieldNames();
for (String field : fields) {
items.add(entity.get(field).toString());
} */
items.add(entity.getStr("name"));
}
for (String item : items) {
List<String> tags = List.of("数据治理", "数据安全管控", "区块链应用", "标签管理", "数据集市");
String shadow = StrUtil.format("大数据湖{}-{}", RandomUtil.randomInt(50), tags.get(RandomUtil.randomInt(tags.size())));
String domain = StrUtil.format("b{}.hdp.dc", RandomUtil.randomInt(10));
String sql = StrUtil.format("update host_assets set {} = '{}', host_name = '{}' where {} = '{}'", fieldName, shadow, domain, fieldName, item);
log.info("{}", sql);
try (Statement statement = session.getConnection().createStatement()) {
statement.execute(sql);
}
}
} catch (SQLException e) {
log.error(e.getMessage(), e);
}
}
}

View File

@@ -1,5 +1,11 @@
### Chat
POST http://localhost:7891/chat/stream
POST http://localhost:7891/chat
Content-Type: application/x-www-form-urlencoded
prompt=tb_app_collect_table_info中src_schema字段各有多少记录&use_database=true&database_url=jdbc:mysql://localhost:3307/main&database_username=test&database_password=test&database_name=main
prompt=你好
### Test local
POST http://132.121.116.79:31348/v1/chat/completions
Content-Type: application/json
{"messages":[{"content":"你好","role":"user"}],"model":"deepseek-coder","stream":true,"temperature":0.7}