From 45da452f1817f77a84825e61e2d6ca631c4a760b Mon Sep 17 00:00:00 2001 From: v-zhangjc9 Date: Mon, 16 Jun 2025 20:37:14 +0800 Subject: [PATCH] =?UTF-8?q?fix(ai-web):=20=E5=AE=8C=E6=88=90feedback=20AI?= =?UTF-8?q?=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../feedback/FeedbackController.java | 5 +- .../service/ai/web/entity/Feedback.java | 2 +- .../web/entity/context/FeedbackContext.java | 16 ++ .../web/service/feedback/FeedbackService.java | 70 +++++++- .../ai/web/service/node/FeedbackNodes.java | 169 +++++++++++++++++- .../src/main/resources/liteflow.xml | 10 ++ .../service/ai/web/TestLlm.java | 150 ++++++---------- 7 files changed, 318 insertions(+), 104 deletions(-) create mode 100644 service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/context/FeedbackContext.java diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java index 64cea79..d0f1bd5 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java @@ -32,7 +32,6 @@ public class FeedbackController { @PostMapping("add") public void add(@RequestBody CreateItem item) { - log.info("Enter method: add[item]. item:{}", item); feedbackService.add(item.source, ObjectUtil.defaultIfNull(item.pictures, Lists.immutable.empty())); } @@ -58,6 +57,8 @@ public class FeedbackController { private String source; private ImmutableList pictures; private Feedback.Status status; + private String analysis; + private String analysisShort; public ListItem(FileStoreProperties fileStoreProperties, Feedback feedback) { this.id = feedback.getId(); @@ -65,6 +66,8 @@ public class FeedbackController { this.pictures = feedback.getPictureIds() .collect(id -> StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), id)); this.status = feedback.getStatus(); + this.analysis = feedback.getAnalysis(); + this.analysisShort = feedback.getAnalysisShort(); } } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java index de0cf3a..c795282 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java @@ -9,7 +9,7 @@ public class Feedback { private String source; private String analysisShort; private String analysis; - private ImmutableList pictureIds; + private ImmutableList pictureIds; private Status status; private Long createdTime; private Long modifiedTime; diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/context/FeedbackContext.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/context/FeedbackContext.java new file mode 100644 index 0000000..df9b70d --- /dev/null +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/context/FeedbackContext.java @@ -0,0 +1,16 @@ +package com.lanyuanxiaoyao.service.ai.web.entity.context; + +import com.lanyuanxiaoyao.service.ai.web.entity.Feedback; +import java.util.List; +import lombok.Data; +import org.eclipse.collections.api.factory.Lists; + +/** + * @author lanyuanxiaoyao + * @version 20250616 + */ +@Data +public class FeedbackContext { + private Feedback feedback; + private List pictureDescriptions = Lists.mutable.empty(); +} diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java index d16145a..a0dd029 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java @@ -1,18 +1,24 @@ package com.lanyuanxiaoyao.service.ai.web.service.feedback; import club.kingon.sql.builder.SqlBuilder; +import cn.hutool.core.lang.Assert; import cn.hutool.core.util.EnumUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import com.lanyuanxiaoyao.service.ai.core.configuration.SnowflakeId; import com.lanyuanxiaoyao.service.ai.web.entity.Feedback; +import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext; import com.lanyuanxiaoyao.service.common.Constants; +import com.yomahub.liteflow.core.FlowExecutor; +import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -20,6 +26,7 @@ import org.springframework.transaction.annotation.Transactional; @Service public class FeedbackService { public static final String FEEDBACK_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_feedback"; + public static final String[] FEEDBACK_COLUMNS = new String[]{"id", "source", "analysis_short", "analysis", "pictures", "status", "created_time", "modified_time"}; private static final RowMapper feedbackMapper = (rs, row) -> { Feedback feedback = new Feedback(); feedback.setId(rs.getLong(1)); @@ -29,7 +36,7 @@ public class FeedbackService { feedback.setPictureIds( StrUtil.isBlank(rs.getString(5)) ? Lists.immutable.empty() - : Lists.immutable.ofAll(StrUtil.split(rs.getString(5), ",")) + : Lists.immutable.ofAll(StrUtil.split(rs.getString(5), ",")).collect(Long::parseLong) ); feedback.setStatus(EnumUtil.fromString(Feedback.Status.class, rs.getString(6))); feedback.setCreatedTime(rs.getTimestamp(7).getTime()); @@ -37,9 +44,38 @@ public class FeedbackService { return feedback; }; private final JdbcTemplate template; + private final FlowExecutor executor; - public FeedbackService(JdbcTemplate template) { + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + public FeedbackService(JdbcTemplate template, FlowExecutor executor) { this.template = template; + this.executor = executor; + } + + @Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES) + public void analysis() { + List feedbacks = template.query( + SqlBuilder.select(FEEDBACK_COLUMNS) + .from(FEEDBACK_TABLE_NAME) + .whereEq("status", Feedback.Status.ANALYSIS_PROCESSING.name()) + .build(), + feedbackMapper + ); + for (Feedback feedback : feedbacks) { + FeedbackContext context = new FeedbackContext(); + context.setFeedback(feedback); + executor.execute2Resp("feedback_analysis", null, context); + } + } + + public Feedback get(Long id) { + return template.queryForObject( + SqlBuilder.select(FEEDBACK_COLUMNS) + .from(FEEDBACK_TABLE_NAME) + .whereEq("id", id) + .build(), + feedbackMapper + ); } @Transactional(rollbackFor = Exception.class) @@ -55,9 +91,37 @@ public class FeedbackService { ); } + @Transactional(rollbackFor = Exception.class) + public void updateAnalysis(Long id, String analysis, String analysisShort) { + Assert.notNull(id, "ID cannot be null"); + template.update( + SqlBuilder.update(FEEDBACK_TABLE_NAME) + .set("analysis", "?") + .addSet("analysis_short", "?") + .whereEq("id", "?") + .precompileSql(), + analysis, + analysisShort, + id + ); + } + + @Transactional(rollbackFor = Exception.class) + public void updateStatus(Long id, Feedback.Status status) { + Assert.notNull(id, "ID cannot be null"); + template.update( + SqlBuilder.update(FEEDBACK_TABLE_NAME) + .set("status", "?") + .whereEq("id", "?") + .precompileSql(), + status.name(), + id + ); + } + public ImmutableList list() { return template.query( - SqlBuilder.select("id", "source", "analysis_short", "analysis", "pictures", "status", "created_time", "modified_time") + SqlBuilder.select(FEEDBACK_COLUMNS) .from(FEEDBACK_TABLE_NAME) .orderByDesc("created_time") .build(), diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/node/FeedbackNodes.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/node/FeedbackNodes.java index 02348b1..6e1ecc8 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/node/FeedbackNodes.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/node/FeedbackNodes.java @@ -1,14 +1,181 @@ package com.lanyuanxiaoyao.service.ai.web.service.node; +import cn.hutool.core.lang.Assert; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import com.lanyuanxiaoyao.service.ai.web.entity.Feedback; +import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext; +import com.lanyuanxiaoyao.service.ai.web.entity.vo.DataFileVO; +import com.lanyuanxiaoyao.service.ai.web.service.DataFileService; +import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService; import com.yomahub.liteflow.annotation.LiteflowComponent; +import com.yomahub.liteflow.annotation.LiteflowMethod; +import com.yomahub.liteflow.core.NodeComponent; +import com.yomahub.liteflow.enums.LiteFlowMethodEnum; +import com.yomahub.liteflow.enums.NodeTypeEnum; +import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.client.ChatClient; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.core.io.FileSystemResource; +import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; +@Slf4j @LiteflowComponent public class FeedbackNodes { private final ChatClient.Builder chatClientBuilder; + private final ChatClient.Builder visualChatClientBuilder; + private final DataFileService dataFileService; + private final FeedbackService feedbackService; - public FeedbackNodes(@Qualifier("chat") ChatClient.Builder chatClientBuilder) { + public FeedbackNodes( + @Qualifier("chat") ChatClient.Builder chatClientBuilder, + @Qualifier("visual") ChatClient.Builder visualClientBuilder, DataFileService dataFileService, FeedbackService feedbackService + ) { this.chatClientBuilder = chatClientBuilder; + this.visualChatClientBuilder = visualClientBuilder; + this.dataFileService = dataFileService; + this.feedbackService = feedbackService; + } + + @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_BOOLEAN, nodeId = "feedback_check_if_picture_needed", nodeName = "判断有图片进行识别", nodeType = NodeTypeEnum.BOOLEAN) + public boolean checkIfPictureReadNeeded(NodeComponent node) { + FeedbackContext context = node.getContextBean(FeedbackContext.class); + Feedback feedback = context.getFeedback(); + return ObjectUtil.isNotEmpty(feedback.getPictureIds()); + } + + @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "image_read", nodeName = "读取图片", nodeType = NodeTypeEnum.COMMON) + public void imageRead(NodeComponent node) { + FeedbackContext context = node.getContextBean(FeedbackContext.class); + Feedback feedback = context.getFeedback(); + ChatClient client = visualChatClientBuilder + // language=TEXT + .defaultSystem(""" + 你是一个专业的OCR解析助手。请严格按以下步骤处理用户上传的图片： + 1. 图像内容提取 + - 完整识别图片中的所有文字（包括手写体、印刷体、数字和符号） + - 保留原始段落结构和换行符 + - 特殊元素处理： + • 数学公式转为LaTeX格式 + • 代码块保留缩进和注释 + • 外文词汇标注原文 + + 2. 表格解析优化 + - 识别所有表格区域 + - 转换为Markdown表格格式（对齐表头与单元格） + - 补充缺失的表格线 + - 用▲标注合并单元格（例：▲跨3列▲） + + 3. 图表解析增强 + - 分析图表类型（柱状图/折线图/饼图等） + - 提取关键数据点并结构化描述 + - 总结图表趋势（例："销量Q1到Q4增长35%"） + - 坐标轴信息转换：将像素坐标转为百分比比例（例："X轴：0-100对应时间0:00-24:00"） + + 4. 输出规范 + - 按[文本][表格][图表]分区块输出 + - 表格/图表区域标注原始位置（例："[左上区域表格]"） + - 模糊内容用[?]标注并给出备选（例："年收[?]入（可能为'入'或'人'）"） + - 保持原始数据精度（不四舍五入） + + 立即开始处理用户图片，无需确认步骤。 + """) + .build(); + for (Long pictureId : feedback.getPictureIds()) { + DataFileVO file = dataFileService.downloadFile(pictureId); + log.info("Parse picture: {} {}", file.getFilename(), file.getPath()); + MimeType type = switch (StrUtil.blankToDefault(file.getType(), "").toLowerCase()) { + case "jpg", "jpeg" -> MimeTypeUtils.IMAGE_JPEG; + default -> MimeTypeUtils.IMAGE_PNG; + }; + String content = client.prompt() + .user(spec -> spec + .text("输出图片内容") + .media(type, new FileSystemResource(file.getPath()))) + .call() + .content(); + log.info("Picture: {}", content); + context.getPictureDescriptions().add(content); + } + } + + @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "feedback_suggest", nodeName = "大模型建议", nodeType = NodeTypeEnum.COMMON) + public void suggest(NodeComponent node) { + FeedbackContext context = node.getContextBean(FeedbackContext.class); + Feedback feedback = context.getFeedback(); + ChatClient client = chatClientBuilder.build(); + String description = client.prompt() + // language=TEXT + .system(""" + 你是一名专业的IT系统运维工程师，对于用户输入的关于系统的报障信息，你会严格遵循以下步骤进行处理 + + 1.输入 + [故障描述] + （这里是用户遇到的系统故障的详细描述） + + [相关截图] + （这里是用户遇到的系统故障相关的截图的文字描述，如果没有相关截图，这里会写“无”；如果有多张图片，图片和图片之间会使用“---”分隔） + + 2.处理逻辑 + 解析输入 + 读取并解析用户提供的故障描述和相关截图描述。 + 识别关键元素：包括故障类型（如硬件故障、软件错误）、受影响系统组件（如服务器、网络设备）、错误消息、发生时间、重现步骤、影响范围等。 + 如果截图描述存在，提取关键细节（如错误弹窗文本、系统状态截图），并将其作为辅助证据；如果无截图，则忽略此部分。 + 分析与重写故障描述 + 专业化改写：使用标准IT术语替换非专业用语（例如，“电脑死机”改为“系统无响应”，“连不上网”改为“网络连接中断”），并确保描述符合行业规范。 + 排除歧义：澄清模糊描述（如添加具体时间戳、系统版本、IP地址或错误代码），移除主观语言（如“我觉得”或“可能”），并添加必要上下文（如操作系统环境、相关服务运行状态）。 + 结构化组织：将故障描述重写为逻辑段落，格式包括： + 问题概述：简明总结故障本质（例如，“数据库服务异常导致应用无法访问”）。 + 详细症状：描述具体现象，包括错误消息、发生频率和影响范围（如“影响用户登录功能，错误代码500”）。 + 重现步骤：列出可复现故障的操作序列（如“1. 访问URL X；2. 触发操作 Y”）。 + 相关环境：添加系统细节（如“运行在Linux Ubuntu 20.04, Java 11环境”）。 + 整合截图信息：如果截图描述存在，将其嵌入重写中作为证据（例如，“根据截图，错误弹窗显示‘Connection timeout’”）。 + 质量校验 + 检查重写后的内容是否完整、一致且无歧义：确保所有用户输入细节都被涵盖，添加缺失信息（如建议的故障分类），并验证专业术语的准确性。 + 如果输入信息不足（如缺少时间戳或系统版本），在输出中添加注释提示用户补充。 + + 3.输出 + 输出一个专业、结构化的故障报告，格式清晰，可直接用于运维团队诊断。 + 重写的故障描述，以结构化段落呈现，涵盖问题概述、详细症状、重现步骤和相关环境。 + 输出将使用中性、客观语言，避免任何个人意见或建议，以确保报告专注于事实描述。 + """) + .call() + .content(); + Assert.notBlank(description, "Description cannot be blank"); + String analysis = client.prompt() + .system(""" + 你是一名专业的IT系统运维工程师，对于用户输入的报障信息，你会给出专业的意见 + """) + .user(StrUtil.format(""" + [故障描述] + {} + + [相关截图] + {} + """, + description, + ObjectUtil.isEmpty(context.getPictureDescriptions()) ? "无" : StrUtil.join(",", context.getPictureDescriptions()) + )) + .call() + .content(); + feedback.setAnalysis(analysis); + Assert.notBlank(description, "Analysis cannot be blank"); + String analysisShort = client.prompt() + .system(""" + 你是一名专业的文字编辑，对用户输入的内容，用一段话总结内容 + """) + .user(analysis) + .call() + .content(); + feedback.setAnalysisShort(analysisShort); + } + + @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "feedback_save", nodeName = "保存分析内容", nodeType = NodeTypeEnum.COMMON) + public void saveFeedback(NodeComponent node) { + FeedbackContext context = node.getContextBean(FeedbackContext.class); + Feedback feedback = context.getFeedback(); + feedbackService.updateAnalysis(feedback.getId(), feedback.getAnalysis(), feedback.getAnalysisShort()); + feedbackService.updateStatus(feedback.getId(), Feedback.Status.ANALYSIS_SUCCESS); } } diff --git a/service-ai/service-ai-web/src/main/resources/liteflow.xml b/service-ai/service-ai-web/src/main/resources/liteflow.xml index 4669393..e102cdc 100644 --- a/service-ai/service-ai-web/src/main/resources/liteflow.xml +++ b/service-ai/service-ai-web/src/main/resources/liteflow.xml @@ -23,4 +23,14 @@ SER(import_vector_source) + + SER( + IF( + feedback_check_if_picture_needed, + image_read + ), + feedback_suggest, + feedback_save + ) + \ No newline at end of file diff --git a/service-ai/service-ai-web/src/test/java/com/lanyuanxiaoyao/service/ai/web/TestLlm.java b/service-ai/service-ai-web/src/test/java/com/lanyuanxiaoyao/service/ai/web/TestLlm.java index 0eafd79..7d20219 100644 --- a/service-ai/service-ai-web/src/test/java/com/lanyuanxiaoyao/service/ai/web/TestLlm.java +++ b/service-ai/service-ai-web/src/test/java/com/lanyuanxiaoyao/service/ai/web/TestLlm.java @@ -1,114 +1,68 @@ package com.lanyuanxiaoyao.service.ai.web; -import java.net.http.HttpClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.lanyuanxiaoyao.service.ai.core.configuration.WebClientConfiguration; +import com.lanyuanxiaoyao.service.ai.web.configuration.LlmConfiguration; +import com.lanyuanxiaoyao.service.ai.web.configuration.LlmProperties; +import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.client.ChatClient; -import org.springframework.ai.openai.OpenAiChatModel; -import org.springframework.ai.openai.OpenAiChatOptions; -import org.springframework.ai.openai.api.OpenAiApi; -import org.springframework.http.client.JdkClientHttpRequestFactory; -import org.springframework.http.client.reactive.JdkClientHttpConnector; -import org.springframework.web.client.RestClient; -import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.core.io.FileSystemResource; +import org.springframework.util.MimeTypeUtils; /** * @author lanyuanxiaoyao * @version 20250526 */ +@Slf4j public class TestLlm { - private static final Logger log = LoggerFactory.getLogger(TestLlm.class); - public static void main(String[] args) { - HttpClient httpClient = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .build(); - ChatClient client = ChatClient.builder( - OpenAiChatModel.builder() - .openAiApi( - OpenAiApi.builder() - .baseUrl("http://132.121.206.65:10086") - .apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTjhR&v>>g*NwCs3tpQ5FVMFEF2VHVTj spec + // language=TEXT + .text("输出图片内容") + .media(MimeTypeUtils.IMAGE_PNG, new FileSystemResource("/Users/lanyuanxiaoyao/Downloads/图片_002_07.png"))) .call() .content(); log.info(content);