refactor(ai): 移除chat，合并chat和knowledge为web
以后有需要再拆分
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
@EnableScheduling
|
||||
public class WebApplication implements ApplicationRunner, ApplicationContextAware {
|
||||
private static ApplicationContext context;
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(WebApplication.class, args);
|
||||
}
|
||||
|
||||
public static <T> T getBean(Class<T> clazz) {
|
||||
return context.getBean(clazz);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
WebApplication.context = context;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "file-store")
|
||||
public class FileStoreConfiguration {
|
||||
private String downloadPrefix;
|
||||
private String uploadPath;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.time.Duration;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.noear.solon.ai.reranking.RerankingModel;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatModel;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatOptions;
|
||||
import org.springframework.ai.deepseek.api.DeepSeekApi;
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
@Slf4j
|
||||
@Configuration
|
||||
public class LlmConfiguration {
|
||||
@Bean("chat")
|
||||
public ChatClient.Builder chatClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getChat(), "chat properties is null");
|
||||
DeepSeekApi.Builder apiBuilder = DeepSeekApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getChat().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getChat().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getChat().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getChat().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
DeepSeekChatModel.builder()
|
||||
.deepSeekApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
DeepSeekChatOptions.builder()
|
||||
.model(llmProperties.getChat().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean("visual")
|
||||
public ChatClient.Builder visualClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getVisual(), "visual properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getVisual().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getVisual().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getVisual().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getVisual().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
OpenAiChatModel.builder()
|
||||
.openAiApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
OpenAiChatOptions.builder()
|
||||
.model(llmProperties.getChat().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EmbeddingModel embeddingModel(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getEmbedding(), "embedding properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getEmbedding().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getEmbedding().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getEmbedding().getEndpoint())) {
|
||||
apiBuilder.embeddingsPath(llmProperties.getEmbedding().getEndpoint());
|
||||
}
|
||||
return new OpenAiEmbeddingModel(
|
||||
apiBuilder.build(),
|
||||
MetadataMode.EMBED,
|
||||
OpenAiEmbeddingOptions.builder()
|
||||
.model(llmProperties.getEmbedding().getModel())
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RerankingModel rerankingModel(LlmProperties llmProperties) {
|
||||
Assert.notNull(llmProperties.getReranker(), "reranker properties is null");
|
||||
String url = llmProperties.getBaseUrl();
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getBaseUrl())) {
|
||||
url = llmProperties.getReranker().getBaseUrl();
|
||||
}
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getEndpoint())) {
|
||||
url += llmProperties.getReranker().getEndpoint();
|
||||
} else {
|
||||
url += "/v1/rerank";
|
||||
}
|
||||
return RerankingModel.of(url)
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getReranker().getApiKey(), llmProperties.getApiKey()))
|
||||
.model(llmProperties.getReranker().getModel())
|
||||
.timeout(Duration.ofMinutes(10))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "spring.llm")
|
||||
public class LlmProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private ChatProperties chat;
|
||||
private ChatProperties visual;
|
||||
private ChatProperties embedding;
|
||||
private ChatProperties reranker;
|
||||
|
||||
@Data
|
||||
public static class ChatProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private String model;
|
||||
private String endpoint;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
|
||||
public interface Prompts {
|
||||
String hudiBase = """
|
||||
Hudi数据同步服务
|
||||
实现从源端数据库（TeleDB、TelePG）到Hudi表的数据实时同步，并通过Hudi-Hive插件将Hudi表封为Hive外表（即目标端），供外部系统进行SQL查询。
|
||||
数据同步任务：基于Flink开发，运行在Hadoop Yarn集群上，该任务常驻运行在集群上，每个任务负责一张或多张表的据同步，同步任务从指定的Pulsar队列中实时读取数据，经过数据转换和业务处理后，写入Hudi表。
|
||||
数据压缩任务：基于Flink开发，运行在Hadoop Yarn集群上，该任务根据指定的调度策略周期性定时启动，每个任务将张指定的Hudi表中的log数据压缩为parquet数据，当压缩完成时，上一次压缩到当前压缩之间被写入的数据才能被外部统查询。
|
||||
服务管理平台：使用Spring Cloud框架开发的微服务集群，用于管理同步任务和压缩任务的运行、停止和状态监控等息，提供友好的前端交互网页。
|
||||
源端（上游）：类似TeleDB、TelePG这些提供原始数据的数据库。
|
||||
目标端（下游）：数据同步到Hudi表后提供给外部系统查询用的Hive表。
|
||||
TeleDB：中国电信自研分布式MySQL集群组件，逻辑上的一张表在物理上被水平分片（Sharding）为多个“set”表存储在个MySQL节点，支持弹性扩容和容灾。
|
||||
TelePG：中国电信自研分布式PostgreSQL集群组件，其架构和功能特性与TeleDB高度相似。
|
||||
逻辑表：一张逻辑表对应一张TeleDB或TelePG上的业务表；逻辑表中包含广东所有地市的业务数据，数据量大的表通常按地市进行分片（分区）。
|
||||
Hudi表：逻辑表数据经同步任务处理后实际存储的位置，根据逻辑表的数据量，数据量大的会根据地市分为A、B表，B也被称为大表，通常包含广深东佛（广州、深圳、东莞、佛山）四个大区的数据，A表包含广东除了广深东佛外的其他市，特大表，如acct_item，会按一个地市对应一个Hudi表；Hudi表统一配置为MOR表，使用Bucket Index索引。
|
||||
Hive表：通过Hudi-Hive插件创建的Hive外表，作为下游系统的唯一查询入口，该表逻辑上对应源端的一张逻辑表，它Hudi服务内部可能存在的多个物理Hudi表（如A表、B表或地市分表）聚合封装成一个逻辑视图，透明地对外提供完整的辑表数据查询。
|
||||
重点表：根据业务系统的要求，对于有的表及时性和数据完整性有更高的要求，这些表被称为重点表，在tb_app_collect_table_info表中的tags字段，包含“FOCUS”字符的是重点表。
|
||||
Flink 任务：即数据同步任务，根据逻辑表的数据量通常有如下规则：
|
||||
大数据量：采用 1逻辑表:1 Flink任务 模式。该Flink任务内聚合处理该逻辑表对应的所有Hudi表（如A表 + B表或多地市表）的同步子任务。
|
||||
小数据量：采用 N逻辑表:1 Flink任务 模式。一个Flink任务内聚合处理多张低数据量逻辑表对应的所有Hudi表同步自务。
|
||||
Pulsar队列（消息队列）：源端TeleDB逻辑表增量日志（包含新增-I、更新-U、删除-D、DDL操作类型）由Canal同步组件实时写入Pulsar队列。TelePG逻辑表增量日志由PGSync组件以相同逻辑同步到对应队列。
|
||||
压缩调度服务：service-scheduler(单实例)：按策略将压缩任务放入预调度队列，定时将预调度任务转移至各集群对应的压缩任务队列；service-launcher(与集群一一对应)：定时轮询对应集群的压缩任务队列，发现任务即调度执行；service-queue：提供队列机制；scheduler与launcher均基于集群资源状态动态调节任务产生与执行速率，利用队列缓冲，避免资源超限。
|
||||
压缩调度：压缩任务耗时长、资源大。为平衡效率与资源，调度服务通常从凌晨2点起，每3小时调度一次全部Hudi表的压缩任务。
|
||||
跨天调度：为确保关键表数据在0点后及时可用（此时Hive最新数据常未达0点），0点至2点间，对重点表进行更高频压缩调度。此高频率调度持续直至目标表被标记为“已跨天”
|
||||
跨天判断：按照先后次序1. 源端同步组件(Canal/PGSync)判断源表数据是否跨天。若跨天，向队列写入跨天消息；2.同步任务接收跨天消息，在tb_app_collect_table_version表插入记录；3.独立程序判断Hudi表数据是否已跨天。若跨天，更新tb_app_collect_table_version对应记录状态。此时逻辑表标记为“已跨天”；一张表必须先接收到跨天标记，然后才能完成跨天。
|
||||
集群：指Hadoop Yarn集群，同步任务仅在b12运行，压缩任务主要在b12运行，部分重点表在b1或a4运行，调度服务根据资源动态在多个集群间分配压缩任务；其中b12集群使用default队列，b1集群使用datalake队列，a4集群使用ten_iap.datalake队列。
|
||||
""";
|
||||
|
||||
String hudiDatabase = """
|
||||
Hudi数据同步服务使用的数据表详情如下
|
||||
hudi_collect_build_b12.tb_app_collect_table_info表：记录同步任务配置信息
|
||||
id：主键
|
||||
alias：表别名，同样可以唯一标识该记录
|
||||
flink_job_id：tb_app_flink_job_config表主键，对应的Flink任务
|
||||
hudi_job_id：hudi：tb_app_hudi_job_config表主键，对应Hudi表的配置
|
||||
sync_yarn_job_id：tb_app_yarn_job_config表主键，同步任务的yarn资源配置
|
||||
compaction_yarn_job_id：tb_app_yarn_job_config表主键，压缩任务的yarn资源配置
|
||||
src_db：源端数据库名称
|
||||
src_type：源端数据库类型，取值有teledb、telepg
|
||||
src_schema：源端数据库schema名称
|
||||
src_table：源端表名
|
||||
src_pulsar_addr：pulsar地址
|
||||
src_topic：pulsar队列名称
|
||||
tgt_hdfs_path：Hudi表对应的hdfs路径
|
||||
status：逻辑删除状态（y为正常，n为删除）
|
||||
filter_field：过滤字段，用于指定源端消息中的某个字段，进行消息过滤，如B表，同步任务会使用CITY_ID字段，并过滤出该字段为广深东佛编码的消息
|
||||
filter_values：过滤值
|
||||
filter_type：过滤类型，INCLUDE表示包含指定过滤值的记录保留，EXCLUDE表示包含指定过滤值的记录丢弃
|
||||
bucket_number：Hudi bucket index的参数值
|
||||
partition_field：Hudi表分区字段，通常使用CITY_ID
|
||||
priority：优先级，整数，数字越大优先级越高，优先级高，该表对应在压缩调度队列中会排在更靠前的位置
|
||||
hive_db：目标端hive库名称
|
||||
hive_table：目标端hive表名
|
||||
tags：标签，使用英文逗号分隔，用于标识表的特殊属性
|
||||
|
||||
hudi_collect_build_b12.tb_app_collect_table_version表：记录跨天版本
|
||||
flink_job_id：tb_app_flink_job_config表主键，对应的Flink任务
|
||||
alias：表别名 tb_app_collect_table_info表alias字段，对应唯一同步任务
|
||||
version：版本，格式为yyyyMMdd，如2025年6月6日跨天版本，则该值为20250605
|
||||
op_ts：操作时间，接收到跨天标记的时间
|
||||
create_time：记录创建时间
|
||||
update_time：记录创建时间
|
||||
scheduled：是否已跨天，1为已跨天，0为未跨天
|
||||
|
||||
hudi_collect_build_b12.tb_app_flink_job_config表：Flink 任务配置
|
||||
id：主键
|
||||
name：Flink任务名称
|
||||
status：逻辑删除状态（y为正常，n为删除）
|
||||
application_id：flink任务对应的yarn任务的application id
|
||||
|
||||
hudi_collect_build_b12.tb_app_hudi_job_config表：Hudi表原生配置
|
||||
id：主键
|
||||
source_tasks：读取Pulsar消息的Flink算子并行度
|
||||
write_tasks：写Hudi表的Flink算子并行度
|
||||
keep_file_version：保留数据文件版本
|
||||
keep_commit_version：保留时间线版本
|
||||
|
||||
hudi_collect_build_b12.tb_app_hudi_sync_state表：同步、压缩任务运行状态
|
||||
id：主键，由flink_job_id和alias使用短横线连接而成，格式为：flink_job_id-alias
|
||||
message_id：最新消费到的Pulsar消息的message id
|
||||
source_start_time：同步任务启动时间
|
||||
source_receive_time：同步任务最新接收到消息队列消息的时间
|
||||
source_checkpoint_time：同步任务对应Flink任务最近一次执行checkpoint的时间，这个时间每15分钟更新一次，可以用来判断flink任务是否还在运行
|
||||
source_publish_time：同步任务接收到的最新一条消息队列的消息被发布到队列中的时间
|
||||
source_op_time：同步任务接收到的最新一条消息队列的消息在源端产生的时间
|
||||
source_application_id：同步任务对应Flink任务对应的yarn任务的application id
|
||||
source_cluster：同步任务运行的yarn集群
|
||||
compaction_start_time：最近一次压缩任务启动时间
|
||||
compaction_finish_time：最近一次压缩任务完成时间
|
||||
compaction_application_id：压缩任务对应的Flink任务对应的yarn任务的application id
|
||||
compaction_cluster：压缩任务运行所在的yarn集群
|
||||
|
||||
hudi_collect_build_b12.tb_app_yarn_job_config表：同步、压缩任务对应的yarn任务资源配置
|
||||
id：主键
|
||||
job_manager_memory：Job Manager内存（MB）
|
||||
task_manager_memory：Task Manager内存（MB）
|
||||
""";
|
||||
|
||||
String hudi = StrUtil.format(
|
||||
"""
|
||||
{}
|
||||
|
||||
{}
|
||||
""",
|
||||
hudiBase,
|
||||
hudiDatabase
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.io.IoUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.FileStoreConfiguration;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.FileChannel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
/**
|
||||
* 文件上传接口
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/upload")
|
||||
public class DataFileController {
|
||||
private final FileStoreConfiguration fileStoreConfiguration;
|
||||
private final DataFileService dataFileService;
|
||||
private final String uploadFolderPath;
|
||||
private final String cacheFolderPath;
|
||||
private final String sliceFolderPath;
|
||||
|
||||
public DataFileController(FileStoreConfiguration fileStoreConfiguration, DataFileService dataFileService) {
|
||||
this.fileStoreConfiguration = fileStoreConfiguration;
|
||||
this.dataFileService = dataFileService;
|
||||
|
||||
this.uploadFolderPath = fileStoreConfiguration.getUploadPath();
|
||||
this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath);
|
||||
this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath);
|
||||
}
|
||||
|
||||
@PostMapping("")
|
||||
public AmisResponse<FinishResponse> upload(@RequestParam("file") MultipartFile file) throws IOException {
|
||||
String filename = file.getOriginalFilename();
|
||||
Long id = dataFileService.initialDataFile(filename);
|
||||
String url = StrUtil.format("{}/upload/download/{}", fileStoreConfiguration.getDownloadPrefix(), id);
|
||||
byte[] bytes = file.getBytes();
|
||||
String originMd5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
|
||||
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, originMd5));
|
||||
if (targetFile.exists()) {
|
||||
dataFileService.updateDataFile(id, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), originMd5, file.getContentType());
|
||||
return AmisResponse.responseSuccess(new FinishResponse(id, filename, String.valueOf(id), url));
|
||||
}
|
||||
File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, id));
|
||||
cacheFile = FileUtil.writeBytes(bytes, cacheFile);
|
||||
String targetMd5 = SecureUtil.md5(cacheFile);
|
||||
if (!StrUtil.equals(originMd5, targetMd5)) {
|
||||
throw new RuntimeException("文件上传失败，校验不匹配");
|
||||
}
|
||||
FileUtil.move(cacheFile, targetFile, true);
|
||||
dataFileService.updateDataFile(id, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), targetMd5, file.getContentType());
|
||||
return AmisResponse.responseSuccess(new FinishResponse(id, filename, String.valueOf(id), url));
|
||||
}
|
||||
|
||||
@GetMapping("/download/{id}")
|
||||
public void download(@PathVariable Long id, HttpServletResponse response) throws IOException {
|
||||
DataFileVO dataFile = dataFileService.downloadFile(id);
|
||||
File targetFile = new File(dataFile.getPath());
|
||||
response.setHeader("Content-Type", dataFile.getType());
|
||||
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
|
||||
response.setHeader("Content-Disposition", StrUtil.format("attachment; filename={}", URLUtil.encodeAll(dataFile.getFilename())));
|
||||
IoUtil.copy(new FileInputStream(targetFile), response.getOutputStream());
|
||||
}
|
||||
|
||||
@PostMapping("/start")
|
||||
public AmisResponse<StartResponse> start(@RequestBody StartRequest request) {
|
||||
Long id = dataFileService.initialDataFile(request.filename);
|
||||
return AmisResponse.responseSuccess(new StartResponse(id.toString()));
|
||||
}
|
||||
|
||||
@PostMapping("/slice")
|
||||
public AmisResponse<SliceResponse> slice(
|
||||
@RequestParam("uploadId")
|
||||
Long uploadId,
|
||||
@RequestParam("partNumber")
|
||||
Integer sequence,
|
||||
@RequestParam("partSize")
|
||||
Long size,
|
||||
@RequestParam("file")
|
||||
MultipartFile file
|
||||
) throws IOException {
|
||||
byte[] bytes = file.getBytes();
|
||||
String md5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
|
||||
String targetFilename = StrUtil.format("{}-{}", sequence, md5);
|
||||
String targetFilePath = sliceFilePath(uploadId, targetFilename);
|
||||
FileUtil.mkParentDirs(targetFilePath);
|
||||
FileUtil.writeBytes(bytes, targetFilePath);
|
||||
return AmisResponse.responseSuccess(new SliceResponse(targetFilename));
|
||||
}
|
||||
|
||||
private String sliceFilePath(Long uploadId, String sliceFilename) {
|
||||
return StrUtil.format("{}/{}/{}", sliceFolderPath, uploadId, sliceFilename);
|
||||
}
|
||||
|
||||
@PostMapping("finish")
|
||||
public AmisResponse<FinishResponse> finish(@RequestBody FinishRequest request) {
|
||||
if (request.partList.anySatisfy(part -> !FileUtil.exist(sliceFilePath(request.uploadId, part.eTag)))) {
|
||||
throw new RuntimeException("文件校验失败，请重新上传");
|
||||
}
|
||||
try {
|
||||
File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId));
|
||||
FileUtil.mkParentDirs(cacheFile);
|
||||
if (cacheFile.createNewFile()) {
|
||||
try (FileOutputStream fos = new FileOutputStream(cacheFile)) {
|
||||
try (FileChannel fosChannel = fos.getChannel()) {
|
||||
for (FinishRequest.Part part : request.partList) {
|
||||
File sliceFile = new File(sliceFilePath(request.uploadId, part.eTag));
|
||||
try (FileInputStream fis = new FileInputStream(sliceFile)) {
|
||||
try (FileChannel fisChannel = fis.getChannel()) {
|
||||
fisChannel.transferTo(0, fisChannel.size(), fosChannel);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String md5 = SecureUtil.md5(cacheFile);
|
||||
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, md5));
|
||||
if (!targetFile.exists()) {
|
||||
FileUtil.move(cacheFile, targetFile, true);
|
||||
}
|
||||
String absolutePath = FileUtil.getAbsolutePath(targetFile);
|
||||
dataFileService.updateDataFile(
|
||||
request.uploadId,
|
||||
absolutePath,
|
||||
FileUtil.size(targetFile),
|
||||
SecureUtil.md5(targetFile),
|
||||
FileUtil.getMimeType(absolutePath)
|
||||
);
|
||||
return AmisResponse.responseSuccess(new FinishResponse(
|
||||
request.uploadId,
|
||||
request.filename,
|
||||
request.uploadId.toString(),
|
||||
StrUtil.format("{}/upload/download/{}", fileStoreConfiguration.getDownloadPrefix(), request.uploadId)
|
||||
));
|
||||
} else {
|
||||
throw new RuntimeException("合并文件失败");
|
||||
}
|
||||
} catch (Throwable throwable) {
|
||||
throw new RuntimeException(throwable);
|
||||
} finally {
|
||||
FileUtil.del(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId));
|
||||
FileUtil.del(StrUtil.format("{}/{}", sliceFolderPath, request.uploadId));
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class StartRequest {
|
||||
private String name;
|
||||
private String filename;
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class StartResponse {
|
||||
private String uploadId;
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class SliceResponse {
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class FinishRequest {
|
||||
private String filename;
|
||||
private Long uploadId;
|
||||
private ImmutableList<Part> partList;
|
||||
|
||||
@Data
|
||||
public static final class Part {
|
||||
private Integer partNumber;
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class FinishResponse {
|
||||
private Long id;
|
||||
private String filename;
|
||||
private String value;
|
||||
private String url;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.caht;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.Prompts;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.MessageVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.ChartTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.TableTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.YarnTool;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
/**
|
||||
* 聊天
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping("chat")
|
||||
public class ChatController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChatController.class);
|
||||
private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
private static final String ROLE_ASSISTANT = "assistant";
|
||||
private static final String ROLE_USER = "user";
|
||||
|
||||
private final ChatClient chatClient;
|
||||
|
||||
public ChatController(@Qualifier("chat") ChatClient.Builder builder) {
|
||||
this.chatClient = builder.build();
|
||||
}
|
||||
|
||||
private ChatClient.ChatClientRequestSpec buildRequest(ImmutableList<MessageVO> messages) {
|
||||
ChatClient.ChatClientRequestSpec spec = chatClient.prompt()
|
||||
.system(
|
||||
StrUtil.format("""
|
||||
你是一名专业的AI运维助手，专职负责“Hudi数据同步服务”的平台运维工作。你的核心职责是：
|
||||
1.友好解答：积极、专业地解答用户（通常是平台管理员或用户）关于该平台运维工作的疑问。
|
||||
2.知识驱动：在解答时，应尽可能通过各种方式（知识库、上下文、外部工具等）全面获取准确知识和数据来支持回答。
|
||||
3.诚实守界：
|
||||
对于无法通过已有知识或数据确认的问题，必须明确告知用户你无法解答，切勿捏造信息或提供不确定的答案。
|
||||
对于与该Hudi数据同步服务平台运维工作无关的问题，需委婉拒绝用户，并明确说明超出你的职责和能力范围。
|
||||
|
||||
对话语言：中文
|
||||
|
||||
{}
|
||||
|
||||
当前时间为：{}
|
||||
|
||||
""",
|
||||
Prompts.hudi,
|
||||
LocalDateTime.now().format(formatter)
|
||||
))
|
||||
.messages(
|
||||
messages
|
||||
.collect(message -> StrUtil.equals(message.getRole(), ROLE_ASSISTANT)
|
||||
? new AssistantMessage(message.getContent())
|
||||
: new UserMessage(message.getContent()))
|
||||
.collect(message -> (Message) message)
|
||||
.toList()
|
||||
);
|
||||
spec.tools(
|
||||
new TableTool(),
|
||||
new YarnTool(),
|
||||
new ChartTool()
|
||||
);
|
||||
return spec;
|
||||
}
|
||||
|
||||
@PostMapping("sync")
|
||||
@ResponseBody
|
||||
public MessageVO chatSync(
|
||||
@RequestBody ImmutableList<MessageVO> messages
|
||||
) {
|
||||
ChatResponse response = buildRequest(messages)
|
||||
.call()
|
||||
.chatResponse();
|
||||
return toMessage(response);
|
||||
}
|
||||
|
||||
@PostMapping("async")
|
||||
public SseEmitter chatAsync(
|
||||
@RequestBody ImmutableList<MessageVO> messages
|
||||
) {
|
||||
SseEmitter emitter = new SseEmitter();
|
||||
buildRequest(messages)
|
||||
.stream()
|
||||
.chatResponse()
|
||||
.subscribe(
|
||||
response -> {
|
||||
try {
|
||||
emitter.send(toMessage(response));
|
||||
} catch (IOException e) {
|
||||
emitter.completeWithError(e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
},
|
||||
emitter::completeWithError,
|
||||
emitter::complete
|
||||
);
|
||||
return emitter;
|
||||
}
|
||||
|
||||
private MessageVO toMessage(ChatResponse response) {
|
||||
AssistantMessage message = Optional.ofNullable(response)
|
||||
.map(ChatResponse::getResult)
|
||||
.map(Generation::getOutput)
|
||||
.orElseThrow(() -> new RuntimeException("ChatResponse is null"));
|
||||
MessageVO vo = new MessageVO();
|
||||
vo.setRole(ROLE_ASSISTANT);
|
||||
vo.setContent(message.getText());
|
||||
if (message instanceof DeepSeekAssistantMessage deepseekMessage) {
|
||||
vo.setReason(deepseekMessage.getReasoningContent());
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge/group")
|
||||
public class GroupController {
|
||||
private final GroupService groupService;
|
||||
|
||||
public GroupController(GroupService groupService) {
|
||||
this.groupService = groupService;
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId) {
|
||||
return AmisResponse.responseCrudData(groupService.list(knowledgeId));
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public AmisResponse<?> delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
|
||||
groupService.remove(id);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.EmbeddingService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge")
|
||||
public class KnowledgeBaseController {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final EmbeddingService embeddingService;
|
||||
|
||||
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.embeddingService = embeddingService;
|
||||
}
|
||||
|
||||
@PostMapping("add")
|
||||
public void add(
|
||||
@RequestParam("name") String name,
|
||||
@RequestParam("strategy") String strategy
|
||||
) throws ExecutionException, InterruptedException {
|
||||
knowledgeBaseService.add(name, strategy);
|
||||
}
|
||||
|
||||
@GetMapping("name")
|
||||
public AmisMapResponse name(@RequestParam("id") Long id) {
|
||||
return AmisResponse.responseMapData()
|
||||
.setData("name", knowledgeBaseService.getName(id));
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list() {
|
||||
return AmisResponse.responseCrudData(knowledgeBaseService.list());
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public void delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
|
||||
knowledgeBaseService.remove(id);
|
||||
}
|
||||
|
||||
@PostMapping("preview_text")
|
||||
public AmisResponse<?> previewText(
|
||||
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
|
||||
@RequestParam(value = "type", defaultValue = "text") String type,
|
||||
@RequestParam(value = "content", required = false) String content,
|
||||
@RequestParam(value = "files", required = false) String files
|
||||
) {
|
||||
if (StrUtil.equals("text", type)) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(mode, content)
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else if (StrUtil.equals("file", type)) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(mode, Lists.immutable.of(files.split(",")))
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + type);
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text")
|
||||
public void submitText(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
|
||||
@RequestParam(value = "type", defaultValue = "text") String type,
|
||||
@RequestParam(value = "content", required = false) String content,
|
||||
@RequestParam(value = "files", required = false) String files
|
||||
) {
|
||||
if (StrUtil.equals("text", type)) {
|
||||
embeddingService.submit(id, mode, content);
|
||||
} else if (StrUtil.equals("file", type)) {
|
||||
embeddingService.submit(id, mode, Lists.immutable.of(files.split(",")));
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + type);
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text_directly")
|
||||
public void submitDirectly(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "name", required = false) String name,
|
||||
@RequestParam(value = "split_key", defaultValue = "\n\n") String splitKey,
|
||||
@RequestBody String content
|
||||
) {
|
||||
embeddingService.submitDirectly(id, name, Lists.immutable.of(content.split(splitKey)));
|
||||
}
|
||||
|
||||
@PostMapping("query")
|
||||
public ImmutableList<String> query(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "limit", defaultValue = "5") Integer limit,
|
||||
@RequestParam(value = "threshold", defaultValue = "0.6") Double threshold,
|
||||
@RequestBody String text
|
||||
) throws ExecutionException, InterruptedException, IOException {
|
||||
return knowledgeBaseService.query(id, text, limit, threshold);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.SegmentService;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge/segment")
|
||||
public class SegmentController {
|
||||
private final SegmentService segmentService;
|
||||
|
||||
public SegmentController(SegmentService segmentService) {
|
||||
this.segmentService = segmentService;
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId, @RequestParam("group_id") Long groupId) throws ExecutionException, InterruptedException {
|
||||
return AmisResponse.responseCrudData(segmentService.list(knowledgeId, groupId));
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public AmisResponse<?> delete(@RequestParam("knowledge_id") Long knowledgeId, @RequestParam("segment_id") Long segmentId) throws ExecutionException, InterruptedException {
|
||||
segmentService.remove(knowledgeId, segmentId);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.springframework.ai.document.Document;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class EmbeddingContext {
|
||||
private Long vectorSourceId;
|
||||
private Long groupId;
|
||||
private Config config;
|
||||
private String content;
|
||||
private String file;
|
||||
private String fileFormat;
|
||||
@Builder.Default
|
||||
private List<Document> documents = Lists.mutable.empty();
|
||||
@Builder.Default
|
||||
private Map<String, Object> metadata = Maps.mutable.empty();
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static final class Config {
|
||||
@Builder.Default
|
||||
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
|
||||
|
||||
public enum SplitStrategy {
|
||||
NORMAL, LLM, QA
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Data
|
||||
public class Group {
|
||||
private String id;
|
||||
private String name;
|
||||
private String status;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Data
|
||||
public class Knowledge {
|
||||
private Long id;
|
||||
private Long vectorSourceId;
|
||||
private String name;
|
||||
private String strategy;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Data
|
||||
public class DataFileVO {
|
||||
private String id;
|
||||
private String filename;
|
||||
private Long size;
|
||||
private String md5;
|
||||
private String path;
|
||||
private String type;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class KnowledgeVO {
|
||||
private String id;
|
||||
private String vectorSourceId;
|
||||
private String name;
|
||||
private String strategy;
|
||||
private Long size;
|
||||
private Long points;
|
||||
private Long segments;
|
||||
private String status;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class MessageVO {
|
||||
private String role;
|
||||
private String content;
|
||||
private String reason;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class SegmentVO {
|
||||
private String id;
|
||||
private String text;
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Service
|
||||
public class DataFileService {
|
||||
private static final String DATA_FILE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_file";
|
||||
|
||||
private final JdbcTemplate template;
|
||||
|
||||
public DataFileService(JdbcTemplate template) {
|
||||
this.template = template;
|
||||
}
|
||||
|
||||
public DataFileVO downloadFile(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "filename", "size", "md5", "path", "type")
|
||||
.from(DATA_FILE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
(rs, row) -> {
|
||||
DataFileVO vo = new DataFileVO();
|
||||
vo.setId(String.valueOf(rs.getLong(1)));
|
||||
vo.setFilename(rs.getString(2));
|
||||
vo.setSize(rs.getLong(3));
|
||||
vo.setMd5(rs.getString(4));
|
||||
vo.setPath(rs.getString(5));
|
||||
vo.setType(rs.getString(6));
|
||||
return vo;
|
||||
},
|
||||
id
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long initialDataFile(String filename) {
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(DATA_FILE_TABLE_NAME, "id", "filename")
|
||||
.values()
|
||||
.addValue("?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
filename
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDataFile(Long id, String path, Long size, String md5, String type) {
|
||||
template.update(
|
||||
SqlBuilder.update(DATA_FILE_TABLE_NAME)
|
||||
.set("size", "?")
|
||||
.addSet("md5", "?")
|
||||
.addSet("path", "?")
|
||||
.addSet("type", "?")
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
size,
|
||||
md5,
|
||||
path,
|
||||
type,
|
||||
id
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Pair;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.EmbeddingContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import com.yomahub.liteflow.core.FlowExecutor;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class EmbeddingService {
|
||||
private final DataFileService dataFileService;
|
||||
private final FlowExecutor executor;
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final GroupService groupService;
|
||||
private final ExecutorService executors = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
|
||||
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
|
||||
public EmbeddingService(DataFileService dataFileService, FlowExecutor executor, KnowledgeBaseService knowledgeBaseService, GroupService groupService) {
|
||||
this.dataFileService = dataFileService;
|
||||
this.executor = executor;
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.groupService = groupService;
|
||||
}
|
||||
|
||||
public ImmutableList<Document> preview(String mode, String content) {
|
||||
if (content.length() > 2000) {
|
||||
content = content.substring(0, 2000);
|
||||
}
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.content(content)
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_preview", null, context);
|
||||
return Lists.immutable.ofAll(context.getDocuments());
|
||||
}
|
||||
|
||||
public ImmutableList<Document> preview(String mode, ImmutableList<String> ids) {
|
||||
DataFileVO vo = dataFileService.downloadFile(Long.parseLong(ids.get(0)));
|
||||
String content = FileUtil.readString(vo.getPath(), StandardCharsets.UTF_8);
|
||||
return preview(mode, content);
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, String content) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
Long groupId = groupService.add(knowledge.getId(), StrUtil.format("文本-{}", IdUtil.nanoId(10)));
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.content(content)
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_submit", null, context);
|
||||
groupService.finish(groupId);
|
||||
});
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, ImmutableList<String> ids) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
List<Pair<Long, DataFileVO>> vos = Lists.mutable.empty();
|
||||
for (String fileId : ids) {
|
||||
DataFileVO vo = dataFileService.downloadFile(Long.parseLong(fileId));
|
||||
Long groupId = groupService.add(id, vo.getFilename());
|
||||
vos.add(Pair.of(groupId, vo));
|
||||
}
|
||||
for (Pair<Long, DataFileVO> pair : vos) {
|
||||
Long groupId = pair.getKey();
|
||||
DataFileVO vo = pair.getValue();
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.file(vo.getPath())
|
||||
.fileFormat(vo.getFilename())
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_submit", null, context);
|
||||
groupService.finish(groupId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void submitDirectly(Long id, String name, ImmutableList<String> contents) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
String groupName = name;
|
||||
if (StrUtil.isBlank(groupName)) {
|
||||
groupName = StrUtil.format("外部-{}", IdUtil.nanoId(10));
|
||||
}
|
||||
Long groupId = groupService.add(knowledge.getId(), groupName);
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.build();
|
||||
context.setDocuments(
|
||||
contents
|
||||
.collect(StrUtil::trim)
|
||||
.collect(content ->
|
||||
Document.builder()
|
||||
.text(content)
|
||||
.metadata(new HashMap<>())
|
||||
.build()
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
executor.execute2Resp("embedding_submit_directly", null, context);
|
||||
groupService.finish(groupId);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import club.kingon.sql.builder.entry.Alias;
|
||||
import club.kingon.sql.builder.entry.Column;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class GroupService {
|
||||
public static final String GROUP_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_group";
|
||||
private static final RowMapper<Group> groupMapper = (rs, row) -> {
|
||||
Group vo = new Group();
|
||||
vo.setId(String.valueOf(rs.getLong(1)));
|
||||
vo.setName(rs.getString(2));
|
||||
vo.setStatus(rs.getString(3));
|
||||
vo.setCreatedTime(rs.getTimestamp(4).getTime());
|
||||
vo.setModifiedTime(rs.getTimestamp(5).getTime());
|
||||
return vo;
|
||||
};
|
||||
|
||||
private final JdbcTemplate template;
|
||||
private final QdrantClient client;
|
||||
|
||||
public GroupService(JdbcTemplate template, VectorStore vectorStore) {
|
||||
this.template = template;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
public Group get(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
|
||||
.from(GROUP_TABLE_NAME)
|
||||
.whereEq("id", id)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
groupMapper
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long add(Long knowledgeId, String name) {
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name", "status")
|
||||
.values()
|
||||
.addValue("?", "?", "?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
knowledgeId,
|
||||
name,
|
||||
"RUNNING"
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
public ImmutableList<Group> list(Long knowledgeId) {
|
||||
return template.query(
|
||||
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
|
||||
.from(GROUP_TABLE_NAME)
|
||||
.whereEq("knowledge_id", knowledgeId)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
groupMapper
|
||||
)
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void finish(Long groupId) {
|
||||
template.update(
|
||||
SqlBuilder.update(GROUP_TABLE_NAME)
|
||||
.set("status", "FINISHED")
|
||||
.whereEq("id", groupId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(Long groupId) throws ExecutionException, InterruptedException {
|
||||
Long vectorSourceId = template.queryForObject(
|
||||
SqlBuilder.select("k.vector_source_id")
|
||||
.from(Alias.of(GROUP_TABLE_NAME, "g"), Alias.of(KnowledgeBaseService.KNOWLEDGE_TABLE_NAME, "k"))
|
||||
.whereEq("g.knowledge_id", Column.as("k.id"))
|
||||
.andEq("g.id", groupId)
|
||||
.precompileSql(),
|
||||
Long.class,
|
||||
groupId
|
||||
);
|
||||
client.deleteAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(vectorSourceId)))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
|
||||
.build()
|
||||
).get();
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("id", groupId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void removeByKnowledgeId(Long knowledgeId) {
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("knowledge_id", "?")
|
||||
.precompileSql(),
|
||||
knowledgeId
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.KnowledgeVO;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseService {
|
||||
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
|
||||
private static final RowMapper<Knowledge> knowledgeMapper = (rs, row) -> {
|
||||
Knowledge knowledge = new Knowledge();
|
||||
knowledge.setId(rs.getLong(1));
|
||||
knowledge.setVectorSourceId(rs.getLong(2));
|
||||
knowledge.setName(rs.getString(3));
|
||||
knowledge.setStrategy(rs.getString(4));
|
||||
knowledge.setCreatedTime(rs.getTimestamp(5).getTime());
|
||||
knowledge.setModifiedTime(rs.getTimestamp(6).getTime());
|
||||
return knowledge;
|
||||
};
|
||||
private final JdbcTemplate template;
|
||||
private final EmbeddingModel model;
|
||||
private final QdrantClient client;
|
||||
private final GroupService groupService;
|
||||
|
||||
public KnowledgeBaseService(JdbcTemplate template, EmbeddingModel model, VectorStore vectorStore, GroupService groupService) {
|
||||
this.template = template;
|
||||
this.model = model;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.groupService = groupService;
|
||||
}
|
||||
|
||||
public Knowledge get(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
knowledgeMapper,
|
||||
id
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void add(String name, String strategy) throws ExecutionException, InterruptedException {
|
||||
Integer count = template.queryForObject(
|
||||
SqlBuilder.select("count(*)")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("name", "?")
|
||||
.precompileSql(),
|
||||
Integer.class,
|
||||
name
|
||||
);
|
||||
if (count > 0) {
|
||||
throw new RuntimeException("名称已存在");
|
||||
}
|
||||
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
long vectorSourceId = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(KNOWLEDGE_TABLE_NAME, "id", "vector_source_id", "name", "strategy")
|
||||
.values()
|
||||
.addValue("?", "?", "?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
vectorSourceId,
|
||||
name,
|
||||
strategy
|
||||
);
|
||||
client.createCollectionAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.valueOf(strategy))
|
||||
.setSize(model.dimensions())
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
|
||||
public String getName(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("name")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", id)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
String.class
|
||||
);
|
||||
}
|
||||
|
||||
public ImmutableList<KnowledgeVO> list() {
|
||||
return template.query(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
knowledgeMapper
|
||||
)
|
||||
.stream()
|
||||
.map(knowledge -> {
|
||||
try {
|
||||
Collections.CollectionInfo info = client.getCollectionInfoAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
KnowledgeVO vo = new KnowledgeVO();
|
||||
vo.setId(String.valueOf(knowledge.getId()));
|
||||
vo.setVectorSourceId(String.valueOf(knowledge.getVectorSourceId()));
|
||||
vo.setName(knowledge.getName());
|
||||
vo.setPoints(info.getPointsCount());
|
||||
vo.setSegments(info.getSegmentsCount());
|
||||
vo.setStatus(info.getStatus().name());
|
||||
Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
|
||||
vo.setStrategy(vectorParams.getDistance().name());
|
||||
vo.setSize(vectorParams.getSize());
|
||||
vo.setCreatedTime(vo.getCreatedTime());
|
||||
vo.setModifiedTime(vo.getModifiedTime());
|
||||
return vo;
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(Long id) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = get(id);
|
||||
if (ObjectUtil.isNull(knowledge)) {
|
||||
throw new RuntimeException(StrUtil.format("{} 不存在"));
|
||||
}
|
||||
template.update(
|
||||
SqlBuilder.delete(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
knowledge.getId()
|
||||
);
|
||||
groupService.removeByKnowledgeId(knowledge.getId());
|
||||
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
}
|
||||
|
||||
public ImmutableList<String> query(
|
||||
Long id,
|
||||
String text,
|
||||
Integer limit,
|
||||
Double threshold
|
||||
) throws ExecutionException, InterruptedException, IOException {
|
||||
Knowledge knowledge = get(id);
|
||||
Boolean exists = client.collectionExistsAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
if (!exists) {
|
||||
throw new RuntimeException(StrUtil.format("{} not exists", id));
|
||||
}
|
||||
VectorStore vs = QdrantVectorStore.builder(client, model)
|
||||
.collectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.initializeSchema(false)
|
||||
.build();
|
||||
List<Document> documents = vs.similaritySearch(
|
||||
SearchRequest.builder()
|
||||
.query(text)
|
||||
.topK(limit)
|
||||
.similarityThreshold(threshold)
|
||||
.build()
|
||||
);
|
||||
// 如果只是一个知识库的话，似乎没有什么rerank的必要...
|
||||
/* List<org.noear.solon.ai.rag.Document> rerankDocuments = rerankingModel.rerank(
|
||||
text,
|
||||
documents.stream()
|
||||
.map(doc -> new org.noear.solon.ai.rag.Document(doc.getId(), doc.getText(), doc.getMetadata(), doc.getScore()))
|
||||
.toList()
|
||||
); */
|
||||
return Lists.immutable.ofAll(documents)
|
||||
.collect(Document::getText);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Service
|
||||
public class SegmentService {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final QdrantClient client;
|
||||
|
||||
public SegmentService(KnowledgeBaseService knowledgeBaseService, VectorStore vectorStore) {
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
public ImmutableList<SegmentVO> list(Long id, Long groupId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
Points.ScrollResponse response = client.scrollAsync(
|
||||
Points.ScrollPoints.newBuilder()
|
||||
.setCollectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.setWithPayload(Points.WithPayloadSelector.newBuilder().setEnable(true).build())
|
||||
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(false).build())
|
||||
.setFilter(
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.get();
|
||||
return response.getResultList()
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.collect(point -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(point.getId().getUuid());
|
||||
vo.setText(point.getPayloadMap().get("doc_content").getStringValue());
|
||||
return vo;
|
||||
})
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
public void remove(Long knowledgeId, Long segmentId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.get(knowledgeId);
|
||||
client.deletePayloadAsync(
|
||||
String.valueOf(knowledgeId),
|
||||
List.of(String.valueOf(segmentId)),
|
||||
null,
|
||||
null,
|
||||
null
|
||||
).get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.node;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.EmbeddingContext;
|
||||
import com.yomahub.liteflow.annotation.LiteflowComponent;
|
||||
import com.yomahub.liteflow.annotation.LiteflowMethod;
|
||||
import com.yomahub.liteflow.core.NodeComponent;
|
||||
import com.yomahub.liteflow.enums.LiteFlowMethodEnum;
|
||||
import com.yomahub.liteflow.enums.NodeTypeEnum;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.document.DocumentReader;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.reader.ExtractedTextFormatter;
|
||||
import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
|
||||
import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
|
||||
import org.springframework.ai.reader.tika.TikaDocumentReader;
|
||||
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.core.io.PathResource;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@LiteflowComponent
|
||||
public class EmbeddingNodes {
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmbeddingNodes.class);
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final QdrantClient qdrantClient;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
public EmbeddingNodes(@Qualifier("chat") ChatClient.Builder builder, VectorStore vectorStore, EmbeddingModel embeddingModel) {
|
||||
this.chatClient = builder.build();
|
||||
this.qdrantClient = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_BOOLEAN, nodeId = "embedding_check_if_file_needed", nodeName = "判断是否需要读取文件", nodeType = NodeTypeEnum.BOOLEAN)
|
||||
public boolean checkIfFileReadNeeded(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
Assert.notNull(context, "EmbeddingContext is null");
|
||||
if (StrUtil.isNotBlank(context.getFile())) {
|
||||
Assert.isTrue(FileUtil.exist(context.getFile()), "File [{}] not exist", context.getFile());
|
||||
return true;
|
||||
}
|
||||
Assert.notBlank(context.getContent(), "Contents is empty");
|
||||
return false;
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_SWITCH, nodeId = "file_reader_switch", nodeName = "判断文件格式", nodeType = NodeTypeEnum.SWITCH)
|
||||
public String fileReaderSwitch(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
String extName = FileUtil.extName(context.getFileFormat());
|
||||
return switch (extName.toLowerCase()) {
|
||||
case "txt", "md", "markdown" -> "txt_file_reader";
|
||||
case "pdf" -> "pdf_file_reader";
|
||||
case "doc", "docx", "xls", "xlsx", "ppt", "pptx", "html", "xml", "wps", "et", "dpt" -> "any_file_reader";
|
||||
default -> throw new IllegalStateException("Unsupported ext: " + extName);
|
||||
};
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "txt_file_reader", nodeName = "读取文本文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void txtFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.setContent(FileUtil.readString(context.getFile(), Charset.defaultCharset()));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "pdf_file_reader", nodeName = "读取pdf文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void pdfFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
PagePdfDocumentReader reader = new PagePdfDocumentReader(
|
||||
new PathResource(context.getFile()),
|
||||
PdfDocumentReaderConfig.builder()
|
||||
.withPageTopMargin(0)
|
||||
.withPageExtractedTextFormatter(ExtractedTextFormatter.builder()
|
||||
.withNumberOfTopTextLinesToDelete(0)
|
||||
.build())
|
||||
.build());
|
||||
context.setContent(readBySpringAiReader(reader));
|
||||
}
|
||||
|
||||
/**
|
||||
* <a href="https://tika.apache.org/3.1.0/formats.html#Database_formats">Tika支持的格式</a>
|
||||
*/
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "any_file_reader", nodeName = "使用Tika尝试读取文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void anyFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.setContent(readBySpringAiReader(new TikaDocumentReader(new PathResource(context.getFile()))));
|
||||
}
|
||||
|
||||
private String readBySpringAiReader(DocumentReader reader) {
|
||||
return reader.get()
|
||||
.stream()
|
||||
.map(Document::getText)
|
||||
.collect(Collectors.joining("\n"))
|
||||
.trim();
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_SWITCH, nodeId = "split_switch", nodeName = "判断使用什么分段方法", nodeType = NodeTypeEnum.SWITCH)
|
||||
public String splitSwitch(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
return switch (context.getConfig().getSplitStrategy()) {
|
||||
case NORMAL -> "normal_split";
|
||||
case LLM -> "llm_split";
|
||||
case QA -> "qa_split";
|
||||
};
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "normal_split", nodeName = "使用普通分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void normalSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
TokenTextSplitter splitter = new TokenTextSplitter(200, 100, 5, 200, true);
|
||||
Document document = Document.builder()
|
||||
.text(context.getContent())
|
||||
.build();
|
||||
context.setDocuments(splitter.split(document));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "llm_split", nodeName = "使用大模型分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void llmSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.getDocuments().addAll(llmSplit(
|
||||
"""
|
||||
请你将用户输入的文本进行语义切分，生成用于知识库检索的文本段。
|
||||
每个文本段要尽可能多地覆盖用户输入文本的各方面知识和细节，包括但不限于主题、概念、关键信息等。对于关键的数字、理论、细节等，要严格遵循原文，不能进行任何虚构和捏造不存在的知识，确保输出内容准确、真实且全面。
|
||||
输出格式为纯文本段，分段之间使用“---”作为分割，方便后续使用代码进行切分。
|
||||
输出文本避免添加markdown格式，保持文本格式紧凑。
|
||||
切分过程中，要注重保持文本的完整性和逻辑性，确保每个文本段都能独立地表达出清晰、准确的信息，以便更好地进行知识库检索。
|
||||
""",
|
||||
context.getContent(),
|
||||
context.getMetadata()
|
||||
));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "qa_split", nodeName = "使用Q/A格式分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void qaSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.getDocuments().addAll(llmSplit(
|
||||
"""
|
||||
对用户输入的文本，生成多组高质量的问答对。请遵循以下指南：
|
||||
1. 问题部分：
|
||||
为同一个主题创建尽可能多的不同表述的问题，确保问题的多样性。
|
||||
每个问题应考虑用户可能的多种问法，例如：
|
||||
直接询问（如“什么是...？”）
|
||||
请求确认（如“是否可以说...？”）
|
||||
寻求解释（如“请解释一下...的含义。”）
|
||||
假设性问题（如“如果...会怎样？”）
|
||||
例子请求（如“能否举个例子说明...？”）
|
||||
问题应涵盖文本中的关键信息、主要概念和细节，确保不遗漏重要内容。
|
||||
2. 答案部分：
|
||||
提供一个全面、信息丰富的答案，涵盖问题的所有可能角度，确保逻辑连贯。
|
||||
答案应直接基于给定文本，确保准确性和一致性。
|
||||
包含相关的细节，如日期、名称、职位等具体信息，必要时提供背景信息以增强理解。
|
||||
3. 格式：
|
||||
使用"问："标记问题集合的开始，所有问题应在一个段落内，问题之间用空格分隔。
|
||||
使用"答："标记答案的开始，答案应清晰分段，便于阅读。
|
||||
问答对之间用“---”分隔，以提高可读性。
|
||||
4. 内容要求：
|
||||
确保问答对紧密围绕文本主题，避免偏离主题。
|
||||
避免添加文本中未提及的信息，确保信息的真实性。
|
||||
一个问题搭配一个答案，避免一组问答对中同时涉及多个问题。
|
||||
如果文本信息不足以回答某个方面，可以在答案中说明 "根据给定信息无法确定"，并尽量提供相关的上下文。
|
||||
格式样例：
|
||||
问：苹果通常是什么颜色的？
|
||||
答：红色。
|
||||
---
|
||||
问：苹果长在树上还是地上？
|
||||
答：苹果长在树上。
|
||||
""",
|
||||
context.getContent(),
|
||||
context.getMetadata()
|
||||
));
|
||||
}
|
||||
|
||||
private List<Document> llmSplit(String prompt, String content, Map<String, Object> metadata) {
|
||||
String response = chatClient.prompt()
|
||||
.system(prompt)
|
||||
.user(content)
|
||||
.call()
|
||||
.content();
|
||||
Assert.notBlank(response, "LLM response is empty");
|
||||
logger.info("LLM response: \n{}", response);
|
||||
// noinspection DataFlowIssue
|
||||
return Arrays.stream(StrUtil.trim(response).split("---"))
|
||||
.map(text -> text.replaceAll("(?!^.+) +$", ""))
|
||||
.map(StrUtil::trim)
|
||||
.map(text -> Document.builder()
|
||||
.text(text)
|
||||
.metadata(Optional.ofNullable(metadata).orElse(new HashMap<>()))
|
||||
.build())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "import_vector_source", nodeName = "导入向量库", nodeType = NodeTypeEnum.COMMON)
|
||||
public void importVectorSource(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
Assert.notNull(context.getVectorSourceId(), "VectorSourceId is null");
|
||||
Assert.notNull(context.getGroupId(), "GroupId is null");
|
||||
if (ObjectUtil.isNotEmpty(context.getDocuments())) {
|
||||
VectorStore vs = QdrantVectorStore.builder(qdrantClient, embeddingModel)
|
||||
.collectionName(String.valueOf(context.getVectorSourceId()))
|
||||
.build();
|
||||
for (Document document : context.getDocuments()) {
|
||||
Map<String, Object> metadata = document.getMetadata();
|
||||
if (StrUtil.isNotBlank(context.getFileFormat()))
|
||||
metadata.put("filename", context.getFileFormat());
|
||||
if (StrUtil.isNotBlank(context.getFile()))
|
||||
metadata.put("filepath", context.getFile());
|
||||
metadata.put("group_id", String.valueOf(context.getGroupId()));
|
||||
metadata.put("vector_source_id", String.valueOf(context.getVectorSourceId()));
|
||||
}
|
||||
vs.add(context.getDocuments());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
/**
|
||||
* 图表工具
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250611
|
||||
*/
|
||||
@Slf4j
|
||||
public class ChartTool {
|
||||
@Tool(description = """
|
||||
根据需求生成mermaid图表代码
|
||||
""")
|
||||
public String mermaid(
|
||||
// language=TEXT
|
||||
@ToolParam(description = """
|
||||
请按以下结构描述图表需求：
|
||||
1.图表类型：[必填] 明确指定图表类型，例如：流程图(flowchart TD)、序列图(sequenceDiagram)、类图(classDiagram)、甘特图(gantt)、状态图(stateDiagram-v2)、用户旅程图(journey)、饼图(pie)、思维导图(mindmap)、实体关系图(erDiagram)等。
|
||||
2.图表标题：[可选] 简述图表的核心主题，例如：“用户登录流程”，“订单处理系统时序交互”，“项目开发计划甘特图”。
|
||||
3.核心内容：[必填] 描述图表逻辑：
|
||||
实体清单：列出所有节点/角色/对象（如“用户、支付系统、数据库”）
|
||||
示例 (流程图)：`用户, 登录界面, 认证服务, 数据库, 主界面, 错误提示`
|
||||
示例 (序列图)：`用户 (User), 客户端 (ClientApp), 认证服务器 (AuthServer), 数据库 (DB)`
|
||||
示例 (甘特图)：`需求分析, 设计, 开发, 测试, 部署`
|
||||
关系描述：说明实体间交互/流程顺序（如“用户提交订单 → 支付系统验证 → 数据库更新记录”）
|
||||
示例 (流程图)：`用户 输入 -> 登录界面 -> 提交 -> 认证服务 -> 验证通过？ --> 是 --> 主界面； 否 --> 错误提示`
|
||||
示例 (序列图)：`用户 -> 客户端：输入凭据； 客户端 -> 认证服务器：发送认证请求； 认证服务器 -> 数据库：查询用户信息； 数据库 --> 认证服务器：返回结果； 认证服务器 --> 客户端：返回认证结果； 客户端 --> 用户：显示结果`
|
||||
示例 (甘特图)：`需求分析 开始于 2023-10-01, 持续 5天； 设计 开始于 需求分析结束, 持续 7天； ...`
|
||||
示例（饼图）：`成功：15.25; 失败：20.22`
|
||||
关键分支：如有条件判断，说明条件及路径（如“验证失败 → 显示错误信息”）
|
||||
示例 (流程图)：`认证服务 --> |验证通过| 主界面； 认证服务 --> |验证失败| 错误提示`
|
||||
4.样式规范：[选填] 指定细节要求：
|
||||
主题：指定预定义主题，例如：default, forest, dark, neutral]` (默认是 default)
|
||||
颜色：实体颜色映射（如“用户节点用#3498db”）
|
||||
形状：特殊节点形状（如“决策节点用菱形”）
|
||||
布局：指定图表方向，例如：从上到下 (TB/TD)， 从左到右 (LR)， 从右到左 (RL)， 从下到上 (BT)]` (默认通常是 TB/TD)
|
||||
其他：箭头类型（实线/虚线）、注释文本等
|
||||
""") String request
|
||||
) {
|
||||
log.info("Enter method: mermaid[request]. request:{}", request);
|
||||
ChatClient.Builder builder = WebApplication.getBean(ChatClient.Builder.class);
|
||||
ChatClient client = builder.build();
|
||||
return client.prompt()
|
||||
// language=TEXT
|
||||
.system("""
|
||||
你是一位专业的Mermaid图表专家，精通各类图表语法（流程图、序列图、类图、甘特图、状态图、饼图等），并能根据需求生成清晰、准确、符合最佳实践的Mermaid代码。
|
||||
请根据我提供的详细描述，为我生成可直接用于渲染的Mermaid图表代码。确保代码语法正确、布局合理、易于理解。
|
||||
输出要求：
|
||||
仅输出纯净的、可立即渲染的 Mermaid 代码。
|
||||
不要包含任何解释性文字、Markdown 标记（除了代码块标识）或额外的说明。
|
||||
代码格式清晰，使用适当的缩进（如果适用）。
|
||||
确保代码语法完全符合 Mermaid 官方文档规范。
|
||||
""")
|
||||
.user(request)
|
||||
.call()
|
||||
.content();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.forest.service.KnowledgeService;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250606
|
||||
*/
|
||||
public class KnowledgeTool {
|
||||
private final Long knowledgeId;
|
||||
|
||||
public KnowledgeTool(Long knowledgeId) {
|
||||
this.knowledgeId = knowledgeId;
|
||||
}
|
||||
|
||||
@Tool(description = """
|
||||
从知识库中检索相关外部知识
|
||||
""")
|
||||
public String queryKnowledge(
|
||||
@ToolParam(description = """
|
||||
精炼准确的知识库关键词，使用逗号分隔，尽可能全面地覆盖需要获取的知识
|
||||
""")
|
||||
String query
|
||||
) {
|
||||
KnowledgeService knowledgeService = WebApplication.getBean(KnowledgeService.class);
|
||||
var documents = knowledgeService.query(knowledgeId, query, 10, 0.5);
|
||||
if (ObjectUtil.isNotEmpty(documents)) {
|
||||
return StrUtil.format("""
|
||||
以下是与用户问题有关的外部知识，优先结合该知识回答用户的提问：
|
||||
{}
|
||||
""", documents.makeString("\n"));
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.forest.service.InfoService;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250605
|
||||
*/
|
||||
@Slf4j
|
||||
public class TableTool {
|
||||
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
@Tool(description = """
|
||||
执行SQL语句，获取查询结果；结果可能是一行或多行，行内以逗号分隔字段。
|
||||
""")
|
||||
public String executeJdbc(
|
||||
@ToolParam(description = """
|
||||
完整的MySQL查询语句，禁止使用除select外的任何语句。
|
||||
""") String sql
|
||||
) {
|
||||
log.info("Enter method: executeJdbc[sql]. sql:{}", sql);
|
||||
InfoService infoService = WebApplication.getBean(InfoService.class);
|
||||
String result = infoService.jdbc(sql)
|
||||
.collect(map -> map.valuesView().makeString(","))
|
||||
.makeString("\n");
|
||||
log.info("SQL result: \n{}", result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Tool(description = """
|
||||
查询表（Hudi表、Hive表、逻辑表等）数量
|
||||
""")
|
||||
private String tableCount(
|
||||
@ToolParam(description = """
|
||||
查询类型，取值如下
|
||||
Hudi表：hudi
|
||||
Hive表：hive
|
||||
逻辑表：logic
|
||||
一次调用只能传一个类型，不支持多个类型同时查询
|
||||
""") String type
|
||||
) {
|
||||
log.info("Enter method: tableCount[type]. type:{}", type);
|
||||
var infoService = WebApplication.getBean(InfoService.class);
|
||||
return switch (type) {
|
||||
case "logic" -> StrUtil.format("""
|
||||
逻辑表共{}张，其中重点表{}张
|
||||
""", infoService.tableCount(), infoService.tableFocusCount());
|
||||
case "hive" -> StrUtil.format("""
|
||||
Hive表共{}张，其中重点表{}张
|
||||
""", infoService.hiveCount(), infoService.hiveFocusCount());
|
||||
case "hudi" -> StrUtil.format("""
|
||||
Hudi表共{}张，其中重点表{}张
|
||||
""", infoService.hudiCount(), infoService.hudiFocusCount());
|
||||
default -> throw new IllegalStateException("Unexpected value: " + type);
|
||||
};
|
||||
}
|
||||
|
||||
@Tool(description = """
|
||||
查询Hudi表跨天情况
|
||||
返回结果包含未接收到跨天标记的表数量和接收到跨天标记但未完成跨天的表数量
|
||||
一张表必须先接收到跨天标记才能完成跨天，两者数量均为0才算所有表完成跨天
|
||||
""")
|
||||
public String version(
|
||||
@ToolParam(description = """
|
||||
查询指定日期的跨天情况，格式为yyyyMMdd（如2025年6月6日取值为20250606）。
|
||||
一次调用只能传一个日期，不支持多个日期同时查询
|
||||
""")
|
||||
String date,
|
||||
@ToolParam(description = """
|
||||
表类型，取值如下
|
||||
普通表：normal
|
||||
重点表：focus
|
||||
一次调用只能传一个类型，不支持多个类型同时查询
|
||||
""")
|
||||
String type
|
||||
) {
|
||||
log.info("Enter method: version[date, type]. date:{},type:{}", date, type);
|
||||
InfoService infoService = WebApplication.getBean(InfoService.class);
|
||||
String version = date;
|
||||
if (StrUtil.isBlank(version)) {
|
||||
version = LocalDateTime.now().minusDays(1).format(FORMATTER);
|
||||
} else {
|
||||
version = LocalDate.parse(version, FORMATTER).minusDays(1).format(FORMATTER);
|
||||
}
|
||||
return switch (type) {
|
||||
case "normal" -> StrUtil.format("""
|
||||
未接收到跨天标记的表数量：{}，接收到跨天标记但未完成跨天的表数量：{}
|
||||
""", infoService.unReceiveVersionNormalTableCount(version), infoService.unScheduledNormalTableCount(version));
|
||||
case "focus" -> StrUtil.format("""
|
||||
未接收到跨天标记的表数量：{}，接收到跨天标记但未完成跨天的表数量：{}
|
||||
""", infoService.unReceiveVersionFocusTableCount(version), infoService.unScheduledFocusTableCount(version));
|
||||
default -> throw new IllegalStateException("Unexpected value: " + type);
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnRootQueue;
|
||||
import com.lanyuanxiaoyao.service.forest.service.YarnService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250606
|
||||
*/
|
||||
@Slf4j
|
||||
public class YarnTool {
|
||||
@Tool(description = """
|
||||
查询yarn集群整体资源情况，返回值为资源占用率（%）
|
||||
""")
|
||||
public Double yarnStatus(
|
||||
@ToolParam(description = """
|
||||
yarn集群名称，取值有b12、b1和a4；
|
||||
一次调用只能查询一个集群，不支持多个集群同时查询
|
||||
""") String cluster
|
||||
) {
|
||||
log.info("Enter method: yarnStatus[cluster]. cluster:{}", cluster);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
YarnRootQueue status = yarnService.cluster(cluster);
|
||||
return (status.getUsedCapacity() * 100.0) / status.getCapacity();
|
||||
}
|
||||
|
||||
@Tool(description = """
|
||||
查询yarn集群中指定队列的资源情况，返回值为资源占用率（%）
|
||||
""")
|
||||
public Double yarnQueueStatus(
|
||||
@ToolParam(description = """
|
||||
yarn集群名称，取值有b12、b1和a4；
|
||||
一次调用只能查询一个集群，不支持多个集群同时查询
|
||||
""") String cluster,
|
||||
@ToolParam(description = """
|
||||
yarn队列名称
|
||||
""") String queue
|
||||
) {
|
||||
log.info("Enter method: yarnQueueStatus[cluster, queue]. cluster:{},queue:{}", cluster, queue);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
YarnQueue status = yarnService.queueDetail(cluster, queue);
|
||||
return (status.getAbsoluteCapacity() * 100.0) / status.getAbsoluteMaxCapacity();
|
||||
}
|
||||
|
||||
@Tool(description = """
|
||||
查询指定集群上同步任务或压缩任务的运行情况
|
||||
""")
|
||||
public String yarnTaskStatus(
|
||||
@ToolParam(description = """
|
||||
yarn集群名称，取值有b12、b1和a4
|
||||
一次调用只能查询一个集群，不支持多个集群同时查询
|
||||
""") String cluster,
|
||||
@ToolParam(description = """
|
||||
查询任务种类，取值如下
|
||||
同步任务：Sync
|
||||
压缩任务：Compaction
|
||||
一次调用只能传一个类型，不支持多个类型同时查询
|
||||
""") String type
|
||||
) {
|
||||
log.info("Enter method: yarnTaskStatus[cluster, type]. cluster:{},type:{}", cluster, type);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
ImmutableList<YarnApplication> applications = yarnService.jobList(cluster).select(app -> StrUtil.isNotBlank(type) && StrUtil.contains(app.getName(), type));
|
||||
return StrUtil.format(
|
||||
"""
|
||||
运行中：{}，调度中：{}
|
||||
""",
|
||||
applications.count(app -> StrUtil.equals(app.getState(), "RUNNING")),
|
||||
applications.count(app -> StrUtil.equals(app.getState(), "ACCEPTED"))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
spring:
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics,forest
|
||||
ai:
|
||||
vectorstore:
|
||||
qdrant:
|
||||
host: 132.121.206.65
|
||||
port: 29463
|
||||
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
|
||||
llm:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
model: 'Qwen3/qwen3-1.7b'
|
||||
visual:
|
||||
model: 'Qwen2.5/qwen2.5-vl-7b'
|
||||
embedding:
|
||||
model: 'Qwen3/qwen3-embedding-4b'
|
||||
reranker:
|
||||
model: 'Bge-reranker-v2'
|
||||
@@ -0,0 +1,39 @@
|
||||
server:
|
||||
port: 8080
|
||||
spring:
|
||||
ai:
|
||||
vectorstore:
|
||||
qdrant:
|
||||
host: 192.168.100.140
|
||||
port: 6334
|
||||
llm:
|
||||
base-url: https://api.siliconflow.cn
|
||||
api-key: sk-xrguybusoqndpqvgzgvllddzgjamksuecyqdaygdwnrnqfwo
|
||||
chat:
|
||||
model: 'Qwen/Qwen3-8B'
|
||||
visual:
|
||||
base-url: https://open.bigmodel.cn/api/paas/v4
|
||||
endpoint: /chat/completions
|
||||
model: 'glm-4v-flash'
|
||||
embedding:
|
||||
model: 'BAAI/bge-m3'
|
||||
reranker:
|
||||
model: 'BAAI/bge-reranker-v2-m3'
|
||||
cloud:
|
||||
discovery:
|
||||
enabled: false
|
||||
zookeeper:
|
||||
enabled: false
|
||||
datasource:
|
||||
url: jdbc:mysql://192.168.100.140:3306/hudi_collect_build_b12?useSSL=false&allowPublicKeyRetrieval=true
|
||||
username: test
|
||||
password: test
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
security:
|
||||
meta:
|
||||
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
|
||||
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
|
||||
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
|
||||
jasypt:
|
||||
encryptor:
|
||||
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
|
||||
19
service-ai/service-ai-web/src/main/resources/application.yml
Normal file
19
service-ai/service-ai-web/src/main/resources/application.yml
Normal file
@@ -0,0 +1,19 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-web
|
||||
mvc:
|
||||
async:
|
||||
request-timeout: 3600000
|
||||
autoconfigure:
|
||||
exclude: |
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiChatAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiAudioSpeechAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiAudioTranscriptionAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiImageAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiEmbeddingAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiModerationAutoConfiguration,
|
||||
org.springframework.ai.model.deepseek.autoconfigure.DeepSeekChatAutoConfiguration
|
||||
liteflow:
|
||||
rule-source: liteflow.xml
|
||||
print-banner: false
|
||||
check-node-exists: false
|
||||
26
service-ai/service-ai-web/src/main/resources/liteflow.xml
Normal file
26
service-ai/service-ai-web/src/main/resources/liteflow.xml
Normal file
@@ -0,0 +1,26 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE flow PUBLIC "liteflow" "https://liteflow.cc/liteflow.dtd">
|
||||
<flow>
|
||||
<chain id="embedding_preview">
|
||||
SER(
|
||||
IF(
|
||||
embedding_check_if_file_needed,
|
||||
SWITCH(file_reader_switch).TO(
|
||||
txt_file_reader,
|
||||
pdf_file_reader
|
||||
).DEFAULT(any_file_reader)
|
||||
),
|
||||
SWITCH(split_switch).TO(
|
||||
normal_split,
|
||||
llm_split,
|
||||
qa_split
|
||||
)
|
||||
)
|
||||
</chain>
|
||||
<chain id="embedding_submit">
|
||||
SER(embedding_preview, import_vector_source)
|
||||
</chain>
|
||||
<chain id="embedding_submit_directly">
|
||||
SER(import_vector_source)
|
||||
</chain>
|
||||
</flow>
|
||||
@@ -0,0 +1,34 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %clr(%5p) %clr([${HOSTNAME}]){yellow} %clr([%t]){magenta} %clr(%logger{40}){cyan} #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
Reference in New Issue
Block a user