Compare commits
39 Commits
506e28c9f7
...
jpa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c9616eb63a | ||
|
|
5b3c27ea48 | ||
| e48d7e8649 | |||
|
|
306c20aa7f | ||
|
|
24d5d10ecb | ||
|
|
4a9a9ec238 | ||
|
|
08aa1d8382 | ||
|
|
1b3045dfd4 | ||
|
|
0f5ae1c4d4 | ||
|
|
48e42ee99a | ||
|
|
0914b458d3 | ||
|
|
368c30676e | ||
|
|
60477f99f5 | ||
|
|
565c530dd5 | ||
|
|
5130885033 | ||
|
|
8e6463845b | ||
|
|
e89bffe289 | ||
|
|
1dd00d329c | ||
| e470a87372 | |||
|
|
45da452f18 | ||
|
|
e6a1bc5383 | ||
|
|
c5916703cd | ||
|
|
807ddbe5cb | ||
|
|
13de694e37 | ||
|
|
1962dd586c | ||
| 138ee140e1 | |||
| e2d69bc6e8 | |||
| b9d707dc8f | |||
| 44d1473c6b | |||
| 9c658afbd7 | |||
| e3f86e6497 | |||
| 256c8c6bd5 | |||
| b627c91acb | |||
| 7fb490778a | |||
| d4d5aede31 | |||
|
|
f11f5e7656 | ||
|
|
bc32a89fea | ||
|
|
2e24bdb90b | ||
|
|
5160c59ab0 |
@@ -77,11 +77,12 @@ export const run_package_batch = async (projects) => {
|
||||
}
|
||||
}
|
||||
|
||||
const upload = async (file_path) => {
|
||||
export const upload = async (file_path) => {
|
||||
let start = new Date().getTime()
|
||||
let basename = path.basename(file_path)
|
||||
let response = await spinner(
|
||||
`Uploading project ${file_path}`,
|
||||
() => fetch(`${upload_url}/file/upload/${path.basename(file_path)}`, {
|
||||
() => fetch(`${upload_url}/file/upload/${basename}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
@@ -98,6 +99,7 @@ const upload = async (file_path) => {
|
||||
console.log(`✅ Finished upload ${file_path} (${millisecondToString((new Date().getTime()) - start)})`)
|
||||
console.log(`📘 Uploaded ${fileSize(fs.statSync(file_path).size)}`)
|
||||
console.log(`📘 MD5 ${md5file.sync(file_path)}`)
|
||||
console.log(`📘 Download curl http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.124:36800/file/download/${basename} -o ${basename}`)
|
||||
fs.rmSync(file_path)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy, run_package, run_upload_normal} from '../../bin/library.js'
|
||||
import {run_deploy} from '../../bin/library.js'
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy('service-ai-core')
|
||||
await run_package('service-ai-chat')
|
||||
await run_upload_normal('service-ai-chat')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy, run_package, run_upload_normal} from '../../bin/library.js'
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy('service-ai-core')
|
||||
await run_package('service-ai-knowledge')
|
||||
await run_upload_normal('service-ai-knowledge')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
21
service-ai/bin/build-ai-web.js
Normal file
21
service-ai/bin/build-ai-web.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import {
|
||||
cd,
|
||||
path,
|
||||
} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {
|
||||
run_deploy,
|
||||
run_package,
|
||||
run_upload_normal,
|
||||
} from '../../bin/library.js'
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy('service-ai-core')
|
||||
await run_package('service-ai-web')
|
||||
await run_upload_normal('service-ai-web')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
18
service-ai/database/service_ai_feedback.sql
Normal file
18
service-ai/database/service_ai_feedback.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE `service_ai_feedback`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`analysis` longtext,
|
||||
`conclusion` longtext,
|
||||
`source` longtext NOT NULL,
|
||||
`status` tinyint NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
|
||||
CREATE TABLE `service_ai_feedback_pictures`
|
||||
(
|
||||
`feedback_id` bigint NOT NULL,
|
||||
`pictures_id` bigint NOT NULL,
|
||||
PRIMARY KEY (`feedback_id`, `pictures_id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
@@ -1,11 +1,12 @@
|
||||
CREATE TABLE `service_ai_file`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`filename` varchar(500) DEFAULT NULL,
|
||||
`size` bigint DEFAULT NULL,
|
||||
`md5` varchar(100) DEFAULT NULL,
|
||||
`path` varchar(500) DEFAULT NULL,
|
||||
`type` varchar(50) DEFAULT NULL,
|
||||
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`filename` varchar(255) DEFAULT NULL,
|
||||
`md5` varchar(255) DEFAULT NULL,
|
||||
`path` varchar(255) DEFAULT NULL,
|
||||
`size` bigint DEFAULT NULL,
|
||||
`type` varchar(255) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
CREATE TABLE `service_ai_group`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`name` varchar(255) NOT NULL,
|
||||
`status` tinyint NOT NULL,
|
||||
`knowledge_id` bigint NOT NULL,
|
||||
`name` varchar(100) NOT NULL,
|
||||
`status` varchar(10) NOT NULL DEFAULT 'RUNNING',
|
||||
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
CREATE TABLE `service_ai_knowledge`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`vector_source_id` varchar(100) NOT NULL,
|
||||
`name` varchar(100) NOT NULL,
|
||||
`strategy` varchar(10) NOT NULL,
|
||||
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`description` longtext NOT NULL,
|
||||
`name` varchar(255) NOT NULL,
|
||||
`strategy` tinyint NOT NULL,
|
||||
`vector_source_id` bigint NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
<description>Hudi AI服务集合</description>
|
||||
<modules>
|
||||
<module>service-ai-core</module>
|
||||
<module>service-ai-chat</module>
|
||||
<module>service-ai-knowledge</module>
|
||||
<module>service-ai-web</module>
|
||||
<module>service-ai-cli</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
@@ -29,8 +29,21 @@
|
||||
<eclipse-collections.version>11.1.0</eclipse-collections.version>
|
||||
<curator.version>5.1.0</curator.version>
|
||||
<hutool.version>5.8.27</hutool.version>
|
||||
<mapstruct.version>1.6.3</mapstruct.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- 当前项目依赖 -->
|
||||
@@ -97,6 +110,11 @@
|
||||
<artifactId>jasypt-spring-boot-starter</artifactId>
|
||||
<version>3.0.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.blinkfox</groupId>
|
||||
<artifactId>fenix-spring-boot-starter</artifactId>
|
||||
<version>3.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 日志相关 -->
|
||||
<dependency>
|
||||
@@ -147,12 +165,22 @@
|
||||
<artifactId>solon-ai-dialect-openai</artifactId>
|
||||
<version>${solon-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.14.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
@@ -161,7 +189,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<version>3.6.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.entity;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class MessageVO {
|
||||
private String role;
|
||||
private String content;
|
||||
private String reason;
|
||||
|
||||
public String getRole() {
|
||||
return role;
|
||||
}
|
||||
|
||||
public void setRole(String role) {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public String getReason() {
|
||||
return reason;
|
||||
}
|
||||
|
||||
public void setReason(String reason) {
|
||||
this.reason = reason;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "MessageVO{" +
|
||||
"role='" + role + '\'' +
|
||||
", content='" + content + '\'' +
|
||||
", reason='" + reason + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-chat
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics,forest
|
||||
ai:
|
||||
deepseek:
|
||||
base-url: http://132.121.206.65:10086/v1
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
options:
|
||||
model: 'Qwen3-1.7-vllm'
|
||||
mvc:
|
||||
async:
|
||||
request-timeout: 3600000
|
||||
@@ -1,16 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250606
|
||||
*/
|
||||
public class TestDatetimeFormat {
|
||||
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(LocalDate.parse("20250606", FORMATTER).format(FORMATTER));
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.TableTool;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.YarnTool;
|
||||
import java.net.http.HttpClient;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatModel;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatOptions;
|
||||
import org.springframework.ai.deepseek.api.DeepSeekApi;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
import org.springframework.http.client.reactive.JdkClientHttpConnector;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
public class TestLlmPlan {
|
||||
public static void main(String[] args) {
|
||||
ChatClient client = ChatClient.builder(
|
||||
DeepSeekChatModel.builder()
|
||||
.deepSeekApi(
|
||||
DeepSeekApi.builder()
|
||||
.baseUrl("http://127.0.0.1:1234/v1")
|
||||
.apiKey("nopassword")
|
||||
.restClientBuilder(restClientBuilder())
|
||||
.webClientBuilder(webClientBuilder())
|
||||
.build()
|
||||
)
|
||||
.defaultOptions(
|
||||
DeepSeekChatOptions.builder()
|
||||
.model("qwen/qwen3-1.7b")
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.defaultSystem(StrUtil.format(
|
||||
"""
|
||||
你是一名专业的AI运维助手,专职负责“Hudi数据同步服务”的平台运维工作。你的核心职责是:
|
||||
1.友好解答:积极、专业地解答用户(通常是平台管理员或用户)关于该平台运维工作的疑问。
|
||||
2.知识驱动:在解答时,应尽可能通过各种方式(知识库、上下文、外部工具等)全面获取准确知识和数据来支持回答。
|
||||
3.诚实守界:
|
||||
对于无法通过已有知识或数据确认的问题,必须明确告知用户你无法解答,切勿捏造信息或提供不确定的答案。
|
||||
对于与该Hudi数据同步服务平台运维工作无关的问题,需委婉拒绝用户,并明确说明超出你的职责和能力范围。
|
||||
|
||||
对话语言:中文
|
||||
|
||||
{}
|
||||
""",
|
||||
Prompts.hudi
|
||||
))
|
||||
.defaultTools(
|
||||
new TableTool(),
|
||||
new YarnTool()
|
||||
)
|
||||
.build();
|
||||
System.out.println(client.prompt("""
|
||||
我需要大模型帮我检查系统整体运行状态,帮我设计详细的指导大模型具体操作的操作步骤
|
||||
避免使用上下文中没有提到的外部工具
|
||||
""").call().content());
|
||||
}
|
||||
|
||||
private static HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static RestClient.Builder restClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
}
|
||||
|
||||
private static WebClient.Builder webClientBuilder() {
|
||||
return WebClient.builder()
|
||||
.clientConnector(new JdkClientHttpConnector(httpClient()));
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatModel;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatOptions;
|
||||
import org.springframework.ai.deepseek.api.DeepSeekApi;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.definition.ToolDefinition;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
import org.springframework.http.client.reactive.JdkClientHttpConnector;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.core.Disposable;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
@SuppressWarnings("NullableProblems")
|
||||
public class TestSpringAIToolChat {
|
||||
public static void main(String[] args) {
|
||||
ChatClient client = ChatClient.builder(
|
||||
DeepSeekChatModel.builder()
|
||||
.deepSeekApi(
|
||||
DeepSeekApi.builder()
|
||||
.baseUrl("http://132.121.206.65:10086/v1")
|
||||
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
|
||||
.restClientBuilder(restClientBuilder())
|
||||
.webClientBuilder(webClientBuilder())
|
||||
.build()
|
||||
)
|
||||
.defaultOptions(
|
||||
DeepSeekChatOptions.builder()
|
||||
.model("Qwen3-1.7-vllm")
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
ToolCallback datetimeTool = new ToolCallback() {
|
||||
@Override
|
||||
public ToolDefinition getToolDefinition() {
|
||||
return ToolDefinition.builder()
|
||||
.name("getCurrentTime")
|
||||
.description("获取当前日期和时间")
|
||||
// language=JSON
|
||||
.inputSchema("""
|
||||
{"type": null}
|
||||
""")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String call(String toolInput) {
|
||||
return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
||||
}
|
||||
};
|
||||
Disposable disposable = client.prompt()
|
||||
.user("当前时间?")
|
||||
.toolCallbacks(datetimeTool)
|
||||
.stream()
|
||||
.content()
|
||||
.subscribe(System.out::println);
|
||||
while (!disposable.isDisposed()) {
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static RestClient.Builder restClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
}
|
||||
|
||||
private static WebClient.Builder webClientBuilder() {
|
||||
return WebClient.builder()
|
||||
.clientConnector(new JdkClientHttpConnector(httpClient()));
|
||||
}
|
||||
}
|
||||
@@ -9,16 +9,20 @@
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-ai-chat</artifactId>
|
||||
<artifactId>service-ai-cli</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai-core</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
||||
<groupId>cn.hutool</groupId>
|
||||
<artifactId>hutool-all</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.freemarker</groupId>
|
||||
<artifactId>freemarker</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -38,4 +42,5 @@
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
@SpringBootApplication
|
||||
public class Generator implements ApplicationRunner {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(Generator.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) throws Exception {
|
||||
new LlamaSwapTool().generate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.extra.template.Template;
|
||||
import cn.hutool.extra.template.TemplateConfig;
|
||||
import cn.hutool.extra.template.TemplateEngine;
|
||||
import cn.hutool.extra.template.TemplateUtil;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
public abstract class GeneratorTool {
|
||||
private final TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
|
||||
|
||||
protected void generateTemplate(String templatePath, Map<?, ?> data, String targetScriptPath) {
|
||||
Template template = engine.getTemplate(templatePath);
|
||||
String script = template.render(data);
|
||||
FileUtil.del(targetScriptPath);
|
||||
FileUtil.writeString(script, targetScriptPath, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public abstract void generate() throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import cn.hutool.core.collection.ListUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
public class LlamaSwapTool extends GeneratorTool {
|
||||
private static final String API_KEY = "*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4";
|
||||
|
||||
public static String displayName(String name) {
|
||||
return name.replaceAll("\\s+", "_")
|
||||
.replaceAll("\\.", "_")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void generate() {
|
||||
generateTemplate(
|
||||
"llama-swap.ftl",
|
||||
Map.of(
|
||||
"models", List.of(
|
||||
llamaCppEmbeddingCmd("BGE/bge-m3-q4km", "bge-m3-Q4_K_M.gguf", 20),
|
||||
vllmEmbeddingCmd("BGE/bge-m3", "bge-m3", 5),
|
||||
llamaCppRerankerCmd("BGE/beg-reranker-v2-q4km", "bge-reranker-v2-m3-Q4_K_M.gguf", 20),
|
||||
vllmRerankerCmd("BGE/beg-reranker-v2", "bge-reranker-v2-m3", 5),
|
||||
|
||||
vllmCmd("Qwen3/qwen3-0.6b", "Qwen3-0.6B", 5, true),
|
||||
vllmCmd("Qwen3/qwen3-1.7b", "Qwen3-1.7B", 5, true),
|
||||
vllmCmd("Qwen3/qwen3-4b", "Qwen3-4B", 8, true),
|
||||
llamaCppCmd("Qwen3/qwen3-4b-q4km", "Qwen3-4B-Q4_K_M.gguf", 35),
|
||||
llamaCppCmd("Qwen3/qwen3-8b-q4km", "Qwen3-8B-Q4_K_M.gguf", 35),
|
||||
|
||||
vllmEmbeddingCmd("Qwen3/qwen3-embedding-0.6b", "Qwen3-Embedding-0.6B", 5),
|
||||
vllmEmbeddingCmd("Qwen3/qwen3-embedding-4b", "Qwen3-Embedding-4B", 8),
|
||||
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-4b-q4km", "Qwen3-Embedding-4B-Q4_K_M.gguf", 35),
|
||||
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-8b-q4km", "Qwen3-Embedding-8B-Q4_K_M.gguf", 35),
|
||||
|
||||
// 0.9.1 vllm还未支持
|
||||
// vllmRerankerCmd("Qwen3/qwen3-reranker-0.6b", "Qwen3-Reranker-0.6B", 5),
|
||||
// vllmRerankerCmd("Qwen3/qwen3-reranker-4b", "Qwen3-Reranker-4B", 8),
|
||||
|
||||
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b", "Qwen2.5-VL-7B-Instruct-BF16.gguf", 35),
|
||||
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b-q4km", "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", 35),
|
||||
vllmCmd("Qwen2.5/qwen2.5-vl-3b-instruct", "Qwen2.5-VL-3B-Instruct", 8, false),
|
||||
vllmCmd("Qwen2.5/qwen2.5-vl-7b-instruct", "Qwen2.5-VL-7B-Instruct", 8, false),
|
||||
|
||||
llamaCppVisualCmd("MiniCPM/minicpm-o-2.6-7.6b-q4km", "MiniCPM-o-2_6-7.6B-Q4_K_M.gguf", 35),
|
||||
vllmCmd("MiniCPM/minicpm-o-2.6-7.6b", "MiniCPM-o-2_6", 10, false)
|
||||
)
|
||||
),
|
||||
"config.yaml"
|
||||
);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppEmbeddingCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, true, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppRerankerCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, true, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppVisualCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, false, true);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppCmd(String name, String model, Integer thread, Boolean isEmbedding, Boolean isReranker, Boolean isVisual) {
|
||||
List<String> arguments = ListUtil.list(
|
||||
false,
|
||||
StrUtil.format("-m /models/{}", model),
|
||||
"--port ${PORT}",
|
||||
StrUtil.format("--api-key {}", API_KEY),
|
||||
"-c 0",
|
||||
"-b 4096",
|
||||
StrUtil.format("-t {}", thread),
|
||||
"-np 5",
|
||||
"--log-disable",
|
||||
"--no-webui"
|
||||
);
|
||||
if (isEmbedding) {
|
||||
arguments.add("--embedding");
|
||||
arguments.add("-ub 8192");
|
||||
arguments.add("--pooling mean");
|
||||
} else if (isReranker) {
|
||||
arguments.add("--reranking");
|
||||
} else if (isVisual) {
|
||||
arguments.add(StrUtil.format("--mmproj /models/{}.mmproj", model));
|
||||
} else {
|
||||
arguments.add("--jinja");
|
||||
}
|
||||
return new DockerCmd(
|
||||
"ghcr.io/ggml-org/llama.cpp:server",
|
||||
name,
|
||||
model,
|
||||
StrUtil.format("http://llamacpp-{}:${PORT}", displayName(model)),
|
||||
List.of(StrUtil.format("--name llamacpp-{}", displayName(model))),
|
||||
arguments
|
||||
);
|
||||
}
|
||||
|
||||
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isReasonable) {
|
||||
return vllmCmd(name, model, cache, false, false, isReasonable);
|
||||
}
|
||||
|
||||
private DockerCmd vllmEmbeddingCmd(String name, String model, Integer cache) {
|
||||
return vllmCmd(name, model, cache, true, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd vllmRerankerCmd(String name, String model, Integer cache) {
|
||||
return vllmCmd(name, model, cache, false, true, false);
|
||||
}
|
||||
|
||||
private DockerCmd vllmVisualCmd(String name, String model, Integer cache, Boolean isReasonable) {
|
||||
return vllmCmd(name, model, cache, false, false, isReasonable);
|
||||
}
|
||||
|
||||
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isEmbedding, Boolean isReranker, Boolean isReasonable) {
|
||||
List<String> arguments = ListUtil.list(
|
||||
false,
|
||||
StrUtil.format("--model /models/{}", model),
|
||||
StrUtil.format("--served-model-name {}", name),
|
||||
"--port ${PORT}",
|
||||
StrUtil.format("--api-key {}", API_KEY),
|
||||
"--disable-log-requests",
|
||||
"--uvicorn-log-level error",
|
||||
"--trust-remote-code"
|
||||
);
|
||||
if (isEmbedding) {
|
||||
arguments.add("--task embedding");
|
||||
} else if (isReranker) {
|
||||
arguments.add("--task score");
|
||||
} else if (isReasonable) {
|
||||
arguments.add("--enable-auto-tool-choice");
|
||||
arguments.add("--tool-call-parser hermes");
|
||||
arguments.add("--enable-reasoning");
|
||||
arguments.add("--reasoning-parser deepseek_r1");
|
||||
}
|
||||
return new DockerCmd(
|
||||
"vllm-server-cpu:0.8.5.post1",
|
||||
name,
|
||||
model,
|
||||
StrUtil.format("http://vllm-{}:${PORT}", displayName(model)),
|
||||
List.of(
|
||||
StrUtil.format("--name vllm-{}", displayName(model)),
|
||||
"--privileged=true",
|
||||
"--shm-size=4g",
|
||||
StrUtil.format("-e VLLM_CPU_KVCACHE_SPACE={}", cache)
|
||||
),
|
||||
arguments
|
||||
);
|
||||
}
|
||||
|
||||
public static class DockerCmd {
|
||||
private String image;
|
||||
private String name;
|
||||
private String model;
|
||||
private String proxy;
|
||||
private List<String> options = ListUtil.list(
|
||||
false,
|
||||
"--rm",
|
||||
"--network llama",
|
||||
"-v /data/models:/models"
|
||||
);
|
||||
private List<String> arguments = ListUtil.list(false);
|
||||
|
||||
public DockerCmd(String image, String name, String model, String proxy, List<String> options, List<String> arguments) {
|
||||
this.image = image;
|
||||
this.name = name;
|
||||
this.model = model;
|
||||
this.proxy = proxy;
|
||||
this.options.addAll(options);
|
||||
this.arguments.addAll(arguments);
|
||||
}
|
||||
|
||||
public String getImage() {
|
||||
return image;
|
||||
}
|
||||
|
||||
public void setImage(String image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public String getProxy() {
|
||||
return proxy;
|
||||
}
|
||||
|
||||
public void setProxy(String proxy) {
|
||||
this.proxy = proxy;
|
||||
}
|
||||
|
||||
public List<String> getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public void setOptions(List<String> options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public List<String> getArguments() {
|
||||
return arguments;
|
||||
}
|
||||
|
||||
public void setArguments(List<String> arguments) {
|
||||
this.arguments = arguments;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DockerCmd{" +
|
||||
"image='" + image + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", model='" + model + '\'' +
|
||||
", proxy='" + proxy + '\'' +
|
||||
", options=" + options +
|
||||
", arguments=" + arguments +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
healthCheckTimeout: 600
|
||||
logLevel: warn
|
||||
models:
|
||||
<#list models as model>
|
||||
"${model.name}":
|
||||
proxy: ${model.proxy}
|
||||
ttl: 86400
|
||||
cmd: |
|
||||
docker run
|
||||
<#list model.options as option>
|
||||
${option}
|
||||
</#list>
|
||||
${model.image}
|
||||
<#list model.arguments as arg>
|
||||
${arg}
|
||||
</#list>
|
||||
</#list>
|
||||
groups:
|
||||
"persistent":
|
||||
swap: false
|
||||
exclusive: false
|
||||
members:
|
||||
<#list models as model>
|
||||
- "${model.name}"
|
||||
</#list>
|
||||
@@ -17,22 +17,30 @@ import org.springframework.web.reactive.function.client.WebClient;
|
||||
*/
|
||||
@Configuration
|
||||
public class WebClientConfiguration {
|
||||
private HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public RestClient.Builder restClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
return generateRestClientBuilder();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public WebClient.Builder webClientBuilder() {
|
||||
return generateWebClientBuilder();
|
||||
}
|
||||
|
||||
private static HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static RestClient.Builder generateRestClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
}
|
||||
|
||||
public static WebClient.Builder generateWebClientBuilder() {
|
||||
return WebClient.builder()
|
||||
.clientConnector(new JdkClientHttpConnector(httpClient()));
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.security.config.Customizer;
|
||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
||||
@@ -20,11 +21,15 @@ import org.springframework.security.web.SecurityFilterChain;
|
||||
@Configuration
|
||||
@EnableWebSecurity
|
||||
public class SecurityConfig {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class);
|
||||
|
||||
@Bean
|
||||
public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
|
||||
return http.authorizeHttpRequests(registry -> registry.anyRequest().authenticated())
|
||||
return http.authorizeHttpRequests(
|
||||
registry -> registry
|
||||
.requestMatchers(HttpMethod.OPTIONS, "/**")
|
||||
.permitAll()
|
||||
.anyRequest()
|
||||
.authenticated()
|
||||
)
|
||||
.httpBasic(Customizer.withDefaults())
|
||||
.cors(AbstractHttpConfigurer::disable)
|
||||
.csrf(AbstractHttpConfigurer::disable)
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge;
|
||||
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
@EnableScheduling
|
||||
public class KnowledgeApplication implements ApplicationRunner {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(KnowledgeApplication.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "knowledge")
|
||||
public class KnowledgeConfiguration {
|
||||
private String downloadPrefix;
|
||||
private String uploadPath;
|
||||
|
||||
public String getDownloadPrefix() {
|
||||
return downloadPrefix;
|
||||
}
|
||||
|
||||
public void setDownloadPrefix(String downloadPrefix) {
|
||||
this.downloadPrefix = downloadPrefix;
|
||||
}
|
||||
|
||||
public String getUploadPath() {
|
||||
return uploadPath;
|
||||
}
|
||||
|
||||
public void setUploadPath(String uploadPath) {
|
||||
this.uploadPath = uploadPath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "KnowledgeConfiguration{" +
|
||||
"downloadPrefix='" + downloadPrefix + '\'' +
|
||||
", uploadPath='" + uploadPath + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import org.noear.solon.ai.reranking.RerankingModel;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250604
|
||||
*/
|
||||
@Configuration
|
||||
public class SolonConfiguration {
|
||||
@Bean
|
||||
public RerankingModel rerankingModel(SolonProperties solonProperties) {
|
||||
return RerankingModel.of(StrUtil.format("{}{}", solonProperties.getBaseUrl(), solonProperties.getRerank().getEndpoint()))
|
||||
.apiKey(solonProperties.getApiKey())
|
||||
.model(solonProperties.getRerank().getModel())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250604
|
||||
*/
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "solon")
|
||||
public class SolonProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private Rerank rerank;
|
||||
|
||||
public String getBaseUrl() {
|
||||
return baseUrl;
|
||||
}
|
||||
|
||||
public void setBaseUrl(String baseUrl) {
|
||||
this.baseUrl = baseUrl;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public void setApiKey(String apiKey) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
public Rerank getRerank() {
|
||||
return rerank;
|
||||
}
|
||||
|
||||
public void setRerank(Rerank rerank) {
|
||||
this.rerank = rerank;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SolonProperties{" +
|
||||
"baseUrl='" + baseUrl + '\'' +
|
||||
", apiKey='" + apiKey + '\'' +
|
||||
", rerank=" + rerank +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Rerank {
|
||||
private String model;
|
||||
private String endpoint;
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public String getEndpoint() {
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
public void setEndpoint(String endpoint) {
|
||||
this.endpoint = endpoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Rerank{" +
|
||||
"model='" + model + '\'' +
|
||||
", endpoint='" + endpoint + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.GroupService;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("group")
|
||||
public class GroupController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(GroupController.class);
|
||||
|
||||
private final GroupService groupService;
|
||||
|
||||
public GroupController(GroupService groupService) {
|
||||
this.groupService = groupService;
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId) {
|
||||
return AmisResponse.responseCrudData(groupService.list(knowledgeId));
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public AmisResponse<?> delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
|
||||
groupService.remove(id);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.SegmentVO;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.EmbeddingService;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.KnowledgeBaseService;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("knowledge")
|
||||
public class KnowledgeBaseController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
|
||||
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final EmbeddingService embeddingService;
|
||||
|
||||
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.embeddingService = embeddingService;
|
||||
}
|
||||
|
||||
@PostMapping("add")
|
||||
public void add(
|
||||
@RequestParam("name") String name,
|
||||
@RequestParam("strategy") String strategy
|
||||
) throws ExecutionException, InterruptedException {
|
||||
knowledgeBaseService.add(name, strategy);
|
||||
}
|
||||
|
||||
@GetMapping("name")
|
||||
public AmisMapResponse name(@RequestParam("id") Long id) {
|
||||
return AmisResponse.responseMapData()
|
||||
.setData("name", knowledgeBaseService.getName(id));
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list() {
|
||||
return AmisResponse.responseCrudData(knowledgeBaseService.list());
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public void delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
|
||||
knowledgeBaseService.remove(id);
|
||||
}
|
||||
|
||||
@PostMapping("preview_text")
|
||||
public AmisResponse<?> previewText(
|
||||
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
|
||||
@RequestParam(value = "type", defaultValue = "text") String type,
|
||||
@RequestParam(value = "content", required = false) String content,
|
||||
@RequestParam(value = "files", required = false) String files
|
||||
) {
|
||||
if (StrUtil.equals("text", type)) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(mode, content)
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else if (StrUtil.equals("file", type)) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(mode, Lists.immutable.of(files.split(",")))
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + type);
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text")
|
||||
public void submitText(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
|
||||
@RequestParam(value = "type", defaultValue = "text") String type,
|
||||
@RequestParam(value = "content", required = false) String content,
|
||||
@RequestParam(value = "files", required = false) String files
|
||||
) {
|
||||
if (StrUtil.equals("text", type)) {
|
||||
embeddingService.submit(id, mode, content);
|
||||
} else if (StrUtil.equals("file", type)) {
|
||||
embeddingService.submit(id, mode, Lists.immutable.of(files.split(",")));
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + type);
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text_directly")
|
||||
public void submitDirectly(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "name", required = false) String name,
|
||||
@RequestParam(value = "split_key", defaultValue = "\n\n") String splitKey,
|
||||
@RequestBody String content
|
||||
) {
|
||||
embeddingService.submitDirectly(id, name, Lists.immutable.of(content.split(splitKey)));
|
||||
}
|
||||
|
||||
@PostMapping("query")
|
||||
public ImmutableList<String> query(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam(value = "limit", defaultValue = "5") Integer limit,
|
||||
@RequestParam(value = "threshold", defaultValue = "0.6") Double threshold,
|
||||
@RequestBody String text
|
||||
) throws ExecutionException, InterruptedException, IOException {
|
||||
return knowledgeBaseService.query(id, text, limit, threshold);
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.springframework.ai.document.Document;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
public class EmbeddingContext {
|
||||
private Long vectorSourceId;
|
||||
private Long groupId;
|
||||
private Config config;
|
||||
private String content;
|
||||
private String file;
|
||||
private String fileFormat;
|
||||
private List<Document> documents = Lists.mutable.empty();
|
||||
private Map<String, Object> metadata = Maps.mutable.empty();
|
||||
|
||||
private EmbeddingContext(Builder builder) {
|
||||
setVectorSourceId(builder.vectorSourceId);
|
||||
setGroupId(builder.groupId);
|
||||
setConfig(builder.config);
|
||||
setContent(builder.content);
|
||||
setFile(builder.file);
|
||||
setFileFormat(builder.fileFormat);
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public Long getGroupId() {
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(Long groupId) {
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
public Long getVectorSourceId() {
|
||||
return vectorSourceId;
|
||||
}
|
||||
|
||||
public void setVectorSourceId(Long vectorSourceId) {
|
||||
this.vectorSourceId = vectorSourceId;
|
||||
}
|
||||
|
||||
public Config getConfig() {
|
||||
return config;
|
||||
}
|
||||
|
||||
public void setConfig(Config config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public String getFile() {
|
||||
return file;
|
||||
}
|
||||
|
||||
public void setFile(String file) {
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
public String getFileFormat() {
|
||||
return fileFormat;
|
||||
}
|
||||
|
||||
public void setFileFormat(String fileFormat) {
|
||||
this.fileFormat = fileFormat;
|
||||
}
|
||||
|
||||
public List<Document> getDocuments() {
|
||||
return documents;
|
||||
}
|
||||
|
||||
public void setDocuments(List<Document> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
|
||||
public Map<String, Object> getMetadata() {
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public void setMetadata(Map<String, Object> metadata) {
|
||||
this.metadata = metadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "EmbeddingContext{" +
|
||||
"vectorSourceId=" + vectorSourceId +
|
||||
", groupId=" + groupId +
|
||||
", config=" + config +
|
||||
", content='" + content + '\'' +
|
||||
", file='" + file + '\'' +
|
||||
", fileFormat='" + fileFormat + '\'' +
|
||||
", documents=" + documents +
|
||||
", metadata=" + metadata +
|
||||
'}';
|
||||
}
|
||||
|
||||
public static final class Config {
|
||||
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
|
||||
|
||||
private Config(Builder builder) {setSplitStrategy(builder.splitStrategy);}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public SplitStrategy getSplitStrategy() {
|
||||
return splitStrategy;
|
||||
}
|
||||
|
||||
public void setSplitStrategy(SplitStrategy splitStrategy) {
|
||||
this.splitStrategy = splitStrategy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Config{" +
|
||||
"splitStrategy=" + splitStrategy +
|
||||
'}';
|
||||
}
|
||||
|
||||
public enum SplitStrategy {
|
||||
NORMAL, LLM, QA
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private SplitStrategy splitStrategy;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder splitStrategy(SplitStrategy val) {
|
||||
splitStrategy = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Config build() {
|
||||
return new Config(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private Long vectorSourceId;
|
||||
private Long groupId;
|
||||
private Config config;
|
||||
private String content;
|
||||
private String file;
|
||||
private String fileFormat;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder vectorSourceId(Long val) {
|
||||
vectorSourceId = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder groupId(Long val) {
|
||||
groupId = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder config(Config val) {
|
||||
config = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder content(String val) {
|
||||
content = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder file(String val) {
|
||||
file = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder fileFormat(String val) {
|
||||
fileFormat = val;
|
||||
return this;
|
||||
}
|
||||
|
||||
public EmbeddingContext build() {
|
||||
return new EmbeddingContext(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
public class Group {
|
||||
private String id;
|
||||
private String name;
|
||||
private String status;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Long getCreatedTime() {
|
||||
return createdTime;
|
||||
}
|
||||
|
||||
public void setCreatedTime(Long createdTime) {
|
||||
this.createdTime = createdTime;
|
||||
}
|
||||
|
||||
public Long getModifiedTime() {
|
||||
return modifiedTime;
|
||||
}
|
||||
|
||||
public void setModifiedTime(Long modifiedTime) {
|
||||
this.modifiedTime = modifiedTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "GroupVO{" +
|
||||
"id='" + id + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", status='" + status + '\'' +
|
||||
", createdTime=" + createdTime +
|
||||
", modifiedTime=" + modifiedTime +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
public class Knowledge {
|
||||
private Long id;
|
||||
private Long vectorSourceId;
|
||||
private String name;
|
||||
private String strategy;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public Long getVectorSourceId() {
|
||||
return vectorSourceId;
|
||||
}
|
||||
|
||||
public void setVectorSourceId(Long vectorSourceId) {
|
||||
this.vectorSourceId = vectorSourceId;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStrategy() {
|
||||
return strategy;
|
||||
}
|
||||
|
||||
public void setStrategy(String strategy) {
|
||||
this.strategy = strategy;
|
||||
}
|
||||
|
||||
public Long getCreatedTime() {
|
||||
return createdTime;
|
||||
}
|
||||
|
||||
public void setCreatedTime(Long createdTime) {
|
||||
this.createdTime = createdTime;
|
||||
}
|
||||
|
||||
public Long getModifiedTime() {
|
||||
return modifiedTime;
|
||||
}
|
||||
|
||||
public void setModifiedTime(Long modifiedTime) {
|
||||
this.modifiedTime = modifiedTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Knowledge{" +
|
||||
"id=" + id +
|
||||
", vectorSourceId=" + vectorSourceId +
|
||||
", name='" + name + '\'' +
|
||||
", strategy='" + strategy + '\'' +
|
||||
", createdTime=" + createdTime +
|
||||
", modifiedTime=" + modifiedTime +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
public class DataFileVO {
|
||||
private String id;
|
||||
private String filename;
|
||||
private Long size;
|
||||
private String md5;
|
||||
private String path;
|
||||
private String type;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public void setFilename(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
public Long getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
public void setSize(Long size) {
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public String getMd5() {
|
||||
return md5;
|
||||
}
|
||||
|
||||
public void setMd5(String md5) {
|
||||
this.md5 = md5;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public void setPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DataFile{" +
|
||||
"id='" + id + '\'' +
|
||||
", filename='" + filename + '\'' +
|
||||
", size=" + size +
|
||||
", md5='" + md5 + '\'' +
|
||||
", path='" + path + '\'' +
|
||||
", type='" + type + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class KnowledgeVO {
|
||||
private String id;
|
||||
private String vectorSourceId;
|
||||
private String name;
|
||||
private String strategy;
|
||||
private Long size;
|
||||
private Long points;
|
||||
private Long segments;
|
||||
private String status;
|
||||
private Long createdTime;
|
||||
private Long modifiedTime;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getVectorSourceId() {
|
||||
return vectorSourceId;
|
||||
}
|
||||
|
||||
public void setVectorSourceId(String vectorSourceId) {
|
||||
this.vectorSourceId = vectorSourceId;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStrategy() {
|
||||
return strategy;
|
||||
}
|
||||
|
||||
public void setStrategy(String strategy) {
|
||||
this.strategy = strategy;
|
||||
}
|
||||
|
||||
public Long getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
public void setSize(Long size) {
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public Long getPoints() {
|
||||
return points;
|
||||
}
|
||||
|
||||
public void setPoints(Long points) {
|
||||
this.points = points;
|
||||
}
|
||||
|
||||
public Long getSegments() {
|
||||
return segments;
|
||||
}
|
||||
|
||||
public void setSegments(Long segments) {
|
||||
this.segments = segments;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Long getCreatedTime() {
|
||||
return createdTime;
|
||||
}
|
||||
|
||||
public void setCreatedTime(Long createdTime) {
|
||||
this.createdTime = createdTime;
|
||||
}
|
||||
|
||||
public Long getModifiedTime() {
|
||||
return modifiedTime;
|
||||
}
|
||||
|
||||
public void setModifiedTime(Long modifiedTime) {
|
||||
this.modifiedTime = modifiedTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "KnowledgeVO{" +
|
||||
"id='" + id + '\'' +
|
||||
", vectorSourceId='" + vectorSourceId + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", strategy='" + strategy + '\'' +
|
||||
", size=" + size +
|
||||
", points=" + points +
|
||||
", segments=" + segments +
|
||||
", status='" + status + '\'' +
|
||||
", createdTime=" + createdTime +
|
||||
", modifiedTime=" + modifiedTime +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class SegmentVO {
|
||||
private String id;
|
||||
private String text;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getText() {
|
||||
return text;
|
||||
}
|
||||
|
||||
public void setText(String text) {
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "PointVO{" +
|
||||
"id='" + id + '\'' +
|
||||
", text='" + text + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Service
|
||||
public class DataFileService {
|
||||
private static final Logger log = LoggerFactory.getLogger(DataFileService.class);
|
||||
private static final String DATA_FILE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_file";
|
||||
|
||||
private final JdbcTemplate template;
|
||||
|
||||
public DataFileService(JdbcTemplate template) {
|
||||
this.template = template;
|
||||
}
|
||||
|
||||
public DataFileVO downloadFile(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "filename", "size", "md5", "path", "type")
|
||||
.from(DATA_FILE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
(rs, row) -> {
|
||||
DataFileVO vo = new DataFileVO();
|
||||
vo.setId(String.valueOf(rs.getLong(1)));
|
||||
vo.setFilename(rs.getString(2));
|
||||
vo.setSize(rs.getLong(3));
|
||||
vo.setMd5(rs.getString(4));
|
||||
vo.setPath(rs.getString(5));
|
||||
vo.setType(rs.getString(6));
|
||||
return vo;
|
||||
},
|
||||
id
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long initialDataFile(String filename) {
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(DATA_FILE_TABLE_NAME, "id", "filename")
|
||||
.values()
|
||||
.addValue("?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
filename
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDataFile(Long id, String path, Long size, String md5, String type) {
|
||||
template.update(
|
||||
SqlBuilder.update(DATA_FILE_TABLE_NAME)
|
||||
.set("size", "?")
|
||||
.addSet("md5", "?")
|
||||
.addSet("path", "?")
|
||||
.addSet("type", "?")
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
size,
|
||||
md5,
|
||||
path,
|
||||
type,
|
||||
id
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import club.kingon.sql.builder.entry.Alias;
|
||||
import club.kingon.sql.builder.entry.Column;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class GroupService {
|
||||
public static final String GROUP_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_group";
|
||||
private static final Logger logger = LoggerFactory.getLogger(GroupService.class);
|
||||
private static final RowMapper<Group> groupMapper = (rs, row) -> {
|
||||
Group vo = new Group();
|
||||
vo.setId(String.valueOf(rs.getLong(1)));
|
||||
vo.setName(rs.getString(2));
|
||||
vo.setStatus(rs.getString(3));
|
||||
vo.setCreatedTime(rs.getTimestamp(4).getTime());
|
||||
vo.setModifiedTime(rs.getTimestamp(5).getTime());
|
||||
return vo;
|
||||
};
|
||||
|
||||
private final JdbcTemplate template;
|
||||
private final QdrantClient client;
|
||||
|
||||
public GroupService(JdbcTemplate template, VectorStore vectorStore) {
|
||||
this.template = template;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
public Group get(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
|
||||
.from(GROUP_TABLE_NAME)
|
||||
.whereEq("id", id)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
groupMapper
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long add(Long knowledgeId, String name) {
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name", "status")
|
||||
.values()
|
||||
.addValue("?", "?", "?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
knowledgeId,
|
||||
name,
|
||||
"RUNNING"
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
public ImmutableList<Group> list(Long knowledgeId) {
|
||||
return template.query(
|
||||
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
|
||||
.from(GROUP_TABLE_NAME)
|
||||
.whereEq("knowledge_id", knowledgeId)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
groupMapper
|
||||
)
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void finish(Long groupId) {
|
||||
template.update(
|
||||
SqlBuilder.update(GROUP_TABLE_NAME)
|
||||
.set("status", "FINISHED")
|
||||
.whereEq("id", groupId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(Long groupId) throws ExecutionException, InterruptedException {
|
||||
Long vectorSourceId = template.queryForObject(
|
||||
SqlBuilder.select("k.vector_source_id")
|
||||
.from(Alias.of(GROUP_TABLE_NAME, "g"), Alias.of(KnowledgeBaseService.KNOWLEDGE_TABLE_NAME, "k"))
|
||||
.whereEq("g.knowledge_id", Column.as("k.id"))
|
||||
.andEq("g.id", groupId)
|
||||
.precompileSql(),
|
||||
Long.class,
|
||||
groupId
|
||||
);
|
||||
client.deleteAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(vectorSourceId)))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
|
||||
.build()
|
||||
).get();
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("id", groupId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void removeByKnowledgeId(Long knowledgeId) {
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("knowledge_id", "?")
|
||||
.precompileSql(),
|
||||
knowledgeId
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.noear.solon.ai.reranking.RerankingModel;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseService {
|
||||
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseService.class);
|
||||
private static final RowMapper<Knowledge> knowledgeMapper = (rs, row) -> {
|
||||
Knowledge knowledge = new Knowledge();
|
||||
knowledge.setId(rs.getLong(1));
|
||||
knowledge.setVectorSourceId(rs.getLong(2));
|
||||
knowledge.setName(rs.getString(3));
|
||||
knowledge.setStrategy(rs.getString(4));
|
||||
knowledge.setCreatedTime(rs.getTimestamp(5).getTime());
|
||||
knowledge.setModifiedTime(rs.getTimestamp(6).getTime());
|
||||
return knowledge;
|
||||
};
|
||||
private final JdbcTemplate template;
|
||||
private final EmbeddingModel model;
|
||||
private final QdrantClient client;
|
||||
private final GroupService groupService;
|
||||
private final RerankingModel rerankingModel;
|
||||
|
||||
public KnowledgeBaseService(JdbcTemplate template, EmbeddingModel model, VectorStore vectorStore, GroupService groupService, RerankingModel rerankingModel) {
|
||||
this.template = template;
|
||||
this.model = model;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.groupService = groupService;
|
||||
this.rerankingModel = rerankingModel;
|
||||
}
|
||||
|
||||
public Knowledge get(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
knowledgeMapper,
|
||||
id
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void add(String name, String strategy) throws ExecutionException, InterruptedException {
|
||||
Integer count = template.queryForObject(
|
||||
SqlBuilder.select("count(*)")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("name", "?")
|
||||
.precompileSql(),
|
||||
Integer.class,
|
||||
name
|
||||
);
|
||||
if (count > 0) {
|
||||
throw new RuntimeException("名称已存在");
|
||||
}
|
||||
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
long vectorSourceId = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(KNOWLEDGE_TABLE_NAME, "id", "vector_source_id", "name", "strategy")
|
||||
.values()
|
||||
.addValue("?", "?", "?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
vectorSourceId,
|
||||
name,
|
||||
strategy
|
||||
);
|
||||
client.createCollectionAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.valueOf(strategy))
|
||||
.setSize(model.dimensions())
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
|
||||
public String getName(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("name")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", id)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
String.class
|
||||
);
|
||||
}
|
||||
|
||||
public ImmutableList<KnowledgeVO> list() {
|
||||
return template.query(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.orderByDesc("created_time")
|
||||
.build(),
|
||||
knowledgeMapper
|
||||
)
|
||||
.stream()
|
||||
.map(knowledge -> {
|
||||
try {
|
||||
Collections.CollectionInfo info = client.getCollectionInfoAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
KnowledgeVO vo = new KnowledgeVO();
|
||||
vo.setId(String.valueOf(knowledge.getId()));
|
||||
vo.setVectorSourceId(String.valueOf(knowledge.getVectorSourceId()));
|
||||
vo.setName(knowledge.getName());
|
||||
vo.setPoints(info.getPointsCount());
|
||||
vo.setSegments(info.getSegmentsCount());
|
||||
vo.setStatus(info.getStatus().name());
|
||||
Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
|
||||
vo.setStrategy(vectorParams.getDistance().name());
|
||||
vo.setSize(vectorParams.getSize());
|
||||
vo.setCreatedTime(vo.getCreatedTime());
|
||||
vo.setModifiedTime(vo.getModifiedTime());
|
||||
return vo;
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(Long id) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = get(id);
|
||||
if (ObjectUtil.isNull(knowledge)) {
|
||||
throw new RuntimeException(StrUtil.format("{} 不存在"));
|
||||
}
|
||||
template.update(
|
||||
SqlBuilder.delete(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
knowledge.getId()
|
||||
);
|
||||
groupService.removeByKnowledgeId(knowledge.getId());
|
||||
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
}
|
||||
|
||||
public ImmutableList<String> query(
|
||||
Long id,
|
||||
String text,
|
||||
Integer limit,
|
||||
Double threshold
|
||||
) throws ExecutionException, InterruptedException, IOException {
|
||||
Knowledge knowledge = get(id);
|
||||
Boolean exists = client.collectionExistsAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
if (!exists) {
|
||||
throw new RuntimeException(StrUtil.format("{} not exists", id));
|
||||
}
|
||||
VectorStore vs = QdrantVectorStore.builder(client, model)
|
||||
.collectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.initializeSchema(false)
|
||||
.build();
|
||||
List<Document> documents = vs.similaritySearch(
|
||||
SearchRequest.builder()
|
||||
.query(text)
|
||||
.topK(limit)
|
||||
.similarityThreshold(threshold)
|
||||
.build()
|
||||
);
|
||||
// 如果只是一个知识库的话,似乎没有什么rerank的必要...
|
||||
/* List<org.noear.solon.ai.rag.Document> rerankDocuments = rerankingModel.rerank(
|
||||
text,
|
||||
documents.stream()
|
||||
.map(doc -> new org.noear.solon.ai.rag.Document(doc.getId(), doc.getText(), doc.getMetadata(), doc.getScore()))
|
||||
.toList()
|
||||
); */
|
||||
return Lists.immutable.ofAll(documents)
|
||||
.collect(Document::getText);
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-knowledge
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics,forest
|
||||
ai:
|
||||
openai:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
options:
|
||||
model: 'Qwen3-1.7-vllm'
|
||||
embedding:
|
||||
options:
|
||||
model: 'Bge-m3-vllm'
|
||||
vectorstore:
|
||||
qdrant:
|
||||
host: 132.121.206.65
|
||||
port: 29463
|
||||
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
|
||||
liteflow:
|
||||
rule-source: config/flow.xml
|
||||
print-banner: false
|
||||
check-node-exists: false
|
||||
solon:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
rerank:
|
||||
model: 'Bge-reranker-v2-vllm'
|
||||
endpoint: '/v1/rerank'
|
||||
@@ -1,34 +0,0 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %clr(%5p) %clr([${HOSTNAME}]){yellow} %clr([%t]){magenta} %clr(%logger{40}){cyan} #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -1,116 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge;
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
import org.springframework.http.client.reactive.JdkClientHttpConnector;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250526
|
||||
*/
|
||||
public class TestLlm {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestLlm.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
HttpClient httpClient = HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
ChatClient client = ChatClient.builder(
|
||||
OpenAiChatModel.builder()
|
||||
.openAiApi(
|
||||
OpenAiApi.builder()
|
||||
.baseUrl("http://132.121.206.65:10086")
|
||||
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
|
||||
.webClientBuilder(WebClient.builder().clientConnector(new JdkClientHttpConnector(httpClient)))
|
||||
.restClientBuilder(RestClient.builder().requestFactory(new JdkClientHttpRequestFactory(httpClient)))
|
||||
.build()
|
||||
)
|
||||
.defaultOptions(
|
||||
OpenAiChatOptions.builder()
|
||||
.model("Qwen3-1.7-vllm")
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
String content = client.prompt()
|
||||
.system("""
|
||||
对用户输入的文本,生成多组高质量的问答对。请遵循以下指南:
|
||||
1. 问题部分:
|
||||
为同一个主题创建尽可能多的不同表述的问题,确保问题的多样性。
|
||||
每个问题应考虑用户可能的多种问法,例如:
|
||||
直接询问(如“什么是...?”)
|
||||
请求确认(如“是否可以说...?”)
|
||||
寻求解释(如“请解释一下...的含义。”)
|
||||
假设性问题(如“如果...会怎样?”)
|
||||
例子请求(如“能否举个例子说明...?”)
|
||||
问题应涵盖文本中的关键信息、主要概念和细节,确保不遗漏重要内容。
|
||||
2. 答案部分:
|
||||
提供一个全面、信息丰富的答案,涵盖问题的所有可能角度,确保逻辑连贯。
|
||||
答案应直接基于给定文本,确保准确性和一致性。
|
||||
包含相关的细节,如日期、名称、职位等具体信息,必要时提供背景信息以增强理解。
|
||||
3. 格式:
|
||||
使用"问:"标记问题集合的开始,所有问题应在一个段落内,问题之间用空格分隔。
|
||||
使用"答:"标记答案的开始,答案应清晰分段,便于阅读。
|
||||
问答对之间用“---”分隔,以提高可读性。
|
||||
4. 内容要求:
|
||||
确保问答对紧密围绕文本主题,避免偏离主题。
|
||||
避免添加文本中未提及的信息,确保信息的真实性。
|
||||
一个问题搭配一个答案,避免一组问答对中同时涉及多个问题。
|
||||
如果文本信息不足以回答某个方面,可以在答案中说明 "根据给定信息无法确定",并尽量提供相关的上下文。
|
||||
""")
|
||||
.user("""
|
||||
Apache Hudi 是一款开源的数据湖管理框架,专注于高效处理大规模数据集的增量更新和实时操作。以下是其核心要点介绍:
|
||||
核心功能与特性
|
||||
|
||||
增量数据处理:支持插入(Insert)、更新(Update)、删除(Delete)操作,并通过“Upsert”(插入或更新)机制高效处理变更数据,避免重写整个数据集。
|
||||
|
||||
事务支持:提供 ACID 事务保证,确保数据一致性和可靠性,支持原子提交和回滚。
|
||||
|
||||
存储优化:自动合并小文件并维护文件最佳大小,减少存储碎片化问题,提升查询性能。
|
||||
|
||||
多查询类型:
|
||||
|
||||
快照查询:获取最新数据状态(对 MoR 表合并基础文件和增量文件)。
|
||||
|
||||
增量查询:捕获自某次提交后的变更数据流,适用于增量管道。
|
||||
|
||||
读取优化查询:针对 MoR 表展示最新压缩后的数据。
|
||||
存储模型
|
||||
|
||||
写入时复制(Copy-on-Write, CoW):数据以列式格式(如 Parquet)存储,每次更新生成新版本文件,适合读多写少的场景。
|
||||
|
||||
读取时合并(Merge-on-Read, MoR):结合列式(Parquet)和行式(Avro)存储,更新写入增量文件后异步合并,适合写频繁的场景。
|
||||
适用场景
|
||||
|
||||
合规性需求:支持 GDPR、CCPA 等法规要求的数据删除和更新。
|
||||
|
||||
实时数据流处理:如 IoT 设备数据、CDC(变更数据捕获)系统,实现近实时分析。
|
||||
|
||||
数据湖管理:优化大规模数据集的存储和查询效率,支持时间旅行查询和历史版本回溯。
|
||||
技术生态与兼容性
|
||||
|
||||
多引擎支持:与 Spark、Flink、Hive、Presto、Trino 等计算和查询引擎集成。
|
||||
|
||||
存储格式:基于 Parquet 和 Avro,兼容 Hadoop 生态及云存储(如 Amazon S3)。
|
||||
优势对比
|
||||
|
||||
相较于同类产品(如 Apache Iceberg、Delta Lake),Hudi 在实时更新和增量处理方面表现突出,尤其适合需要频繁数据变更的场景。其独特的 MoR 模型在写入性能上优于传统批处理方案。
|
||||
|
||||
总结
|
||||
|
||||
Apache Hudi 通过灵活的存储模型、高效的事务管理和广泛的生态系统集成,成为构建现代化数据湖的核心工具,适用于金融、物联网、实时分析等对数据新鲜度和操作效率要求高的领域。
|
||||
""")
|
||||
.call()
|
||||
.content();
|
||||
log.info(content);
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-ai-knowledge</artifactId>
|
||||
<artifactId>service-ai-web</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -20,12 +20,24 @@
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jetty</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-vector-store-qdrant</artifactId>
|
||||
@@ -36,7 +48,11 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
<artifactId>spring-boot-starter-data-jpa</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.blinkfox</groupId>
|
||||
<artifactId>fenix-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
@@ -66,6 +82,37 @@
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<annotationProcessorPaths>
|
||||
<path>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct-processor</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok-mapstruct-binding</artifactId>
|
||||
<version>0.2.0</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.hibernate</groupId>
|
||||
<artifactId>hibernate-jpamodelgen</artifactId>
|
||||
<version>6.6.8.Final</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
<compilerArgs>
|
||||
<arg>-Amapstruct.defaultComponentModel=spring</arg>
|
||||
<arg>-Amapstruct.defaultInjectionStrategy=constructor</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
@@ -1,37 +1,53 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import com.blinkfox.fenix.EnableFenix;
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
* @version 20250515
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
public class AiChatApplication implements ApplicationContextAware {
|
||||
@EnableScheduling
|
||||
@EnableFenix
|
||||
@EnableJpaAuditing
|
||||
public class WebApplication implements ApplicationRunner, ApplicationContextAware {
|
||||
private static ApplicationContext context;
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(AiChatApplication.class, args);
|
||||
SpringApplication.run(WebApplication.class, args);
|
||||
}
|
||||
|
||||
public static <T> T getBean(Class<T> clazz) {
|
||||
return context.getBean(clazz);
|
||||
}
|
||||
|
||||
public static <T> T getBean(String name, Class<T> clazz) {
|
||||
return context.getBean(name, clazz);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
AiChatApplication.context = context;
|
||||
WebApplication.context = context;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface DetailController<DETAIL_ITEM> {
|
||||
String DETAIL = "/detail/{id}";
|
||||
|
||||
AmisResponse<DETAIL_ITEM> detail(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface ListController {
|
||||
String LIST = "/list";
|
||||
|
||||
AmisCrudResponse list() throws Exception;
|
||||
|
||||
AmisCrudResponse list(Query query) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface RemoveController {
|
||||
String REMOVE = "/remove/{id}";
|
||||
|
||||
AmisResponse<Object> remove(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SaveController<SAVE_ITEM> {
|
||||
String SAVE = "/save";
|
||||
|
||||
AmisResponse<Long> save(SAVE_ITEM item) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> extends SaveController<SAVE_ITEM>, ListController, DetailController<DETAIL_ITEM>, RemoveController {
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-26
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class SimpleControllerSupport<ENTITY extends SimpleEntity, SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> implements SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> {
|
||||
protected final SimpleServiceSupport<ENTITY> service;
|
||||
|
||||
public SimpleControllerSupport(SimpleServiceSupport<ENTITY> service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
@PostMapping(SAVE)
|
||||
@Override
|
||||
public AmisResponse<Long> save(@RequestBody SAVE_ITEM item) throws Exception {
|
||||
SaveItemMapper<ENTITY, SAVE_ITEM> mapper = saveItemMapper();
|
||||
return AmisResponse.responseSuccess(service.save(mapper.from(item)));
|
||||
}
|
||||
|
||||
@GetMapping(LIST)
|
||||
@Override
|
||||
public AmisCrudResponse list() throws Exception {
|
||||
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
|
||||
return AmisCrudResponse.responseCrudData(
|
||||
service.list()
|
||||
.collect(entity -> {
|
||||
try {
|
||||
return mapper.from(entity);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@PostMapping(LIST)
|
||||
@Override
|
||||
public AmisCrudResponse list(@RequestBody Query query) throws Exception {
|
||||
if (ObjectUtil.isNull(query)) {
|
||||
return AmisCrudResponse.responseCrudData(List.of(), 0);
|
||||
}
|
||||
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
|
||||
Page<ENTITY> result = service.list(query);
|
||||
return AmisCrudResponse.responseCrudData(
|
||||
result.get()
|
||||
.map(entity -> {
|
||||
try {
|
||||
return mapper.from(entity);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.toList(),
|
||||
result.getTotalElements()
|
||||
);
|
||||
}
|
||||
|
||||
@GetMapping(DETAIL)
|
||||
@Override
|
||||
public AmisResponse<DETAIL_ITEM> detail(@PathVariable("id") Long id) throws Exception {
|
||||
DetailItemMapper<ENTITY, DETAIL_ITEM> mapper = detailItemMapper();
|
||||
return AmisResponse.responseSuccess(mapper.from(service.detailOrThrow(id)));
|
||||
}
|
||||
|
||||
@GetMapping(REMOVE)
|
||||
@Override
|
||||
public AmisResponse<Object> remove(@PathVariable("id") Long id) throws Exception {
|
||||
service.remove(id);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
|
||||
protected abstract SaveItemMapper<ENTITY, SAVE_ITEM> saveItemMapper();
|
||||
|
||||
protected abstract ListItemMapper<ENTITY, LIST_ITEM> listItemMapper();
|
||||
|
||||
protected abstract DetailItemMapper<ENTITY, DETAIL_ITEM> detailItemMapper();
|
||||
|
||||
public interface SaveItemMapper<ENTITY, SAVE_ITEM> {
|
||||
ENTITY from(SAVE_ITEM item) throws Exception;
|
||||
}
|
||||
|
||||
public interface ListItemMapper<ENTITY, LIST_ITEM> {
|
||||
LIST_ITEM from(ENTITY entity) throws Exception;
|
||||
}
|
||||
|
||||
public interface DetailItemMapper<ENTITY, DETAIL_ITEM> {
|
||||
DETAIL_ITEM from(ENTITY entity) throws Exception;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller.query;
|
||||
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
|
||||
/**
|
||||
* 统一查询
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-12-03
|
||||
*/
|
||||
@Data
|
||||
public class Query {
|
||||
private Queryable query;
|
||||
private ImmutableList<Sortable> sort;
|
||||
private Pageable page;
|
||||
|
||||
@Data
|
||||
public static class Queryable {
|
||||
private ImmutableList<String> nullEqual;
|
||||
private ImmutableList<String> notNullEqual;
|
||||
private ImmutableList<String> empty;
|
||||
private ImmutableList<String> notEmpty;
|
||||
private ImmutableMap<String, ?> equal;
|
||||
private ImmutableMap<String, ?> notEqual;
|
||||
private ImmutableMap<String, String> like;
|
||||
private ImmutableMap<String, String> notLike;
|
||||
private ImmutableMap<String, ?> great;
|
||||
private ImmutableMap<String, ?> less;
|
||||
private ImmutableMap<String, ?> greatEqual;
|
||||
private ImmutableMap<String, ?> lessEqual;
|
||||
private ImmutableMap<String, ImmutableList<?>> in;
|
||||
private ImmutableMap<String, ImmutableList<?>> notIn;
|
||||
private ImmutableMap<String, Between> between;
|
||||
private ImmutableMap<String, Between> notBetween;
|
||||
|
||||
@Data
|
||||
public static class Between {
|
||||
private String start;
|
||||
private String end;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Sortable {
|
||||
private String column;
|
||||
private Direction direction;
|
||||
|
||||
public enum Direction {
|
||||
ASC,
|
||||
DESC,
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Pageable {
|
||||
private Integer index;
|
||||
private Integer size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.MappedSuperclass;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.GenericGenerator;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* 实体类公共字段
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-20
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@MappedSuperclass
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
public class IdOnlyEntity {
|
||||
@Id
|
||||
@GeneratedValue(generator = "snowflake")
|
||||
@GenericGenerator(name = "snowflake", strategy = "com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator")
|
||||
private Long id;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.MappedSuperclass;
|
||||
import java.time.LocalDateTime;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.springframework.data.annotation.CreatedDate;
|
||||
import org.springframework.data.annotation.LastModifiedDate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* 实体类公共字段
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-20
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString(callSuper = true)
|
||||
@MappedSuperclass
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
public class SimpleEntity extends IdOnlyEntity {
|
||||
@CreatedDate
|
||||
private LocalDateTime createdTime;
|
||||
@LastModifiedDate
|
||||
private LocalDateTime modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 创建对象使用的接收类
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-26
|
||||
*/
|
||||
@Data
|
||||
public class SimpleItem {
|
||||
private Long id;
|
||||
private LocalDateTime createdTime;
|
||||
private LocalDateTime modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.repository;
|
||||
|
||||
import com.blinkfox.fenix.jpa.FenixJpaRepository;
|
||||
import com.blinkfox.fenix.specification.FenixJpaSpecificationExecutor;
|
||||
import org.springframework.data.repository.NoRepositoryBean;
|
||||
import org.springframework.data.repository.query.QueryByExampleExecutor;
|
||||
|
||||
/**
|
||||
* 整合一下
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@NoRepositoryBean
|
||||
public interface SimpleRepository<E, ID> extends FenixJpaRepository<E, ID>, FenixJpaSpecificationExecutor<E>, QueryByExampleExecutor<E> {
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.service;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import java.util.Optional;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SimpleService<ENTITY extends SimpleEntity> {
|
||||
Long save(ENTITY entity) throws Exception;
|
||||
|
||||
Long count() throws Exception;
|
||||
|
||||
ImmutableList<ENTITY> list() throws Exception;
|
||||
|
||||
ImmutableList<ENTITY> list(ImmutableSet<Long> ids) throws Exception;
|
||||
|
||||
Page<ENTITY> list(Query query) throws Exception;
|
||||
|
||||
Optional<ENTITY> detailOptional(Long id) throws Exception;
|
||||
|
||||
ENTITY detail(Long id) throws Exception;
|
||||
|
||||
ENTITY detailOrThrow(Long id) throws Exception;
|
||||
|
||||
ENTITY detailOrNull(Long id) throws Exception;
|
||||
|
||||
void remove(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.service;
|
||||
|
||||
import cn.hutool.core.bean.BeanUtil;
|
||||
import cn.hutool.core.bean.copier.CopyOptions;
|
||||
import cn.hutool.core.util.EnumUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import jakarta.persistence.criteria.CriteriaBuilder;
|
||||
import jakarta.persistence.criteria.CriteriaQuery;
|
||||
import jakarta.persistence.criteria.Path;
|
||||
import jakarta.persistence.criteria.Predicate;
|
||||
import jakarta.persistence.criteria.Root;
|
||||
import jakarta.transaction.Transactional;
|
||||
import java.util.Optional;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.list.MutableList;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Sort;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implements SimpleService<ENTITY> {
|
||||
private static final Integer DEFAULT_PAGE_INDEX = 1;
|
||||
private static final Integer DEFAULT_PAGE_SIZE = 10;
|
||||
protected final SimpleRepository<ENTITY, Long> repository;
|
||||
|
||||
public SimpleServiceSupport(SimpleRepository<ENTITY, Long> repository) {
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
@Transactional(rollbackOn = Throwable.class)
|
||||
@Override
|
||||
public Long save(ENTITY entity) {
|
||||
if (ObjectUtil.isNotNull(entity.getId())) {
|
||||
Long id = entity.getId();
|
||||
ENTITY targetEntity = repository.findById(entity.getId()).orElseThrow(() -> new IdNotFoundException(id));
|
||||
BeanUtil.copyProperties(
|
||||
entity,
|
||||
targetEntity,
|
||||
CopyOptions.create()
|
||||
.setIgnoreProperties(
|
||||
IdOnlyEntity_.ID,
|
||||
SimpleEntity_.CREATED_TIME,
|
||||
SimpleEntity_.MODIFIED_TIME
|
||||
)
|
||||
);
|
||||
entity = targetEntity;
|
||||
}
|
||||
entity = repository.save(entity);
|
||||
return entity.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long count() {
|
||||
return repository.count(
|
||||
(root, query, builder) ->
|
||||
builder.and(
|
||||
listPredicate(root, query, builder)
|
||||
.reject(ObjectUtil::isNull)
|
||||
.toArray(new Predicate[]{})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<ENTITY> list() {
|
||||
return Lists.immutable.ofAll(repository.findAll(
|
||||
(root, query, builder) ->
|
||||
builder.and(
|
||||
listPredicate(root, query, builder)
|
||||
.reject(ObjectUtil::isNull)
|
||||
.toArray(new Predicate[]{})
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<ENTITY> list(ImmutableSet<Long> ids) {
|
||||
return Lists.immutable.ofAll(repository.findAll(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.add(builder.in(root.get("id")).value(ids));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
private <Y> Path<Y> column(Root<ENTITY> root, String column) {
|
||||
String[] columns = StrUtil.splitToArray(column, "/");
|
||||
Path<Y> path = root.get(columns[0]);
|
||||
for (int i = 1; i < columns.length; i++) {
|
||||
path = path.get(columns[i]);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private <Y> Object value(Path<Y> column, Object value) {
|
||||
Class<?> javaType = column.getJavaType();
|
||||
if (EnumUtil.isEnum(javaType)) {
|
||||
return EnumUtil.fromString((Class<Enum>) javaType, (String) value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected ImmutableList<Predicate> queryPredicates(Query.Queryable queryable, Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
|
||||
MutableList<Predicate> predicates = Lists.mutable.empty();
|
||||
if (ObjectUtil.isEmpty(queryable)) {
|
||||
return predicates.toImmutable();
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNullEqual())) {
|
||||
queryable.getNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotNullEqual())) {
|
||||
queryable.getNotNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getEmpty())) {
|
||||
queryable.getEmpty().forEach(column -> predicates.add(builder.isEmpty(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotEmpty())) {
|
||||
queryable.getNotEmpty().forEach(column -> predicates.add(builder.isNotEmpty(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getEqual())) {
|
||||
queryable.getEqual().forEachKeyValue((column, value) -> {
|
||||
Path<Object> path = column(root, column);
|
||||
predicates.add(builder.equal(path, value(path, value)));
|
||||
});
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotEqual())) {
|
||||
queryable.getEqual().forEachKeyValue((column, value) -> {
|
||||
Path<Object> path = column(root, column);
|
||||
predicates.add(builder.notEqual(path, value(path, value)));
|
||||
});
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLike())) {
|
||||
queryable.getLike().forEachKeyValue((column, value) -> predicates.add(builder.like(column(root, column), value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotLike())) {
|
||||
queryable.getNotLike().forEachKeyValue((column, value) -> predicates.add(builder.notLike(column(root, column), value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getGreat())) {
|
||||
queryable.getGreat().forEachKeyValue((column, value) -> predicates.add(builder.greaterThan(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLess())) {
|
||||
queryable.getLess().forEachKeyValue((column, value) -> predicates.add(builder.lessThan(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getGreatEqual())) {
|
||||
queryable.getGreatEqual().forEachKeyValue((column, value) -> predicates.add(builder.greaterThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLessEqual())) {
|
||||
queryable.getLessEqual().forEachKeyValue((column, value) -> predicates.add(builder.lessThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getIn())) {
|
||||
queryable.getIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotIn())) {
|
||||
queryable.getNotIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value).not()));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getBetween())) {
|
||||
queryable.getBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotBetween())) {
|
||||
queryable.getNotBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
|
||||
}
|
||||
return predicates.toImmutable();
|
||||
}
|
||||
|
||||
protected ImmutableList<Predicate> listPredicate(Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
|
||||
return Lists.immutable.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<ENTITY> list(Query listQuery) {
|
||||
PageRequest pageRequest = PageRequest.of(DEFAULT_PAGE_INDEX - 1, DEFAULT_PAGE_SIZE, Sort.by(SimpleEntity_.CREATED_TIME).descending());
|
||||
if (ObjectUtil.isNotNull(listQuery.getPage())) {
|
||||
pageRequest = PageRequest.of(
|
||||
ObjectUtil.defaultIfNull(listQuery.getPage().getIndex(), DEFAULT_PAGE_INDEX) - 1,
|
||||
ObjectUtil.defaultIfNull(listQuery.getPage().getSize(), DEFAULT_PAGE_SIZE),
|
||||
Sort.by(SimpleEntity_.CREATED_TIME).descending()
|
||||
);
|
||||
}
|
||||
return repository.findAll(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.addAllIterable(queryPredicates(listQuery.getQuery(), root, query, builder));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
},
|
||||
pageRequest
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ENTITY> detailOptional(Long id) {
|
||||
if (ObjectUtil.isNull(id)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return repository.findOne(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.add(builder.equal(root.get(IdOnlyEntity_.id), id));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detail(Long id) {
|
||||
return detailOrNull(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detailOrThrow(Long id) {
|
||||
return detailOptional(id).orElseThrow(() -> new IdNotFoundException(id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detailOrNull(Long id) {
|
||||
return detailOptional(id).orElse(null);
|
||||
}
|
||||
|
||||
@Transactional(rollbackOn = Throwable.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
if (ObjectUtil.isNotNull(id)) {
|
||||
repository.deleteById(id);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class IdNotFoundException extends RuntimeException {
|
||||
public IdNotFoundException() {
|
||||
super("资源不存在");
|
||||
}
|
||||
|
||||
public IdNotFoundException(Long id) {
|
||||
super(StrUtil.format("ID为{}的资源不存在", id));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "file-store")
|
||||
public class FileStoreProperties {
|
||||
private String downloadPrefix;
|
||||
private String uploadPath;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.time.Duration;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.noear.solon.ai.reranking.RerankingModel;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatModel;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatOptions;
|
||||
import org.springframework.ai.deepseek.api.DeepSeekApi;
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
@Slf4j
|
||||
@Configuration
|
||||
public class LlmConfiguration {
|
||||
@Bean("chat")
|
||||
public ChatClient.Builder chatClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getChat(), "chat properties is null");
|
||||
DeepSeekApi.Builder apiBuilder = DeepSeekApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getChat().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getChat().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getChat().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getChat().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
DeepSeekChatModel.builder()
|
||||
.deepSeekApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
DeepSeekChatOptions.builder()
|
||||
.model(llmProperties.getChat().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean("visual")
|
||||
public ChatClient.Builder visualClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getVisual(), "visual properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getVisual().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getVisual().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getVisual().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getVisual().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
OpenAiChatModel.builder()
|
||||
.openAiApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
OpenAiChatOptions.builder()
|
||||
.model(llmProperties.getVisual().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EmbeddingModel embeddingModel(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getEmbedding(), "embedding properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getEmbedding().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getEmbedding().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getEmbedding().getEndpoint())) {
|
||||
apiBuilder.embeddingsPath(llmProperties.getEmbedding().getEndpoint());
|
||||
}
|
||||
return new OpenAiEmbeddingModel(
|
||||
apiBuilder.build(),
|
||||
MetadataMode.EMBED,
|
||||
OpenAiEmbeddingOptions.builder()
|
||||
.model(llmProperties.getEmbedding().getModel())
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RerankingModel rerankingModel(LlmProperties llmProperties) {
|
||||
Assert.notNull(llmProperties.getReranker(), "reranker properties is null");
|
||||
String url = llmProperties.getBaseUrl();
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getBaseUrl())) {
|
||||
url = llmProperties.getReranker().getBaseUrl();
|
||||
}
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getEndpoint())) {
|
||||
url += llmProperties.getReranker().getEndpoint();
|
||||
} else {
|
||||
url += "/v1/rerank";
|
||||
}
|
||||
return RerankingModel.of(url)
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getReranker().getApiKey(), llmProperties.getApiKey()))
|
||||
.model(llmProperties.getReranker().getModel())
|
||||
.timeout(Duration.ofMinutes(10))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "spring.llm")
|
||||
public class LlmProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private ChatProperties chat;
|
||||
private ChatProperties visual;
|
||||
private ChatProperties embedding;
|
||||
private ChatProperties reranker;
|
||||
|
||||
@Data
|
||||
public static class ChatProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private String model;
|
||||
private String endpoint;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.hibernate.engine.spi.SharedSessionContractImplementor;
|
||||
import org.hibernate.id.IdentifierGenerator;
|
||||
|
||||
/**
|
||||
* 使用雪花算法作为ID生成器
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-14
|
||||
*/
|
||||
@Slf4j
|
||||
public class SnowflakeIdGenerator implements IdentifierGenerator {
|
||||
|
||||
@Override
|
||||
public Serializable generate(SharedSessionContractImplementor session, Object object) {
|
||||
try {
|
||||
return Snowflake.next();
|
||||
} catch (Exception e) {
|
||||
log.error("Generate snowflake id failed", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Snowflake {
|
||||
/**
|
||||
* 起始的时间戳
|
||||
*/
|
||||
private final static long START_TIMESTAMP = 1;
|
||||
|
||||
/**
|
||||
* 序列号占用的位数
|
||||
*/
|
||||
private final static long SEQUENCE_BIT = 11;
|
||||
|
||||
/**
|
||||
* 序列号最大值
|
||||
*/
|
||||
private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
|
||||
|
||||
/**
|
||||
* 时间戳值向左位移
|
||||
*/
|
||||
private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
|
||||
|
||||
/**
|
||||
* 序列号
|
||||
*/
|
||||
private static long sequence = 0;
|
||||
/**
|
||||
* 上一次时间戳
|
||||
*/
|
||||
private static long lastTimestamp = -1;
|
||||
|
||||
public static synchronized long next() {
|
||||
long currentTimestamp = nowTimestamp();
|
||||
if (currentTimestamp < lastTimestamp) {
|
||||
throw new RuntimeException("Clock have moved backwards.");
|
||||
}
|
||||
|
||||
if (currentTimestamp == lastTimestamp) {
|
||||
// 相同毫秒内, 序列号自增
|
||||
sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
|
||||
// 同一毫秒的序列数已经达到最大
|
||||
if (sequence == 0) {
|
||||
currentTimestamp = nextTimestamp();
|
||||
}
|
||||
} else {
|
||||
// 不同毫秒内, 序列号置为0
|
||||
sequence = 0;
|
||||
}
|
||||
|
||||
lastTimestamp = currentTimestamp;
|
||||
return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
|
||||
}
|
||||
|
||||
private static long nextTimestamp() {
|
||||
long milli = nowTimestamp();
|
||||
while (milli <= lastTimestamp) {
|
||||
milli = nowTimestamp();
|
||||
}
|
||||
return milli;
|
||||
}
|
||||
|
||||
private static long nowTimestamp() {
|
||||
return Instant.now().toEpochMilli();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.io.IoUtil;
|
||||
@@ -7,9 +7,9 @@ import cn.hutool.core.util.URLUtil;
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.configuration.KnowledgeConfiguration;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.DataFileService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.FileStoreProperties;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
@@ -17,9 +17,11 @@ import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.FileChannel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
@@ -35,22 +37,21 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/upload")
|
||||
public class DataFileController {
|
||||
private static final Logger log = LoggerFactory.getLogger(DataFileController.class);
|
||||
|
||||
private final KnowledgeConfiguration knowledgeConfiguration;
|
||||
private final FileStoreProperties fileStoreProperties;
|
||||
private final DataFileService dataFileService;
|
||||
private final String uploadFolderPath;
|
||||
private final String cacheFolderPath;
|
||||
private final String sliceFolderPath;
|
||||
|
||||
public DataFileController(KnowledgeConfiguration knowledgeConfiguration, DataFileService dataFileService) {
|
||||
this.knowledgeConfiguration = knowledgeConfiguration;
|
||||
public DataFileController(FileStoreProperties fileStoreProperties, DataFileService dataFileService) {
|
||||
this.fileStoreProperties = fileStoreProperties;
|
||||
this.dataFileService = dataFileService;
|
||||
|
||||
this.uploadFolderPath = knowledgeConfiguration.getUploadPath();
|
||||
this.uploadFolderPath = fileStoreProperties.getUploadPath();
|
||||
this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath);
|
||||
this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath);
|
||||
}
|
||||
@@ -59,7 +60,7 @@ public class DataFileController {
|
||||
public AmisResponse<FinishResponse> upload(@RequestParam("file") MultipartFile file) throws IOException {
|
||||
String filename = file.getOriginalFilename();
|
||||
Long id = dataFileService.initialDataFile(filename);
|
||||
String url = StrUtil.format("{}/upload/download/{}", knowledgeConfiguration.getDownloadPrefix(), id);
|
||||
String url = StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), id);
|
||||
byte[] bytes = file.getBytes();
|
||||
String originMd5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
|
||||
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, originMd5));
|
||||
@@ -79,8 +80,8 @@ public class DataFileController {
|
||||
}
|
||||
|
||||
@GetMapping("/download/{id}")
|
||||
public void download(@PathVariable Long id, HttpServletResponse response) throws IOException {
|
||||
DataFileVO dataFile = dataFileService.downloadFile(id);
|
||||
public void download(@PathVariable("id") Long id, HttpServletResponse response) throws IOException {
|
||||
DataFile dataFile = dataFileService.downloadFile(id);
|
||||
File targetFile = new File(dataFile.getPath());
|
||||
response.setHeader("Content-Type", dataFile.getType());
|
||||
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
|
||||
@@ -156,7 +157,7 @@ public class DataFileController {
|
||||
request.uploadId,
|
||||
request.filename,
|
||||
request.uploadId.toString(),
|
||||
StrUtil.format("{}/upload/download/{}", knowledgeConfiguration.getDownloadPrefix(), request.uploadId)
|
||||
StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), request.uploadId)
|
||||
));
|
||||
} else {
|
||||
throw new RuntimeException("合并文件失败");
|
||||
@@ -169,213 +170,48 @@ public class DataFileController {
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class StartRequest {
|
||||
private String name;
|
||||
private String filename;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public void setFilename(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "StartRequest{" +
|
||||
"name='" + name + '\'' +
|
||||
", filename='" + filename + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class StartResponse {
|
||||
private String uploadId;
|
||||
|
||||
public StartResponse() {
|
||||
}
|
||||
|
||||
public StartResponse(String uploadId) {
|
||||
this.uploadId = uploadId;
|
||||
}
|
||||
|
||||
public String getUploadId() {
|
||||
return uploadId;
|
||||
}
|
||||
|
||||
public void setUploadId(String uploadId) {
|
||||
this.uploadId = uploadId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "StartResponse{" +
|
||||
"uploadId='" + uploadId + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class SliceResponse {
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
|
||||
public SliceResponse() {
|
||||
}
|
||||
|
||||
public SliceResponse(String eTag) {
|
||||
this.eTag = eTag;
|
||||
}
|
||||
|
||||
public String geteTag() {
|
||||
return eTag;
|
||||
}
|
||||
|
||||
public void seteTag(String eTag) {
|
||||
this.eTag = eTag;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SliceResponse{" +
|
||||
"eTag='" + eTag + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class FinishRequest {
|
||||
private String filename;
|
||||
private Long uploadId;
|
||||
private ImmutableList<Part> partList;
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public void setFilename(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
public Long getUploadId() {
|
||||
return uploadId;
|
||||
}
|
||||
|
||||
public void setUploadId(Long uploadId) {
|
||||
this.uploadId = uploadId;
|
||||
}
|
||||
|
||||
public ImmutableList<Part> getPartList() {
|
||||
return partList;
|
||||
}
|
||||
|
||||
public void setPartList(ImmutableList<Part> partList) {
|
||||
this.partList = partList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FinishRequest{" +
|
||||
"filename='" + filename + '\'' +
|
||||
", uploadId=" + uploadId +
|
||||
", partList=" + partList +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class Part {
|
||||
private Integer partNumber;
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
|
||||
public Integer getPartNumber() {
|
||||
return partNumber;
|
||||
}
|
||||
|
||||
public void setPartNumber(Integer partNumber) {
|
||||
this.partNumber = partNumber;
|
||||
}
|
||||
|
||||
public String geteTag() {
|
||||
return eTag;
|
||||
}
|
||||
|
||||
public void seteTag(String eTag) {
|
||||
this.eTag = eTag;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Part{" +
|
||||
"partNumber=" + partNumber +
|
||||
", eTag='" + eTag + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class FinishResponse {
|
||||
private Long id;
|
||||
private String filename;
|
||||
private String value;
|
||||
private String url;
|
||||
|
||||
public FinishResponse() {
|
||||
}
|
||||
|
||||
public FinishResponse(Long id, String filename, String value, String url) {
|
||||
this.id = id;
|
||||
this.filename = filename;
|
||||
this.value = value;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public void setFilename(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FinishResponse{" +
|
||||
"id=" + id +
|
||||
", filename='" + filename + '\'' +
|
||||
", value='" + value + '\'' +
|
||||
", url='" + url + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,18 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.controller;
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.chat;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.Prompts;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.entity.MessageVO;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.ChartTool;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.KnowledgeTool;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.TableTool;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.YarnTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.Prompts;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.MessageVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.ChartTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.TableTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.YarnTool;
|
||||
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -22,11 +23,11 @@ import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
@@ -46,11 +47,11 @@ public class ChatController {
|
||||
|
||||
private final ChatClient chatClient;
|
||||
|
||||
public ChatController(ChatClient.Builder builder) {
|
||||
public ChatController(@Qualifier("chat") ChatClient.Builder builder) {
|
||||
this.chatClient = builder.build();
|
||||
}
|
||||
|
||||
private ChatClient.ChatClientRequestSpec buildRequest(Long knowledgeId, ImmutableList<MessageVO> messages) {
|
||||
private ChatClient.ChatClientRequestSpec buildRequest(ImmutableList<MessageVO> messages) {
|
||||
ChatClient.ChatClientRequestSpec spec = chatClient.prompt()
|
||||
.system(
|
||||
StrUtil.format("""
|
||||
@@ -79,9 +80,6 @@ public class ChatController {
|
||||
.collect(message -> (Message) message)
|
||||
.toList()
|
||||
);
|
||||
if (ObjectUtil.isNotNull(knowledgeId)) {
|
||||
spec.tools(new KnowledgeTool(knowledgeId));
|
||||
}
|
||||
spec.tools(
|
||||
new TableTool(),
|
||||
new YarnTool(),
|
||||
@@ -93,10 +91,9 @@ public class ChatController {
|
||||
@PostMapping("sync")
|
||||
@ResponseBody
|
||||
public MessageVO chatSync(
|
||||
@RequestParam(value = "knowledge_id", required = false) Long knowledgeId,
|
||||
@RequestBody ImmutableList<MessageVO> messages
|
||||
) {
|
||||
ChatResponse response = buildRequest(knowledgeId, messages)
|
||||
ChatResponse response = buildRequest(messages)
|
||||
.call()
|
||||
.chatResponse();
|
||||
return toMessage(response);
|
||||
@@ -104,25 +101,34 @@ public class ChatController {
|
||||
|
||||
@PostMapping("async")
|
||||
public SseEmitter chatAsync(
|
||||
@RequestParam(value = "knowledge_id", required = false) Long knowledgeId,
|
||||
@RequestBody ImmutableList<MessageVO> messages
|
||||
@RequestBody ImmutableList<MessageVO> messages,
|
||||
HttpServletResponse httpResponse
|
||||
) {
|
||||
SseEmitter emitter = new SseEmitter();
|
||||
buildRequest(knowledgeId, messages)
|
||||
.stream()
|
||||
.chatResponse()
|
||||
.subscribe(
|
||||
response -> {
|
||||
try {
|
||||
emitter.send(toMessage(response));
|
||||
} catch (IOException e) {
|
||||
emitter.completeWithError(e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
},
|
||||
emitter::completeWithError,
|
||||
emitter::complete
|
||||
);
|
||||
httpResponse.setHeader("X-Accel-Buffering", "no");
|
||||
|
||||
SseEmitter emitter = new SseEmitter(20 * 60 * 1000L);
|
||||
ExecutorProvider.EXECUTORS.submit(() -> {
|
||||
buildRequest(messages)
|
||||
.stream()
|
||||
.chatResponse()
|
||||
.subscribe(
|
||||
response -> {
|
||||
try {
|
||||
emitter.send(
|
||||
SseEmitter.event()
|
||||
.data(toMessage(response))
|
||||
.reconnectTime(5 * 1000L)
|
||||
.build()
|
||||
);
|
||||
} catch (IOException e) {
|
||||
emitter.completeWithError(e);
|
||||
}
|
||||
},
|
||||
emitter::completeWithError,
|
||||
emitter::complete
|
||||
);
|
||||
});
|
||||
emitter.onTimeout(() -> emitter.completeWithError(new TimeoutException("SseEmitter Timeout")));
|
||||
return emitter;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.feedback;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Sets;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("feedback")
|
||||
public class FeedbackController extends SimpleControllerSupport<Feedback, FeedbackController.SaveItem, FeedbackController.ListItem, FeedbackController.ListItem> {
|
||||
private final FeedbackService feedbackService;
|
||||
private final DataFileService dataFileService;
|
||||
|
||||
public FeedbackController(FeedbackService feedbackService, DataFileService dataFileService) {
|
||||
super(feedbackService);
|
||||
this.feedbackService = feedbackService;
|
||||
this.dataFileService = dataFileService;
|
||||
}
|
||||
|
||||
@GetMapping("reanalysis/{id}")
|
||||
public void reanalysis(@PathVariable("id") Long id) {
|
||||
feedbackService.reanalysis(id);
|
||||
}
|
||||
|
||||
@PostMapping("conclude")
|
||||
public void conclude(@RequestBody ConcludeItem item) {
|
||||
feedbackService.updateConclusion(item.getId(), item.getConclusion());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Feedback, SaveItem> saveItemMapper() {
|
||||
return item -> {
|
||||
Feedback feedback = new Feedback();
|
||||
feedback.setSource(item.getSource());
|
||||
feedback.setPictures(dataFileService.downloadFile(item.getPictures()).toSet());
|
||||
return feedback;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Feedback, ListItem> listItemMapper() {
|
||||
return Mappers.getMapper(ListItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Feedback, ListItem> detailItemMapper() {
|
||||
ListItemMapper<Feedback, ListItem> mapper = listItemMapper();
|
||||
return mapper::from;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String source;
|
||||
private ImmutableSet<Long> pictures;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class ConcludeItem {
|
||||
private Long id;
|
||||
private String conclusion;
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static class ListItem extends SimpleItem {
|
||||
private String source;
|
||||
private ImmutableSet<Long> pictures;
|
||||
private String analysis;
|
||||
private String conclusion;
|
||||
private Feedback.Status status;
|
||||
|
||||
@org.mapstruct.Mapper(
|
||||
imports = {
|
||||
IdOnlyEntity.class,
|
||||
Sets.class
|
||||
}
|
||||
)
|
||||
public interface Mapper extends ListItemMapper<Feedback, ListItem> {
|
||||
@Mapping(target = "pictures", expression = "java(Sets.immutable.ofAll(feedback.getPictures()).collect(IdOnlyEntity::getId))")
|
||||
@Override
|
||||
ListItem from(Feedback feedback);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge/group")
|
||||
public class GroupController extends SimpleControllerSupport<Group, GroupController.SaveItem, GroupController.ListItem, GroupController.ListItem> {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
|
||||
public GroupController(GroupService groupService, KnowledgeBaseService knowledgeBaseService) {
|
||||
super(groupService);
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Group, SaveItem> saveItemMapper() {
|
||||
return item -> {
|
||||
Group group = new Group();
|
||||
group.setName(item.getName());
|
||||
group.setKnowledge(knowledgeBaseService.detailOrThrow(item.getKnowledgeId()));
|
||||
return group;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Group, ListItem> listItemMapper() {
|
||||
return Mappers.getMapper(ListItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Group, ListItem> detailItemMapper() {
|
||||
ListItemMapper<Group, ListItem> mapper = listItemMapper();
|
||||
return mapper::from;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String name;
|
||||
private Long knowledgeId;
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static final class ListItem extends SimpleItem {
|
||||
private String name;
|
||||
private Group.Status status;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends ListItemMapper<Group, ListItem> {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.EmbeddingService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge")
|
||||
public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge, KnowledgeBaseController.SaveItem, KnowledgeBaseController.ListItem, KnowledgeBaseController.ListItem> {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final EmbeddingService embeddingService;
|
||||
|
||||
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
|
||||
super(knowledgeBaseService);
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.embeddingService = embeddingService;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Knowledge, SaveItem> saveItemMapper() {
|
||||
return Mappers.getMapper(SaveItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Knowledge, ListItem> listItemMapper() {
|
||||
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
|
||||
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Knowledge, ListItem> detailItemMapper() {
|
||||
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
|
||||
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
|
||||
}
|
||||
|
||||
@GetMapping("{id}/name")
|
||||
public AmisMapResponse name(@PathVariable("id") Long id) {
|
||||
return AmisResponse.responseMapData()
|
||||
.setData("name", knowledgeBaseService.getName(id));
|
||||
}
|
||||
|
||||
@PostMapping("update_description")
|
||||
public void updateDescription(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam("description") String description
|
||||
) throws Exception {
|
||||
knowledgeBaseService.updateDescription(id, description);
|
||||
}
|
||||
|
||||
@PostMapping("preview_text")
|
||||
public AmisResponse<?> previewText(@RequestBody SubmitItem item) {
|
||||
if (StrUtil.equals("text", item.getType())) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(item.getMode(), item.getContent())
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else if (StrUtil.equals("file", item.getType())) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(item.getMode(), item.getFiles())
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + item.getType());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text")
|
||||
public void submitText(@RequestBody SubmitItem item) {
|
||||
if (StrUtil.equals("text", item.getMode())) {
|
||||
embeddingService.submit(item.getId(), item.getMode(), item.getContent());
|
||||
} else if (StrUtil.equals("file", item.getType())) {
|
||||
embeddingService.submit(item.getId(), item.getMode(), item.getFiles());
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + item.getType());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text_directly")
|
||||
public void submitDirectly(@RequestBody SubmitDirectlyItem item) {
|
||||
embeddingService.submitDirectly(item.getId(), item.getName(), Lists.immutable.ofAll(StrUtil.split(item.getContent(), item.getSplitKey())));
|
||||
}
|
||||
|
||||
@PostMapping("query")
|
||||
public ImmutableList<String> query(@RequestBody QueryItem item) throws ExecutionException, InterruptedException, IOException {
|
||||
return knowledgeBaseService.query(item.getId(), item.getText(), item.getLimit(), item.getThreshold());
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String name;
|
||||
private String strategy;
|
||||
private String description;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends SaveItemMapper<Knowledge, SaveItem> {
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static final class ListItem extends SimpleItem {
|
||||
private Long vectorSourceId;
|
||||
private String name;
|
||||
private String description;
|
||||
private String strategy;
|
||||
private Long size;
|
||||
private Long points;
|
||||
private Long segments;
|
||||
private String status;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends ListItemMapper<Knowledge, ListItem> {
|
||||
@Mapping(source = "info.config.params.vectorsConfig.params.distance", target = "strategy")
|
||||
@Mapping(source = "info.config.params.vectorsConfig.params.size", target = "size")
|
||||
@Mapping(source = "info.pointsCount", target = "points")
|
||||
@Mapping(source = "info.segmentsCount", target = "segments")
|
||||
@Mapping(source = "info.status", target = "status")
|
||||
ListItem from(Knowledge knowledge, Collections.CollectionInfo info);
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SubmitItem {
|
||||
private Long id;
|
||||
private String mode;
|
||||
private String type;
|
||||
private String content;
|
||||
private ImmutableList<Long> files;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SubmitDirectlyItem {
|
||||
private Long id;
|
||||
private String name;
|
||||
private String splitKey = "\n\n";
|
||||
private String content;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class QueryItem {
|
||||
private Long id;
|
||||
private Integer limit = 5;
|
||||
private Double threshold = 0.6;
|
||||
private String text;
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.SegmentService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.SegmentService;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
@@ -14,11 +13,10 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("segment")
|
||||
@RequestMapping("knowledge/segment")
|
||||
public class SegmentController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SegmentController.class);
|
||||
|
||||
private final SegmentService segmentService;
|
||||
|
||||
public SegmentController(SegmentService segmentService) {
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
|
||||
/**
|
||||
* 上传文件
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_file")
|
||||
@NoArgsConstructor
|
||||
public class DataFile extends SimpleEntity {
|
||||
private String filename;
|
||||
private Long size;
|
||||
private String md5;
|
||||
private String path;
|
||||
private String type;
|
||||
|
||||
public DataFile(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.ConstraintMode;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.ForeignKey;
|
||||
import jakarta.persistence.JoinTable;
|
||||
import jakarta.persistence.NamedAttributeNode;
|
||||
import jakarta.persistence.NamedEntityGraph;
|
||||
import jakarta.persistence.OneToMany;
|
||||
import jakarta.persistence.Table;
|
||||
import java.util.Set;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_feedback")
|
||||
@NamedEntityGraph(name = "feedback.detail", attributeNodes = {
|
||||
@NamedAttributeNode("pictures")
|
||||
})
|
||||
public class Feedback extends SimpleEntity {
|
||||
@Column(nullable = false, columnDefinition = "longtext")
|
||||
private String source;
|
||||
@OneToMany(fetch = FetchType.EAGER)
|
||||
@JoinTable(catalog = Constants.DATABASE_NAME, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
|
||||
@ToString.Exclude
|
||||
private Set<DataFile> pictures;
|
||||
@Column(columnDefinition = "longtext")
|
||||
private String analysis;
|
||||
@Column(columnDefinition = "longtext")
|
||||
private String conclusion;
|
||||
@Column(nullable = false)
|
||||
private Status status = Status.ANALYSIS_PROCESSING;
|
||||
|
||||
public enum Status {
|
||||
ANALYSIS_PROCESSING,
|
||||
ANALYSIS_SUCCESS,
|
||||
FINISHED,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.ConstraintMode;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.ForeignKey;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_group")
|
||||
public class Group extends SimpleEntity {
|
||||
@Column(nullable = false)
|
||||
private String name;
|
||||
@Column(nullable = false)
|
||||
private Status status = Status.RUNNING;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(nullable = false, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
|
||||
@ToString.Exclude
|
||||
private Knowledge knowledge;
|
||||
|
||||
public enum Status {
|
||||
RUNNING,
|
||||
FINISHED,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.CascadeType;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.OneToMany;
|
||||
import jakarta.persistence.Table;
|
||||
import java.util.Set;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_knowledge")
|
||||
public class Knowledge extends SimpleEntity {
|
||||
@Column(nullable = false)
|
||||
private Long vectorSourceId;
|
||||
@Column(nullable = false)
|
||||
private String name;
|
||||
@Column(nullable = false, columnDefinition = "longtext")
|
||||
private String description;
|
||||
@Column(nullable = false)
|
||||
private Strategy strategy = Strategy.Cosine;
|
||||
|
||||
@OneToMany(fetch = FetchType.LAZY, mappedBy = "knowledge", cascade = CascadeType.ALL)
|
||||
@ToString.Exclude
|
||||
private Set<Group> groups;
|
||||
|
||||
public enum Strategy {
|
||||
Cosine,
|
||||
Euclid,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.context;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.springframework.ai.document.Document;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class EmbeddingContext {
|
||||
private Long vectorSourceId;
|
||||
private Long groupId;
|
||||
private Config config;
|
||||
private String content;
|
||||
private String file;
|
||||
private String fileFormat;
|
||||
@Builder.Default
|
||||
private List<Document> documents = Lists.mutable.empty();
|
||||
@Builder.Default
|
||||
private Map<String, Object> metadata = Maps.mutable.empty();
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static final class Config {
|
||||
@Builder.Default
|
||||
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
|
||||
|
||||
public enum SplitStrategy {
|
||||
NORMAL, LLM, QA
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.context;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250616
|
||||
*/
|
||||
@Data
|
||||
public class FeedbackContext {
|
||||
private Feedback feedback;
|
||||
private String optimizedSource;
|
||||
private List<String> pictureDescriptions = Lists.mutable.empty();
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class MessageVO {
|
||||
private String role;
|
||||
private String content;
|
||||
private String reason;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class SegmentVO {
|
||||
private String id;
|
||||
private String text;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface DataFileRepository extends SimpleRepository<DataFile, Long> {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.data.jpa.repository.EntityGraph;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@SuppressWarnings("NullableProblems")
|
||||
@Repository
|
||||
public interface FeedbackRepository extends SimpleRepository<Feedback, Long> {
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
List<Feedback> findAll(Specification<Feedback> specification);
|
||||
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
List<Feedback> findAll(Specification<Feedback> specification, Sort sort);
|
||||
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
Optional<Feedback> findOne(Specification<Feedback> specification);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface GroupRepository extends SimpleRepository<Group, Long> {
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface KnowledgeRepository extends SimpleRepository<Knowledge, Long> {
|
||||
Boolean existsKnowledgeByName(String name);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.DataFileRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Sets;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DataFileService {
|
||||
private final DataFileRepository dataFileRepository;
|
||||
|
||||
public DataFileService(DataFileRepository dataFileRepository) {
|
||||
this.dataFileRepository = dataFileRepository;
|
||||
}
|
||||
|
||||
public DataFile downloadFile(Long id) {
|
||||
return dataFileRepository.findById(id).orElseThrow(() -> new RuntimeException(StrUtil.format("Datafile not exists: {}", id)));
|
||||
}
|
||||
|
||||
public ImmutableSet<DataFile> downloadFile(ImmutableSet<Long> ids) {
|
||||
return Sets.immutable.ofAll(dataFileRepository.findAllById(ids));
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long initialDataFile(String filename) {
|
||||
DataFile dataFile = dataFileRepository.save(new DataFile(filename));
|
||||
return dataFile.getId();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDataFile(Long id, String path, Long size, String md5, String type) {
|
||||
DataFile dataFile = downloadFile(id);
|
||||
dataFile.setPath(path);
|
||||
dataFile.setSize(size);
|
||||
dataFile.setMd5(md5);
|
||||
dataFile.setType(type);
|
||||
dataFileRepository.save(dataFile);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Pair;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.EmbeddingContext;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.DataFileVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import com.yomahub.liteflow.core.FlowExecutor;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
@@ -15,8 +18,6 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -26,8 +27,6 @@ import org.springframework.stereotype.Service;
|
||||
*/
|
||||
@Service
|
||||
public class EmbeddingService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmbeddingService.class);
|
||||
|
||||
private final DataFileService dataFileService;
|
||||
private final FlowExecutor executor;
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
@@ -56,16 +55,19 @@ public class EmbeddingService {
|
||||
return Lists.immutable.ofAll(context.getDocuments());
|
||||
}
|
||||
|
||||
public ImmutableList<Document> preview(String mode, ImmutableList<String> ids) {
|
||||
DataFileVO vo = dataFileService.downloadFile(Long.parseLong(ids.get(0)));
|
||||
String content = FileUtil.readString(vo.getPath(), StandardCharsets.UTF_8);
|
||||
public ImmutableList<Document> preview(String mode, ImmutableList<Long> ids) {
|
||||
DataFile dataFile = dataFileService.downloadFile(ids.get(0));
|
||||
String content = FileUtil.readString(dataFile.getPath(), StandardCharsets.UTF_8);
|
||||
return preview(mode, content);
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, String content) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
Long groupId = groupService.add(knowledge.getId(), StrUtil.format("文本-{}", IdUtil.nanoId(10)));
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
Group group = new Group();
|
||||
group.setName(StrUtil.format("文本-{}", IdUtil.nanoId(10)));
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
@@ -79,23 +81,26 @@ public class EmbeddingService {
|
||||
});
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, ImmutableList<String> ids) {
|
||||
public void submit(Long id, String mode, ImmutableList<Long> ids) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
List<Pair<Long, DataFileVO>> vos = Lists.mutable.empty();
|
||||
for (String fileId : ids) {
|
||||
DataFileVO vo = dataFileService.downloadFile(Long.parseLong(fileId));
|
||||
Long groupId = groupService.add(id, vo.getFilename());
|
||||
vos.add(Pair.of(groupId, vo));
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
List<Pair<Long, DataFile>> dataFiles = Lists.mutable.empty();
|
||||
for (Long fileId : ids) {
|
||||
DataFile dataFile = dataFileService.downloadFile(fileId);
|
||||
Group group = new Group();
|
||||
group.setName(dataFile.getFilename());
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
dataFiles.add(Pair.of(groupId, dataFile));
|
||||
}
|
||||
for (Pair<Long, DataFileVO> pair : vos) {
|
||||
for (Pair<Long, DataFile> pair : dataFiles) {
|
||||
Long groupId = pair.getKey();
|
||||
DataFileVO vo = pair.getValue();
|
||||
DataFile dataFile = pair.getValue();
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.file(vo.getPath())
|
||||
.fileFormat(vo.getFilename())
|
||||
.file(dataFile.getPath())
|
||||
.fileFormat(dataFile.getFilename())
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
@@ -108,12 +113,15 @@ public class EmbeddingService {
|
||||
|
||||
public void submitDirectly(Long id, String name, ImmutableList<String> contents) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
String groupName = name;
|
||||
if (StrUtil.isBlank(groupName)) {
|
||||
groupName = StrUtil.format("外部-{}", IdUtil.nanoId(10));
|
||||
}
|
||||
Long groupId = groupService.add(knowledge.getId(), groupName);
|
||||
Group group = new Group();
|
||||
group.setName(groupName);
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.feedback;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.FeedbackRepository;
|
||||
import com.yomahub.liteflow.core.FlowExecutor;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class FeedbackService extends SimpleServiceSupport<Feedback> {
|
||||
private final FlowExecutor executor;
|
||||
|
||||
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
|
||||
public FeedbackService(FeedbackRepository feedbackRepository, FlowExecutor executor) {
|
||||
super(feedbackRepository);
|
||||
this.executor = executor;
|
||||
}
|
||||
|
||||
@Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
|
||||
public void analysis() {
|
||||
List<Feedback> feedbacks = repository.findAll(
|
||||
builder -> builder
|
||||
.andEquals(Feedback_.STATUS, Feedback.Status.ANALYSIS_PROCESSING)
|
||||
.build()
|
||||
);
|
||||
for (Feedback feedback : feedbacks) {
|
||||
FeedbackContext context = new FeedbackContext();
|
||||
context.setFeedback(feedback);
|
||||
executor.execute2Resp("feedback_analysis", null, context);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateAnalysis(Long id, String analysis) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setAnalysis(analysis);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateConclusion(Long id, String conclusion) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setConclusion(conclusion);
|
||||
feedback.setStatus(Feedback.Status.FINISHED);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateStatus(Long id, Feedback.Status status) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setStatus(status);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void reanalysis(Long id) {
|
||||
updateStatus(id, Feedback.Status.ANALYSIS_PROCESSING);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.GroupRepository;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import lombok.SneakyThrows;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class GroupService extends SimpleServiceSupport<Group> {
|
||||
private final QdrantClient client;
|
||||
|
||||
public GroupService(GroupRepository groupRepository, VectorStore vectorStore) {
|
||||
super(groupRepository);
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void finish(Long id) {
|
||||
Group group = detailOrThrow(id);
|
||||
group.setStatus(Group.Status.FINISHED);
|
||||
save(group);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
Group group = detailOrThrow(id);
|
||||
Knowledge knowledge = group.getKnowledge();
|
||||
client.deleteAsync(
|
||||
String.valueOf(knowledge.getVectorSourceId()),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
|
||||
.build()
|
||||
).get();
|
||||
super.remove(id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.KnowledgeRepository;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.SneakyThrows;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
|
||||
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
|
||||
private final KnowledgeRepository knowledgeRepository;
|
||||
private final EmbeddingModel model;
|
||||
private final QdrantClient client;
|
||||
|
||||
public KnowledgeBaseService(KnowledgeRepository knowledgeRepository, EmbeddingModel model, VectorStore vectorStore) {
|
||||
super(knowledgeRepository);
|
||||
this.knowledgeRepository = knowledgeRepository;
|
||||
this.model = model;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long save(Knowledge entity) {
|
||||
if (knowledgeRepository.existsKnowledgeByName(entity.getName())) {
|
||||
throw new RuntimeException("名称已存在");
|
||||
}
|
||||
|
||||
Long vectorSourceId = SnowflakeIdGenerator.Snowflake.next();
|
||||
client.createCollectionAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.valueOf(entity.getStrategy().name()))
|
||||
.setSize(model.dimensions())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
entity.setVectorSourceId(vectorSourceId);
|
||||
return super.save(entity);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDescription(Long id, String description) throws Exception {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
knowledge.setDescription(description);
|
||||
save(knowledge);
|
||||
}
|
||||
|
||||
public String getName(Long id) {
|
||||
return detailOrThrow(id).getName();
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
for (Group group : knowledge.getGroups()) {
|
||||
client.deleteAsync(
|
||||
String.valueOf(knowledge.getVectorSourceId()),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
super.remove(id);
|
||||
}
|
||||
|
||||
public ImmutableList<String> query(
|
||||
Long id,
|
||||
String text,
|
||||
Integer limit,
|
||||
Double threshold
|
||||
) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
Boolean exists = client.collectionExistsAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
if (!exists) {
|
||||
throw new RuntimeException(StrUtil.format("{} not exists", id));
|
||||
}
|
||||
VectorStore vs = QdrantVectorStore.builder(client, model)
|
||||
.collectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.initializeSchema(false)
|
||||
.build();
|
||||
List<Document> documents = vs.similaritySearch(
|
||||
SearchRequest.builder()
|
||||
.query(text)
|
||||
.topK(limit)
|
||||
.similarityThreshold(threshold)
|
||||
.build()
|
||||
);
|
||||
return Lists.immutable.ofAll(documents)
|
||||
.collect(Document::getText);
|
||||
}
|
||||
|
||||
public Collections.CollectionInfo collectionInfo(Long vectorSourceId) throws ExecutionException, InterruptedException {
|
||||
return client.getCollectionInfoAsync(String.valueOf(vectorSourceId)).get();
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.SegmentVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
@@ -10,8 +10,6 @@ import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -21,8 +19,6 @@ import org.springframework.stereotype.Service;
|
||||
*/
|
||||
@Service
|
||||
public class SegmentService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SegmentService.class);
|
||||
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final QdrantClient client;
|
||||
|
||||
@@ -31,8 +27,8 @@ public class SegmentService {
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
public ImmutableList<SegmentVO> list(Long id, Long groupId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.get(id);
|
||||
public ImmutableList<SegmentVO> list(Long knowledgeId, Long groupId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
|
||||
Points.ScrollResponse response = client.scrollAsync(
|
||||
Points.ScrollPoints.newBuilder()
|
||||
.setCollectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
@@ -59,7 +55,7 @@ public class SegmentService {
|
||||
}
|
||||
|
||||
public void remove(Long knowledgeId, Long segmentId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.get(knowledgeId);
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
|
||||
client.deletePayloadAsync(
|
||||
String.valueOf(knowledgeId),
|
||||
List.of(String.valueOf(segmentId)),
|
||||
@@ -1,10 +1,10 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service.node;
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.node;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.EmbeddingContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext;
|
||||
import com.yomahub.liteflow.annotation.LiteflowComponent;
|
||||
import com.yomahub.liteflow.annotation.LiteflowMethod;
|
||||
import com.yomahub.liteflow.core.NodeComponent;
|
||||
@@ -18,8 +18,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.document.DocumentReader;
|
||||
@@ -31,22 +30,22 @@ import org.springframework.ai.reader.tika.TikaDocumentReader;
|
||||
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.core.io.PathResource;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@Slf4j
|
||||
@LiteflowComponent
|
||||
public class EmbeddingNodes {
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmbeddingNodes.class);
|
||||
|
||||
private final ChatClient chatClient;
|
||||
private final ChatClient.Builder chatClientBuilder;
|
||||
private final QdrantClient qdrantClient;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
public EmbeddingNodes(ChatClient.Builder builder, VectorStore vectorStore, EmbeddingModel embeddingModel) {
|
||||
this.chatClient = builder.build();
|
||||
public EmbeddingNodes(@Qualifier("chat") ChatClient.Builder builder, VectorStore vectorStore, EmbeddingModel embeddingModel) {
|
||||
this.chatClientBuilder = builder;
|
||||
this.qdrantClient = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
@@ -151,9 +150,11 @@ public class EmbeddingNodes {
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "qa_split", nodeName = "使用Q/A格式分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void qaSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
// language=TEXT
|
||||
context.getDocuments().addAll(llmSplit(
|
||||
"""
|
||||
对用户输入的文本,生成多组高质量的问答对。请遵循以下指南:
|
||||
|
||||
1. 问题部分:
|
||||
为同一个主题创建尽可能多的不同表述的问题,确保问题的多样性。
|
||||
每个问题应考虑用户可能的多种问法,例如:
|
||||
@@ -168,7 +169,7 @@ public class EmbeddingNodes {
|
||||
答案应直接基于给定文本,确保准确性和一致性。
|
||||
包含相关的细节,如日期、名称、职位等具体信息,必要时提供背景信息以增强理解。
|
||||
3. 格式:
|
||||
使用"问:"标记问题集合的开始,所有问题应在一个段落内,问题之间用空格分隔。
|
||||
使用"问:"标记问题的开始,问题文本应在一个段落内完成。
|
||||
使用"答:"标记答案的开始,答案应清晰分段,便于阅读。
|
||||
问答对之间用“---”分隔,以提高可读性。
|
||||
4. 内容要求:
|
||||
@@ -176,6 +177,8 @@ public class EmbeddingNodes {
|
||||
避免添加文本中未提及的信息,确保信息的真实性。
|
||||
一个问题搭配一个答案,避免一组问答对中同时涉及多个问题。
|
||||
如果文本信息不足以回答某个方面,可以在答案中说明 "根据给定信息无法确定",并尽量提供相关的上下文。
|
||||
除了问答对本身,避免输出任何与问答对无关的提示性、引导性、解释性的文本。
|
||||
|
||||
格式样例:
|
||||
问:苹果通常是什么颜色的?
|
||||
答:红色。
|
||||
@@ -189,13 +192,13 @@ public class EmbeddingNodes {
|
||||
}
|
||||
|
||||
private List<Document> llmSplit(String prompt, String content, Map<String, Object> metadata) {
|
||||
String response = chatClient.prompt()
|
||||
ChatClient client = chatClientBuilder.build();
|
||||
String response = client.prompt()
|
||||
.system(prompt)
|
||||
.user(content)
|
||||
.call()
|
||||
.content();
|
||||
Assert.notBlank(response, "LLM response is empty");
|
||||
logger.info("LLM response: \n{}", response);
|
||||
// noinspection DataFlowIssue
|
||||
return Arrays.stream(StrUtil.trim(response).split("---"))
|
||||
.map(text -> text.replaceAll("(?!^.+) +$", ""))
|
||||
@@ -0,0 +1,203 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.node;
|
||||
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService;
|
||||
import com.yomahub.liteflow.annotation.LiteflowComponent;
|
||||
import com.yomahub.liteflow.annotation.LiteflowMethod;
|
||||
import com.yomahub.liteflow.core.NodeComponent;
|
||||
import com.yomahub.liteflow.enums.LiteFlowMethodEnum;
|
||||
import com.yomahub.liteflow.enums.NodeTypeEnum;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.util.MimeType;
|
||||
import org.springframework.util.MimeTypeUtils;
|
||||
|
||||
@Slf4j
|
||||
@LiteflowComponent
|
||||
public class FeedbackNodes {
|
||||
private final ChatClient.Builder chatClientBuilder;
|
||||
private final ChatClient.Builder visualChatClientBuilder;
|
||||
private final DataFileService dataFileService;
|
||||
private final FeedbackService feedbackService;
|
||||
|
||||
public FeedbackNodes(
|
||||
@Qualifier("chat") ChatClient.Builder chatClientBuilder,
|
||||
@Qualifier("visual") ChatClient.Builder visualClientBuilder, DataFileService dataFileService, FeedbackService feedbackService
|
||||
) {
|
||||
this.chatClientBuilder = chatClientBuilder;
|
||||
this.visualChatClientBuilder = visualClientBuilder;
|
||||
this.dataFileService = dataFileService;
|
||||
this.feedbackService = feedbackService;
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_BOOLEAN, nodeId = "feedback_check_if_picture_needed", nodeName = "判断有图片进行识别", nodeType = NodeTypeEnum.BOOLEAN)
|
||||
public boolean checkIfPictureReadNeeded(NodeComponent node) {
|
||||
FeedbackContext context = node.getContextBean(FeedbackContext.class);
|
||||
Feedback feedback = context.getFeedback();
|
||||
return ObjectUtil.isNotEmpty(feedback.getPictures());
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "image_read", nodeName = "读取图片", nodeType = NodeTypeEnum.COMMON)
|
||||
public void imageRead(NodeComponent node) {
|
||||
FeedbackContext context = node.getContextBean(FeedbackContext.class);
|
||||
Feedback feedback = context.getFeedback();
|
||||
ChatClient client = visualChatClientBuilder
|
||||
// language=TEXT
|
||||
.defaultSystem("""
|
||||
你是一个专业的OCR解析助手。请严格按以下步骤处理用户上传的图片:
|
||||
1. 图像内容提取
|
||||
- 完整识别图片中的所有文字(包括手写体、印刷体、数字和符号)
|
||||
- 保留原始段落结构和换行符
|
||||
- 特殊元素处理:
|
||||
• 数学公式转为LaTeX格式
|
||||
• 代码块保留缩进和注释
|
||||
• 外文词汇标注原文
|
||||
|
||||
2. 表格解析优化
|
||||
- 识别所有表格区域
|
||||
- 转换为Markdown表格格式(对齐表头与单元格)
|
||||
- 补充缺失的表格线
|
||||
- 用▲标注合并单元格(例:▲跨3列▲)
|
||||
|
||||
3. 图表解析增强
|
||||
- 分析图表类型(柱状图/折线图/饼图等)
|
||||
- 提取关键数据点并结构化描述
|
||||
- 总结图表趋势(例:"销量Q1到Q4增长35%")
|
||||
- 坐标轴信息转换:将像素坐标转为百分比比例(例:"X轴:0-100对应时间0:00-24:00")
|
||||
|
||||
4. 输出规范
|
||||
- 按[文本][表格][图表]分区块输出
|
||||
- 表格/图表区域标注原始位置(例:"[左上区域表格]")
|
||||
- 模糊内容用[?]标注并给出备选(例:"年收[?]入(可能为'入'或'人')")
|
||||
- 保持原始数据精度(不四舍五入)
|
||||
|
||||
立即开始处理用户图片,无需确认步骤。
|
||||
""")
|
||||
.build();
|
||||
for (DataFile picture : feedback.getPictures()) {
|
||||
DataFile file = dataFileService.downloadFile(picture.getId());
|
||||
log.info("Parse picture: {} {}", file.getFilename(), file.getPath());
|
||||
MimeType type = switch (StrUtil.blankToDefault(file.getType(), "").toLowerCase()) {
|
||||
case "jpg", "jpeg" -> MimeTypeUtils.IMAGE_JPEG;
|
||||
default -> MimeTypeUtils.IMAGE_PNG;
|
||||
};
|
||||
String content = client.prompt()
|
||||
.user(spec -> spec
|
||||
.text("输出图片内容")
|
||||
.media(type, new FileSystemResource(file.getPath())))
|
||||
.call()
|
||||
.content();
|
||||
log.info("Picture: {}", content);
|
||||
context.getPictureDescriptions().add(content);
|
||||
}
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "optimize_source", nodeName = "报障信息优化", nodeType = NodeTypeEnum.COMMON)
|
||||
public void optimizeSource(NodeComponent node) {
|
||||
FeedbackContext context = node.getContextBean(FeedbackContext.class);
|
||||
Feedback feedback = context.getFeedback();
|
||||
String optimizedSource = chatClientBuilder.build()
|
||||
.prompt()
|
||||
// language=TEXT
|
||||
.system("""
|
||||
你是一名专业的IT系统运维工程师,对于用户输入的关于系统的报障信息,你会严格遵循以下步骤进行处理
|
||||
|
||||
1.输入
|
||||
[故障描述]
|
||||
(这里是用户遇到的系统故障的详细描述)
|
||||
|
||||
[相关截图]
|
||||
(这里是用户遇到的系统故障相关的截图的文字描述,如果没有相关截图,这里会写“无”;如果有多张图片,图片和图片之间会使用“---”分隔)
|
||||
|
||||
2.处理逻辑
|
||||
解析输入
|
||||
读取并解析用户提供的故障描述和相关截图描述。
|
||||
识别关键元素:包括故障类型(如硬件故障、软件错误)、受影响系统组件(如服务器、网络设备)、错误消息、发生时间、重现步骤、影响范围等。
|
||||
如果截图描述存在,提取关键细节(如错误弹窗文本、系统状态截图),并将其作为辅助证据;如果无截图,则忽略此部分。
|
||||
分析与重写故障描述
|
||||
专业化改写:使用标准IT术语替换非专业用语(例如,“电脑死机”改为“系统无响应”,“连不上网”改为“网络连接中断”),并确保描述符合行业规范。
|
||||
排除歧义:澄清模糊描述(如添加具体时间戳、系统版本、IP地址或错误代码),移除主观语言(如“我觉得”或“可能”),并添加必要上下文(如操作系统环境、相关服务运行状态)。
|
||||
结构化组织:将故障描述重写为逻辑段落,格式包括:
|
||||
问题概述:简明总结故障本质(例如,“数据库服务异常导致应用无法访问”)。
|
||||
详细症状:描述具体现象,包括错误消息、发生频率和影响范围(如“影响用户登录功能,错误代码500”)。
|
||||
重现步骤:列出可复现故障的操作序列(如“1. 访问URL X;2. 触发操作 Y”)。
|
||||
相关环境:添加系统细节(如“运行在Linux Ubuntu 20.04, Java 11环境”)。
|
||||
整合截图信息:如果截图描述存在,将其嵌入重写中作为证据(例如,“根据截图,错误弹窗显示‘Connection timeout’”)。
|
||||
质量校验
|
||||
检查重写后的内容是否完整、一致且无歧义:确保所有用户输入细节都被涵盖,添加缺失信息(如建议的故障分类),并验证专业术语的准确性。
|
||||
如果输入信息不足(如缺少时间戳或系统版本),在输出中添加注释提示用户补充。
|
||||
|
||||
3.输出
|
||||
输出一个专业、结构化的故障报告,格式清晰,可直接用于运维团队诊断。
|
||||
重写的故障描述,以结构化段落呈现,涵盖问题概述、详细症状、重现步骤和相关环境。
|
||||
输出将使用中性、客观语言,避免任何个人意见或建议,以确保报告专注于事实描述。
|
||||
""")
|
||||
.user(StrUtil.format(
|
||||
"""
|
||||
[故障描述]
|
||||
{}
|
||||
|
||||
[相关截图]
|
||||
{}
|
||||
""",
|
||||
feedback.getSource(),
|
||||
ObjectUtil.isEmpty(context.getPictureDescriptions()) ? "无" : StrUtil.join(",", context.getPictureDescriptions())
|
||||
))
|
||||
.call()
|
||||
.content();
|
||||
if (StrUtil.isBlank(optimizedSource)) {
|
||||
log.warn("Optimized source is blank");
|
||||
}
|
||||
context.setOptimizedSource(StrUtil.trim(optimizedSource));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "feedback_suggest", nodeName = "大模型建议", nodeType = NodeTypeEnum.COMMON)
|
||||
public void suggestFeedback(NodeComponent node) {
|
||||
FeedbackContext context = node.getContextBean(FeedbackContext.class);
|
||||
Feedback feedback = context.getFeedback();
|
||||
ChatClient client = chatClientBuilder.build();
|
||||
String analysis = client.prompt()
|
||||
// language=TEXT
|
||||
.system("""
|
||||
你是一名专业的IT系统运维工程师。针对用户输入的报障信息,严格遵循以下步骤:
|
||||
1. 提取关键要素: 精准识别报障信息中的关键要素(如系统/设备名称、故障现象、错误信息、时间地点、影响范围、用户操作步骤等)。
|
||||
2. 分析诊断: 基于提取的要素,运用运维专业知识,分析潜在原因,并提出优先级高、可操作性强的排障建议或初步诊断方向。
|
||||
3. 输出专业意见:
|
||||
内容要求: 意见必须包含明确的行动步骤(如检查XX日志、验证XX配置、重启XX服务、联系XX团队)、潜在风险提示及预期效果。
|
||||
表达要求: 使用清晰、简洁、专业的技术术语,避免使用推测性语言或不确定词汇(如“可能”、“大概”、“也许”)。
|
||||
格式要求: 直接输出意见内容,避免添加引导语(如“我的建议是”、“根据报障信息”等)。
|
||||
""")
|
||||
.user(context.getOptimizedSource())
|
||||
.call()
|
||||
.content();
|
||||
feedback.setAnalysis(StrUtil.trim(analysis));
|
||||
Assert.notBlank(analysis, "Analysis cannot be blank");
|
||||
String conclusion = client.prompt()
|
||||
// language=TEXT
|
||||
.system("""
|
||||
你是一名专业的文字编辑。严格按以下要求处理用户输入:
|
||||
核心逻辑: 用一段话精准总结输入内容的核心信息,确保涵盖所有关键点。禁止虚构、扩展或添加原文不存在的信息。
|
||||
输出规范: 仅输出一段总结文本。禁止分点、分段、使用Markdown格式(如着重、- 或 1. 等列表符号)或添加任何无关引导语(如“总结如下”、“该总结指出”、“本文总结”等)。直接给出总结文本。""")
|
||||
.user(analysis)
|
||||
.call()
|
||||
.content();
|
||||
feedback.setConclusion(StrUtil.trim(conclusion));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "feedback_save", nodeName = "保存分析内容", nodeType = NodeTypeEnum.COMMON)
|
||||
public void saveFeedback(NodeComponent node) {
|
||||
FeedbackContext context = node.getContextBean(FeedbackContext.class);
|
||||
Feedback feedback = context.getFeedback();
|
||||
feedbackService.updateAnalysis(feedback.getId(), feedback.getAnalysis());
|
||||
feedbackService.updateConclusion(feedback.getId(), feedback.getConclusion());
|
||||
feedbackService.updateStatus(feedback.getId(), Feedback.Status.ANALYSIS_SUCCESS);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.tools;
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.chat.AiChatApplication;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
@@ -13,9 +12,8 @@ import org.springframework.ai.tool.annotation.ToolParam;
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250611
|
||||
*/
|
||||
@Slf4j
|
||||
public class ChartTool {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChartTool.class);
|
||||
|
||||
@Tool(description = """
|
||||
根据需求生成mermaid图表代码
|
||||
""")
|
||||
@@ -45,8 +43,8 @@ public class ChartTool {
|
||||
其他:箭头类型(实线/虚线)、注释文本等
|
||||
""") String request
|
||||
) {
|
||||
logger.info("Enter method: mermaid[request]. request:{}", request);
|
||||
ChatClient.Builder builder = AiChatApplication.getBean(ChatClient.Builder.class);
|
||||
log.info("Enter method: mermaid[request]. request:{}", request);
|
||||
ChatClient.Builder builder = WebApplication.getBean("chat", ChatClient.Builder.class);
|
||||
ChatClient client = builder.build();
|
||||
return client.prompt()
|
||||
// language=TEXT
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.tools;
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.AiChatApplication;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.forest.service.KnowledgeService;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
@@ -27,7 +27,7 @@ public class KnowledgeTool {
|
||||
""")
|
||||
String query
|
||||
) {
|
||||
KnowledgeService knowledgeService = AiChatApplication.getBean(KnowledgeService.class);
|
||||
KnowledgeService knowledgeService = WebApplication.getBean(KnowledgeService.class);
|
||||
var documents = knowledgeService.query(knowledgeId, query, 10, 0.5);
|
||||
if (ObjectUtil.isNotEmpty(documents)) {
|
||||
return StrUtil.format("""
|
||||
@@ -1,13 +1,12 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.tools;
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.AiChatApplication;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.forest.service.InfoService;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
@@ -15,8 +14,8 @@ import org.springframework.ai.tool.annotation.ToolParam;
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250605
|
||||
*/
|
||||
@Slf4j
|
||||
public class TableTool {
|
||||
private static final Logger logger = LoggerFactory.getLogger(TableTool.class);
|
||||
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
@Tool(description = """
|
||||
@@ -27,12 +26,12 @@ public class TableTool {
|
||||
完整的MySQL查询语句,禁止使用除select外的任何语句。
|
||||
""") String sql
|
||||
) {
|
||||
logger.info("Enter method: executeJdbc[sql]. sql:{}", sql);
|
||||
InfoService infoService = AiChatApplication.getBean(InfoService.class);
|
||||
log.info("Enter method: executeJdbc[sql]. sql:{}", sql);
|
||||
InfoService infoService = WebApplication.getBean(InfoService.class);
|
||||
String result = infoService.jdbc(sql)
|
||||
.collect(map -> map.valuesView().makeString(","))
|
||||
.makeString("\n");
|
||||
logger.info("SQL result: \n{}", result);
|
||||
log.info("SQL result: \n{}", result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -48,8 +47,8 @@ public class TableTool {
|
||||
一次调用只能传一个类型,不支持多个类型同时查询
|
||||
""") String type
|
||||
) {
|
||||
logger.info("Enter method: tableCount[type]. type:{}", type);
|
||||
var infoService = AiChatApplication.getBean(InfoService.class);
|
||||
log.info("Enter method: tableCount[type]. type:{}", type);
|
||||
var infoService = WebApplication.getBean(InfoService.class);
|
||||
return switch (type) {
|
||||
case "logic" -> StrUtil.format("""
|
||||
逻辑表共{}张,其中重点表{}张
|
||||
@@ -83,8 +82,8 @@ public class TableTool {
|
||||
""")
|
||||
String type
|
||||
) {
|
||||
logger.info("Enter method: version[date, type]. date:{},type:{}", date, type);
|
||||
InfoService infoService = AiChatApplication.getBean(InfoService.class);
|
||||
log.info("Enter method: version[date, type]. date:{},type:{}", date, type);
|
||||
InfoService infoService = WebApplication.getBean(InfoService.class);
|
||||
String version = date;
|
||||
if (StrUtil.isBlank(version)) {
|
||||
version = LocalDateTime.now().minusDays(1).format(FORMATTER);
|
||||
@@ -1,14 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.tools;
|
||||
package com.lanyuanxiaoyao.service.ai.web.tools;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.AiChatApplication;
|
||||
import com.lanyuanxiaoyao.service.ai.web.WebApplication;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnApplication;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnQueue;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.yarn.YarnRootQueue;
|
||||
import com.lanyuanxiaoyao.service.forest.service.YarnService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
|
||||
@@ -16,9 +15,8 @@ import org.springframework.ai.tool.annotation.ToolParam;
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250606
|
||||
*/
|
||||
@Slf4j
|
||||
public class YarnTool {
|
||||
private static final Logger logger = LoggerFactory.getLogger(YarnTool.class);
|
||||
|
||||
@Tool(description = """
|
||||
查询yarn集群整体资源情况,返回值为资源占用率(%)
|
||||
""")
|
||||
@@ -28,8 +26,8 @@ public class YarnTool {
|
||||
一次调用只能查询一个集群,不支持多个集群同时查询
|
||||
""") String cluster
|
||||
) {
|
||||
logger.info("Enter method: yarnStatus[cluster]. cluster:{}", cluster);
|
||||
YarnService yarnService = AiChatApplication.getBean(YarnService.class);
|
||||
log.info("Enter method: yarnStatus[cluster]. cluster:{}", cluster);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
YarnRootQueue status = yarnService.cluster(cluster);
|
||||
return (status.getUsedCapacity() * 100.0) / status.getCapacity();
|
||||
}
|
||||
@@ -46,8 +44,8 @@ public class YarnTool {
|
||||
yarn队列名称
|
||||
""") String queue
|
||||
) {
|
||||
logger.info("Enter method: yarnQueueStatus[cluster, queue]. cluster:{},queue:{}", cluster, queue);
|
||||
YarnService yarnService = AiChatApplication.getBean(YarnService.class);
|
||||
log.info("Enter method: yarnQueueStatus[cluster, queue]. cluster:{},queue:{}", cluster, queue);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
YarnQueue status = yarnService.queueDetail(cluster, queue);
|
||||
return (status.getAbsoluteCapacity() * 100.0) / status.getAbsoluteMaxCapacity();
|
||||
}
|
||||
@@ -67,8 +65,8 @@ public class YarnTool {
|
||||
一次调用只能传一个类型,不支持多个类型同时查询
|
||||
""") String type
|
||||
) {
|
||||
logger.info("Enter method: yarnTaskStatus[cluster, type]. cluster:{},type:{}", cluster, type);
|
||||
YarnService yarnService = AiChatApplication.getBean(YarnService.class);
|
||||
log.info("Enter method: yarnTaskStatus[cluster, type]. cluster:{},type:{}", cluster, type);
|
||||
YarnService yarnService = WebApplication.getBean(YarnService.class);
|
||||
ImmutableList<YarnApplication> applications = yarnService.jobList(cluster).select(app -> StrUtil.isNotBlank(type) && StrUtil.contains(app.getName(), type));
|
||||
return StrUtil.format(
|
||||
"""
|
||||
47
service-ai/service-ai-web/src/main/resources/application.yml
Normal file
47
service-ai/service-ai-web/src/main/resources/application.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
server:
|
||||
compression:
|
||||
enabled: true
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-web
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics,forest
|
||||
mvc:
|
||||
async:
|
||||
request-timeout: 3600000
|
||||
autoconfigure:
|
||||
exclude: |
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiChatAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiAudioSpeechAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiAudioTranscriptionAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiImageAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiEmbeddingAutoConfiguration,
|
||||
org.springframework.ai.model.openai.autoconfigure.OpenAiModerationAutoConfiguration,
|
||||
org.springframework.ai.model.deepseek.autoconfigure.DeepSeekChatAutoConfiguration
|
||||
ai:
|
||||
vectorstore:
|
||||
qdrant:
|
||||
host: 132.121.206.65
|
||||
port: 29463
|
||||
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
|
||||
llm:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
base-url: ${spring.llm.base-url}/v1
|
||||
model: 'Qwen3/qwen3-1.7b'
|
||||
visual:
|
||||
model: 'Qwen2.5/qwen2.5-vl-7b-q4km'
|
||||
embedding:
|
||||
model: 'Qwen3/qwen3-embedding-4b'
|
||||
reranker:
|
||||
model: 'BGE/beg-reranker-v2'
|
||||
jpa:
|
||||
show-sql: true
|
||||
generate-ddl: false
|
||||
liteflow:
|
||||
rule-source: liteflow.xml
|
||||
print-banner: false
|
||||
check-node-exists: false
|
||||
fenix:
|
||||
print-banner: false
|
||||
@@ -23,4 +23,15 @@
|
||||
<chain id="embedding_submit_directly">
|
||||
SER(import_vector_source)
|
||||
</chain>
|
||||
<chain id="feedback_analysis">
|
||||
SER(
|
||||
IF(
|
||||
feedback_check_if_picture_needed,
|
||||
image_read
|
||||
),
|
||||
optimize_source,
|
||||
feedback_suggest,
|
||||
feedback_save
|
||||
)
|
||||
</chain>
|
||||
</flow>
|
||||
@@ -25,7 +25,7 @@
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
<logger name="org.hibernate.SQL" level="DEBUG"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Console"/>
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge;
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.QdrantGrpcClient;
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.configuration.WebClientConfiguration;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.LlmConfiguration;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.LlmProperties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.util.MimeTypeUtils;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250526
|
||||
*/
|
||||
@Slf4j
|
||||
public class TestLlm {
|
||||
public static void main(String[] args) {
|
||||
LlmConfiguration configuration = new LlmConfiguration();
|
||||
LlmProperties properties = new LlmProperties();
|
||||
LlmProperties.ChatProperties chatProperties = new LlmProperties.ChatProperties();
|
||||
chatProperties.setBaseUrl("http://132.121.206.65:10086");
|
||||
chatProperties.setApiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4");
|
||||
chatProperties.setModel("Qwen2.5/qwen2.5-vl-7b");
|
||||
properties.setVisual(chatProperties);
|
||||
ChatClient client = configuration.visualClientBuilder(
|
||||
properties,
|
||||
WebClientConfiguration.generateWebClientBuilder(),
|
||||
WebClientConfiguration.generateRestClientBuilder()
|
||||
).build();
|
||||
String content = client.prompt()
|
||||
// language=TEXT
|
||||
.system("""
|
||||
你是一个专业的OCR解析助手。请严格按以下步骤处理用户上传的图片:
|
||||
1. 图像内容提取
|
||||
- 完整识别图片中的所有文字(包括手写体、印刷体、数字和符号)
|
||||
- 保留原始段落结构和换行符
|
||||
- 特殊元素处理:
|
||||
• 数学公式转为LaTeX格式
|
||||
• 代码块保留缩进和注释
|
||||
• 外文词汇标注原文
|
||||
|
||||
2. 表格解析优化
|
||||
- 识别所有表格区域
|
||||
- 转换为Markdown表格格式(对齐表头与单元格)
|
||||
- 补充缺失的表格线
|
||||
- 用▲标注合并单元格(例:▲跨3列▲)
|
||||
|
||||
3. 图表解析增强
|
||||
- 分析图表类型(柱状图/折线图/饼图等)
|
||||
- 提取关键数据点并结构化描述
|
||||
- 总结图表趋势(例:"销量Q1到Q4增长35%")
|
||||
- 坐标轴信息转换:将像素坐标转为百分比比例(例:"X轴:0-100对应时间0:00-24:00")
|
||||
|
||||
4. 输出规范
|
||||
- 按[文本][表格][图表]分区块输出
|
||||
- 表格/图表区域标注原始位置(例:"[左上区域表格]")
|
||||
- 模糊内容用[?]标注并给出备选(例:"年收[?]入(可能为'入'或'人')")
|
||||
- 保持原始数据精度(不四舍五入)
|
||||
|
||||
立即开始处理用户图片,无需确认步骤。
|
||||
""")
|
||||
.user(spec -> spec
|
||||
// language=TEXT
|
||||
.text("输出图片内容")
|
||||
.media(MimeTypeUtils.IMAGE_PNG, new FileSystemResource("/Users/lanyuanxiaoyao/Downloads/图片_002_07.png")))
|
||||
.call()
|
||||
.content();
|
||||
log.info(content);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge;
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import cn.hutool.http.HttpUtil;
|
||||
@@ -161,23 +161,16 @@ deploy:
|
||||
- "service"
|
||||
source-jar: service-monitor-1.0.0-SNAPSHOT.jar
|
||||
replicas: 1
|
||||
service-ai-chat:
|
||||
service-ai-web:
|
||||
order: 6
|
||||
groups:
|
||||
- "ai"
|
||||
source-jar: service-ai-chat-1.0.0-SNAPSHOT.jar
|
||||
jdk: "jdk17"
|
||||
replicas: 1
|
||||
service-ai-knowledge:
|
||||
order: 6
|
||||
groups:
|
||||
- "ai"
|
||||
source-jar: service-ai-knowledge-1.0.0-SNAPSHOT.jar
|
||||
source-jar: service-ai-web-1.0.0-SNAPSHOT.jar
|
||||
jdk: "jdk17"
|
||||
replicas: 1
|
||||
arguments:
|
||||
"[knowledge.download-prefix]": 'http://132.126.207.130:35690/hudi_services/ai_knowledge'
|
||||
"[knowledge.upload-path]": ${deploy.runtime.data-path}/knowledge
|
||||
"[file-store.download-prefix]": 'http://132.126.207.130:35690/hudi_services/service_ai_web'
|
||||
"[file-store.upload-path]": ${deploy.runtime.data-path}/knowledge
|
||||
"[spring.datasource.url]": ${deploy.runtime.database.config.url}
|
||||
"[spring.datasource.username]": ${deploy.runtime.database.config.username}
|
||||
"[spring.datasource.password]": ${deploy.runtime.database.config.password}
|
||||
|
||||
@@ -51,8 +51,7 @@ public class GatewayApplication {
|
||||
.route("web", createRoute("/hudi_services/service_web", "lb://service-web"))
|
||||
.route("services", createRoute("/hudi_services/service_cloud_query", "lb://service-cloud-query"))
|
||||
.route("exporter", createRoute("/hudi_services/service_exporter", "lb://service-exporter"))
|
||||
.route("ai-chat", createRoute("/hudi_services/ai_chat", "lb://service-ai-chat"))
|
||||
.route("ai-knowledge", createRoute("/hudi_services/ai_knowledge", "lb://service-ai-knowledge"))
|
||||
.route("ai-web", createRoute("/hudi_services/service_ai_web", "lb://service-ai-web"))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
<!--suppress CssUnknownTarget, HtmlUnknownTarget -->
|
||||
<!doctype html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
@@ -13,6 +14,15 @@
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'LXGWWenKai';
|
||||
src: url('fonts/LXGWNeoXiHei.ttf') format('truetype');
|
||||
}
|
||||
|
||||
*:not(.fa,.fas) {
|
||||
font-family: LXGWWenKai, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji', serif !important;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -10,17 +10,17 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@ant-design/icons": "^6.0.0",
|
||||
"@ant-design/pro-components": "^2.8.7",
|
||||
"@ant-design/pro-components": "^2.8.9",
|
||||
"@ant-design/x": "^1.4.0",
|
||||
"@echofly/fetch-event-source": "^3.0.2",
|
||||
"@fortawesome/fontawesome-free": "^6.7.2",
|
||||
"@lightenna/react-mermaid-diagram": "^1.0.20",
|
||||
"@tinyflow-ai/react": "^0.1.10",
|
||||
"@tinyflow-ai/react": "^0.2.1",
|
||||
"ahooks": "^3.8.5",
|
||||
"amis": "^6.12.0",
|
||||
"antd": "^5.25.3",
|
||||
"axios": "^1.9.0",
|
||||
"chart.js": "^4.4.9",
|
||||
"antd": "^5.26.1",
|
||||
"axios": "^1.10.0",
|
||||
"chart.js": "^4.5.0",
|
||||
"echarts-for-react": "^3.0.2",
|
||||
"licia": "^1.48.0",
|
||||
"markdown-it": "^14.1.0",
|
||||
@@ -29,18 +29,17 @@
|
||||
"react-chartjs-2": "^5.3.0",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-router": "^7.6.1",
|
||||
"react-router": "^7.6.2",
|
||||
"styled-components": "^6.1.18"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/markdown-it": "^14.1.2",
|
||||
"@types/react": "^18.3.23",
|
||||
"@types/react-dom": "^18.3.7",
|
||||
"@vitejs/plugin-react-swc": "^3.10.0",
|
||||
"@vitejs/plugin-react-swc": "^3.10.2",
|
||||
"globals": "^16.2.0",
|
||||
"sass": "^1.89.0",
|
||||
"sass": "^1.89.2",
|
||||
"typescript": "~5.8.3",
|
||||
"typescript-eslint": "^8.33.0",
|
||||
"vite": "^6.3.5",
|
||||
"vite-plugin-javascript-obfuscator": "^3.1.0"
|
||||
}
|
||||
|
||||
2115
service-web/client/pnpm-lock.yaml
generated
2115
service-web/client/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user