141 Commits

Author SHA1 Message Date
v-zhangjc9
a2a8c39145 feat(ai-web): 输出节点改为多变量输出 2025-07-21 18:33:45 +08:00
v-zhangjc9
82ad3e7854 fix(web): 修复api引用错误 2025-07-21 10:08:37 +08:00
1e3e14c590 refactor(ai-web): 优化inputSchema可以为空,方便各处判断流程是否有入参 2025-07-21 00:05:28 +08:00
267eecbf45 refactor(ai-web): 使用输入节点代替inputSchema
将输入表单放在流程图中一起定义,方便统一参数的处理,不需要单独为输入表单的变量进行合并和操作
2025-07-20 19:27:09 +08:00
a5282762ed refractor(web): 增加输入节点 2025-07-20 17:17:22 +08:00
v-zhangjc9
77a09472aa feat(ai-web): 完成代码节点的执行 2025-07-18 15:48:35 +08:00
v-zhangjc9
4cfa110f2f feat(web): 增加数组的判断 2025-07-18 11:07:48 +08:00
v-zhangjc9
ad9dade6a1 feat(web): 完成子流程节点和代码节点的运行 2025-07-17 18:38:19 +08:00
v-zhangjc9
3a61da054e feat(web): 增加循环节点参数 2025-07-17 18:37:43 +08:00
fbc6144d82 refractor(web): 优化选择节点的选择器 2025-07-16 23:08:25 +08:00
72a9d58f4c fix(web): 修复节点入参不更新 2025-07-16 22:54:48 +08:00
v-zhangjc9
a3c2250285 feat(web): 完成分支节点动态改变handle 2025-07-16 19:54:48 +08:00
91e6f49342 feat(web): 增加判断节点对不同类型的适配和处理 2025-07-15 23:19:33 +08:00
v-zhangjc9
35c5150a1f feat(web): 入参增加类型 2025-07-15 17:32:23 +08:00
v-zhangjc9
a7b245a670 refractor(web): 修复入参解析 2025-07-15 16:27:00 +08:00
v-zhangjc9
df8270676a feat(web): 增加循环节点的基本参数 2025-07-15 11:11:36 +08:00
v-zhangjc9
b0e4f5853e feat(web): 增加循环节点的输出变量 2025-07-15 09:09:23 +08:00
0efb041c71 fix(web): 不太优雅地修复循环节点初始化高度的问题 2025-07-14 23:34:58 +08:00
fa06207af8 feat(web): 实现循环节点中新增节点 2025-07-14 23:12:43 +08:00
c7e0b56850 feat(web): 增加循环节点子节点的循环入参 2025-07-14 22:41:27 +08:00
v-zhangjc9
04bc9a2c16 feat(web): 完成循环节点的基本配置 2025-07-14 19:10:58 +08:00
v-zhangjc9
c77395fec4 feat(web): 升级部份依赖 2025-07-14 12:16:13 +08:00
21f02660e9 refractor(web): 优化节点和流程图显示效果 2025-07-13 22:42:37 +08:00
f6e9c9bc70 fix(web): 修复模板节点没有输出 2025-07-13 22:24:40 +08:00
v-zhangjc9
e798332828 feat(ai-web): 增加模板渲染的处理 2025-07-12 23:24:22 +08:00
v-zhangjc9
b0d41e0d88 feat(web): 增加模板节点 2025-07-12 22:56:15 +08:00
v-zhangjc9
02b2d44ccc feat(ai-web): 修复出现遗留边的时候导致出现的null错误 2025-07-12 21:46:00 +08:00
v-zhangjc9
47de3cc376 feat(web): 输出节点增加输出类型 2025-07-12 21:16:06 +08:00
v-zhangjc9
5b9920449d feat(web): 增加节点图标 2025-07-12 20:46:12 +08:00
v-zhangjc9
60f6b79167 feat(web): 增加节点分组 2025-07-12 19:09:47 +08:00
v-zhangjc9
528e66c497 feat(ai-web): 实现入参解析 2025-07-11 16:50:11 +08:00
v-zhangjc9
707f538213 feat(web): 增加文本段输入 2025-07-11 14:43:48 +08:00
v-zhangjc9
eae0d8dacd fix(web): 修复变量校验没有包含入参 2025-07-11 14:43:26 +08:00
v-zhangjc9
bf37c163fb feat(ai-web): 完成流程任务的前后端拉通 2025-07-11 14:26:30 +08:00
v-zhangjc9
ac2b6b1611 feat(web): 增加表单数据校验 2025-07-11 11:12:06 +08:00
v-zhangjc9
863638deaa feat(web): 优化节点展现 2025-07-11 09:46:12 +08:00
v-zhangjc9
fad190567b feat(web): 节点描述和名称直接放在节点数据中 2025-07-10 16:34:47 +08:00
v-zhangjc9
333da7ef88 style(web): 移除未使用的引入 2025-07-10 14:35:29 +08:00
v-zhangjc9
d0ca36e9d7 style(web): 优化命名 2025-07-10 14:33:36 +08:00
v-zhangjc9
f70b3b2a32 fix(web): 移除节点未移除节点数据 2025-07-10 14:28:32 +08:00
v-zhangjc9
f707a0d2b5 refactor(web): 优化节点展现 2025-07-10 12:44:37 +08:00
v-zhangjc9
5e763637da refactor(web): 优化流程节点的定义和实现 2025-07-10 12:08:13 +08:00
v-zhangjc9
898e20d5d7 fix(web): 修复节点option key错误 2025-07-08 10:29:55 +08:00
605cfb7182 feat(web): 尝试增加校验 2025-07-07 23:31:04 +08:00
v-zhangjc9
3afdff0a05 feat(web): 完成前序节点输出变量注入 2025-07-07 19:50:22 +08:00
v-zhangjc9
f523fc7638 feat(web): 优化节点编辑性能 2025-07-07 18:51:35 +08:00
03d0d9d85b feat(web): 尝试优化流程图性能 2025-07-06 22:39:26 +08:00
187c565da4 feat(ai-web): 将任务模板信息固化到任务对象中
保证每个任务对象都有对应的模板信息可以追溯,不会随着模板被修改而失效
2025-07-05 18:45:24 +08:00
f3dfff5075 fix(web): 修复删除api的路径 2025-07-05 17:41:27 +08:00
69420094ec feat(web): 实现任务显示文件列表 2025-07-05 17:39:44 +08:00
v-zhangjc9
051b3dbad2 feat(ai-web): 完成执行任务的创建(提交漏的文件) 2025-07-05 14:48:53 +08:00
v-zhangjc9
64ef5514b4 feat(ai-web): 完成执行任务的创建 2025-07-04 19:31:24 +08:00
v-zhangjc9
4f1dc84405 feat(web): 调整侧边栏实现 2025-07-04 10:54:45 +08:00
d979b3941d feat(ai-web): 完成自研流程图的保存 2025-07-03 23:40:41 +08:00
abdbb5ed03 feat(web): 增加任务和流程图CRUD 2025-07-03 22:21:31 +08:00
v-zhangjc9
8ebaf5de8e feat(web): 优化流程任务编辑 2025-07-03 19:18:17 +08:00
v-zhangjc9
5f7eeb3596 feat(ai-web): 完成任务模板的CRUD 2025-07-03 18:01:36 +08:00
v-zhangjc9
064443f740 feat(ai-web): 移除liteflow入参,该用自己实现的流程引擎 2025-07-03 16:10:02 +08:00
v-zhangjc9
6809750cfa fix(ai-web): 不能设置私有网络访问,会拒绝所有的跨域访问 2025-07-03 10:37:29 +08:00
v-zhangjc9
49cb62b287 fix(bin): 修复pom文件名错误 2025-07-03 10:36:57 +08:00
v-zhangjc9
5cbda28594 fix(gateway): 网关导致重复跨域标记,关闭网关的跨域设置 2025-07-03 10:36:33 +08:00
v-zhangjc9
11c5481287 fix(bin): 修复项目编译时间不正确导致无法识别项目修改状态 2025-07-03 10:01:47 +08:00
v-zhangjc9
67e0cada00 feat(bin): 优化打包脚本,遇到没有修改的模块就跳过发布流程 2025-07-02 17:29:28 +08:00
v-zhangjc9
cdf51cc85f fix(configuration): 优化安全配置 2025-07-02 16:25:06 +08:00
v-zhangjc9
9277d1690c feat(web): 增加常用app list 2025-07-02 15:48:22 +08:00
v-zhangjc9
24ac681cb3 feat(web): 增加装饰 2025-07-02 13:06:10 +08:00
v-zhangjc9
c46da52942 fix(web): 修复访问地址改到了本地测试 2025-07-02 11:28:44 +08:00
v-zhangjc9
7209b52e3d feat(ai-web): 增加JPA建表语句自动生成 2025-07-02 10:54:32 +08:00
v-zhangjc9
959d6fb5c7 feat(ai-web): 完成选择节点 2025-07-01 18:16:05 +08:00
v-zhangjc9
c5c62ab713 feat(ai-web): 完成流程引擎雏形 2025-07-01 15:33:07 +08:00
e59e89a5ad feat(ai-web): 尝试设计流程引擎 2025-06-30 23:31:01 +08:00
b7626180c1 refactor(ai-web): 统一bean工具类 2025-06-30 22:54:40 +08:00
v-zhangjc9
68e54d5110 feat(ai-web): 尝试自建流程引擎 2025-06-30 19:15:19 +08:00
v-zhangjc9
5f133fbfc3 refactor(web): 无聊的改动 2025-06-30 17:15:35 +08:00
v-zhangjc9
d28fbbbba8 fix(web): 优化冗余边检测导致分支节点不正常 2025-06-30 11:14:45 +08:00
v-zhangjc9
67f41c08a0 feat(web): 优化冗余边检测 2025-06-30 10:41:40 +08:00
111ca49815 feat(web): 引入dify的流程检查算法 2025-06-30 00:12:51 +08:00
779fd0eb18 feat(web): 增加分支节点,增加流程检查的测试 2025-06-29 19:02:19 +08:00
8884495a89 fix(ai-web): 恢复配置文件 2025-06-29 14:27:42 +08:00
d08a6babbe feat(ai-web): 增加任务和任务模板 2025-06-28 21:08:31 +08:00
9a3375bd03 fix(ai-web): 修复跨域错误 2025-06-28 17:23:19 +08:00
v-zhangjc9
2c808a5bc9 feat(web): 增加节点的参数展示 2025-06-27 11:31:02 +08:00
v-zhangjc9
6e667c45e1 feat(web): 节点增加入参变量 2025-06-27 10:04:48 +08:00
v-zhangjc9
635c6537ed feat(web): 实现EL表达式的解析 2025-06-27 10:04:00 +08:00
v-zhangjc9
d6b70b1750 feat(web): 尝试增加并行节点解决解析问题 2025-06-26 20:48:49 +08:00
v-zhangjc9
c92a374591 feat(web): 优化UI展示 2025-06-26 20:48:20 +08:00
v-zhangjc9
a2aba82b6e feat(web): 调整节点入参形式 2025-06-26 14:38:09 +08:00
873c1a1d20 feat(web): 增加EL表达式转换 2025-06-26 00:31:04 +08:00
v-zhangjc9
f6bd7e52e1 feat(ai-web): 尝试解析流程图 2025-06-25 17:55:05 +08:00
v-zhangjc9
6f7f7cea67 feat(web): 增加代码节点 2025-06-25 17:54:43 +08:00
v-zhangjc9
33df256863 feat(web): 增加知识库节点 2025-06-25 17:25:50 +08:00
v-zhangjc9
3a51d1e33f feat(web): 升级依赖 2025-06-25 12:36:43 +08:00
v-zhangjc9
d3c7457889 fix(web): 调整样式 2025-06-25 10:43:26 +08:00
2d2eaafcd4 feat(web): 优化样式 2025-06-25 00:04:27 +08:00
v-zhangjc9
566dfef208 feat(web): 增加流程图连线限制 2025-06-24 14:07:07 +08:00
1cba0f4422 feat(web): 完成流程图外部加载 2025-06-24 10:44:18 +08:00
ab56385c8a feat(web): 增加节点输出编辑 2025-06-24 10:15:08 +08:00
b58c34443f feat(web): 增加跨站导航列表 2025-06-24 09:29:48 +08:00
53638a8a6d feat(web): 优化流程界面显示 2025-06-23 23:55:31 +08:00
dc55605c99 feat(web): 正式提取流程设计能力到AI目录下,做代码拆分 2025-06-23 22:51:40 +08:00
7345774258 feat(web): 增加编辑侧边栏关闭时保存节点数据 2025-06-23 22:27:23 +08:00
fcf5f8ad18 feat(web): 优化编辑界面 2025-06-23 22:27:23 +08:00
v-zhangjc9
b53ee57dc3 feat(web): 替换amis渲染,amis渲染太慢,导致卡顿 2025-06-23 22:27:23 +08:00
v-zhangjc9
b916acb1c3 feat(web): 增加流程定义基本能力 2025-06-23 22:27:23 +08:00
v-zhangjc9
c9616eb63a fix(web): 提交遗漏文件 2025-06-23 16:55:36 +08:00
v-zhangjc9
5b3c27ea48 feat(ai-web): 完成JPA存储适配 2025-06-23 16:53:34 +08:00
e48d7e8649 feat(ai-web): 尝试使用jpa作为通用数据库后端 2025-06-23 00:26:31 +08:00
v-zhangjc9
306c20aa7f fix(web): 修复删除按钮hover不是红色 2025-06-19 11:31:07 +08:00
v-zhangjc9
24d5d10ecb fix(web): 优化字号 2025-06-19 11:14:59 +08:00
v-zhangjc9
4a9a9ec238 fix(ai-web): 优化qa嵌入提示词 2025-06-19 11:10:14 +08:00
v-zhangjc9
08aa1d8382 fix(ai-web): 修复chartjs工具获取bean错误 2025-06-19 11:09:52 +08:00
v-zhangjc9
1b3045dfd4 feat(web): 统一字体展示 2025-06-19 11:09:05 +08:00
v-zhangjc9
0f5ae1c4d4 fix(ai-web): 修复超时时间设置过短导致反复重连 2025-06-18 16:09:37 +08:00
v-zhangjc9
48e42ee99a fix(ai-web): 升级部份依赖版本 2025-06-18 14:53:45 +08:00
v-zhangjc9
0914b458d3 fix(ai-web): 修复页面失去焦点的时候没有断开对话的连接 2025-06-18 10:34:56 +08:00
v-zhangjc9
368c30676e feat(ai-web): 尝试优化对话连接的稳定性 2025-06-18 10:33:44 +08:00
v-zhangjc9
60477f99f5 fix(ai-web): 修复错别字 2025-06-17 17:43:18 +08:00
v-zhangjc9
565c530dd5 feat(ai-web): 知识库增加描述 2025-06-17 17:19:04 +08:00
v-zhangjc9
5130885033 fix(ai-web): 改正包名 2025-06-17 16:18:38 +08:00
v-zhangjc9
8e6463845b feat(ai-web): 开启gzip 2025-06-17 16:16:01 +08:00
v-zhangjc9
e89bffe289 feat(ai-web): 增加Feedback详情展示和处理情况确认 2025-06-17 16:15:42 +08:00
v-zhangjc9
1dd00d329c refactor(ai-web): 优化流程编排 2025-06-17 10:35:39 +08:00
e470a87372 feat(web): 修复yarn页面查看资源队列错误 2025-06-16 23:51:36 +08:00
v-zhangjc9
45da452f18 fix(ai-web): 完成feedback AI流程 2025-06-16 20:37:14 +08:00
v-zhangjc9
e6a1bc5383 fix(ai-web): 修复取值错误 2025-06-16 17:13:16 +08:00
v-zhangjc9
c5916703cd fix(web): 修复base url错误 2025-06-16 16:12:39 +08:00
v-zhangjc9
807ddbe5cb feat(ai-web): 完成图片上传和显示 2025-06-16 13:38:42 +08:00
v-zhangjc9
13de694e37 fix(all): 修复配置错误 2025-06-16 12:13:42 +08:00
v-zhangjc9
1962dd586c feat(ai-web): 增加辅助插件 2025-06-16 11:07:56 +08:00
138ee140e1 feat(ai-web): 增加feedback 2025-06-15 23:42:26 +08:00
e2d69bc6e8 refactor(ai-web): 优化id的生成 2025-06-15 22:19:05 +08:00
b9d707dc8f refactor(gateway): 适配ai web的路由 2025-06-15 20:10:54 +08:00
44d1473c6b refactor(ai): 移除chat,合并chat和knowledge为web
以后有需要再拆分
2025-06-15 20:08:51 +08:00
9c658afbd7 refactor(ai): 迁移chat到知识库中 2025-06-15 19:59:35 +08:00
e3f86e6497 refactor(knowledge): 优化配置文件 2025-06-15 19:15:40 +08:00
256c8c6bd5 refactor(knowledge): 修改代码结构,增加多环境配置文件方便本地开发 2025-06-15 19:10:38 +08:00
b627c91acb refactor(knowledge): 重构大模型配置
Spring AI默认大模型配置不支持同时配置两个文本大模型,比如一个文本大模型和一个图像大模型,改用自定义的配置
2025-06-15 17:57:09 +08:00
7fb490778a refactor(knowledge): 优化接口结构,统一到一个路径下,为合并做准备 2025-06-15 17:02:53 +08:00
d4d5aede31 feat(ai): restClient和webClient提供给其他类使用 2025-06-14 17:45:56 +08:00
v-zhangjc9
f11f5e7656 feat(ai): 调整模型 2025-06-13 19:07:13 +08:00
v-zhangjc9
bc32a89fea fix(ai): 移除不支持的模型 2025-06-13 16:11:37 +08:00
183 changed files with 8700 additions and 4660 deletions

1
.gitignore vendored
View File

@@ -110,3 +110,4 @@ Network Trash Folder
Temporary Items
.apdisk
**/temp/
.build

View File

@@ -1,5 +1,5 @@
import {$, fetch, fs, glob, os, path, spinner, syncProcessCwd, usePowerShell} from 'zx'
import {isEqual, trim, fileSize} from "licia";
import {fileSize, isEqual, trim} from "licia";
import md5file from 'md5-file'
syncProcessCwd(true)
@@ -41,20 +41,70 @@ const millisecondToString = (timestamp) => {
return parts.join('')
}
const dotBuildPath = () => `.build`
const modifiedDataPath = () => `${dotBuildPath()}/modified_time.json`
const readModifiedTimeData = async () => {
if (!fs.existsSync(dotBuildPath())) {
fs.mkdirSync(dotBuildPath(), {recursive: true})
}
if (!(await fs.exists(modifiedDataPath()))) {
fs.writeFileSync(modifiedDataPath(), '{}')
}
return JSON.parse(await fs.readFile(modifiedDataPath(), 'utf-8'))
}
const updateModifiedTimeData = (data) => {
fs.writeFileSync(modifiedDataPath(), JSON.stringify(data, null, 2))
}
const isModified = async (target) => {
if (!target || !(await fs.exists(target))) {
throw new Error("Target 不存在")
}
let stat = fs.statSync(target)
let currentModifiedTime = stat.mtimeMs
let lastModifiedTime = (await readModifiedTimeData())[target]
return !(lastModifiedTime && isEqual(currentModifiedTime, lastModifiedTime));
}
const updateModifiedTime = async (target) => {
if (!target || !(await fs.exists(target))) {
throw new Error("Target 不存在")
}
let stat = fs.statSync(target)
let currentModifiedTime = stat.mtimeMs
let modifiedTimeData = await readModifiedTimeData()
modifiedTimeData[target] = currentModifiedTime
updateModifiedTimeData(modifiedTimeData)
}
export const run_deploy = async (project) => {
if (!(await isModified(project))) {
console.log(`✅ Skip deploy ${project}`)
return
}
let output = await spinner(
`Deploying project ${project}`,
() => $`mvn -pl ${project} clean deploy -D skipTests -s ${maven_setting}`
)
console.log(`✅ Finished deploy ${project} (${millisecondToString(output['duration'])})`)
await updateModifiedTime(project)
}
export const run_deploy_root = async () => {
if (!(await isModified('pom.xml'))) {
console.log(`✅ Skip deploy root`)
return
}
let output = await spinner(
`Deploying root`,
() => $`mvn clean deploy -N -D skipTests -s ${maven_setting}`
)
console.log(`✅ Finished deploy root (${millisecondToString(output['duration'])})`)
await updateModifiedTime('pom.xml')
}
export const run_deploy_batch = async (projects) => {

3
bin/test.js Normal file
View File

@@ -0,0 +1,3 @@
import {isModified} from './library.js'
console.log(await isModified('/Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/pom.xml'))

View File

@@ -1,14 +1,12 @@
import {cd, path} from 'zx'
import {trim} from "licia";
import {run_deploy, run_package, run_upload_normal} from '../../bin/library.js'
import {run_deploy} from '../../bin/library.js'
// 切换目录
cd(trim(path.dirname(import.meta.dirname)))
// 执行流程
try {
await run_deploy('service-ai-core')
await run_package('service-ai-chat')
await run_upload_normal('service-ai-chat')
} catch (e) {
console.error(e)
}

View File

@@ -1,14 +0,0 @@
import {cd, path} from 'zx'
import {trim} from "licia";
import {run_deploy, run_package, run_upload_normal} from '../../bin/library.js'
// 切换目录
cd(trim(path.dirname(import.meta.dirname)))
// 执行流程
try {
await run_deploy('service-ai-core')
await run_package('service-ai-knowledge')
await run_upload_normal('service-ai-knowledge')
} catch (e) {
console.error(e)
}

View File

@@ -0,0 +1,21 @@
import {
cd,
path,
} from 'zx'
import {trim} from "licia";
import {
run_deploy,
run_package,
run_upload_normal,
} from '../../bin/library.js'
// 切换目录
cd(trim(path.dirname(import.meta.dirname)))
// 执行流程
try {
await run_deploy('service-ai-core')
await run_package('service-ai-web')
await run_upload_normal('service-ai-web')
} catch (e) {
console.error(e)
}

View File

@@ -0,0 +1,86 @@
create table hudi_collect_build_b12.service_ai_feedback
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
analysis longtext comment 'AI的分析结果',
conclusion longtext comment 'AI的解决方案',
source longtext not null comment '原始报障说明',
status enum ('ANALYSIS_PROCESSING','ANALYSIS_SUCCESS','FINISHED') not null comment '报障处理状态',
primary key (id)
) comment ='报障信息记录' charset = utf8mb4;
create table hudi_collect_build_b12.service_ai_feedback_pictures
(
feedback_id bigint not null,
pictures_id bigint not null,
primary key (feedback_id, pictures_id)
) comment ='报障相关截图' charset = utf8mb4;
alter table hudi_collect_build_b12.service_ai_feedback_pictures
add constraint UK3npjcyjyqfbdlf2v5tj64j2g3 unique (pictures_id);
create table hudi_collect_build_b12.service_ai_file
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
filename varchar(255) comment '文件名称',
md5 varchar(255) comment '文件的md5编码用于校验文件的完整性',
path varchar(255) comment '文件在主机上存储的实际路径',
size bigint comment '文件大小单位是byte',
type varchar(255) comment '文件类型,通常记录的是文件的后缀名',
primary key (id)
) comment ='记录上传的文件存储信息' charset = utf8mb4;
create table hudi_collect_build_b12.service_ai_flow_task
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
comment text comment '任务注释,用于额外说明',
error longtext comment '任务运行产生的报错',
input longtext comment '任务输入',
result longtext comment '任务运行结果',
status enum ('ERROR','FINISHED','RUNNING') not null comment '任务运行状态',
template_description varchar(255) comment '任务对应的模板功能、内容说明',
template_flow_graph longtext not null comment '任务对应的模板前端流程图数据',
template_input_schema longtext comment '任务对应的模板入参Schema',
template_name varchar(255) not null comment '任务对应的模板名称',
primary key (id)
) comment ='流程任务记录' charset = utf8mb4;
create table hudi_collect_build_b12.service_ai_flow_task_template
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
description varchar(255) comment '模板功能、内容说明',
flow_graph longtext not null comment '前端流程图数据',
input_schema longtext comment '模板入参Schema',
name varchar(255) not null comment '模板名称',
primary key (id)
) comment ='流程任务模板' charset = utf8mb4;
create table hudi_collect_build_b12.service_ai_group
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
name varchar(255) not null comment '分组名称',
status enum ('FINISHED','RUNNING') not null comment '分组处理状态',
knowledge_id bigint not null,
primary key (id)
) comment ='知识库下包含的分组' charset = utf8mb4;
create table hudi_collect_build_b12.service_ai_knowledge
(
id bigint not null comment '记录唯一标记',
created_time datetime(6) comment '记录创建时间',
modified_time datetime(6) comment '记录更新时间',
description longtext not null comment '知识库说明',
name varchar(255) not null comment '知识库名称',
strategy enum ('Cosine','Euclid') not null comment '知识库策略',
vector_source_id bigint not null comment '知识库对应的向量库名',
primary key (id)
) comment ='知识库' charset = utf8mb4;

View File

@@ -1,11 +0,0 @@
CREATE TABLE `service_ai_file`
(
`id` bigint NOT NULL,
`filename` varchar(500) DEFAULT NULL,
`size` bigint DEFAULT NULL,
`md5` varchar(100) DEFAULT NULL,
`path` varchar(500) DEFAULT NULL,
`type` varchar(50) DEFAULT NULL,
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) DEFAULT CHARSET = utf8mb4;

View File

@@ -1,9 +0,0 @@
CREATE TABLE `service_ai_group`
(
`id` bigint NOT NULL,
`knowledge_id` bigint NOT NULL,
`name` varchar(100) NOT NULL,
`status` varchar(10) NOT NULL DEFAULT 'RUNNING',
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) DEFAULT CHARSET = utf8mb4;

View File

@@ -1,10 +0,0 @@
CREATE TABLE `service_ai_knowledge`
(
`id` bigint NOT NULL,
`vector_source_id` varchar(100) NOT NULL,
`name` varchar(100) NOT NULL,
`strategy` varchar(10) NOT NULL,
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) DEFAULT CHARSET = utf8mb4;

View File

@@ -11,8 +11,7 @@
<description>Hudi AI服务集合</description>
<modules>
<module>service-ai-core</module>
<module>service-ai-chat</module>
<module>service-ai-knowledge</module>
<module>service-ai-web</module>
<module>service-ai-cli</module>
</modules>
@@ -30,8 +29,22 @@
<eclipse-collections.version>11.1.0</eclipse-collections.version>
<curator.version>5.1.0</curator.version>
<hutool.version>5.8.27</hutool.version>
<mapstruct.version>1.6.3</mapstruct.version>
<liteflow.version>2.13.2</liteflow.version>
</properties>
<dependencies>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<!-- 当前项目依赖 -->
@@ -98,6 +111,11 @@
<artifactId>jasypt-spring-boot-starter</artifactId>
<version>3.0.5</version>
</dependency>
<dependency>
<groupId>com.blinkfox</groupId>
<artifactId>fenix-spring-boot-starter</artifactId>
<version>3.0.0</version>
</dependency>
<!-- 日志相关 -->
<dependency>
@@ -136,7 +154,27 @@
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-spring-boot-starter</artifactId>
<version>2.13.2</version>
<version>${liteflow.version}</version>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-el-builder</artifactId>
<version>${liteflow.version}</version>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-script-graaljs</artifactId>
<version>${liteflow.version}</version>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-script-python</artifactId>
<version>${liteflow.version}</version>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-script-lua</artifactId>
<version>${liteflow.version}</version>
</dependency>
<dependency>
<groupId>org.noear</groupId>
@@ -148,12 +186,22 @@
<artifactId>solon-ai-dialect-openai</artifactId>
<version>${solon-ai.version}</version>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
<version>${mapstruct.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.14.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
@@ -162,7 +210,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.3.0</version>
<version>3.6.0</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>

View File

@@ -1,56 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>service-ai-chat</artifactId>
<dependencies>
<dependency>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-deepseek</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai-dialect-openai</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,37 +0,0 @@
package com.lanyuanxiaoyao.service.ai.chat;
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
import org.springframework.beans.BeansException;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.retry.annotation.EnableRetry;
/**
* @author lanyuanxiaoyao
* @version 20250514
*/
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
@EnableDiscoveryClient
@EnableConfigurationProperties
@EnableEncryptableProperties
@EnableRetry
public class AiChatApplication implements ApplicationContextAware {
private static ApplicationContext context;
public static void main(String[] args) {
SpringApplication.run(AiChatApplication.class, args);
}
public static <T> T getBean(Class<T> clazz) {
return context.getBean(clazz);
}
@Override
public void setApplicationContext(ApplicationContext context) throws BeansException {
AiChatApplication.context = context;
}
}

View File

@@ -1,44 +0,0 @@
package com.lanyuanxiaoyao.service.ai.chat.entity;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
public class MessageVO {
private String role;
private String content;
private String reason;
public String getRole() {
return role;
}
public void setRole(String role) {
this.role = role;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getReason() {
return reason;
}
public void setReason(String reason) {
this.reason = reason;
}
@Override
public String toString() {
return "MessageVO{" +
"role='" + role + '\'' +
", content='" + content + '\'' +
", reason='" + reason + '\'' +
'}';
}
}

View File

@@ -1,15 +0,0 @@
spring:
application:
name: service-ai-chat
profiles:
include: random-port,common,discovery,metrics,forest
ai:
deepseek:
base-url: http://132.121.206.65:10086/v1
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
chat:
options:
model: 'Qwen3-1.7-vllm'
mvc:
async:
request-timeout: 3600000

View File

@@ -1,34 +0,0 @@
<configuration>
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %clr(%5p) %clr([${HOSTNAME}]){yellow} %clr([%t]){magenta} %clr(%logger{40}){cyan} #@# %m%n%wEx</pattern>
</encoder>
</appender>
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
<MaxHistory>7</MaxHistory>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx</pattern>
</encoder>
</appender>
<logger name="com.zaxxer.hikari" level="ERROR"/>
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
<root level="INFO">
<appender-ref ref="Console"/>
<appender-ref ref="RollingFile"/>
</root>
</configuration>

View File

@@ -1,16 +0,0 @@
package com.lanyuanxiaoyao.service.ai.chat;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
/**
* @author lanyuanxiaoyao
* @version 20250606
*/
public class TestDatetimeFormat {
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
public static void main(String[] args) {
System.out.println(LocalDate.parse("20250606", FORMATTER).format(FORMATTER));
}
}

View File

@@ -1,76 +0,0 @@
package com.lanyuanxiaoyao.service.ai.chat;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.chat.tools.TableTool;
import com.lanyuanxiaoyao.service.ai.chat.tools.YarnTool;
import java.net.http.HttpClient;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.deepseek.DeepSeekChatModel;
import org.springframework.ai.deepseek.DeepSeekChatOptions;
import org.springframework.ai.deepseek.api.DeepSeekApi;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.http.client.reactive.JdkClientHttpConnector;
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;
public class TestLlmPlan {
public static void main(String[] args) {
ChatClient client = ChatClient.builder(
DeepSeekChatModel.builder()
.deepSeekApi(
DeepSeekApi.builder()
.baseUrl("http://127.0.0.1:1234/v1")
.apiKey("nopassword")
.restClientBuilder(restClientBuilder())
.webClientBuilder(webClientBuilder())
.build()
)
.defaultOptions(
DeepSeekChatOptions.builder()
.model("qwen/qwen3-1.7b")
.build()
)
.build()
)
.defaultSystem(StrUtil.format(
"""
你是一名专业的AI运维助手专职负责“Hudi数据同步服务”的平台运维工作。你的核心职责是
1.友好解答:积极、专业地解答用户(通常是平台管理员或用户)关于该平台运维工作的疑问。
2.知识驱动:在解答时,应尽可能通过各种方式(知识库、上下文、外部工具等)全面获取准确知识和数据来支持回答。
3.诚实守界:
对于无法通过已有知识或数据确认的问题,必须明确告知用户你无法解答,切勿捏造信息或提供不确定的答案。
对于与该Hudi数据同步服务平台运维工作无关的问题需委婉拒绝用户并明确说明超出你的职责和能力范围。
对话语言:中文
{}
""",
Prompts.hudi
))
.defaultTools(
new TableTool(),
new YarnTool()
)
.build();
System.out.println(client.prompt("""
我需要大模型帮我检查系统整体运行状态,帮我设计详细的指导大模型具体操作的操作步骤
避免使用上下文中没有提到的外部工具
""").call().content());
}
private static HttpClient httpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build();
}
private static RestClient.Builder restClientBuilder() {
return RestClient.builder()
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
}
private static WebClient.Builder webClientBuilder() {
return WebClient.builder()
.clientConnector(new JdkClientHttpConnector(httpClient()));
}
}

View File

@@ -1,170 +0,0 @@
package com.lanyuanxiaoyao.service.ai.chat;
import java.io.IOException;
import java.net.http.HttpClient;
import java.util.Arrays;
import java.util.List;
import org.noear.solon.ai.rag.Document;
import org.noear.solon.ai.reranking.RerankingModel;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.document.MetadataMode;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.http.client.reactive.JdkClientHttpConnector;
import org.springframework.util.MimeTypeUtils;
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;
/**
* @author lanyuanxiaoyao
* @version 20250514
*/
public class TestSpringAIToolChat {
public static void main(String[] args) throws IOException {
testChatModel();
testVisualModel();
// testEmbeddingModel();
// testRerankingModel();
}
private static void testChatModel() {
for (String model : List.of(
"Qwen3/qwen3-0.6b",
"Qwen3/qwen3-1.7b",
"Qwen3/qwen3-4b",
"Qwen3/qwen3-4b-q4km",
"Qwen3/qwen3-8b-q4km"
)) {
ChatClient client = chatClient(model);
String content = client.prompt()
.user("你好")
.call()
.content();
System.out.println(content);
}
}
private static void testVisualModel() {
for (String model : List.of(
"Qwen2.5/qwen2.5-vl-7b",
"Qwen2.5/qwen2.5-vl-7b-q4km",
"Qwen2.5/qwen2.5-vl-3b-instruct",
"Qwen2.5/qwen2.5-vl-3b-instruct-awq",
"Qwen2.5/qwen2.5-vl-7b-instruct",
"Qwen2.5/qwen2.5-vl-7b-instruct-awq",
"MiniCPM/minicpm-o-2.6-7.6b",
"MiniCPM/minicpm-o-2.6-7.6b-q4km"
)) {
ChatClient client = chatClient(model);
String content = client.prompt()
.user(spec -> spec.text("图片中有什么").media(MimeTypeUtils.IMAGE_PNG, new FileSystemResource("/Users/lanyuanxiaoyao/Pictures/deepseek.png")))
.call()
.content();
System.out.println(content);
}
}
private static void testEmbeddingModel() {
for (String model : List.of(
"Qwen3/qwen3-embedding-0.6b",
"Qwen3/qwen3-embedding-4b",
"Qwen3/qwen3-embedding-4b-q4km",
"Qwen3/qwen3-embedding-8b-q4km",
"BGE/bge-m3",
"BGE/bge-m3-q4km",
"MiniCPM/minicpm-embedding",
"MiniCPM/minicpm-embedding-light"
)) {
EmbeddingModel embeddingModel = embeddingModel(model);
float[] worlds = embeddingModel.embed("Hello world");
System.out.println(Arrays.toString(worlds));
}
}
private static void testRerankingModel() throws IOException {
for (String model : List.of(
"BGE/beg-reranker-v2",
"MiniCPM/minicpm-reranker",
"MiniCPM/minicpm-reranker-light",
"BGE/beg-reranker-v2",
"BGE/beg-reranker-v2-q4km"
)) {
RerankingModel rerankingModel = rerankingModel(model);
List<Document> list = rerankingModel.rerank(
"你好",
List.of(
new Document("go go go滚犊子"),
new Document("我是tom你最近过得好吗"),
new Document("666你就是大聪明")
)
);
list.forEach(System.out::println);
}
}
private static ChatClient chatClient(String model) {
return ChatClient.builder(
OpenAiChatModel.builder()
.openAiApi(
OpenAiApi.builder()
.baseUrl("http://132.121.206.65:10086")
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
.restClientBuilder(restClientBuilder())
.webClientBuilder(webClientBuilder())
.build()
)
.defaultOptions(
OpenAiChatOptions.builder()
.model(model)
.build()
)
.build()
)
.build();
}
private static EmbeddingModel embeddingModel(String model) {
return new OpenAiEmbeddingModel(
OpenAiApi.builder()
.baseUrl("http://132.121.206.65:10086")
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
.restClientBuilder(restClientBuilder())
.webClientBuilder(webClientBuilder())
.build(),
MetadataMode.EMBED,
OpenAiEmbeddingOptions.builder()
.model(model)
.build()
);
}
private static RerankingModel rerankingModel(String model) {
return RerankingModel.of("http://132.121.206.65:10086/v1/rerank")
.model(model)
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
.build();
}
private static HttpClient httpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build();
}
private static RestClient.Builder restClientBuilder() {
return RestClient.builder()
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
}
private static WebClient.Builder webClientBuilder() {
return WebClient.builder()
.clientConnector(new JdkClientHttpConnector(httpClient()));
}
}

View File

@@ -25,39 +25,32 @@ public class LlamaSwapTool extends GeneratorTool {
Map.of(
"models", List.of(
llamaCppEmbeddingCmd("BGE/bge-m3-q4km", "bge-m3-Q4_K_M.gguf", 20),
vllmEmbeddingCmd("BGE/bge-m3", "bge-m3", 20, 5),
vllmEmbeddingCmd("BGE/bge-m3", "bge-m3", 5),
llamaCppRerankerCmd("BGE/beg-reranker-v2-q4km", "bge-reranker-v2-m3-Q4_K_M.gguf", 20),
llamaCppRerankerCmd("BGE/beg-reranker-v2", "bge-reranker-v2-m3", 20),
vllmRerankerCmd("BGE/beg-reranker-v2", "bge-reranker-v2-m3", 5),
vllmCmd("Qwen3/qwen3-0.6b", "Qwen3-0.6B", 35, 5, true),
vllmCmd("Qwen3/qwen3-1.7b", "Qwen3-1.7B", 35, 5, true),
vllmCmd("Qwen3/qwen3-4b", "Qwen3-4B", 35, 8, true),
vllmCmd("Qwen3/qwen3-0.6b", "Qwen3-0.6B", 5, true),
vllmCmd("Qwen3/qwen3-1.7b", "Qwen3-1.7B", 5, true),
vllmCmd("Qwen3/qwen3-4b", "Qwen3-4B", 8, true),
llamaCppCmd("Qwen3/qwen3-4b-q4km", "Qwen3-4B-Q4_K_M.gguf", 35),
llamaCppCmd("Qwen3/qwen3-8b-q4km", "Qwen3-8B-Q4_K_M.gguf", 35),
vllmEmbeddingCmd("Qwen3/qwen3-embedding-0.6b", "Qwen3-Embedding-0.6B", 35, 5),
vllmEmbeddingCmd("Qwen3/qwen3-embedding-4b", "Qwen3-Embedding-4B", 35, 8),
vllmEmbeddingCmd("Qwen3/qwen3-embedding-0.6b", "Qwen3-Embedding-0.6B", 5),
vllmEmbeddingCmd("Qwen3/qwen3-embedding-4b", "Qwen3-Embedding-4B", 8),
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-4b-q4km", "Qwen3-Embedding-4B-Q4_K_M.gguf", 35),
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-8b-q4km", "Qwen3-Embedding-8B-Q4_K_M.gguf", 35),
vllmRerankerCmd("Qwen3/qwen3-reranker-0.6b", "Qwen3-Reranker-0.6B", 35, 5),
vllmRerankerCmd("Qwen3/qwen3-reranker-4b", "Qwen3-Reranker-4B", 35, 8),
// 0.9.1 vllm还未支持
// vllmRerankerCmd("Qwen3/qwen3-reranker-0.6b", "Qwen3-Reranker-0.6B", 5),
// vllmRerankerCmd("Qwen3/qwen3-reranker-4b", "Qwen3-Reranker-4B", 8),
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b", "Qwen2.5-VL-7B-Instruct-BF16.gguf", 35),
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b-q4km", "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", 35),
vllmCmd("Qwen2.5/qwen2.5-vl-7b-instruct", "Qwen2.5-VL-7B-Instruct", 35, 8, false),
vllmCmd("Qwen2.5/qwen2.5-vl-7b-instruct-awq", "Qwen2.5-VL-7B-Instruct-AWQ", 35, 8, false),
vllmCmd("Qwen2.5/qwen2.5-vl-3b-instruct", "Qwen2.5-VL-3B-Instruct", 35, 8, false),
vllmCmd("Qwen2.5/qwen2.5-vl-3b-instruct-awq", "Qwen2.5-VL-3B-Instruct-AWQ", 35, 8, false),
vllmCmd("Qwen2.5/qwen2.5-vl-3b-instruct", "Qwen2.5-VL-3B-Instruct", 8, false),
vllmCmd("Qwen2.5/qwen2.5-vl-7b-instruct", "Qwen2.5-VL-7B-Instruct", 8, false),
llamaCppVisualCmd("MiniCPM/minicpm-o-2.6-7.6b-q4km", "MiniCPM-o-2_6-7.6B-Q4_K_M.gguf", 35),
vllmCmd("MiniCPM/minicpm-o-2.6-7.6b", "MiniCPM-o-2_6", 35, 10, false),
vllmEmbeddingCmd("MiniCPM/minicpm-embedding", "MiniCPM-Embedding", 20, 8),
vllmEmbeddingCmd("MiniCPM/minicpm-embedding-light", "MiniCPM-Embedding-Light", 20, 5),
vllmEmbeddingCmd("MiniCPM/minicpm-reranker", "MiniCPM-Reranker", 20, 8),
vllmEmbeddingCmd("MiniCPM/minicpm-reranker-light", "MiniCPM-Reranker-Light", 20, 5)
vllmCmd("MiniCPM/minicpm-o-2.6-7.6b", "MiniCPM-o-2_6", 10, false)
)
),
"config.yaml"
@@ -114,23 +107,23 @@ public class LlamaSwapTool extends GeneratorTool {
);
}
private DockerCmd vllmCmd(String name, String model, Integer thread, Integer cache, Boolean isReasonable) {
return vllmCmd(name, model, thread, cache, false, false, isReasonable, false);
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isReasonable) {
return vllmCmd(name, model, cache, false, false, isReasonable);
}
private DockerCmd vllmEmbeddingCmd(String name, String model, Integer thread, Integer cache) {
return vllmCmd(name, model, thread, cache, true, false, false, false);
private DockerCmd vllmEmbeddingCmd(String name, String model, Integer cache) {
return vllmCmd(name, model, cache, true, false, false);
}
private DockerCmd vllmRerankerCmd(String name, String model, Integer thread, Integer cache) {
return vllmCmd(name, model, thread, cache, false, true, false, false);
private DockerCmd vllmRerankerCmd(String name, String model, Integer cache) {
return vllmCmd(name, model, cache, false, true, false);
}
private DockerCmd vllmVisualCmd(String name, String model, Integer thread, Integer cache, Boolean isReasonable) {
return vllmCmd(name, model, thread, cache, false, false, isReasonable, true);
private DockerCmd vllmVisualCmd(String name, String model, Integer cache, Boolean isReasonable) {
return vllmCmd(name, model, cache, false, false, isReasonable);
}
private DockerCmd vllmCmd(String name, String model, Integer thread, Integer cache, Boolean isEmbedding, Boolean isReranker, Boolean isReasonable, Boolean isVisual) {
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isEmbedding, Boolean isReranker, Boolean isReasonable) {
List<String> arguments = ListUtil.list(
false,
StrUtil.format("--model /models/{}", model),
@@ -138,11 +131,13 @@ public class LlamaSwapTool extends GeneratorTool {
"--port ${PORT}",
StrUtil.format("--api-key {}", API_KEY),
"--disable-log-requests",
"--uvicorn-log-level error"
"--uvicorn-log-level error",
"--trust-remote-code"
);
if (isEmbedding) {
arguments.add("--task embedding");
} else if (isReranker) {
arguments.add("--task score");
} else if (isReasonable) {
arguments.add("--enable-auto-tool-choice");
arguments.add("--tool-call-parser hermes");
@@ -150,7 +145,7 @@ public class LlamaSwapTool extends GeneratorTool {
arguments.add("--reasoning-parser deepseek_r1");
}
return new DockerCmd(
"vllm-server-cpu:0.9.1",
"vllm-server-cpu:0.8.5.post1",
name,
model,
StrUtil.format("http://vllm-{}:${PORT}", displayName(model)),
@@ -158,9 +153,7 @@ public class LlamaSwapTool extends GeneratorTool {
StrUtil.format("--name vllm-{}", displayName(model)),
"--privileged=true",
"--shm-size=4g",
StrUtil.format("-e VLLM_CPU_KVCACHE_SPACE={}", cache),
StrUtil.format("-e VLLM_CPU_OMP_THREADS_BIND=0-{}", thread - 1),
"-e VLLM_CPU_MOE_PREPACK=0"
StrUtil.format("-e VLLM_CPU_KVCACHE_SPACE={}", cache)
),
arguments
);

View File

@@ -1,4 +1,4 @@
healthCheckTimeout: 120
healthCheckTimeout: 600
logLevel: warn
models:
<#list models as model>

View File

@@ -17,22 +17,30 @@ import org.springframework.web.reactive.function.client.WebClient;
*/
@Configuration
public class WebClientConfiguration {
private HttpClient httpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build();
}
@Bean
@Primary
public RestClient.Builder restClientBuilder() {
return RestClient.builder()
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
return generateRestClientBuilder();
}
@Bean
@Primary
public WebClient.Builder webClientBuilder() {
return generateWebClientBuilder();
}
private static HttpClient httpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build();
}
public static RestClient.Builder generateRestClientBuilder() {
return RestClient.builder()
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
}
public static WebClient.Builder generateWebClientBuilder() {
return WebClient.builder()
.clientConnector(new JdkClientHttpConnector(httpClient()));
}

View File

@@ -1,9 +1,8 @@
package com.lanyuanxiaoyao.service.configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpMethod;
import org.springframework.security.config.Customizer;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
@@ -12,6 +11,9 @@ import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.provisioning.InMemoryUserDetailsManager;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
/**
* @author lanyuanxiaoyao
@@ -20,13 +22,30 @@ import org.springframework.security.web.SecurityFilterChain;
@Configuration
@EnableWebSecurity
public class SecurityConfig {
private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class);
@Bean
public CorsFilter corsFilter() {
CorsConfiguration configuration = new CorsConfiguration();
configuration.setAllowCredentials(true);
configuration.addAllowedOriginPattern("*");
configuration.addAllowedHeader("*");
configuration.addAllowedMethod("*");
configuration.setMaxAge(7200L);
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
source.registerCorsConfiguration("/**", configuration);
return new CorsFilter(source);
}
@Bean
public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
return http.authorizeHttpRequests(registry -> registry.anyRequest().authenticated())
return http.authorizeHttpRequests(
registry -> registry
.requestMatchers(HttpMethod.OPTIONS, "/**")
.permitAll()
.anyRequest()
.authenticated()
)
.httpBasic(Customizer.withDefaults())
.cors(AbstractHttpConfigurer::disable)
.cors(Customizer.withDefaults())
.csrf(AbstractHttpConfigurer::disable)
.formLogin(AbstractHttpConfigurer::disable)
.build();

View File

@@ -1,83 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>service-ai-knowledge</artifactId>
<dependencies>
<dependency>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai-core</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-vector-store-qdrant</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-markdown-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-tika-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-pdf-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai</artifactId>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai-dialect-openai</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,39 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
@Configuration
@ConfigurationProperties(prefix = "knowledge")
public class KnowledgeConfiguration {
private String downloadPrefix;
private String uploadPath;
public String getDownloadPrefix() {
return downloadPrefix;
}
public void setDownloadPrefix(String downloadPrefix) {
this.downloadPrefix = downloadPrefix;
}
public String getUploadPath() {
return uploadPath;
}
public void setUploadPath(String uploadPath) {
this.uploadPath = uploadPath;
}
@Override
public String toString() {
return "KnowledgeConfiguration{" +
"downloadPrefix='" + downloadPrefix + '\'' +
", uploadPath='" + uploadPath + '\'' +
'}';
}
}

View File

@@ -1,21 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
import cn.hutool.core.util.StrUtil;
import org.noear.solon.ai.reranking.RerankingModel;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @version 20250604
*/
@Configuration
public class SolonConfiguration {
@Bean
public RerankingModel rerankingModel(SolonProperties solonProperties) {
return RerankingModel.of(StrUtil.format("{}{}", solonProperties.getBaseUrl(), solonProperties.getRerank().getEndpoint()))
.apiKey(solonProperties.getApiKey())
.model(solonProperties.getRerank().getModel())
.build();
}
}

View File

@@ -1,78 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.configuration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @version 20250604
*/
@Configuration
@ConfigurationProperties(prefix = "solon")
public class SolonProperties {
private String baseUrl;
private String apiKey;
private Rerank rerank;
public String getBaseUrl() {
return baseUrl;
}
public void setBaseUrl(String baseUrl) {
this.baseUrl = baseUrl;
}
public String getApiKey() {
return apiKey;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
}
public Rerank getRerank() {
return rerank;
}
public void setRerank(Rerank rerank) {
this.rerank = rerank;
}
@Override
public String toString() {
return "SolonProperties{" +
"baseUrl='" + baseUrl + '\'' +
", apiKey='" + apiKey + '\'' +
", rerank=" + rerank +
'}';
}
public static final class Rerank {
private String model;
private String endpoint;
public String getModel() {
return model;
}
public void setModel(String model) {
this.model = model;
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
@Override
public String toString() {
return "Rerank{" +
"model='" + model + '\'' +
", endpoint='" + endpoint + '\'' +
'}';
}
}
}

View File

@@ -1,38 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.knowledge.service.GroupService;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* @author lanyuanxiaoyao
* @version 20250528
*/
@RestController
@RequestMapping("group")
public class GroupController {
private static final Logger logger = LoggerFactory.getLogger(GroupController.class);
private final GroupService groupService;
public GroupController(GroupService groupService) {
this.groupService = groupService;
}
@GetMapping("list")
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId) {
return AmisResponse.responseCrudData(groupService.list(knowledgeId));
}
@GetMapping("delete")
public AmisResponse<?> delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
groupService.remove(id);
return AmisResponse.responseSuccess();
}
}

View File

@@ -1,131 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.SegmentVO;
import com.lanyuanxiaoyao.service.ai.knowledge.service.EmbeddingService;
import com.lanyuanxiaoyao.service.ai.knowledge.service.KnowledgeBaseService;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* @author lanyuanxiaoyao
* @version 20250515
*/
@RestController
@RequestMapping("knowledge")
public class KnowledgeBaseController {
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
private final KnowledgeBaseService knowledgeBaseService;
private final EmbeddingService embeddingService;
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
this.knowledgeBaseService = knowledgeBaseService;
this.embeddingService = embeddingService;
}
@PostMapping("add")
public void add(
@RequestParam("name") String name,
@RequestParam("strategy") String strategy
) throws ExecutionException, InterruptedException {
knowledgeBaseService.add(name, strategy);
}
@GetMapping("name")
public AmisMapResponse name(@RequestParam("id") Long id) {
return AmisResponse.responseMapData()
.setData("name", knowledgeBaseService.getName(id));
}
@GetMapping("list")
public AmisResponse<?> list() {
return AmisResponse.responseCrudData(knowledgeBaseService.list());
}
@GetMapping("delete")
public void delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
knowledgeBaseService.remove(id);
}
@PostMapping("preview_text")
public AmisResponse<?> previewText(
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
@RequestParam(value = "type", defaultValue = "text") String type,
@RequestParam(value = "content", required = false) String content,
@RequestParam(value = "files", required = false) String files
) {
if (StrUtil.equals("text", type)) {
return AmisResponse.responseCrudData(
embeddingService.preview(mode, content)
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
vo.setText(doc.getText());
return vo;
})
);
} else if (StrUtil.equals("file", type)) {
return AmisResponse.responseCrudData(
embeddingService.preview(mode, Lists.immutable.of(files.split(",")))
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
vo.setText(doc.getText());
return vo;
})
);
} else {
throw new IllegalArgumentException("Unsupported type: " + type);
}
}
@PostMapping("submit_text")
public void submitText(
@RequestParam("id") Long id,
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
@RequestParam(value = "type", defaultValue = "text") String type,
@RequestParam(value = "content", required = false) String content,
@RequestParam(value = "files", required = false) String files
) {
if (StrUtil.equals("text", type)) {
embeddingService.submit(id, mode, content);
} else if (StrUtil.equals("file", type)) {
embeddingService.submit(id, mode, Lists.immutable.of(files.split(",")));
} else {
throw new IllegalArgumentException("Unsupported type: " + type);
}
}
@PostMapping("submit_text_directly")
public void submitDirectly(
@RequestParam("id") Long id,
@RequestParam(value = "name", required = false) String name,
@RequestParam(value = "split_key", defaultValue = "\n\n") String splitKey,
@RequestBody String content
) {
embeddingService.submitDirectly(id, name, Lists.immutable.of(content.split(splitKey)));
}
@PostMapping("query")
public ImmutableList<String> query(
@RequestParam("id") Long id,
@RequestParam(value = "limit", defaultValue = "5") Integer limit,
@RequestParam(value = "threshold", defaultValue = "0.6") Double threshold,
@RequestBody String text
) throws ExecutionException, InterruptedException, IOException {
return knowledgeBaseService.query(id, text, limit, threshold);
}
}

View File

@@ -1,202 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
import java.util.List;
import java.util.Map;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.springframework.ai.document.Document;
/**
* @author lanyuanxiaoyao
* @version 20250523
*/
public class EmbeddingContext {
private Long vectorSourceId;
private Long groupId;
private Config config;
private String content;
private String file;
private String fileFormat;
private List<Document> documents = Lists.mutable.empty();
private Map<String, Object> metadata = Maps.mutable.empty();
private EmbeddingContext(Builder builder) {
setVectorSourceId(builder.vectorSourceId);
setGroupId(builder.groupId);
setConfig(builder.config);
setContent(builder.content);
setFile(builder.file);
setFileFormat(builder.fileFormat);
}
public static Builder builder() {
return new Builder();
}
public Long getGroupId() {
return groupId;
}
public void setGroupId(Long groupId) {
this.groupId = groupId;
}
public Long getVectorSourceId() {
return vectorSourceId;
}
public void setVectorSourceId(Long vectorSourceId) {
this.vectorSourceId = vectorSourceId;
}
public Config getConfig() {
return config;
}
public void setConfig(Config config) {
this.config = config;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getFile() {
return file;
}
public void setFile(String file) {
this.file = file;
}
public String getFileFormat() {
return fileFormat;
}
public void setFileFormat(String fileFormat) {
this.fileFormat = fileFormat;
}
public List<Document> getDocuments() {
return documents;
}
public void setDocuments(List<Document> documents) {
this.documents = documents;
}
public Map<String, Object> getMetadata() {
return metadata;
}
public void setMetadata(Map<String, Object> metadata) {
this.metadata = metadata;
}
@Override
public String toString() {
return "EmbeddingContext{" +
"vectorSourceId=" + vectorSourceId +
", groupId=" + groupId +
", config=" + config +
", content='" + content + '\'' +
", file='" + file + '\'' +
", fileFormat='" + fileFormat + '\'' +
", documents=" + documents +
", metadata=" + metadata +
'}';
}
public static final class Config {
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
private Config(Builder builder) {setSplitStrategy(builder.splitStrategy);}
public static Builder builder() {
return new Builder();
}
public SplitStrategy getSplitStrategy() {
return splitStrategy;
}
public void setSplitStrategy(SplitStrategy splitStrategy) {
this.splitStrategy = splitStrategy;
}
@Override
public String toString() {
return "Config{" +
"splitStrategy=" + splitStrategy +
'}';
}
public enum SplitStrategy {
NORMAL, LLM, QA
}
public static final class Builder {
private SplitStrategy splitStrategy;
private Builder() {}
public Builder splitStrategy(SplitStrategy val) {
splitStrategy = val;
return this;
}
public Config build() {
return new Config(this);
}
}
}
public static final class Builder {
private Long vectorSourceId;
private Long groupId;
private Config config;
private String content;
private String file;
private String fileFormat;
private Builder() {}
public Builder vectorSourceId(Long val) {
vectorSourceId = val;
return this;
}
public Builder groupId(Long val) {
groupId = val;
return this;
}
public Builder config(Config val) {
config = val;
return this;
}
public Builder content(String val) {
content = val;
return this;
}
public Builder file(String val) {
file = val;
return this;
}
public Builder fileFormat(String val) {
fileFormat = val;
return this;
}
public EmbeddingContext build() {
return new EmbeddingContext(this);
}
}
}

View File

@@ -1,64 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
public class Group {
private String id;
private String name;
private String status;
private Long createdTime;
private Long modifiedTime;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Long getCreatedTime() {
return createdTime;
}
public void setCreatedTime(Long createdTime) {
this.createdTime = createdTime;
}
public Long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(Long modifiedTime) {
this.modifiedTime = modifiedTime;
}
@Override
public String toString() {
return "GroupVO{" +
"id='" + id + '\'' +
", name='" + name + '\'' +
", status='" + status + '\'' +
", createdTime=" + createdTime +
", modifiedTime=" + modifiedTime +
'}';
}
}

View File

@@ -1,74 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
/**
* @author lanyuanxiaoyao
* @version 20250522
*/
public class Knowledge {
private Long id;
private Long vectorSourceId;
private String name;
private String strategy;
private Long createdTime;
private Long modifiedTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public Long getVectorSourceId() {
return vectorSourceId;
}
public void setVectorSourceId(Long vectorSourceId) {
this.vectorSourceId = vectorSourceId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStrategy() {
return strategy;
}
public void setStrategy(String strategy) {
this.strategy = strategy;
}
public Long getCreatedTime() {
return createdTime;
}
public void setCreatedTime(Long createdTime) {
this.createdTime = createdTime;
}
public Long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(Long modifiedTime) {
this.modifiedTime = modifiedTime;
}
@Override
public String toString() {
return "Knowledge{" +
"id=" + id +
", vectorSourceId=" + vectorSourceId +
", name='" + name + '\'' +
", strategy='" + strategy + '\'' +
", createdTime=" + createdTime +
", modifiedTime=" + modifiedTime +
'}';
}
}

View File

@@ -1,74 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
public class DataFileVO {
private String id;
private String filename;
private Long size;
private String md5;
private String path;
private String type;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
}
public String getMd5() {
return md5;
}
public void setMd5(String md5) {
this.md5 = md5;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
@Override
public String toString() {
return "DataFile{" +
"id='" + id + '\'' +
", filename='" + filename + '\'' +
", size=" + size +
", md5='" + md5 + '\'' +
", path='" + path + '\'' +
", type='" + type + '\'' +
'}';
}
}

View File

@@ -1,114 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
public class KnowledgeVO {
private String id;
private String vectorSourceId;
private String name;
private String strategy;
private Long size;
private Long points;
private Long segments;
private String status;
private Long createdTime;
private Long modifiedTime;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getVectorSourceId() {
return vectorSourceId;
}
public void setVectorSourceId(String vectorSourceId) {
this.vectorSourceId = vectorSourceId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStrategy() {
return strategy;
}
public void setStrategy(String strategy) {
this.strategy = strategy;
}
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
}
public Long getPoints() {
return points;
}
public void setPoints(Long points) {
this.points = points;
}
public Long getSegments() {
return segments;
}
public void setSegments(Long segments) {
this.segments = segments;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Long getCreatedTime() {
return createdTime;
}
public void setCreatedTime(Long createdTime) {
this.createdTime = createdTime;
}
public Long getModifiedTime() {
return modifiedTime;
}
public void setModifiedTime(Long modifiedTime) {
this.modifiedTime = modifiedTime;
}
@Override
public String toString() {
return "KnowledgeVO{" +
"id='" + id + '\'' +
", vectorSourceId='" + vectorSourceId + '\'' +
", name='" + name + '\'' +
", strategy='" + strategy + '\'' +
", size=" + size +
", points=" + points +
", segments=" + segments +
", status='" + status + '\'' +
", createdTime=" + createdTime +
", modifiedTime=" + modifiedTime +
'}';
}
}

View File

@@ -1,34 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
public class SegmentVO {
private String id;
private String text;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getText() {
return text;
}
public void setText(String text) {
this.text = text;
}
@Override
public String toString() {
return "PointVO{" +
"id='" + id + '\'' +
", text='" + text + '\'' +
'}';
}
}

View File

@@ -1,79 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.service;
import club.kingon.sql.builder.SqlBuilder;
import cn.hutool.core.util.IdUtil;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.DataFileVO;
import com.lanyuanxiaoyao.service.common.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
@Service
public class DataFileService {
private static final Logger log = LoggerFactory.getLogger(DataFileService.class);
private static final String DATA_FILE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_file";
private final JdbcTemplate template;
public DataFileService(JdbcTemplate template) {
this.template = template;
}
public DataFileVO downloadFile(Long id) {
return template.queryForObject(
SqlBuilder.select("id", "filename", "size", "md5", "path", "type")
.from(DATA_FILE_TABLE_NAME)
.whereEq("id", "?")
.precompileSql(),
(rs, row) -> {
DataFileVO vo = new DataFileVO();
vo.setId(String.valueOf(rs.getLong(1)));
vo.setFilename(rs.getString(2));
vo.setSize(rs.getLong(3));
vo.setMd5(rs.getString(4));
vo.setPath(rs.getString(5));
vo.setType(rs.getString(6));
return vo;
},
id
);
}
@Transactional(rollbackFor = Exception.class)
public Long initialDataFile(String filename) {
long id = IdUtil.getSnowflakeNextId();
template.update(
SqlBuilder.insertInto(DATA_FILE_TABLE_NAME, "id", "filename")
.values()
.addValue("?", "?")
.precompileSql(),
id,
filename
);
return id;
}
@Transactional(rollbackFor = Exception.class)
public void updateDataFile(Long id, String path, Long size, String md5, String type) {
template.update(
SqlBuilder.update(DATA_FILE_TABLE_NAME)
.set("size", "?")
.addSet("md5", "?")
.addSet("path", "?")
.addSet("type", "?")
.whereEq("id", "?")
.precompileSql(),
size,
md5,
path,
type,
id
);
}
}

View File

@@ -1,135 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.service;
import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.entry.Alias;
import club.kingon.sql.builder.entry.Column;
import cn.hutool.core.util.IdUtil;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Group;
import com.lanyuanxiaoyao.service.common.Constants;
import io.qdrant.client.ConditionFactory;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.grpc.Points;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* @author lanyuanxiaoyao
* @version 20250522
*/
@Service
public class GroupService {
public static final String GROUP_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_group";
private static final Logger logger = LoggerFactory.getLogger(GroupService.class);
private static final RowMapper<Group> groupMapper = (rs, row) -> {
Group vo = new Group();
vo.setId(String.valueOf(rs.getLong(1)));
vo.setName(rs.getString(2));
vo.setStatus(rs.getString(3));
vo.setCreatedTime(rs.getTimestamp(4).getTime());
vo.setModifiedTime(rs.getTimestamp(5).getTime());
return vo;
};
private final JdbcTemplate template;
private final QdrantClient client;
public GroupService(JdbcTemplate template, VectorStore vectorStore) {
this.template = template;
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
}
public Group get(Long id) {
return template.queryForObject(
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
.from(GROUP_TABLE_NAME)
.whereEq("id", id)
.orderByDesc("created_time")
.build(),
groupMapper
);
}
@Transactional(rollbackFor = Exception.class)
public Long add(Long knowledgeId, String name) {
long id = IdUtil.getSnowflakeNextId();
template.update(
SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name", "status")
.values()
.addValue("?", "?", "?", "?")
.precompileSql(),
id,
knowledgeId,
name,
"RUNNING"
);
return id;
}
public ImmutableList<Group> list(Long knowledgeId) {
return template.query(
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
.from(GROUP_TABLE_NAME)
.whereEq("knowledge_id", knowledgeId)
.orderByDesc("created_time")
.build(),
groupMapper
)
.stream()
.collect(Collectors.toCollection(Lists.mutable::empty))
.toImmutable();
}
@Transactional(rollbackFor = Exception.class)
public void finish(Long groupId) {
template.update(
SqlBuilder.update(GROUP_TABLE_NAME)
.set("status", "FINISHED")
.whereEq("id", groupId)
.build()
);
}
@Transactional(rollbackFor = Exception.class)
public void remove(Long groupId) throws ExecutionException, InterruptedException {
Long vectorSourceId = template.queryForObject(
SqlBuilder.select("k.vector_source_id")
.from(Alias.of(GROUP_TABLE_NAME, "g"), Alias.of(KnowledgeBaseService.KNOWLEDGE_TABLE_NAME, "k"))
.whereEq("g.knowledge_id", Column.as("k.id"))
.andEq("g.id", groupId)
.precompileSql(),
Long.class,
groupId
);
client.deleteAsync(
String.valueOf(vectorSourceId),
Points.Filter.newBuilder()
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(vectorSourceId)))
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
.build()
).get();
template.update(
SqlBuilder.delete(GROUP_TABLE_NAME)
.whereEq("id", groupId)
.build()
);
}
@Transactional(rollbackFor = Exception.class)
public void removeByKnowledgeId(Long knowledgeId) {
template.update(
SqlBuilder.delete(GROUP_TABLE_NAME)
.whereEq("knowledge_id", "?")
.precompileSql(),
knowledgeId
);
}
}

View File

@@ -1,201 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge.service;
import club.kingon.sql.builder.SqlBuilder;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
import com.lanyuanxiaoyao.service.common.Constants;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.grpc.Collections;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.noear.solon.ai.reranking.RerankingModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* @author lanyuanxiaoyao
* @version 20250522
*/
@Service
public class KnowledgeBaseService {
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseService.class);
private static final RowMapper<Knowledge> knowledgeMapper = (rs, row) -> {
Knowledge knowledge = new Knowledge();
knowledge.setId(rs.getLong(1));
knowledge.setVectorSourceId(rs.getLong(2));
knowledge.setName(rs.getString(3));
knowledge.setStrategy(rs.getString(4));
knowledge.setCreatedTime(rs.getTimestamp(5).getTime());
knowledge.setModifiedTime(rs.getTimestamp(6).getTime());
return knowledge;
};
private final JdbcTemplate template;
private final EmbeddingModel model;
private final QdrantClient client;
private final GroupService groupService;
private final RerankingModel rerankingModel;
public KnowledgeBaseService(JdbcTemplate template, EmbeddingModel model, VectorStore vectorStore, GroupService groupService, RerankingModel rerankingModel) {
this.template = template;
this.model = model;
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
this.groupService = groupService;
this.rerankingModel = rerankingModel;
}
public Knowledge get(Long id) {
return template.queryForObject(
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
.from(KNOWLEDGE_TABLE_NAME)
.whereEq("id", "?")
.precompileSql(),
knowledgeMapper,
id
);
}
@Transactional(rollbackFor = Exception.class)
public void add(String name, String strategy) throws ExecutionException, InterruptedException {
Integer count = template.queryForObject(
SqlBuilder.select("count(*)")
.from(KNOWLEDGE_TABLE_NAME)
.whereEq("name", "?")
.precompileSql(),
Integer.class,
name
);
if (count > 0) {
throw new RuntimeException("名称已存在");
}
long id = IdUtil.getSnowflakeNextId();
long vectorSourceId = IdUtil.getSnowflakeNextId();
template.update(
SqlBuilder.insertInto(KNOWLEDGE_TABLE_NAME, "id", "vector_source_id", "name", "strategy")
.values()
.addValue("?", "?", "?", "?")
.precompileSql(),
id,
vectorSourceId,
name,
strategy
);
client.createCollectionAsync(
String.valueOf(vectorSourceId),
Collections.VectorParams.newBuilder()
.setDistance(Collections.Distance.valueOf(strategy))
.setSize(model.dimensions())
.build()
).get();
}
public String getName(Long id) {
return template.queryForObject(
SqlBuilder.select("name")
.from(KNOWLEDGE_TABLE_NAME)
.whereEq("id", id)
.orderByDesc("created_time")
.build(),
String.class
);
}
public ImmutableList<KnowledgeVO> list() {
return template.query(
SqlBuilder.select("id", "vector_source_id", "name", "strategy", "created_time", "modified_time")
.from(KNOWLEDGE_TABLE_NAME)
.orderByDesc("created_time")
.build(),
knowledgeMapper
)
.stream()
.map(knowledge -> {
try {
Collections.CollectionInfo info = client.getCollectionInfoAsync(String.valueOf(knowledge.getVectorSourceId())).get();
KnowledgeVO vo = new KnowledgeVO();
vo.setId(String.valueOf(knowledge.getId()));
vo.setVectorSourceId(String.valueOf(knowledge.getVectorSourceId()));
vo.setName(knowledge.getName());
vo.setPoints(info.getPointsCount());
vo.setSegments(info.getSegmentsCount());
vo.setStatus(info.getStatus().name());
Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
vo.setStrategy(vectorParams.getDistance().name());
vo.setSize(vectorParams.getSize());
vo.setCreatedTime(vo.getCreatedTime());
vo.setModifiedTime(vo.getModifiedTime());
return vo;
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
})
.collect(Collectors.toCollection(Lists.mutable::empty))
.toImmutable();
}
@Transactional(rollbackFor = Exception.class)
public void remove(Long id) throws ExecutionException, InterruptedException {
Knowledge knowledge = get(id);
if (ObjectUtil.isNull(knowledge)) {
throw new RuntimeException(StrUtil.format("{} 不存在"));
}
template.update(
SqlBuilder.delete(KNOWLEDGE_TABLE_NAME)
.whereEq("id", "?")
.precompileSql(),
knowledge.getId()
);
groupService.removeByKnowledgeId(knowledge.getId());
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
}
public ImmutableList<String> query(
Long id,
String text,
Integer limit,
Double threshold
) throws ExecutionException, InterruptedException, IOException {
Knowledge knowledge = get(id);
Boolean exists = client.collectionExistsAsync(String.valueOf(knowledge.getVectorSourceId())).get();
if (!exists) {
throw new RuntimeException(StrUtil.format("{} not exists", id));
}
VectorStore vs = QdrantVectorStore.builder(client, model)
.collectionName(String.valueOf(knowledge.getVectorSourceId()))
.initializeSchema(false)
.build();
List<Document> documents = vs.similaritySearch(
SearchRequest.builder()
.query(text)
.topK(limit)
.similarityThreshold(threshold)
.build()
);
// 如果只是一个知识库的话似乎没有什么rerank的必要...
/* List<org.noear.solon.ai.rag.Document> rerankDocuments = rerankingModel.rerank(
text,
documents.stream()
.map(doc -> new org.noear.solon.ai.rag.Document(doc.getId(), doc.getText(), doc.getMetadata(), doc.getScore()))
.toList()
); */
return Lists.immutable.ofAll(documents)
.collect(Document::getText);
}
}

View File

@@ -1,30 +0,0 @@
spring:
application:
name: service-ai-knowledge
profiles:
include: random-port,common,discovery,metrics,forest
ai:
openai:
base-url: http://132.121.206.65:10086
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
chat:
options:
model: 'Qwen3-1.7-vllm'
embedding:
options:
model: 'Bge-m3-vllm'
vectorstore:
qdrant:
host: 132.121.206.65
port: 29463
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
liteflow:
rule-source: config/flow.xml
print-banner: false
check-node-exists: false
solon:
base-url: http://132.121.206.65:10086
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
rerank:
model: 'Bge-reranker-v2-vllm'
endpoint: '/v1/rerank'

View File

@@ -1,116 +0,0 @@
package com.lanyuanxiaoyao.service.ai.knowledge;
import java.net.http.HttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.http.client.reactive.JdkClientHttpConnector;
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;
/**
* @author lanyuanxiaoyao
* @version 20250526
*/
public class TestLlm {
private static final Logger log = LoggerFactory.getLogger(TestLlm.class);
public static void main(String[] args) {
HttpClient httpClient = HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.build();
ChatClient client = ChatClient.builder(
OpenAiChatModel.builder()
.openAiApi(
OpenAiApi.builder()
.baseUrl("http://132.121.206.65:10086")
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
.webClientBuilder(WebClient.builder().clientConnector(new JdkClientHttpConnector(httpClient)))
.restClientBuilder(RestClient.builder().requestFactory(new JdkClientHttpRequestFactory(httpClient)))
.build()
)
.defaultOptions(
OpenAiChatOptions.builder()
.model("Qwen3-1.7-vllm")
.build()
)
.build()
)
.build();
String content = client.prompt()
.system("""
对用户输入的文本,生成多组高质量的问答对。请遵循以下指南:
1. 问题部分:
为同一个主题创建尽可能多的不同表述的问题,确保问题的多样性。
每个问题应考虑用户可能的多种问法,例如:
直接询问(如“什么是...?”)
请求确认(如“是否可以说...?”)
寻求解释(如“请解释一下...的含义。”)
假设性问题(如“如果...会怎样?”)
例子请求(如“能否举个例子说明...?”)
问题应涵盖文本中的关键信息、主要概念和细节,确保不遗漏重要内容。
2. 答案部分:
提供一个全面、信息丰富的答案,涵盖问题的所有可能角度,确保逻辑连贯。
答案应直接基于给定文本,确保准确性和一致性。
包含相关的细节,如日期、名称、职位等具体信息,必要时提供背景信息以增强理解。
3. 格式:
使用"问:"标记问题集合的开始,所有问题应在一个段落内,问题之间用空格分隔。
使用"答:"标记答案的开始,答案应清晰分段,便于阅读。
问答对之间用“---”分隔,以提高可读性。
4. 内容要求:
确保问答对紧密围绕文本主题,避免偏离主题。
避免添加文本中未提及的信息,确保信息的真实性。
一个问题搭配一个答案,避免一组问答对中同时涉及多个问题。
如果文本信息不足以回答某个方面,可以在答案中说明 "根据给定信息无法确定",并尽量提供相关的上下文。
""")
.user("""
Apache Hudi 是一款开源的数据湖管理框架,专注于高效处理大规模数据集的增量更新和实时操作。以下是其核心要点介绍:
核心功能与特性
增量数据处理支持插入Insert、更新Update、删除Delete操作并通过“Upsert”插入或更新机制高效处理变更数据避免重写整个数据集。
事务支持:提供 ACID 事务保证,确保数据一致性和可靠性,支持原子提交和回滚。
存储优化:自动合并小文件并维护文件最佳大小,减少存储碎片化问题,提升查询性能。
多查询类型:
快照查询:获取最新数据状态(对 MoR 表合并基础文件和增量文件)。
增量查询:捕获自某次提交后的变更数据流,适用于增量管道。
读取优化查询:针对 MoR 表展示最新压缩后的数据。
存储模型
写入时复制Copy-on-Write, CoW数据以列式格式如 Parquet存储每次更新生成新版本文件适合读多写少的场景。
读取时合并Merge-on-Read, MoR结合列式Parquet和行式Avro存储更新写入增量文件后异步合并适合写频繁的场景。
适用场景
合规性需求:支持 GDPR、CCPA 等法规要求的数据删除和更新。
实时数据流处理:如 IoT 设备数据、CDC变更数据捕获系统实现近实时分析。
数据湖管理:优化大规模数据集的存储和查询效率,支持时间旅行查询和历史版本回溯。
技术生态与兼容性
多引擎支持:与 Spark、Flink、Hive、Presto、Trino 等计算和查询引擎集成。
存储格式:基于 Parquet 和 Avro兼容 Hadoop 生态及云存储(如 Amazon S3
优势对比
相较于同类产品(如 Apache Iceberg、Delta LakeHudi 在实时更新和增量处理方面表现突出,尤其适合需要频繁数据变更的场景。其独特的 MoR 模型在写入性能上优于传统批处理方案。
总结
Apache Hudi 通过灵活的存储模型、高效的事务管理和广泛的生态系统集成,成为构建现代化数据湖的核心工具,适用于金融、物联网、实时分析等对数据新鲜度和操作效率要求高的领域。
""")
.call()
.content();
log.info(content);
}
}

View File

@@ -0,0 +1,153 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>service-ai-web</artifactId>
<dependencies>
<dependency>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>service-ai-core</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jetty</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-deepseek</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-vector-store-qdrant</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-markdown-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
</dependency>
<dependency>
<groupId>com.blinkfox</groupId>
<artifactId>fenix-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-el-builder</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-tika-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-pdf-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai</artifactId>
</dependency>
<dependency>
<groupId>org.noear</groupId>
<artifactId>solon-ai-dialect-openai</artifactId>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-script-graaljs</artifactId>
</dependency>
<dependency>
<groupId>com.yomahub</groupId>
<artifactId>liteflow-script-python</artifactId>
</dependency>
<dependency>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-ant</artifactId>
<version>6.6.8.Final</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct-processor</artifactId>
<version>${mapstruct.version}</version>
</path>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</path>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-mapstruct-binding</artifactId>
<version>0.2.0</version>
</path>
<path>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-jpamodelgen</artifactId>
<version>6.6.8.Final</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Amapstruct.defaultComponentModel=spring</arg>
<arg>-Amapstruct.defaultInjectionStrategy=constructor</arg>
</compilerArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,5 +1,6 @@
package com.lanyuanxiaoyao.service.ai.knowledge;
package com.lanyuanxiaoyao.service.ai.web;
import com.blinkfox.fenix.EnableFenix;
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
@@ -7,6 +8,7 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.retry.annotation.EnableRetry;
import org.springframework.scheduling.annotation.EnableScheduling;
@@ -20,9 +22,14 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@EnableEncryptableProperties
@EnableRetry
@EnableScheduling
public class KnowledgeApplication implements ApplicationRunner {
@EnableFenix
@EnableJpaAuditing
public class WebApplication implements ApplicationRunner {
public static void main(String[] args) {
SpringApplication.run(KnowledgeApplication.class, args);
System.setProperty("polyglot.engine.WarnInterpreterOnly", "false");
SpringApplication.run(WebApplication.class, args);
}
@Override

View File

@@ -0,0 +1,13 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface DetailController<DETAIL_ITEM> {
String DETAIL = "/detail/{id}";
AmisResponse<DETAIL_ITEM> detail(Long id) throws Exception;
}

View File

@@ -0,0 +1,16 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface ListController {
String LIST = "/list";
AmisCrudResponse list() throws Exception;
AmisCrudResponse list(Query query) throws Exception;
}

View File

@@ -0,0 +1,13 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface RemoveController {
String REMOVE = "/remove/{id}";
AmisResponse<Object> remove(Long id) throws Exception;
}

View File

@@ -0,0 +1,13 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface SaveController<SAVE_ITEM> {
String SAVE = "/save";
AmisResponse<Long> save(SAVE_ITEM item) throws Exception;
}

View File

@@ -0,0 +1,8 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> extends SaveController<SAVE_ITEM>, ListController, DetailController<DETAIL_ITEM>, RemoveController {
}

View File

@@ -0,0 +1,105 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import cn.hutool.core.util.ObjectUtil;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Page;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
/**
* @author lanyuanxiaoyao
* @date 2024-11-26
*/
@Slf4j
public abstract class SimpleControllerSupport<ENTITY extends SimpleEntity, SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> implements SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> {
protected final SimpleServiceSupport<ENTITY> service;
public SimpleControllerSupport(SimpleServiceSupport<ENTITY> service) {
this.service = service;
}
@PostMapping(SAVE)
@Override
public AmisResponse<Long> save(@RequestBody SAVE_ITEM item) throws Exception {
SaveItemMapper<ENTITY, SAVE_ITEM> mapper = saveItemMapper();
return AmisResponse.responseSuccess(service.save(mapper.from(item)));
}
@GetMapping(LIST)
@Override
public AmisCrudResponse list() throws Exception {
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
return AmisCrudResponse.responseCrudData(
service.list()
.collect(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
);
}
@PostMapping(LIST)
@Override
public AmisCrudResponse list(@RequestBody Query query) throws Exception {
if (ObjectUtil.isNull(query)) {
return AmisCrudResponse.responseCrudData(List.of(), 0);
}
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
Page<ENTITY> result = service.list(query);
return AmisCrudResponse.responseCrudData(
result.get()
.map(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
.toList(),
result.getTotalElements()
);
}
@GetMapping(DETAIL)
@Override
public AmisResponse<DETAIL_ITEM> detail(@PathVariable("id") Long id) throws Exception {
DetailItemMapper<ENTITY, DETAIL_ITEM> mapper = detailItemMapper();
return AmisResponse.responseSuccess(mapper.from(service.detailOrThrow(id)));
}
@GetMapping(REMOVE)
@Override
public AmisResponse<Object> remove(@PathVariable("id") Long id) throws Exception {
service.remove(id);
return AmisResponse.responseSuccess();
}
protected abstract SaveItemMapper<ENTITY, SAVE_ITEM> saveItemMapper();
protected abstract ListItemMapper<ENTITY, LIST_ITEM> listItemMapper();
protected abstract DetailItemMapper<ENTITY, DETAIL_ITEM> detailItemMapper();
public interface SaveItemMapper<ENTITY, SAVE_ITEM> {
ENTITY from(SAVE_ITEM item) throws Exception;
}
public interface ListItemMapper<ENTITY, LIST_ITEM> {
LIST_ITEM from(ENTITY entity) throws Exception;
}
public interface DetailItemMapper<ENTITY, DETAIL_ITEM> {
DETAIL_ITEM from(ENTITY entity) throws Exception;
}
}

View File

@@ -0,0 +1,61 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller.query;
import lombok.Data;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
/**
* 统一查询
*
* @author lanyuanxiaoyao
* @date 2024-12-03
*/
@Data
public class Query {
private Queryable query;
private ImmutableList<Sortable> sort;
private Pageable page;
@Data
public static class Queryable {
private ImmutableList<String> nullEqual;
private ImmutableList<String> notNullEqual;
private ImmutableList<String> empty;
private ImmutableList<String> notEmpty;
private ImmutableMap<String, ?> equal;
private ImmutableMap<String, ?> notEqual;
private ImmutableMap<String, String> like;
private ImmutableMap<String, String> notLike;
private ImmutableMap<String, ?> great;
private ImmutableMap<String, ?> less;
private ImmutableMap<String, ?> greatEqual;
private ImmutableMap<String, ?> lessEqual;
private ImmutableMap<String, ImmutableList<?>> in;
private ImmutableMap<String, ImmutableList<?>> notIn;
private ImmutableMap<String, Between> between;
private ImmutableMap<String, Between> notBetween;
@Data
public static class Between {
private String start;
private String end;
}
}
@Data
public static class Sortable {
private String column;
private Direction direction;
public enum Direction {
ASC,
DESC,
}
}
@Data
public static class Pageable {
private Integer index;
private Integer size;
}
}

View File

@@ -0,0 +1,31 @@
package com.lanyuanxiaoyao.service.ai.web.base.entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* 实体类公共字段
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public class IdOnlyEntity {
@Comment("记录唯一标记")
@Id
@GeneratedValue(generator = "snowflake")
@GenericGenerator(name = "snowflake", strategy = "com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator")
private Long id;
}

View File

@@ -0,0 +1,32 @@
package com.lanyuanxiaoyao.service.ai.web.base.entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.MappedSuperclass;
import java.time.LocalDateTime;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* 实体类公共字段
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString(callSuper = true)
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public class SimpleEntity extends IdOnlyEntity {
@Comment("记录创建时间")
@CreatedDate
private LocalDateTime createdTime;
@Comment("记录更新时间")
@LastModifiedDate
private LocalDateTime modifiedTime;
}

View File

@@ -0,0 +1,17 @@
package com.lanyuanxiaoyao.service.ai.web.base.entity;
import java.time.LocalDateTime;
import lombok.Data;
/**
* 创建对象使用的接收类
*
* @author lanyuanxiaoyao
* @date 2024-11-26
*/
@Data
public class SimpleItem {
private Long id;
private LocalDateTime createdTime;
private LocalDateTime modifiedTime;
}

View File

@@ -0,0 +1,16 @@
package com.lanyuanxiaoyao.service.ai.web.base.repository;
import com.blinkfox.fenix.jpa.FenixJpaRepository;
import com.blinkfox.fenix.specification.FenixJpaSpecificationExecutor;
import org.springframework.data.repository.NoRepositoryBean;
import org.springframework.data.repository.query.QueryByExampleExecutor;
/**
* 整合一下
*
* @author lanyuanxiaoyao
* @date 2024-11-21
*/
@NoRepositoryBean
public interface SimpleRepository<E, ID> extends FenixJpaRepository<E, ID>, FenixJpaSpecificationExecutor<E>, QueryByExampleExecutor<E> {
}

View File

@@ -0,0 +1,34 @@
package com.lanyuanxiaoyao.service.ai.web.base.service;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import java.util.Optional;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.set.ImmutableSet;
import org.springframework.data.domain.Page;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface SimpleService<ENTITY extends SimpleEntity> {
Long save(ENTITY entity) throws Exception;
Long count() throws Exception;
ImmutableList<ENTITY> list() throws Exception;
ImmutableList<ENTITY> list(ImmutableSet<Long> ids) throws Exception;
Page<ENTITY> list(Query query) throws Exception;
Optional<ENTITY> detailOptional(Long id) throws Exception;
ENTITY detail(Long id) throws Exception;
ENTITY detailOrThrow(Long id) throws Exception;
ENTITY detailOrNull(Long id) throws Exception;
void remove(Long id) throws Exception;
}

View File

@@ -0,0 +1,251 @@
package com.lanyuanxiaoyao.service.ai.web.base.service;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.bean.copier.CopyOptions;
import cn.hutool.core.util.EnumUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity_;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity_;
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaQuery;
import jakarta.persistence.criteria.Path;
import jakarta.persistence.criteria.Predicate;
import jakarta.persistence.criteria.Root;
import jakarta.transaction.Transactional;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.set.ImmutableSet;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
/**
* @author lanyuanxiaoyao
* @date 2024-11-21
*/
@Slf4j
public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implements SimpleService<ENTITY> {
private static final Integer DEFAULT_PAGE_INDEX = 1;
private static final Integer DEFAULT_PAGE_SIZE = 10;
protected final SimpleRepository<ENTITY, Long> repository;
public SimpleServiceSupport(SimpleRepository<ENTITY, Long> repository) {
this.repository = repository;
}
@Transactional(rollbackOn = Throwable.class)
@Override
public Long save(ENTITY entity) {
if (ObjectUtil.isNotNull(entity.getId())) {
Long id = entity.getId();
ENTITY targetEntity = repository.findById(entity.getId()).orElseThrow(() -> new IdNotFoundException(id));
BeanUtil.copyProperties(
entity,
targetEntity,
CopyOptions.create()
.setIgnoreProperties(
IdOnlyEntity_.ID,
SimpleEntity_.CREATED_TIME,
SimpleEntity_.MODIFIED_TIME
)
);
entity = targetEntity;
}
entity = repository.save(entity);
return entity.getId();
}
@Override
public Long count() {
return repository.count(
(root, query, builder) ->
builder.and(
listPredicate(root, query, builder)
.reject(ObjectUtil::isNull)
.toArray(new Predicate[]{})
)
);
}
@Override
public ImmutableList<ENTITY> list() {
return Lists.immutable.ofAll(repository.findAll(
(root, query, builder) ->
builder.and(
listPredicate(root, query, builder)
.reject(ObjectUtil::isNull)
.toArray(new Predicate[]{})
)
));
}
@Override
public ImmutableList<ENTITY> list(ImmutableSet<Long> ids) {
return Lists.immutable.ofAll(repository.findAll(
(root, query, builder) -> {
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
predicates.add(builder.in(root.get("id")).value(ids));
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
}
));
}
private <Y> Path<Y> column(Root<ENTITY> root, String column) {
String[] columns = StrUtil.splitToArray(column, "/");
Path<Y> path = root.get(columns[0]);
for (int i = 1; i < columns.length; i++) {
path = path.get(columns[i]);
}
return path;
}
@SuppressWarnings({"unchecked", "rawtypes"})
private <Y> Object value(Path<Y> column, Object value) {
Class<?> javaType = column.getJavaType();
if (EnumUtil.isEnum(javaType)) {
return EnumUtil.fromString((Class<Enum>) javaType, (String) value);
}
return value;
}
@SuppressWarnings("unchecked")
protected ImmutableList<Predicate> queryPredicates(Query.Queryable queryable, Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
MutableList<Predicate> predicates = Lists.mutable.empty();
if (ObjectUtil.isEmpty(queryable)) {
return predicates.toImmutable();
}
if (ObjectUtil.isNotEmpty(queryable.getNullEqual())) {
queryable.getNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
}
if (ObjectUtil.isNotEmpty(queryable.getNotNullEqual())) {
queryable.getNotNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
}
if (ObjectUtil.isNotEmpty(queryable.getEmpty())) {
queryable.getEmpty().forEach(column -> predicates.add(builder.isEmpty(column(root, column))));
}
if (ObjectUtil.isNotEmpty(queryable.getNotEmpty())) {
queryable.getNotEmpty().forEach(column -> predicates.add(builder.isNotEmpty(column(root, column))));
}
if (ObjectUtil.isNotEmpty(queryable.getEqual())) {
queryable.getEqual().forEachKeyValue((column, value) -> {
Path<Object> path = column(root, column);
predicates.add(builder.equal(path, value(path, value)));
});
}
if (ObjectUtil.isNotEmpty(queryable.getNotEqual())) {
queryable.getEqual().forEachKeyValue((column, value) -> {
Path<Object> path = column(root, column);
predicates.add(builder.notEqual(path, value(path, value)));
});
}
if (ObjectUtil.isNotEmpty(queryable.getLike())) {
queryable.getLike().forEachKeyValue((column, value) -> predicates.add(builder.like(column(root, column), value)));
}
if (ObjectUtil.isNotEmpty(queryable.getNotLike())) {
queryable.getNotLike().forEachKeyValue((column, value) -> predicates.add(builder.notLike(column(root, column), value)));
}
if (ObjectUtil.isNotEmpty(queryable.getGreat())) {
queryable.getGreat().forEachKeyValue((column, value) -> predicates.add(builder.greaterThan(column(root, column), (Comparable<Object>) value)));
}
if (ObjectUtil.isNotEmpty(queryable.getLess())) {
queryable.getLess().forEachKeyValue((column, value) -> predicates.add(builder.lessThan(column(root, column), (Comparable<Object>) value)));
}
if (ObjectUtil.isNotEmpty(queryable.getGreatEqual())) {
queryable.getGreatEqual().forEachKeyValue((column, value) -> predicates.add(builder.greaterThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
}
if (ObjectUtil.isNotEmpty(queryable.getLessEqual())) {
queryable.getLessEqual().forEachKeyValue((column, value) -> predicates.add(builder.lessThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
}
if (ObjectUtil.isNotEmpty(queryable.getIn())) {
queryable.getIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value)));
}
if (ObjectUtil.isNotEmpty(queryable.getNotIn())) {
queryable.getNotIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value).not()));
}
if (ObjectUtil.isNotEmpty(queryable.getBetween())) {
queryable.getBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
}
if (ObjectUtil.isNotEmpty(queryable.getNotBetween())) {
queryable.getNotBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
}
return predicates.toImmutable();
}
protected ImmutableList<Predicate> listPredicate(Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
return Lists.immutable.empty();
}
@Override
public Page<ENTITY> list(Query listQuery) {
PageRequest pageRequest = PageRequest.of(DEFAULT_PAGE_INDEX - 1, DEFAULT_PAGE_SIZE, Sort.by(SimpleEntity_.CREATED_TIME).descending());
if (ObjectUtil.isNotNull(listQuery.getPage())) {
pageRequest = PageRequest.of(
ObjectUtil.defaultIfNull(listQuery.getPage().getIndex(), DEFAULT_PAGE_INDEX) - 1,
ObjectUtil.defaultIfNull(listQuery.getPage().getSize(), DEFAULT_PAGE_SIZE),
Sort.by(SimpleEntity_.CREATED_TIME).descending()
);
}
return repository.findAll(
(root, query, builder) -> {
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
predicates.addAllIterable(queryPredicates(listQuery.getQuery(), root, query, builder));
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
},
pageRequest
);
}
@Override
public Optional<ENTITY> detailOptional(Long id) {
if (ObjectUtil.isNull(id)) {
return Optional.empty();
}
return repository.findOne(
(root, query, builder) -> {
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
predicates.add(builder.equal(root.get(IdOnlyEntity_.id), id));
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
}
);
}
@Override
public ENTITY detail(Long id) {
return detailOrNull(id);
}
@Override
public ENTITY detailOrThrow(Long id) {
return detailOptional(id).orElseThrow(() -> new IdNotFoundException(id));
}
@Override
public ENTITY detailOrNull(Long id) {
return detailOptional(id).orElse(null);
}
@Transactional(rollbackOn = Throwable.class)
@Override
public void remove(Long id) {
if (ObjectUtil.isNotNull(id)) {
repository.deleteById(id);
}
}
public static final class IdNotFoundException extends RuntimeException {
public IdNotFoundException() {
super("资源不存在");
}
public IdNotFoundException(Long id) {
super(StrUtil.format("ID为{}的资源不存在", id));
}
}
}

View File

@@ -0,0 +1,17 @@
package com.lanyuanxiaoyao.service.ai.web.configuration;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
@Data
@Configuration
@ConfigurationProperties(prefix = "file-store")
public class FileStoreProperties {
private String downloadPrefix;
private String uploadPath;
}

View File

@@ -0,0 +1,111 @@
package com.lanyuanxiaoyao.service.ai.web.configuration;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.StrUtil;
import java.time.Duration;
import lombok.extern.slf4j.Slf4j;
import org.noear.solon.ai.reranking.RerankingModel;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.deepseek.DeepSeekChatModel;
import org.springframework.ai.deepseek.DeepSeekChatOptions;
import org.springframework.ai.deepseek.api.DeepSeekApi;
import org.springframework.ai.document.MetadataMode;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;
@Slf4j
@Configuration
public class LlmConfiguration {
@Bean("chat")
public ChatClient.Builder chatClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
Assert.notNull(llmProperties.getChat(), "chat properties is null");
DeepSeekApi.Builder apiBuilder = DeepSeekApi.builder()
.baseUrl(StrUtil.firstNonBlank(llmProperties.getChat().getBaseUrl(), llmProperties.getBaseUrl()))
.apiKey(StrUtil.firstNonBlank(llmProperties.getChat().getApiKey(), llmProperties.getApiKey()))
.webClientBuilder(webClientBuilder)
.restClientBuilder(restClientBuilder);
if (StrUtil.isNotBlank(llmProperties.getChat().getEndpoint())) {
apiBuilder.completionsPath(llmProperties.getChat().getEndpoint());
}
return ChatClient.builder(
DeepSeekChatModel.builder()
.deepSeekApi(apiBuilder.build())
.defaultOptions(
DeepSeekChatOptions.builder()
.model(llmProperties.getChat().getModel())
.build()
)
.build()
);
}
@Bean("visual")
public ChatClient.Builder visualClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
Assert.notNull(llmProperties.getVisual(), "visual properties is null");
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
.baseUrl(StrUtil.firstNonBlank(llmProperties.getVisual().getBaseUrl(), llmProperties.getBaseUrl()))
.apiKey(StrUtil.firstNonBlank(llmProperties.getVisual().getApiKey(), llmProperties.getApiKey()))
.webClientBuilder(webClientBuilder)
.restClientBuilder(restClientBuilder);
if (StrUtil.isNotBlank(llmProperties.getVisual().getEndpoint())) {
apiBuilder.completionsPath(llmProperties.getVisual().getEndpoint());
}
return ChatClient.builder(
OpenAiChatModel.builder()
.openAiApi(apiBuilder.build())
.defaultOptions(
OpenAiChatOptions.builder()
.model(llmProperties.getVisual().getModel())
.build()
)
.build()
);
}
@Bean
public EmbeddingModel embeddingModel(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
Assert.notNull(llmProperties.getEmbedding(), "embedding properties is null");
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
.baseUrl(StrUtil.firstNonBlank(llmProperties.getEmbedding().getBaseUrl(), llmProperties.getBaseUrl()))
.apiKey(StrUtil.firstNonBlank(llmProperties.getEmbedding().getApiKey(), llmProperties.getApiKey()))
.webClientBuilder(webClientBuilder)
.restClientBuilder(restClientBuilder);
if (StrUtil.isNotBlank(llmProperties.getEmbedding().getEndpoint())) {
apiBuilder.embeddingsPath(llmProperties.getEmbedding().getEndpoint());
}
return new OpenAiEmbeddingModel(
apiBuilder.build(),
MetadataMode.EMBED,
OpenAiEmbeddingOptions.builder()
.model(llmProperties.getEmbedding().getModel())
.build()
);
}
@Bean
public RerankingModel rerankingModel(LlmProperties llmProperties) {
Assert.notNull(llmProperties.getReranker(), "reranker properties is null");
String url = llmProperties.getBaseUrl();
if (StrUtil.isNotBlank(llmProperties.getReranker().getBaseUrl())) {
url = llmProperties.getReranker().getBaseUrl();
}
if (StrUtil.isNotBlank(llmProperties.getReranker().getEndpoint())) {
url += llmProperties.getReranker().getEndpoint();
} else {
url += "/v1/rerank";
}
return RerankingModel.of(url)
.apiKey(StrUtil.firstNonBlank(llmProperties.getReranker().getApiKey(), llmProperties.getApiKey()))
.model(llmProperties.getReranker().getModel())
.timeout(Duration.ofMinutes(10))
.build();
}
}

View File

@@ -0,0 +1,25 @@
package com.lanyuanxiaoyao.service.ai.web.configuration;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "spring.llm")
public class LlmProperties {
private String baseUrl;
private String apiKey;
private ChatProperties chat;
private ChatProperties visual;
private ChatProperties embedding;
private ChatProperties reranker;
@Data
public static class ChatProperties {
private String baseUrl;
private String apiKey;
private String model;
private String endpoint;
}
}

View File

@@ -1,4 +1,4 @@
package com.lanyuanxiaoyao.service.ai.chat;
package com.lanyuanxiaoyao.service.ai.web.configuration;
import cn.hutool.core.util.StrUtil;

View File

@@ -0,0 +1,92 @@
package com.lanyuanxiaoyao.service.ai.web.configuration;
import java.io.Serializable;
import java.time.Instant;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.id.IdentifierGenerator;
/**
* 使用雪花算法作为ID生成器
*
* @author lanyuanxiaoyao
* @date 2024-11-14
*/
@Slf4j
public class SnowflakeIdGenerator implements IdentifierGenerator {
@Override
public Serializable generate(SharedSessionContractImplementor session, Object object) {
try {
return Snowflake.next();
} catch (Exception e) {
log.error("Generate snowflake id failed", e);
throw new RuntimeException(e);
}
}
public static class Snowflake {
/**
* 起始的时间戳
*/
private final static long START_TIMESTAMP = 1;
/**
* 序列号占用的位数
*/
private final static long SEQUENCE_BIT = 11;
/**
* 序列号最大值
*/
private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
/**
* 时间戳值向左位移
*/
private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
/**
* 序列号
*/
private static long sequence = 0;
/**
* 上一次时间戳
*/
private static long lastTimestamp = -1;
public static synchronized long next() {
long currentTimestamp = nowTimestamp();
if (currentTimestamp < lastTimestamp) {
throw new RuntimeException("Clock have moved backwards.");
}
if (currentTimestamp == lastTimestamp) {
// 相同毫秒内, 序列号自增
sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
// 同一毫秒的序列数已经达到最大
if (sequence == 0) {
currentTimestamp = nextTimestamp();
}
} else {
// 不同毫秒内, 序列号置为0
sequence = 0;
}
lastTimestamp = currentTimestamp;
return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
}
private static long nextTimestamp() {
long milli = nowTimestamp();
while (milli <= lastTimestamp) {
milli = nowTimestamp();
}
return milli;
}
private static long nowTimestamp() {
return Instant.now().toEpochMilli();
}
}
}

View File

@@ -0,0 +1,24 @@
package com.lanyuanxiaoyao.service.ai.web.configuration;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class SpringBeanGetter implements ApplicationContextAware {
private static ApplicationContext context;
public static <T> T getBean(Class<T> clazz) {
return context.getBean(clazz);
}
public static <T> T getBean(String name, Class<T> clazz) {
return context.getBean(name, clazz);
}
@Override
public void setApplicationContext(ApplicationContext context) throws BeansException {
SpringBeanGetter.context = context;
}
}

View File

@@ -1,4 +1,4 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
package com.lanyuanxiaoyao.service.ai.web.controller;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
@@ -7,9 +7,9 @@ import cn.hutool.core.util.URLUtil;
import cn.hutool.crypto.SecureUtil;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.knowledge.configuration.KnowledgeConfiguration;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.DataFileVO;
import com.lanyuanxiaoyao.service.ai.knowledge.service.DataFileService;
import com.lanyuanxiaoyao.service.ai.web.configuration.FileStoreProperties;
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
import jakarta.servlet.http.HttpServletResponse;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -17,9 +17,14 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.eclipse.collections.api.set.ImmutableSet;
import org.mapstruct.factory.Mappers;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@@ -35,31 +40,50 @@ import org.springframework.web.multipart.MultipartFile;
* @author lanyuanxiaoyao
* @date 2024-11-21
*/
@Slf4j
@RestController
@RequestMapping("/upload")
public class DataFileController {
private static final Logger log = LoggerFactory.getLogger(DataFileController.class);
private final KnowledgeConfiguration knowledgeConfiguration;
private final FileStoreProperties fileStoreProperties;
private final DataFileService dataFileService;
private final String uploadFolderPath;
private final String cacheFolderPath;
private final String sliceFolderPath;
public DataFileController(KnowledgeConfiguration knowledgeConfiguration, DataFileService dataFileService) {
this.knowledgeConfiguration = knowledgeConfiguration;
public DataFileController(FileStoreProperties fileStoreProperties, DataFileService dataFileService) {
this.fileStoreProperties = fileStoreProperties;
this.dataFileService = dataFileService;
this.uploadFolderPath = knowledgeConfiguration.getUploadPath();
this.uploadFolderPath = fileStoreProperties.getUploadPath();
this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath);
this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath);
}
@PostMapping("/detail")
public AmisResponse<?> detail(@RequestBody DetailRequest request) {
var mapper = Mappers.getMapper(DetailResponse.Mapper.class);
return AmisResponse.responseCrudData(dataFileService.downloadFile(request.ids).collect(mapper::from));
}
@GetMapping("/detail")
public AmisResponse<?> detail(@RequestParam("ids") String ids) {
if (StrUtil.isBlank(ids)) {
return AmisResponse.responseCrudData(Sets.immutable.empty());
}
var mapper = Mappers.getMapper(DetailResponse.Mapper.class);
return AmisResponse.responseCrudData(
dataFileService.downloadFile(
Sets.immutable.ofAll(StrUtil.split(ids, ","))
.collect(Long::parseLong)
).collect(mapper::from)
);
}
@PostMapping("")
public AmisResponse<FinishResponse> upload(@RequestParam("file") MultipartFile file) throws IOException {
String filename = file.getOriginalFilename();
Long id = dataFileService.initialDataFile(filename);
String url = StrUtil.format("{}/upload/download/{}", knowledgeConfiguration.getDownloadPrefix(), id);
String url = StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), id);
byte[] bytes = file.getBytes();
String originMd5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, originMd5));
@@ -79,8 +103,8 @@ public class DataFileController {
}
@GetMapping("/download/{id}")
public void download(@PathVariable Long id, HttpServletResponse response) throws IOException {
DataFileVO dataFile = dataFileService.downloadFile(id);
public void download(@PathVariable("id") Long id, HttpServletResponse response) throws IOException {
DataFile dataFile = dataFileService.downloadFile(id);
File targetFile = new File(dataFile.getPath());
response.setHeader("Content-Type", dataFile.getType());
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
@@ -156,7 +180,7 @@ public class DataFileController {
request.uploadId,
request.filename,
request.uploadId.toString(),
StrUtil.format("{}/upload/download/{}", knowledgeConfiguration.getDownloadPrefix(), request.uploadId)
StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), request.uploadId)
));
} else {
throw new RuntimeException("合并文件失败");
@@ -169,213 +193,66 @@ public class DataFileController {
}
}
@Data
public static final class DetailRequest {
private ImmutableSet<Long> ids;
}
@Data
public static final class DetailResponse {
private Long id;
private String filename;
private Long size;
private String md5;
@org.mapstruct.Mapper
public interface Mapper {
DetailResponse from(DataFile file);
}
}
@Data
public static final class StartRequest {
private String name;
private String filename;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
@Override
public String toString() {
return "StartRequest{" +
"name='" + name + '\'' +
", filename='" + filename + '\'' +
'}';
}
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static final class StartResponse {
private String uploadId;
public StartResponse() {
}
public StartResponse(String uploadId) {
this.uploadId = uploadId;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
@Override
public String toString() {
return "StartResponse{" +
"uploadId='" + uploadId + '\'' +
'}';
}
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static final class SliceResponse {
@JsonProperty("eTag")
private String eTag;
public SliceResponse() {
}
public SliceResponse(String eTag) {
this.eTag = eTag;
}
public String geteTag() {
return eTag;
}
public void seteTag(String eTag) {
this.eTag = eTag;
}
@Override
public String toString() {
return "SliceResponse{" +
"eTag='" + eTag + '\'' +
'}';
}
}
@Data
public static final class FinishRequest {
private String filename;
private Long uploadId;
private ImmutableList<Part> partList;
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
public Long getUploadId() {
return uploadId;
}
public void setUploadId(Long uploadId) {
this.uploadId = uploadId;
}
public ImmutableList<Part> getPartList() {
return partList;
}
public void setPartList(ImmutableList<Part> partList) {
this.partList = partList;
}
@Override
public String toString() {
return "FinishRequest{" +
"filename='" + filename + '\'' +
", uploadId=" + uploadId +
", partList=" + partList +
'}';
}
@Data
public static final class Part {
private Integer partNumber;
@JsonProperty("eTag")
private String eTag;
public Integer getPartNumber() {
return partNumber;
}
public void setPartNumber(Integer partNumber) {
this.partNumber = partNumber;
}
public String geteTag() {
return eTag;
}
public void seteTag(String eTag) {
this.eTag = eTag;
}
@Override
public String toString() {
return "Part{" +
"partNumber=" + partNumber +
", eTag='" + eTag + '\'' +
'}';
}
}
}
@Data
@NoArgsConstructor
@AllArgsConstructor
public static final class FinishResponse {
private Long id;
private String filename;
private String value;
private String url;
public FinishResponse() {
}
public FinishResponse(Long id, String filename, String value, String url) {
this.id = id;
this.filename = filename;
this.value = value;
this.url = url;
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
@Override
public String toString() {
return "FinishResponse{" +
"id=" + id +
", filename='" + filename + '\'' +
", value='" + value + '\'' +
", url='" + url + '\'' +
'}';
}
}
}

View File

@@ -1,17 +1,18 @@
package com.lanyuanxiaoyao.service.ai.chat.controller;
package com.lanyuanxiaoyao.service.ai.web.controller.chat;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.chat.Prompts;
import com.lanyuanxiaoyao.service.ai.chat.entity.MessageVO;
import com.lanyuanxiaoyao.service.ai.chat.tools.ChartTool;
import com.lanyuanxiaoyao.service.ai.chat.tools.KnowledgeTool;
import com.lanyuanxiaoyao.service.ai.chat.tools.TableTool;
import com.lanyuanxiaoyao.service.ai.chat.tools.YarnTool;
import com.lanyuanxiaoyao.service.ai.web.configuration.Prompts;
import com.lanyuanxiaoyao.service.ai.web.entity.vo.MessageVO;
import com.lanyuanxiaoyao.service.ai.web.tools.ChartTool;
import com.lanyuanxiaoyao.service.ai.web.tools.TableTool;
import com.lanyuanxiaoyao.service.ai.web.tools.YarnTool;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.concurrent.TimeoutException;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -22,11 +23,11 @@ import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
@@ -46,11 +47,11 @@ public class ChatController {
private final ChatClient chatClient;
public ChatController(ChatClient.Builder builder) {
public ChatController(@Qualifier("chat") ChatClient.Builder builder) {
this.chatClient = builder.build();
}
private ChatClient.ChatClientRequestSpec buildRequest(Long knowledgeId, ImmutableList<MessageVO> messages) {
private ChatClient.ChatClientRequestSpec buildRequest(ImmutableList<MessageVO> messages) {
ChatClient.ChatClientRequestSpec spec = chatClient.prompt()
.system(
StrUtil.format("""
@@ -79,9 +80,6 @@ public class ChatController {
.collect(message -> (Message) message)
.toList()
);
if (ObjectUtil.isNotNull(knowledgeId)) {
spec.tools(new KnowledgeTool(knowledgeId));
}
spec.tools(
new TableTool(),
new YarnTool(),
@@ -93,10 +91,9 @@ public class ChatController {
@PostMapping("sync")
@ResponseBody
public MessageVO chatSync(
@RequestParam(value = "knowledge_id", required = false) Long knowledgeId,
@RequestBody ImmutableList<MessageVO> messages
) {
ChatResponse response = buildRequest(knowledgeId, messages)
ChatResponse response = buildRequest(messages)
.call()
.chatResponse();
return toMessage(response);
@@ -104,25 +101,34 @@ public class ChatController {
@PostMapping("async")
public SseEmitter chatAsync(
@RequestParam(value = "knowledge_id", required = false) Long knowledgeId,
@RequestBody ImmutableList<MessageVO> messages
@RequestBody ImmutableList<MessageVO> messages,
HttpServletResponse httpResponse
) {
SseEmitter emitter = new SseEmitter();
buildRequest(knowledgeId, messages)
.stream()
.chatResponse()
.subscribe(
response -> {
try {
emitter.send(toMessage(response));
} catch (IOException e) {
emitter.completeWithError(e);
throw new RuntimeException(e);
}
},
emitter::completeWithError,
emitter::complete
);
httpResponse.setHeader("X-Accel-Buffering", "no");
SseEmitter emitter = new SseEmitter(20 * 60 * 1000L);
ExecutorProvider.EXECUTORS.submit(() -> {
buildRequest(messages)
.stream()
.chatResponse()
.subscribe(
response -> {
try {
emitter.send(
SseEmitter.event()
.data(toMessage(response))
.reconnectTime(5 * 1000L)
.build()
);
} catch (IOException e) {
emitter.completeWithError(e);
}
},
emitter::completeWithError,
emitter::complete
);
});
emitter.onTimeout(() -> emitter.completeWithError(new TimeoutException("SseEmitter Timeout")));
return emitter;
}

View File

@@ -0,0 +1,100 @@
package com.lanyuanxiaoyao.service.ai.web.controller.feedback;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.set.ImmutableSet;
import org.mapstruct.Mapping;
import org.mapstruct.factory.Mappers;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("feedback")
public class FeedbackController extends SimpleControllerSupport<Feedback, FeedbackController.SaveItem, FeedbackController.ListItem, FeedbackController.ListItem> {
private final FeedbackService feedbackService;
private final DataFileService dataFileService;
public FeedbackController(FeedbackService feedbackService, DataFileService dataFileService) {
super(feedbackService);
this.feedbackService = feedbackService;
this.dataFileService = dataFileService;
}
@GetMapping("reanalysis/{id}")
public void reanalysis(@PathVariable("id") Long id) {
feedbackService.reanalysis(id);
}
@PostMapping("conclude")
public void conclude(@RequestBody ConcludeItem item) {
feedbackService.updateConclusion(item.getId(), item.getConclusion());
}
@Override
protected SaveItemMapper<Feedback, SaveItem> saveItemMapper() {
return item -> {
Feedback feedback = new Feedback();
feedback.setSource(item.getSource());
feedback.setPictures(dataFileService.downloadFile(item.getPictures()).toSet());
return feedback;
};
}
@Override
protected ListItemMapper<Feedback, ListItem> listItemMapper() {
return Mappers.getMapper(ListItem.Mapper.class);
}
@Override
protected DetailItemMapper<Feedback, ListItem> detailItemMapper() {
ListItemMapper<Feedback, ListItem> mapper = listItemMapper();
return mapper::from;
}
@Data
public static final class SaveItem {
private String source;
private ImmutableSet<Long> pictures;
}
@Data
public static final class ConcludeItem {
private Long id;
private String conclusion;
}
@Data
@EqualsAndHashCode(callSuper = true)
public static class ListItem extends SimpleItem {
private String source;
private ImmutableSet<Long> pictures;
private String analysis;
private String conclusion;
private Feedback.Status status;
@org.mapstruct.Mapper(
imports = {
IdOnlyEntity.class,
Sets.class
}
)
public interface Mapper extends ListItemMapper<Feedback, ListItem> {
@Mapping(target = "pictures", expression = "java(Sets.immutable.ofAll(feedback.getPictures()).collect(IdOnlyEntity::getId))")
@Override
ListItem from(Feedback feedback);
}
}
}

View File

@@ -0,0 +1,67 @@
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.mapstruct.factory.Mappers;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author lanyuanxiaoyao
* @version 20250528
*/
@Slf4j
@RestController
@RequestMapping("knowledge/group")
public class GroupController extends SimpleControllerSupport<Group, GroupController.SaveItem, GroupController.ListItem, GroupController.ListItem> {
private final KnowledgeBaseService knowledgeBaseService;
public GroupController(GroupService groupService, KnowledgeBaseService knowledgeBaseService) {
super(groupService);
this.knowledgeBaseService = knowledgeBaseService;
}
@Override
protected SaveItemMapper<Group, SaveItem> saveItemMapper() {
return item -> {
Group group = new Group();
group.setName(item.getName());
group.setKnowledge(knowledgeBaseService.detailOrThrow(item.getKnowledgeId()));
return group;
};
}
@Override
protected ListItemMapper<Group, ListItem> listItemMapper() {
return Mappers.getMapper(ListItem.Mapper.class);
}
@Override
protected DetailItemMapper<Group, ListItem> detailItemMapper() {
ListItemMapper<Group, ListItem> mapper = listItemMapper();
return mapper::from;
}
@Data
public static final class SaveItem {
private String name;
private Long knowledgeId;
}
@Data
@EqualsAndHashCode(callSuper = true)
public static final class ListItem extends SimpleItem {
private String name;
private Group.Status status;
@org.mapstruct.Mapper
public interface Mapper extends ListItemMapper<Group, ListItem> {
}
}
}

View File

@@ -0,0 +1,184 @@
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
import com.lanyuanxiaoyao.service.ai.web.service.EmbeddingService;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
import io.qdrant.client.grpc.Collections;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.mapstruct.Mapping;
import org.mapstruct.factory.Mappers;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* @author lanyuanxiaoyao
* @version 20250515
*/
@Slf4j
@RestController
@RequestMapping("knowledge")
public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge, KnowledgeBaseController.SaveItem, KnowledgeBaseController.ListItem, KnowledgeBaseController.ListItem> {
private final KnowledgeBaseService knowledgeBaseService;
private final EmbeddingService embeddingService;
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
super(knowledgeBaseService);
this.knowledgeBaseService = knowledgeBaseService;
this.embeddingService = embeddingService;
}
@Override
protected SaveItemMapper<Knowledge, SaveItem> saveItemMapper() {
return Mappers.getMapper(SaveItem.Mapper.class);
}
@Override
protected ListItemMapper<Knowledge, ListItem> listItemMapper() {
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
}
@Override
protected DetailItemMapper<Knowledge, ListItem> detailItemMapper() {
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
}
@GetMapping("{id}/name")
public AmisMapResponse name(@PathVariable("id") Long id) {
return AmisResponse.responseMapData()
.setData("name", knowledgeBaseService.getName(id));
}
@PostMapping("update_description")
public void updateDescription(
@RequestParam("id") Long id,
@RequestParam("description") String description
) throws Exception {
knowledgeBaseService.updateDescription(id, description);
}
@PostMapping("preview_text")
public AmisResponse<?> previewText(@RequestBody SubmitItem item) {
if (StrUtil.equals("text", item.getType())) {
return AmisResponse.responseCrudData(
embeddingService.preview(item.getMode(), item.getContent())
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
vo.setText(doc.getText());
return vo;
})
);
} else if (StrUtil.equals("file", item.getType())) {
return AmisResponse.responseCrudData(
embeddingService.preview(item.getMode(), item.getFiles())
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
vo.setText(doc.getText());
return vo;
})
);
} else {
throw new IllegalArgumentException("Unsupported type: " + item.getType());
}
}
@PostMapping("submit_text")
public void submitText(@RequestBody SubmitItem item) {
if (StrUtil.equals("text", item.getMode())) {
embeddingService.submit(item.getId(), item.getMode(), item.getContent());
} else if (StrUtil.equals("file", item.getType())) {
embeddingService.submit(item.getId(), item.getMode(), item.getFiles());
} else {
throw new IllegalArgumentException("Unsupported type: " + item.getType());
}
}
@PostMapping("submit_text_directly")
public void submitDirectly(@RequestBody SubmitDirectlyItem item) {
embeddingService.submitDirectly(item.getId(), item.getName(), Lists.immutable.ofAll(StrUtil.split(item.getContent(), item.getSplitKey())));
}
@PostMapping("query")
public ImmutableList<String> query(@RequestBody QueryItem item) throws ExecutionException, InterruptedException, IOException {
return knowledgeBaseService.query(item.getId(), item.getText(), item.getLimit(), item.getThreshold());
}
@Data
public static final class SaveItem {
private String name;
private String strategy;
private String description;
@org.mapstruct.Mapper
public interface Mapper extends SaveItemMapper<Knowledge, SaveItem> {
}
}
@Data
@EqualsAndHashCode(callSuper = true)
public static final class ListItem extends SimpleItem {
private Long vectorSourceId;
private String name;
private String description;
private String strategy;
private Long size;
private Long points;
private Long segments;
private String status;
@org.mapstruct.Mapper
public interface Mapper extends ListItemMapper<Knowledge, ListItem> {
@Mapping(source = "info.config.params.vectorsConfig.params.distance", target = "strategy")
@Mapping(source = "info.config.params.vectorsConfig.params.size", target = "size")
@Mapping(source = "info.pointsCount", target = "points")
@Mapping(source = "info.segmentsCount", target = "segments")
@Mapping(source = "info.status", target = "status")
ListItem from(Knowledge knowledge, Collections.CollectionInfo info);
}
}
@Data
public static final class SubmitItem {
private Long id;
private String mode;
private String type;
private String content;
private ImmutableList<Long> files;
}
@Data
public static final class SubmitDirectlyItem {
private Long id;
private String name;
private String splitKey = "\n\n";
private String content;
}
@Data
public static final class QueryItem {
private Long id;
private Integer limit = 5;
private Double threshold = 0.6;
private String text;
}
}

View File

@@ -1,10 +1,9 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.knowledge.service.SegmentService;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.SegmentService;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@@ -14,11 +13,10 @@ import org.springframework.web.bind.annotation.RestController;
* @author lanyuanxiaoyao
* @version 20250528
*/
@Slf4j
@RestController
@RequestMapping("segment")
@RequestMapping("knowledge/segment")
public class SegmentController {
private static final Logger logger = LoggerFactory.getLogger(SegmentController.class);
private final SegmentService segmentService;
public SegmentController(SegmentService segmentService) {

View File

@@ -0,0 +1,128 @@
package com.lanyuanxiaoyao.service.ai.web.controller.task;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.FlowTask;
import com.lanyuanxiaoyao.service.ai.web.entity.FlowTaskTemplate;
import com.lanyuanxiaoyao.service.ai.web.service.task.FlowTaskService;
import com.lanyuanxiaoyao.service.ai.web.service.task.FlowTaskTemplateService;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.mapstruct.Mapping;
import org.mapstruct.factory.Mappers;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("flow_task")
public class TaskController extends SimpleControllerSupport<FlowTask, TaskController.SaveItem, TaskController.ListItem, TaskController.DetailItem> {
private final FlowTaskService flowTaskService;
private final FlowTaskTemplateService flowTaskTemplateService;
private final ObjectMapper mapper;
public TaskController(FlowTaskService flowTaskService, FlowTaskTemplateService flowTaskTemplateService, Jackson2ObjectMapperBuilder builder) {
super(flowTaskService);
this.flowTaskService = flowTaskService;
this.flowTaskTemplateService = flowTaskTemplateService;
this.mapper = builder.build();
}
@GetMapping("input_data/{id}")
public AmisResponse<?> getInputData(@PathVariable("id") Long id) throws JsonProcessingException {
var task = flowTaskService.detailOrThrow(id);
if (ObjectUtil.isEmpty(task.getInput())) {
return AmisResponse.responseSuccess();
}
return AmisResponse.responseSuccess(mapper.readValue(task.getInput(), Map.class));
}
@GetMapping("input_schema/{id}")
public AmisResponse<?> getInputSchema(@PathVariable("id") Long id) throws JsonProcessingException {
var task = flowTaskService.detailOrThrow(id);
if (ObjectUtil.isEmpty(task.getTemplateInputSchema())) {
return AmisResponse.responseSuccess();
}
return AmisResponse.responseSuccess(mapper.readValue(task.getTemplateInputSchema(), Map.class));
}
@GetMapping("execute/{id}")
public AmisResponse<?> execute(@PathVariable("id") Long id) throws JsonProcessingException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
flowTaskService.execute(id);
return AmisResponse.responseSuccess();
}
@Override
protected SaveItemMapper<FlowTask, SaveItem> saveItemMapper() {
return item -> {
FlowTask task = new FlowTask();
FlowTaskTemplate template = flowTaskTemplateService.detailOrThrow(item.getTemplateId());
task.setTemplateName(template.getName());
task.setTemplateDescription(template.getDescription());
task.setTemplateInputSchema(template.getInputSchema());
task.setTemplateFlowGraph(template.getFlowGraph());
task.setInput(ObjectUtil.isEmpty(item.getInput()) ? null : mapper.writeValueAsString(item.getInput()));
return task;
};
}
@Override
protected ListItemMapper<FlowTask, ListItem> listItemMapper() {
ListItem.Mapper map = Mappers.getMapper(ListItem.Mapper.class);
return map::from;
}
@Override
protected DetailItemMapper<FlowTask, DetailItem> detailItemMapper() {
DetailItem.Mapper map = Mappers.getMapper(DetailItem.Mapper.class);
return map::from;
}
@Data
public static final class SaveItem {
private Long templateId;
private Object input;
}
@Data
@EqualsAndHashCode(callSuper = true)
public static class ListItem extends SimpleItem {
private String templateName;
private FlowTask.Status status;
private Boolean hasInput;
@org.mapstruct.Mapper(imports = {
StrUtil.class
})
public static abstract class Mapper {
@Mapping(target = "hasInput", expression = "java(StrUtil.isNotBlank(task.getInput()))")
public abstract ListItem from(FlowTask task);
}
}
@Data
@EqualsAndHashCode(callSuper = true)
public static class DetailItem extends ListItem {
private String error;
private String result;
@org.mapstruct.Mapper(imports = {
StrUtil.class
})
public static abstract class Mapper extends ListItem.Mapper {
@Mapping(target = "hasInput", expression = "java(StrUtil.isNotBlank(task.getInput()))")
public abstract DetailItem from(FlowTask task);
}
}
}

View File

@@ -0,0 +1,125 @@
package com.lanyuanxiaoyao.service.ai.web.controller.task;
import cn.hutool.core.util.ObjectUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.FlowTaskTemplate;
import com.lanyuanxiaoyao.service.ai.web.service.task.FlowTaskTemplateService;
import java.util.Map;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.mapstruct.Context;
import org.mapstruct.factory.Mappers;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("flow_task/template")
public class TaskTemplateController extends SimpleControllerSupport<FlowTaskTemplate, TaskTemplateController.SaveItem, TaskTemplateController.ListItem, TaskTemplateController.DetailItem> {
private final FlowTaskTemplateService flowTaskTemplateService;
private final ObjectMapper mapper;
public TaskTemplateController(FlowTaskTemplateService flowTaskTemplateService, Jackson2ObjectMapperBuilder builder) {
super(flowTaskTemplateService);
this.flowTaskTemplateService = flowTaskTemplateService;
this.mapper = builder.build();
}
@GetMapping("input_schema/{id}")
public AmisResponse<?> getInputSchema(@PathVariable("id") Long id) throws JsonProcessingException {
var template = flowTaskTemplateService.detailOrThrow(id);
if (ObjectUtil.isEmpty(template.getInputSchema())) {
return AmisResponse.responseSuccess();
}
return AmisResponse.responseSuccess(mapper.readValue(template.getInputSchema(), Map.class));
}
@GetMapping("flow_graph/{id}")
public AmisResponse<?> getFlowGraph(@PathVariable("id") Long id) throws JsonProcessingException {
var template = flowTaskTemplateService.detailOrThrow(id);
return AmisResponse.responseSuccess(mapper.readValue(template.getFlowGraph(), Map.class));
}
@PostMapping("update_flow_graph")
public AmisResponse<?> updateFlowGraph(@RequestBody UpdateGraphItem item) throws JsonProcessingException {
flowTaskTemplateService.updateFlowGraph(item.getId(), mapper.writeValueAsString(item.getGraph()));
return AmisResponse.responseSuccess();
}
@Override
protected SaveItemMapper<FlowTaskTemplate, SaveItem> saveItemMapper() {
return Mappers.getMapper(SaveItem.Mapper.class);
}
@Override
protected ListItemMapper<FlowTaskTemplate, ListItem> listItemMapper() {
return Mappers.getMapper(ListItem.Mapper.class);
}
@Override
protected DetailItemMapper<FlowTaskTemplate, DetailItem> detailItemMapper() {
var map = Mappers.getMapper(DetailItem.Mapper.class);
return template -> map.from(template, mapper);
}
@Data
public static final class SaveItem {
private Long id;
private String name;
private String description;
@org.mapstruct.Mapper
public interface Mapper extends SaveItemMapper<FlowTaskTemplate, SaveItem> {
}
}
@Data
@EqualsAndHashCode(callSuper = true)
public static class ListItem extends SimpleItem {
private String name;
private String description;
@org.mapstruct.Mapper
public interface Mapper extends ListItemMapper<FlowTaskTemplate, ListItem> {
}
}
@Data
@EqualsAndHashCode(callSuper = true)
public static class DetailItem extends SimpleItem {
private String name;
private String description;
private Map<String, Object> inputSchema;
private Map<String, Object> flowGraph;
@org.mapstruct.Mapper
public static abstract class Mapper {
public abstract DetailItem from(FlowTaskTemplate template, @Context ObjectMapper mapper) throws Exception;
public Map<String, Object> mapJson(String source, @Context ObjectMapper mapper) throws Exception {
if (ObjectUtil.isNull(source)) {
return null;
}
return mapper.readValue(source, new TypeReference<>() {
});
}
}
}
@Data
public static class UpdateGraphItem {
private Long id;
private Map<String, Object> graph;
}
}

View File

@@ -0,0 +1,32 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowContext;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowGraph;
import com.lanyuanxiaoyao.service.ai.web.engine.store.FlowStore;
import java.lang.reflect.InvocationTargetException;
import org.eclipse.collections.api.map.ImmutableMap;
/**
* 流程执行器
*
* @author lanyuanxiaoyao
* @version 20250630
*/
public class FlowExecutor {
private final FlowStore flowStore;
private final ImmutableMap<String, Class<? extends FlowNodeRunner>> runnerMap;
public FlowExecutor(FlowStore flowStore, ImmutableMap<String, Class<? extends FlowNodeRunner>> runnerMap) {
this.flowStore = flowStore;
this.runnerMap = runnerMap;
}
public void execute(FlowGraph graph) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
execute(graph, new FlowContext());
}
public void execute(FlowGraph graph, FlowContext context) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
var runner = new FlowGraphRunner(graph, context, flowStore, runnerMap);
runner.run();
}
}

View File

@@ -0,0 +1,124 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowContext;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowEdge;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowGraph;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowNode;
import com.lanyuanxiaoyao.service.ai.web.engine.store.FlowStore;
import java.lang.reflect.InvocationTargetException;
import java.util.LinkedList;
import java.util.Queue;
import org.eclipse.collections.api.map.ImmutableMap;
import org.eclipse.collections.api.multimap.set.ImmutableSetMultimap;
/**
* Graph执行器
*
* @author lanyuanxiaoyao
* @version 20250701
*/
public final class FlowGraphRunner {
private final FlowGraph flowGraph;
private final FlowContext flowContext;
private final FlowStore flowStore;
private final ImmutableMap<String, Class<? extends FlowNodeRunner>> nodeRunnerClass;
private final Queue<FlowNode> executionQueue = new LinkedList<>();
private final ImmutableSetMultimap<String, FlowEdge> nodeInputMap;
private final ImmutableSetMultimap<String, FlowEdge> nodeOutputMap;
private final ImmutableMap<String, FlowNode> nodeMap;
public FlowGraphRunner(FlowGraph flowGraph, FlowContext flowContext, FlowStore flowStore, ImmutableMap<String, Class<? extends FlowNodeRunner>> nodeRunnerClass) {
this.flowGraph = flowGraph;
this.flowContext = flowContext;
this.flowStore = flowStore;
this.nodeRunnerClass = nodeRunnerClass;
nodeInputMap = flowGraph.edges().groupBy(FlowEdge::target);
nodeOutputMap = flowGraph.edges().groupBy(FlowEdge::source);
nodeMap = flowGraph.nodes().toImmutableMap(FlowNode::id, node -> node);
}
public void run() throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
flowStore.init(flowGraph);
for (FlowNode node : flowGraph.nodes()) {
if (ObjectUtil.isNull(node.parentId())) {
executionQueue.offer(node);
}
}
while (!executionQueue.isEmpty()) {
var node = executionQueue.poll();
if (ObjectUtil.isNull(node)) {
continue;
}
process(node, flowContext);
}
}
private void process(FlowNode node, FlowContext context) throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
if (
(
// 没有入节点,即开始节点
!nodeInputMap.containsKey(node.id())
// 或者所有入的边状态都已经完成
|| nodeInputMap.get(node.id()).allSatisfy(edge -> flowStore.checkEdgeStatus(flowGraph.id(), edge.id(), FlowEdge.Status.EXECUTE, FlowEdge.Status.SKIP))
)
// 当前节点还未执行
&& flowStore.checkNodeStatus(flowGraph.id(), node.id(), FlowNode.Status.INITIAL)
) {
// 是开始节点或入的边有至少一条是「执行」
if (
!nodeInputMap.containsKey(node.id())
|| nodeInputMap.get(node.id()).anySatisfy(edge -> flowStore.checkEdgeStatus(flowGraph.id(), edge.id(), FlowEdge.Status.EXECUTE))
) {
flowStore.updateNodeToRunning(flowGraph.id(), node.id());
var runnerClazz = nodeRunnerClass.get(node.type());
var runner = runnerClazz.getDeclaredConstructor().newInstance();
runner.setNodeId(node.id());
runner.setContext(context);
// 处理子流程节点的逻辑
if (runner instanceof FlowNodeSubflowRunner subflowRunner) {
var subflowNodes = flowGraph.nodes().select(n -> StrUtil.equals(n.parentId(), node.id()));
var subGraph = new FlowGraph(IdUtil.fastUUID(), subflowNodes, flowGraph.edges());
subflowRunner.setSubGraph(subGraph);
}
runner.run();
// 处理选择节点的逻辑
if (runner instanceof FlowNodeOptionalRunner) {
var targetPoint = ((FlowNodeOptionalRunner) runner).getTargetPoint();
for (FlowEdge edge : nodeOutputMap.get(node.id())) {
if (StrUtil.equals(targetPoint, edge.sourcePoint())) {
flowStore.updateEdgeToExecute(flowGraph.id(), edge.id());
} else {
flowStore.updateEdgeToSkip(flowGraph.id(), edge.id());
}
executionQueue.offer(nodeMap.get(edge.target()));
}
} else {
for (FlowEdge edge : nodeOutputMap.get(node.id())) {
flowStore.updateEdgeToExecute(flowGraph.id(), edge.id());
executionQueue.offer(nodeMap.get(edge.target()));
}
}
flowStore.updateNodeToFinished(flowGraph.id(), node.id());
}
// 所有入的边都是跳过,当前节点就跳过
else {
flowStore.updateNodeToSkipped(flowGraph.id(), node.id());
for (FlowEdge edge : nodeOutputMap.get(node.id())) {
flowStore.updateEdgeToSkip(flowGraph.id(), edge.id());
executionQueue.offer(nodeMap.get(edge.target()));
}
}
}
}
}

View File

@@ -0,0 +1,58 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.extra.template.TemplateEngine;
import cn.hutool.extra.template.TemplateUtil;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowContext;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.map.ImmutableMap;
/**
* @author lanyuanxiaoyao
* @version 20250711
*/
@Slf4j
public class FlowHelper {
private static final TemplateEngine TEMPLATE_ENGINE = TemplateUtil.createEngine();
public static ImmutableMap<String, Object> generateInputVariablesMap(String nodeId, FlowContext context) {
var variableMap = Maps.mutable.<String, Object>empty();
var currentNodeData = context.get(nodeId);
if (currentNodeData.containsKey("inputs")) {
var inputsMap = (Map<String, Map<String, String>>) currentNodeData.get("inputs");
for (String variableName : inputsMap.keySet()) {
var expression = inputsMap.get(variableName).get("variable");
var targetVariable = generateVariable(expression, context);
if (ObjectUtil.isNotNull(targetVariable)) {
variableMap.put(variableName, targetVariable);
}
}
}
return variableMap.toImmutable();
}
public static Object generateVariable(String expression, FlowContext context) {
if (StrUtil.contains(expression, ".")) {
var splits = StrUtil.splitTrim(expression, ".", 2);
var targetNodeId = splits.get(0);
var targetVariableName = splits.get(1);
if (!context.getData().containsKey(targetNodeId)) {
throw new RuntimeException(StrUtil.format("Target node id not found: {}", targetNodeId));
}
var targetNodeData = context.getData().get(targetNodeId);
if (!targetNodeData.containsKey(targetVariableName)) {
throw new RuntimeException(StrUtil.format("Target node variable not found: {}.{}", targetNodeId, targetVariableName));
}
return targetNodeData.get(targetVariableName);
}
return null;
}
public static String renderTemplateText(String templateText, Map<?, ?> data) {
var template = TEMPLATE_ENGINE.getTemplate(templateText);
return template.render(data);
}
}

View File

@@ -0,0 +1,15 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import lombok.Getter;
public abstract class FlowNodeOptionalRunner extends FlowNodeRunner {
@Getter
private String targetPoint;
public abstract String runOptional();
@Override
public void run() {
this.targetPoint = runOptional();
}
}

View File

@@ -0,0 +1,36 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowContext;
import lombok.Getter;
import org.eclipse.collections.api.map.MutableMap;
public abstract class FlowNodeRunner {
@Getter
private String nodeId;
@Getter
private FlowContext context;
public abstract void run();
void setNodeId(String nodeId) {
this.nodeId = nodeId;
}
void setContext(FlowContext context) {
this.context = context;
}
protected MutableMap<String, Object> getData() {
return context.get(nodeId);
}
protected <T> T getData(String key) {
var data = context.get(nodeId);
return (T) data.get(key);
}
protected <T> void setData(String key, T value) {
var data = context.get(nodeId);
data.put(key, value);
}
}

View File

@@ -0,0 +1,17 @@
package com.lanyuanxiaoyao.service.ai.web.engine;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowGraph;
import lombok.Getter;
import lombok.Setter;
/**
* 包含子流程的流程
*
* @author lanyuanxiaoyao
* @version 20250717
*/
public abstract class FlowNodeSubflowRunner extends FlowNodeRunner {
@Getter
@Setter
private FlowGraph subGraph;
}

View File

@@ -0,0 +1,17 @@
package com.lanyuanxiaoyao.service.ai.web.engine.entity;
import lombok.Data;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.map.MutableMap;
@Data
public class FlowContext {
private MutableMap<String, MutableMap<String, Object>> data = Maps.mutable.<String, MutableMap<String, Object>>empty().asSynchronized();
public MutableMap<String, Object> get(String key) {
if (!data.containsKey(key)) {
data.put(key, Maps.mutable.<String, Object>empty().asSynchronized());
}
return data.get(key);
}
}

View File

@@ -0,0 +1,39 @@
package com.lanyuanxiaoyao.service.ai.web.engine.entity;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.LocalDateTime;
/**
* 流程图中的边
*
* @author lanyuanxiaoyao
* @version 20250630
*/
public record FlowEdge(
String id,
String source,
String target,
@JsonProperty("sourceHandle")
String sourcePoint,
@JsonProperty("targetHandle")
String targetPoint
) {
public enum Status {
INITIAL, EXECUTE, SKIP
}
public record State(
String id,
Status status,
LocalDateTime startingTime,
LocalDateTime finishedTime
) {
public State(String edgeId) {
this(edgeId, Status.INITIAL, LocalDateTime.now(), null);
}
public State(String edgeId, Status status) {
this(edgeId, status, LocalDateTime.now(), null);
}
}
}

View File

@@ -0,0 +1,16 @@
package com.lanyuanxiaoyao.service.ai.web.engine.entity;
import org.eclipse.collections.api.set.ImmutableSet;
/**
* 流程图
*
* @author lanyuanxiaoyao
* @version 20250630
*/
public record FlowGraph(
String id,
ImmutableSet<FlowNode> nodes,
ImmutableSet<FlowEdge> edges
) {
}

View File

@@ -0,0 +1,34 @@
package com.lanyuanxiaoyao.service.ai.web.engine.entity;
import java.time.LocalDateTime;
/**
* 流程图中的节点
*
* @author lanyuanxiaoyao
* @version 20250630
*/
public record FlowNode(
String id,
String type,
String parentId
) {
public enum Status {
INITIAL, RUNNING, FINISHED, SKIPPED
}
public record State(
String id,
Status status,
LocalDateTime startingTime,
LocalDateTime finishedTime
) {
public State(String nodeId) {
this(nodeId, Status.INITIAL, LocalDateTime.now(), null);
}
public State(String nodeId, Status status) {
this(nodeId, status, LocalDateTime.now(), null);
}
}
}

View File

@@ -0,0 +1,37 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lanyuanxiaoyao.service.ai.web.configuration.SpringBeanGetter;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowHelper;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeRunner;
import com.lanyuanxiaoyao.service.ai.web.engine.node.code.CodeExecutor;
import com.lanyuanxiaoyao.service.ai.web.engine.node.code.JavaScriptCodeExecutor;
import com.lanyuanxiaoyao.service.ai.web.engine.node.code.PythonCodeExecutor;
import lombok.extern.slf4j.Slf4j;
/**
* @author lanyuanxiaoyao
* @version 20250717
*/
@Slf4j
public class CodeNode extends FlowNodeRunner {
@Override
public void run() {
var mapper = SpringBeanGetter.getBean(ObjectMapper.class);
var inputVariablesMap = FlowHelper.generateInputVariablesMap(getNodeId(), getContext());
var type = this.<String>getData("type");
var script = this.<String>getData("content");
CodeExecutor executor = switch (type) {
case "javascript" -> new JavaScriptCodeExecutor(mapper);
case "python" -> new PythonCodeExecutor(mapper);
default -> null;
};
if (ObjectUtil.isNull(executor)) {
throw new RuntimeException(StrUtil.format("Unsupported type: {}", type));
}
var result = executor.execute(script, inputVariablesMap);
result.forEachKeyValue(this::setData);
}
}

View File

@@ -0,0 +1,25 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeRunner;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
/**
* @author lanyuanxiaoyao
* @version 20250711
*/
@Slf4j
public class InputNode extends FlowNodeRunner {
public static final String KEY = "flow_inputs";
@Override
public void run() {
var inputData = getContext().getData().get(KEY);
var inputs = this.<Map<String, Object>>getData("inputs");
for (String variable : inputs.keySet()) {
if (inputData.containsKey(variable)) {
setData(variable, inputData.get(variable));
}
}
}
}

View File

@@ -0,0 +1,31 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.configuration.SpringBeanGetter;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowHelper;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeRunner;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.client.ChatClient;
/**
* @author lanyuanxiaoyao
* @version 20250711
*/
@Slf4j
public class LlmNode extends FlowNodeRunner {
@Override
public void run() {
var variableMap = FlowHelper.generateInputVariablesMap(getNodeId(), getContext());
var sourcePrompt = (String) getData("systemPrompt");
if (StrUtil.isNotBlank(sourcePrompt)) {
var prompt = FlowHelper.renderTemplateText(sourcePrompt, variableMap.toMap());
var builder = SpringBeanGetter.getBean("chat", ChatClient.Builder.class);
var client = builder.build();
var content = client.prompt()
.user(prompt)
.call()
.content();
setData("text", content);
}
}
}

View File

@@ -0,0 +1,18 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeSubflowRunner;
import lombok.extern.slf4j.Slf4j;
/**
* 循环节点
*
* @author lanyuanxiaoyao
* @version 20250717
*/
@Slf4j
public class LoopNode extends FlowNodeSubflowRunner {
@Override
public void run() {
log.info("{}", getSubGraph());
}
}

View File

@@ -0,0 +1,20 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowHelper;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeRunner;
import lombok.extern.slf4j.Slf4j;
/**
* @author lanyuanxiaoyao
* @version 20250711
*/
@Slf4j
public class OutputNode extends FlowNodeRunner {
private static final String KEY = "flow_outputs";
@Override
public void run() {
var variableMap = FlowHelper.generateInputVariablesMap(getNodeId(), getContext());
getContext().getData().put(KEY, variableMap.toMap());
}
}

View File

@@ -0,0 +1,87 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowHelper;
import com.lanyuanxiaoyao.service.ai.web.engine.FlowNodeOptionalRunner;
import java.math.BigDecimal;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
/**
* @author lanyuanxiaoyao
* @version 20250717
*/
@SuppressWarnings("unchecked")
@Slf4j
public class SwitchNode extends FlowNodeOptionalRunner {
@Override
public String runOptional() {
var conditions = this.<List<Map<String, Object>>>getData("conditions");
for (Map<String, Object> item : conditions) {
var condition = (Map<String, Object>) item.getOrDefault("condition", Map.of());
var id = (String) condition.getOrDefault("id", "");
var conjunction = (String) condition.getOrDefault("conjunction", "and");
var conditionChildren = ((List<Map<String, Object>>) condition.getOrDefault("children", List.<Map<String, Object>>of()));
if (
StrUtil.equals(conjunction, "and")
&& conditionChildren.stream().allMatch(this::check)
) {
return id;
} else if (
StrUtil.equals(conjunction, "or")
&& conditionChildren.stream().anyMatch(this::check)
) {
return id;
}
}
return "";
}
private Boolean check(Map<String, Object> condition) {
var leftVariable = (String) BeanUtil.getProperty(condition, "left.field");
var left = FlowHelper.generateVariable(leftVariable, getContext());
var operator = (String) condition.get("op");
var right = condition.get("right");
if (left instanceof CharSequence || left instanceof Boolean) {
String source = StrUtil.toStringOrNull(left);
String target = StrUtil.toStringOrNull(right);
return switch (operator) {
case "equal" -> StrUtil.equals(source, target);
case "not_equal" -> !StrUtil.equals(source, target);
case "is_empty" -> StrUtil.isBlank(source);
case "is_not_empty" -> StrUtil.isNotBlank(source);
case "like" -> StrUtil.contains(source, target);
case "not_like" -> !StrUtil.contains(source, target);
case "starts_with" -> StrUtil.startWith(source, target);
case "ends_with" -> StrUtil.endWith(source, target);
default -> false;
};
} else if (left instanceof Number source) {
var sourceNumber = new BigDecimal(StrUtil.toString(source));
var targetNumber = new BigDecimal(StrUtil.toString(right));
return switch (operator) {
case "equal" -> NumberUtil.equals(sourceNumber, targetNumber);
case "not_equal" -> !NumberUtil.equals(sourceNumber, targetNumber);
case "greater" -> NumberUtil.isGreater(sourceNumber, targetNumber);
case "greater_equal" -> NumberUtil.isGreaterOrEqual(sourceNumber, targetNumber);
case "less" -> NumberUtil.isLess(sourceNumber, targetNumber);
case "less_equal" -> NumberUtil.isLessOrEqual(sourceNumber, targetNumber);
default -> false;
};
} else if (left instanceof Collection<?> source) {
return switch (operator) {
case "is_empty" -> CollectionUtil.isEmpty(source);
case "is_not_empty" -> CollectionUtil.isNotEmpty(source);
case "contain" -> CollectionUtil.safeContains(source, right);
case "not_contain" -> !CollectionUtil.safeContains(source, right);
default -> false;
};
}
return false;
}
}

View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node.code;
import org.eclipse.collections.api.map.ImmutableMap;
/**
* @author lanyuanxiaoyao
* @version 20250717
*/
public interface CodeExecutor {
ImmutableMap<String, Object> execute(String script, ImmutableMap<String, Object> inputVariablesMap);
}

View File

@@ -0,0 +1,57 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node.code;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.map.ImmutableMap;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.Source;
/**
* @author lanyuanxiaoyao
* @version 20250717
*/
@Slf4j
public class JavaScriptCodeExecutor implements CodeExecutor {
private final ObjectMapper mapper;
public JavaScriptCodeExecutor(ObjectMapper mapper) {
this.mapper = mapper;
}
@SneakyThrows
@Override
public ImmutableMap<String, Object> execute(String script, ImmutableMap<String, Object> inputVariablesMap) {
if (StrUtil.isBlank(script)) {
return Maps.immutable.empty();
}
try (var engin = Engine.create()) {
try (
var context = Context.newBuilder()
.allowAllAccess(true)
.engine(engin)
.build()
) {
var bindings = context.getBindings("js");
bindings.putMember("context", inputVariablesMap);
var result = context.eval(
Source.create(
"js",
"""
function process() {
%s
}
var result = process();
JSON.stringify(result? result: {})
""".formatted(script)
)
);
return mapper.readValue(result.asString(), new TypeReference<>() {});
}
}
}
}

View File

@@ -0,0 +1,50 @@
package com.lanyuanxiaoyao.service.ai.web.engine.node.code;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Arrays;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.map.ImmutableMap;
import org.python.core.PySystemState;
import org.python.util.PythonInterpreter;
/**
* @author lanyuanxiaoyao
* @version 20250718
*/
@Slf4j
public class PythonCodeExecutor implements CodeExecutor {
private final ObjectMapper mapper;
public PythonCodeExecutor(ObjectMapper mapper) {
this.mapper = mapper;
}
@SneakyThrows
@Override
public ImmutableMap<String, Object> execute(String script, ImmutableMap<String, Object> inputVariablesMap) {
try (var systemState = new PySystemState()) {
systemState.setdefaultencoding("UTF-8");
var interpreter = new PythonInterpreter(null, systemState);
script = Arrays.stream(script.split("\n"))
.map(line -> " " + line)
.collect(Collectors.joining("\n"));
interpreter.set("context", inputVariablesMap);
var pythonScript = interpreter.compile(
"""
import json
def process():
%s
result = json.dumps(process())
""".formatted(script)
);
interpreter.exec(pythonScript);
var result = interpreter.get("result");
return mapper.readValue((String) result.__tojava__(String.class), new TypeReference<>() {});
}
}
}

View File

@@ -0,0 +1,31 @@
package com.lanyuanxiaoyao.service.ai.web.engine.store;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowEdge;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowGraph;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowNode;
/**
* 存储状态
*
* @author lanyuanxiaoyao
* @version 20250701
*/
public interface FlowStore {
void init(FlowGraph flowGraph);
void updateNodeToRunning(String graphId, String nodeId);
void updateNodeToSkipped(String graphId, String nodeId);
void updateNodeToFinished(String graphId, String nodeId);
void updateEdgeToExecute(String graphId, String edgeId);
void updateEdgeToSkip(String graphId, String edgeId);
boolean checkNodeStatus(String graphId, String nodeId, FlowNode.Status... statuses);
boolean checkEdgeStatus(String graphId, String edgeId, FlowEdge.Status... statuses);
void print();
}

View File

@@ -0,0 +1,107 @@
package com.lanyuanxiaoyao.service.ai.web.engine.store;
import cn.hutool.core.util.ArrayUtil;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowEdge;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowGraph;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowNode;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.map.MutableMap;
/**
* 基于内存的流程状态存储
*
* @author lanyuanxiaoyao
* @version 20250701
*/
@Slf4j
public class InMemoryFlowStore implements FlowStore {
private static final MutableMap<String, FlowNode.State> flowNodeStateMap = Maps.mutable.<String, FlowNode.State>empty().asSynchronized();
private static final MutableMap<String, FlowEdge.State> flowEdgeStateMap = Maps.mutable.<String, FlowEdge.State>empty().asSynchronized();
private String multiKey(String... key) {
return String.join("-", key);
}
@Override
public void init(FlowGraph flowGraph) {
for (FlowNode node : flowGraph.nodes()) {
flowNodeStateMap.put(multiKey(flowGraph.id(), node.id()), new FlowNode.State(node.id()));
}
for (FlowEdge edge : flowGraph.edges()) {
flowEdgeStateMap.put(multiKey(flowGraph.id(), edge.id()), new FlowEdge.State(edge.id()));
}
}
@Override
public void updateNodeToRunning(String graphId, String nodeId) {
flowNodeStateMap.updateValue(
multiKey(graphId, nodeId),
() -> new FlowNode.State(nodeId, FlowNode.Status.RUNNING),
old -> new FlowNode.State(nodeId, FlowNode.Status.RUNNING, old.startingTime(), old.finishedTime())
);
}
@Override
public void updateNodeToSkipped(String graphId, String nodeId) {
flowNodeStateMap.updateValue(
multiKey(graphId, nodeId),
() -> new FlowNode.State(nodeId, FlowNode.Status.SKIPPED),
old -> new FlowNode.State(nodeId, FlowNode.Status.SKIPPED, old.startingTime(), old.finishedTime())
);
}
@Override
public void updateNodeToFinished(String graphId, String nodeId) {
flowNodeStateMap.updateValue(
multiKey(graphId, nodeId),
() -> new FlowNode.State(nodeId, FlowNode.Status.FINISHED),
old -> new FlowNode.State(nodeId, FlowNode.Status.FINISHED, old.startingTime(), old.finishedTime())
);
}
@Override
public void updateEdgeToExecute(String graphId, String edgeId) {
flowEdgeStateMap.updateValue(
multiKey(graphId, edgeId),
() -> new FlowEdge.State(edgeId, FlowEdge.Status.EXECUTE),
old -> new FlowEdge.State(edgeId, FlowEdge.Status.EXECUTE, old.startingTime(), old.finishedTime())
);
}
@Override
public void updateEdgeToSkip(String graphId, String edgeId) {
flowEdgeStateMap.updateValue(
multiKey(graphId, edgeId),
() -> new FlowEdge.State(edgeId, FlowEdge.Status.SKIP),
old -> new FlowEdge.State(edgeId, FlowEdge.Status.SKIP, old.startingTime(), old.finishedTime())
);
}
@Override
public boolean checkNodeStatus(String graphId, String nodeId, FlowNode.Status... statuses) {
String key = multiKey(graphId, nodeId);
if (flowNodeStateMap.containsKey(key)) {
return ArrayUtil.contains(statuses, flowNodeStateMap.get(key).status());
}
return false;
}
@Override
public boolean checkEdgeStatus(String graphId, String edgeId, FlowEdge.Status... statuses) {
String key = multiKey(graphId, edgeId);
if (flowEdgeStateMap.containsKey(key)) {
return ArrayUtil.contains(statuses, flowEdgeStateMap.get(key).status());
}
return false;
}
@Override
public void print() {
log.info("====== Flow Store ======");
log.info("====== Flow Node ======");
flowNodeStateMap.forEachKeyValue((key, value) -> log.info("{}: {}", key, value.status()));
log.info("====== Flow Edge ======");
flowEdgeStateMap.forEachKeyValue((key, value) -> log.info("{}: {}", key, value.status()));
}
}

View File

@@ -0,0 +1,43 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
/**
* 上传文件
*
* @author lanyuanxiaoyao
* @date 2024-11-21
*/
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_file")
@NoArgsConstructor
@Comment("记录上传的文件存储信息")
public class DataFile extends SimpleEntity {
@Comment("文件名称")
private String filename;
@Comment("文件大小单位是byte")
private Long size;
@Comment("文件的md5编码用于校验文件的完整性")
private String md5;
@Comment("文件在主机上存储的实际路径")
private String path;
@Comment("文件类型,通常记录的是文件的后缀名")
private String type;
public DataFile(String filename) {
this.filename = filename;
}
}

View File

@@ -0,0 +1,62 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.ConstraintMode;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.JoinTable;
import jakarta.persistence.NamedAttributeNode;
import jakarta.persistence.NamedEntityGraph;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_feedback")
@NamedEntityGraph(name = "feedback.detail", attributeNodes = {
@NamedAttributeNode("pictures")
})
@Comment("报障信息记录")
public class Feedback extends SimpleEntity {
@Comment("原始报障说明")
@Column(nullable = false, columnDefinition = "longtext")
private String source;
@Comment("报障相关截图")
@OneToMany(fetch = FetchType.EAGER)
@JoinTable(catalog = Constants.DATABASE_NAME, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
@ToString.Exclude
private Set<DataFile> pictures;
@Comment("AI的分析结果")
@Column(columnDefinition = "longtext")
private String analysis;
@Comment("AI的解决方案")
@Column(columnDefinition = "longtext")
private String conclusion;
@Comment("报障处理状态")
@Column(nullable = false)
@Enumerated(EnumType.STRING)
private Status status = Status.ANALYSIS_PROCESSING;
public enum Status {
ANALYSIS_PROCESSING,
ANALYSIS_SUCCESS,
FINISHED,
}
}

View File

@@ -0,0 +1,63 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_flow_task")
@Comment("流程任务记录")
public class FlowTask extends SimpleEntity {
// 每个任务对应的模板都是唯一,避免模板修改之后任务的状态、运行等状态都无法展示
// 不管允许不允许任务重跑,这些都要保存下来
@Comment("任务对应的模板名称")
@Column(nullable = false)
private String templateName;
@Comment("任务对应的模板功能、内容说明")
private String templateDescription;
@Comment("任务对应的模板入参Schema")
@Column(columnDefinition = "longtext")
private String templateInputSchema;
@Comment("任务对应的模板前端流程图数据")
@Column(nullable = false, columnDefinition = "longtext")
private String templateFlowGraph = "{}";
@Comment("任务注释,用于额外说明")
@Column(columnDefinition = "text")
private String comment;
@Comment("任务输入")
@Column(columnDefinition = "longtext")
private String input;
@Comment("任务运行状态")
@Column(nullable = false)
@Enumerated(EnumType.STRING)
private Status status = Status.RUNNING;
@Comment("任务运行产生的报错")
@Column(columnDefinition = "longtext")
private String error;
@Comment("任务运行结果")
@Column(columnDefinition = "longtext")
private String result;
public enum Status {
RUNNING,
ERROR,
FINISHED,
}
}

View File

@@ -0,0 +1,36 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_flow_task_template")
@Comment("流程任务模板")
public class FlowTaskTemplate extends SimpleEntity {
@Comment("模板名称")
@Column(nullable = false)
private String name;
@Comment("模板功能、内容说明")
private String description;
@Comment("模板入参Schema")
@Column(columnDefinition = "longtext")
private String inputSchema;
@Comment("前端流程图数据")
@Column(nullable = false, columnDefinition = "longtext")
private String flowGraph = "{}";
}

View File

@@ -0,0 +1,53 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.ConstraintMode;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* @author lanyuanxiaoyao
* @version 20250527
*/
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_group")
@Comment("知识库内的逻辑分组,比如一个文件是一个分组或一次上传的所有文本是一个分组,可以自由使用而不是限于文件范畴")
public class Group extends SimpleEntity {
@Comment("分组名称")
@Column(nullable = false)
private String name;
@Comment("分组处理状态")
@Column(nullable = false)
@Enumerated(EnumType.STRING)
private Status status = Status.RUNNING;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
@ToString.Exclude
private Knowledge knowledge;
public enum Status {
RUNNING,
FINISHED,
}
}

View File

@@ -0,0 +1,58 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.Comment;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* @author lanyuanxiaoyao
* @version 20250522
*/
@Getter
@Setter
@ToString
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_knowledge")
@Comment("知识库")
public class Knowledge extends SimpleEntity {
@Comment("知识库对应的向量库名")
@Column(nullable = false)
private Long vectorSourceId;
@Comment("知识库名称")
@Column(nullable = false)
private String name;
@Comment("知识库说明")
@Column(nullable = false, columnDefinition = "longtext")
private String description;
@Comment("知识库策略")
@Column(nullable = false)
@Enumerated(EnumType.STRING)
private Strategy strategy = Strategy.Cosine;
@Comment("知识库下包含的分组")
@OneToMany(fetch = FetchType.LAZY, mappedBy = "knowledge", cascade = CascadeType.ALL)
@ToString.Exclude
private Set<Group> groups;
public enum Strategy {
Cosine,
Euclid,
}
}

View File

@@ -0,0 +1,39 @@
package com.lanyuanxiaoyao.service.ai.web.entity.context;
import java.util.List;
import java.util.Map;
import lombok.Builder;
import lombok.Data;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.springframework.ai.document.Document;
/**
* @author lanyuanxiaoyao
* @version 20250523
*/
@Data
@Builder
public class EmbeddingContext {
private Long vectorSourceId;
private Long groupId;
private Config config;
private String content;
private String file;
private String fileFormat;
@Builder.Default
private List<Document> documents = Lists.mutable.empty();
@Builder.Default
private Map<String, Object> metadata = Maps.mutable.empty();
@Data
@Builder
public static final class Config {
@Builder.Default
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
public enum SplitStrategy {
NORMAL, LLM, QA
}
}
}

View File

@@ -0,0 +1,17 @@
package com.lanyuanxiaoyao.service.ai.web.entity.context;
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
import java.util.List;
import lombok.Data;
import org.eclipse.collections.api.factory.Lists;
/**
* @author lanyuanxiaoyao
* @version 20250616
*/
@Data
public class FeedbackContext {
private Feedback feedback;
private String optimizedSource;
private List<String> pictureDescriptions = Lists.mutable.empty();
}

View File

@@ -0,0 +1,14 @@
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowEdge;
import com.lanyuanxiaoyao.service.ai.web.engine.entity.FlowNode;
import lombok.Data;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.set.ImmutableSet;
@Data
public class FlowGraphVo {
private ImmutableSet<FlowNode> nodes;
private ImmutableSet<FlowEdge> edges;
private MutableMap<String, MutableMap<String, Object>> data;
}

View File

@@ -0,0 +1,14 @@
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
import lombok.Data;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
@Data
public class MessageVO {
private String role;
private String content;
private String reason;
}

Some files were not shown because too many files have changed in this diff Show More