Compare commits
87 Commits
fce4816880
...
jpa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c9616eb63a | ||
|
|
5b3c27ea48 | ||
| e48d7e8649 | |||
|
|
306c20aa7f | ||
|
|
24d5d10ecb | ||
|
|
4a9a9ec238 | ||
|
|
08aa1d8382 | ||
|
|
1b3045dfd4 | ||
|
|
0f5ae1c4d4 | ||
|
|
48e42ee99a | ||
|
|
0914b458d3 | ||
|
|
368c30676e | ||
|
|
60477f99f5 | ||
|
|
565c530dd5 | ||
|
|
5130885033 | ||
|
|
8e6463845b | ||
|
|
e89bffe289 | ||
|
|
1dd00d329c | ||
| e470a87372 | |||
|
|
45da452f18 | ||
|
|
e6a1bc5383 | ||
|
|
c5916703cd | ||
|
|
807ddbe5cb | ||
|
|
13de694e37 | ||
|
|
1962dd586c | ||
| 138ee140e1 | |||
| e2d69bc6e8 | |||
| b9d707dc8f | |||
| 44d1473c6b | |||
| 9c658afbd7 | |||
| e3f86e6497 | |||
| 256c8c6bd5 | |||
| b627c91acb | |||
| 7fb490778a | |||
| d4d5aede31 | |||
|
|
f11f5e7656 | ||
|
|
bc32a89fea | ||
|
|
2e24bdb90b | ||
|
|
5160c59ab0 | ||
|
|
506e28c9f7 | ||
| 9076bd4c09 | |||
| 69f0bed9a1 | |||
| c04269c3fa | |||
| 4fe21f3d8b | |||
| cb42376e46 | |||
| 34bdb59501 | |||
|
|
72c23d916a | ||
|
|
e01a883d37 | ||
|
|
951075fc9f | ||
|
|
90fea22de5 | ||
|
|
a35980a5f4 | ||
|
|
e359bed97c | ||
|
|
8b4827b164 | ||
|
|
577834568b | ||
|
|
c4d5a7b300 | ||
|
|
4124a8a851 | ||
|
|
6d4dedc3f4 | ||
|
|
b8aea3bdf0 | ||
|
|
d36ad95a85 | ||
|
|
fdec62b56e | ||
| 1217d114bd | |||
| 2d7b30bb7a | |||
|
|
c2af2d6365 | ||
|
|
536c4e9cab | ||
|
|
c9a1ea2be5 | ||
|
|
602a337923 | ||
|
|
fe9e185a9a | ||
|
|
3901a47da0 | ||
|
|
dc5998cf72 | ||
|
|
993940e810 | ||
|
|
b8cc8fee67 | ||
|
|
2cac589b0f | ||
|
|
29859664e3 | ||
|
|
0ceb5d7fc3 | ||
|
|
947c831609 | ||
|
|
ce95ec7444 | ||
|
|
95214f7af3 | ||
|
|
0f49c91fde | ||
|
|
0262c573ae | ||
|
|
8c9cb6f21d | ||
|
|
e6e24dff27 | ||
|
|
fc2ea107d2 | ||
|
|
6f9c898d51 | ||
|
|
7fd484eeab | ||
|
|
3ee6303cf5 | ||
|
|
f7ed3bd270 | ||
|
|
e57c81ce75 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -109,3 +109,4 @@ Icon
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
**/temp/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
@@ -9,7 +9,7 @@ try {
|
||||
await run_deploy_root()
|
||||
await run_deploy_batch(['service-cli', 'service-cli/service-cli-core'])
|
||||
await run_package('service-cli/service-cli-runner')
|
||||
await run_upload_normal('service-cli-runner', 'service-cli')
|
||||
await run_upload('**/service-cli-runner-1.0.0-SNAPSHOT.jar')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
@@ -9,7 +9,7 @@ try {
|
||||
await run_deploy_root()
|
||||
await run_deploy_batch(['service-common', 'service-dependencies', 'service-configuration', 'service-forest', 'service-executor', 'service-executor/service-executor-core'])
|
||||
await run_package('service-executor/service-executor-manager')
|
||||
await run_upload_normal('service-executor-manager', 'service-executor')
|
||||
await run_upload('**/service-executor-manager-1.0.0-SNAPSHOT.jar')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
@@ -9,7 +9,7 @@ try {
|
||||
await run_deploy_root()
|
||||
await run_deploy_batch(['service-common', 'service-dependencies', 'service-configuration', 'service-forest', 'service-executor', 'service-executor/service-executor-core'])
|
||||
await run_package('service-executor/service-executor-task')
|
||||
await run_upload_normal('service-executor-task', 'service-executor')
|
||||
await run_upload('**/service-executor-task-1.0.0-SNAPSHOT.jar')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
13
bin/build-forest.js
Normal file
13
bin/build-forest.js
Normal file
@@ -0,0 +1,13 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy_root()
|
||||
await run_deploy_batch(['service-common', 'service-dependencies', 'service-configuration', 'service-forest'])
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
@@ -9,7 +9,7 @@ try {
|
||||
await run_deploy_root()
|
||||
await run_deploy_batch(['service-common'])
|
||||
await run_package('utils/patch')
|
||||
await run_upload_normal('**/patch-1.0.0-SNAPSHOT.jar')
|
||||
await run_upload('**/patch-1.0.0-SNAPSHOT.jar')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload, run_upload_normal} from "./library.js";
|
||||
import {run_deploy_batch, run_deploy_root, run_package, run_upload_normal} from "./library.js";
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
|
||||
@@ -4,7 +4,7 @@ jars_path=/data/datalake/jars
|
||||
jdk_path=/opt/jdk1.8.0_162/bin/java
|
||||
|
||||
arguments=$@
|
||||
# 手动上传jar包则注释掉这行,更显神通吧反正是
|
||||
# 手动上传jar包则注释掉这行,各显神通吧反正是
|
||||
curl http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.124:36800/file/download/service-cli-runner-1.0.0-SNAPSHOT.jar -o ${jars_path}/service-cli-runner.jar
|
||||
${jdk_path} -jar ${jars_path}/service-cli-runner.jar \
|
||||
--spring.profiles.active=b12 \
|
||||
|
||||
@@ -1,17 +1,24 @@
|
||||
import {$, fetch, fs, glob, os, path, spinner, syncProcessCwd, usePowerShell} from 'zx'
|
||||
import {isEqual, trim} from "licia";
|
||||
import {isEqual, trim, fileSize} from "licia";
|
||||
import md5file from 'md5-file'
|
||||
|
||||
syncProcessCwd(true)
|
||||
if (isEqual(os.platform(), 'win32')) {
|
||||
usePowerShell()
|
||||
}
|
||||
|
||||
const maven_setting = '/Users/lanyuanxiaoyao/.m2/settings-nas.xml'
|
||||
const maven_setting = path.join(os.homedir(), '.m2', 'settings-nas.xml')
|
||||
|
||||
const upload_url = 'http://132.126.207.124:36800'
|
||||
const upload_username = 'AxhEbscwsJDbYMH2'
|
||||
const upload_password = 'cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4'
|
||||
|
||||
/**
|
||||
* 时间戳转自然语言
|
||||
*
|
||||
* @param timestamp 时间戳
|
||||
* @returns {string} 自然语言描述的时间
|
||||
*/
|
||||
const millisecondToString = (timestamp) => {
|
||||
const totalSeconds = Math.floor(parseFloat(timestamp) / 1000)
|
||||
if (isNaN(totalSeconds) || totalSeconds < 0) {
|
||||
@@ -39,7 +46,7 @@ export const run_deploy = async (project) => {
|
||||
`Deploying project ${project}`,
|
||||
() => $`mvn -pl ${project} clean deploy -D skipTests -s ${maven_setting}`
|
||||
)
|
||||
console.log(`✅ Finish deploy ${project} (${millisecondToString(output['duration'])})`)
|
||||
console.log(`✅ Finished deploy ${project} (${millisecondToString(output['duration'])})`)
|
||||
}
|
||||
|
||||
export const run_deploy_root = async () => {
|
||||
@@ -47,7 +54,7 @@ export const run_deploy_root = async () => {
|
||||
`Deploying root`,
|
||||
() => $`mvn clean deploy -N -D skipTests -s ${maven_setting}`
|
||||
)
|
||||
console.log(`✅ Finish deploy root (${millisecondToString(output['duration'])})`)
|
||||
console.log(`✅ Finished deploy root (${millisecondToString(output['duration'])})`)
|
||||
}
|
||||
|
||||
export const run_deploy_batch = async (projects) => {
|
||||
@@ -61,7 +68,7 @@ export const run_package = async (project, profile = 'b2b12') => {
|
||||
`Packaging project ${project}${isEqual(profile, 'b2b12') ? '' : ` ${profile}`}`,
|
||||
() => $`mvn -pl ${project} clean package -D skipTests -P ${profile} -s ${maven_setting}`
|
||||
)
|
||||
console.log(`✅ Finish package ${project}${isEqual(profile, 'b2b12') ? '' : ` ${profile}`} (${millisecondToString(output['duration'])})`)
|
||||
console.log(`✅ Finished package ${project}${isEqual(profile, 'b2b12') ? '' : ` ${profile}`} (${millisecondToString(output['duration'])})`)
|
||||
}
|
||||
|
||||
export const run_package_batch = async (projects) => {
|
||||
@@ -70,10 +77,12 @@ export const run_package_batch = async (projects) => {
|
||||
}
|
||||
}
|
||||
|
||||
const upload = async (file_path) => {
|
||||
export const upload = async (file_path) => {
|
||||
let start = new Date().getTime()
|
||||
let basename = path.basename(file_path)
|
||||
let response = await spinner(
|
||||
`Uploading project ${file_path}`,
|
||||
() => fetch(`${upload_url}/file/upload/${path.basename(file_path)}`, {
|
||||
() => fetch(`${upload_url}/file/upload/${basename}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
@@ -87,7 +96,10 @@ const upload = async (file_path) => {
|
||||
if (!isEqual(response.status, 200)) {
|
||||
throw response
|
||||
}
|
||||
console.log(`✅ Finish upload ${file_path}`)
|
||||
console.log(`✅ Finished upload ${file_path} (${millisecondToString((new Date().getTime()) - start)})`)
|
||||
console.log(`📘 Uploaded ${fileSize(fs.statSync(file_path).size)}`)
|
||||
console.log(`📘 MD5 ${md5file.sync(file_path)}`)
|
||||
console.log(`📘 Download curl http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.124:36800/file/download/${basename} -o ${basename}`)
|
||||
fs.rmSync(file_path)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
maven_setting=/Users/lanyuanxiaoyao/.m2/settings-nas.xml
|
||||
|
||||
build_profile=b2b12
|
||||
|
||||
upload_username=AxhEbscwsJDbYMH2
|
||||
upload_password=cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
|
||||
upload_url=http://$upload_username:$upload_password@132.126.207.124:36800
|
||||
|
||||
root_path=$(dirname $(cd $(dirname $0);pwd))
|
||||
|
||||
function upload() {
|
||||
source_file_path=$(realpath $1)
|
||||
file_name=$(basename $source_file_path)
|
||||
echo "↪ Source md5: $(md5sum $source_file_path | awk '{print $1}')"
|
||||
echo "↪ Uploading $source_file_path"
|
||||
curl $upload_url/file/upload/$file_name -T $source_file_path
|
||||
echo "↪ Upload ytp success"
|
||||
echo "↪ Download: curl $upload_url/file/download/$file_name -o $file_name"
|
||||
echo "↪ Delete source"
|
||||
rm $source_file_path
|
||||
}
|
||||
|
||||
function joining {
|
||||
local d=${1-} f=${2-}
|
||||
if shift 2; then
|
||||
printf %s "$f" "${@/#/$d}"
|
||||
fi
|
||||
}
|
||||
|
||||
function deploy_root() {
|
||||
mvn deploy -N -D skipTests -s $maven_setting
|
||||
}
|
||||
|
||||
function deploy() {
|
||||
mvn -pl $(joining , $@) clean deploy -D skipTests -s $maven_setting
|
||||
}
|
||||
|
||||
function package() {
|
||||
mvn -pl $(joining , $@) clean package -D skipTests -P $build_profile -s $maven_setting
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
"fs-extra": "^11.3.0",
|
||||
"globby": "^14.1.0",
|
||||
"licia": "^1.48.0",
|
||||
"md5-file": "^5.0.0",
|
||||
"minimist": "^1.2.8",
|
||||
"node-fetch-native": "^1.6.6",
|
||||
"which": "^5.0.0",
|
||||
|
||||
10
bin/pnpm-lock.yaml
generated
10
bin/pnpm-lock.yaml
generated
@@ -26,6 +26,9 @@ importers:
|
||||
licia:
|
||||
specifier: ^1.48.0
|
||||
version: 1.48.0
|
||||
md5-file:
|
||||
specifier: ^5.0.0
|
||||
version: 5.0.0
|
||||
minimist:
|
||||
specifier: ^1.2.8
|
||||
version: 1.2.8
|
||||
@@ -129,6 +132,11 @@ packages:
|
||||
licia@1.48.0:
|
||||
resolution: {integrity: sha512-bBWiT5CSdEtwuAHiYTJ74yItCjIFdHi4xiFk6BRDfKa+sdCpkUHp69YKb5udNOJlHDzFjNjcMgNZ/+wQIHrB8A==}
|
||||
|
||||
md5-file@5.0.0:
|
||||
resolution: {integrity: sha512-xbEFXCYVWrSx/gEKS1VPlg84h/4L20znVIulKw6kMfmBUAZNAnF00eczz9ICMl+/hjQGo5KSXRxbL/47X3rmMw==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
hasBin: true
|
||||
|
||||
merge2@1.4.1:
|
||||
resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
|
||||
engines: {node: '>= 8'}
|
||||
@@ -283,6 +291,8 @@ snapshots:
|
||||
|
||||
licia@1.48.0: {}
|
||||
|
||||
md5-file@5.0.0: {}
|
||||
|
||||
merge2@1.4.1: {}
|
||||
|
||||
micromatch@4.0.8:
|
||||
|
||||
10
pom.xml
10
pom.xml
@@ -417,6 +417,16 @@
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.4.2</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-invoker-plugin</artifactId>
|
||||
<version>3.9.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import {cd, path} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {run_deploy, run_package, run_upload} from '../../bin/library.js'
|
||||
import {run_deploy} from '../../bin/library.js'
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy('service-ai-core')
|
||||
await run_package('service-ai-chat')
|
||||
await run_upload('service-ai-chat')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
21
service-ai/bin/build-ai-web.js
Normal file
21
service-ai/bin/build-ai-web.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import {
|
||||
cd,
|
||||
path,
|
||||
} from 'zx'
|
||||
import {trim} from "licia";
|
||||
import {
|
||||
run_deploy,
|
||||
run_package,
|
||||
run_upload_normal,
|
||||
} from '../../bin/library.js'
|
||||
|
||||
// 切换目录
|
||||
cd(trim(path.dirname(import.meta.dirname)))
|
||||
// 执行流程
|
||||
try {
|
||||
await run_deploy('service-ai-core')
|
||||
await run_package('service-ai-web')
|
||||
await run_upload_normal('service-ai-web')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
"fs-extra": "^11.3.0",
|
||||
"globby": "^14.1.0",
|
||||
"licia": "^1.48.0",
|
||||
"md5-file": "^5.0.0",
|
||||
"minimist": "^1.2.8",
|
||||
"node-fetch-native": "^1.6.6",
|
||||
"which": "^5.0.0",
|
||||
|
||||
10
service-ai/bin/pnpm-lock.yaml
generated
10
service-ai/bin/pnpm-lock.yaml
generated
@@ -26,6 +26,9 @@ importers:
|
||||
licia:
|
||||
specifier: ^1.48.0
|
||||
version: 1.48.0
|
||||
md5-file:
|
||||
specifier: ^5.0.0
|
||||
version: 5.0.0
|
||||
minimist:
|
||||
specifier: ^1.2.8
|
||||
version: 1.2.8
|
||||
@@ -129,6 +132,11 @@ packages:
|
||||
licia@1.48.0:
|
||||
resolution: {integrity: sha512-bBWiT5CSdEtwuAHiYTJ74yItCjIFdHi4xiFk6BRDfKa+sdCpkUHp69YKb5udNOJlHDzFjNjcMgNZ/+wQIHrB8A==}
|
||||
|
||||
md5-file@5.0.0:
|
||||
resolution: {integrity: sha512-xbEFXCYVWrSx/gEKS1VPlg84h/4L20znVIulKw6kMfmBUAZNAnF00eczz9ICMl+/hjQGo5KSXRxbL/47X3rmMw==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
hasBin: true
|
||||
|
||||
merge2@1.4.1:
|
||||
resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
|
||||
engines: {node: '>= 8'}
|
||||
@@ -283,6 +291,8 @@ snapshots:
|
||||
|
||||
licia@1.48.0: {}
|
||||
|
||||
md5-file@5.0.0: {}
|
||||
|
||||
merge2@1.4.1: {}
|
||||
|
||||
micromatch@4.0.8:
|
||||
|
||||
18
service-ai/database/service_ai_feedback.sql
Normal file
18
service-ai/database/service_ai_feedback.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE `service_ai_feedback`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`analysis` longtext,
|
||||
`conclusion` longtext,
|
||||
`source` longtext NOT NULL,
|
||||
`status` tinyint NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
|
||||
CREATE TABLE `service_ai_feedback_pictures`
|
||||
(
|
||||
`feedback_id` bigint NOT NULL,
|
||||
`pictures_id` bigint NOT NULL,
|
||||
PRIMARY KEY (`feedback_id`, `pictures_id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
12
service-ai/database/service_ai_file.sql
Normal file
12
service-ai/database/service_ai_file.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE `service_ai_file`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`filename` varchar(255) DEFAULT NULL,
|
||||
`md5` varchar(255) DEFAULT NULL,
|
||||
`path` varchar(255) DEFAULT NULL,
|
||||
`size` bigint DEFAULT NULL,
|
||||
`type` varchar(255) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
10
service-ai/database/service_ai_group.sql
Normal file
10
service-ai/database/service_ai_group.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE `service_ai_group`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`name` varchar(255) NOT NULL,
|
||||
`status` tinyint NOT NULL,
|
||||
`knowledge_id` bigint NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET=utf8mb4;
|
||||
11
service-ai/database/service_ai_knowledge.sql
Normal file
11
service-ai/database/service_ai_knowledge.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE `service_ai_knowledge`
|
||||
(
|
||||
`id` bigint NOT NULL,
|
||||
`created_time` datetime(6) DEFAULT NULL,
|
||||
`modified_time` datetime(6) DEFAULT NULL,
|
||||
`description` longtext NOT NULL,
|
||||
`name` varchar(255) NOT NULL,
|
||||
`strategy` tinyint NOT NULL,
|
||||
`vector_source_id` bigint NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARSET = utf8mb4;
|
||||
@@ -11,8 +11,8 @@
|
||||
<description>Hudi AI服务集合</description>
|
||||
<modules>
|
||||
<module>service-ai-core</module>
|
||||
<module>service-ai-chat</module>
|
||||
<module>service-ai-knowledge</module>
|
||||
<module>service-ai-web</module>
|
||||
<module>service-ai-cli</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
@@ -24,12 +24,26 @@
|
||||
|
||||
<spring-boot.version>3.4.3</spring-boot.version>
|
||||
<spring-cloud.version>2024.0.1</spring-cloud.version>
|
||||
<spring-ai.version>1.0.0-RC1</spring-ai.version>
|
||||
<spring-ai.version>1.0.0</spring-ai.version>
|
||||
<solon-ai.version>3.3.1</solon-ai.version>
|
||||
<eclipse-collections.version>11.1.0</eclipse-collections.version>
|
||||
<curator.version>5.1.0</curator.version>
|
||||
<hutool.version>5.8.27</hutool.version>
|
||||
<mapstruct.version>1.6.3</mapstruct.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- 当前项目依赖 -->
|
||||
@@ -96,6 +110,11 @@
|
||||
<artifactId>jasypt-spring-boot-starter</artifactId>
|
||||
<version>3.0.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.blinkfox</groupId>
|
||||
<artifactId>fenix-spring-boot-starter</artifactId>
|
||||
<version>3.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- 日志相关 -->
|
||||
<dependency>
|
||||
@@ -136,12 +155,32 @@
|
||||
<artifactId>liteflow-spring-boot-starter</artifactId>
|
||||
<version>2.13.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.noear</groupId>
|
||||
<artifactId>solon-ai</artifactId>
|
||||
<version>${solon-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.noear</groupId>
|
||||
<artifactId>solon-ai-dialect-openai</artifactId>
|
||||
<version>${solon-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.14.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
@@ -150,7 +189,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<version>3.6.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
public class AiChatApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(AiChatApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.controller;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.entity.MessageVO;
|
||||
import com.lanyuanxiaoyao.service.ai.chat.tools.DatetimeTools;
|
||||
import java.io.IOException;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
/**
|
||||
* 聊天
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping("chat")
|
||||
public class ChatController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChatController.class);
|
||||
|
||||
private final ChatClient chatClient;
|
||||
|
||||
public ChatController(ChatClient.Builder builder) {
|
||||
this.chatClient = builder.build();
|
||||
}
|
||||
|
||||
private ChatClient.ChatClientRequestSpec buildRequest(ImmutableList<MessageVO> messages) {
|
||||
return chatClient.prompt()
|
||||
.system("""
|
||||
你是一名专业的AI运维助手,负责“Hudi数据同步服务平台”的运维工作;
|
||||
你将会友好地帮助用户解答关于该平台运维工作的问题,你会尽可能通过各种方式获取知识和数据来解答;
|
||||
对于无法通过已有知识回答的问题,你会提示用户你无法解答该问题,而不是虚构不存在的数据或答案;
|
||||
对于与该平台无关的问题,你会委婉地拒绝用户,并提示无法回答;
|
||||
你将始终在中文语境下进行对话。
|
||||
""")
|
||||
.tools(new DatetimeTools())
|
||||
.messages(
|
||||
messages
|
||||
.collect(message -> StrUtil.equals(message.getRole(), "assistant")
|
||||
? new AssistantMessage(message.getContent())
|
||||
: new UserMessage(message.getContent()))
|
||||
.collect(message -> (Message) message)
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
|
||||
@PostMapping("sync")
|
||||
@ResponseBody
|
||||
public String chatSync(@RequestBody ImmutableList<MessageVO> messages) {
|
||||
String content = buildRequest(messages)
|
||||
.call()
|
||||
.content();
|
||||
return StrUtil.trimToEmpty(content);
|
||||
}
|
||||
|
||||
@PostMapping("async")
|
||||
public SseEmitter chatAsync(@RequestBody ImmutableList<MessageVO> messages) {
|
||||
SseEmitter emitter = new SseEmitter();
|
||||
buildRequest(messages)
|
||||
.stream()
|
||||
.content()
|
||||
.subscribe(
|
||||
content -> {
|
||||
try {
|
||||
emitter.send(content);
|
||||
} catch (IOException e) {
|
||||
emitter.completeWithError(e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
},
|
||||
emitter::completeWithError,
|
||||
emitter::complete
|
||||
);
|
||||
return emitter;
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.entity;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class MessageVO {
|
||||
private String role;
|
||||
private String content;
|
||||
|
||||
public String getRole() {
|
||||
return role;
|
||||
}
|
||||
|
||||
public void setRole(String role) {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public void setContent(String content) {
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "MessageVO{" +
|
||||
"role='" + role + '\'' +
|
||||
", content='" + content + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat.tools;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class DatetimeTools {
|
||||
private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
@Tool(description = "获取当前日期和时间")
|
||||
public String getCurrentDateTime() {
|
||||
return LocalDateTime.now().format(formatter);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-chat
|
||||
profiles:
|
||||
include: random-port,common,discovery,metrics,forest
|
||||
ai:
|
||||
openai:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
options:
|
||||
model: 'Qwen3-1.7-vllm'
|
||||
mvc:
|
||||
async:
|
||||
request-timeout: 300000
|
||||
@@ -1,37 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.chat;
|
||||
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import reactor.core.Disposable;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
public class TestChat {
|
||||
public static void main(String[] args) {
|
||||
ChatClient client = ChatClient.builder(
|
||||
OpenAiChatModel.builder()
|
||||
.openAiApi(
|
||||
OpenAiApi.builder()
|
||||
.baseUrl("http://132.121.206.65:10086")
|
||||
.apiKey("*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4")
|
||||
.build()
|
||||
)
|
||||
.defaultOptions(
|
||||
OpenAiChatOptions.builder()
|
||||
.model("Qwen3-1.7")
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
String content = client.prompt()
|
||||
.user("你好")
|
||||
.call()
|
||||
.content();
|
||||
System.out.println(content);
|
||||
}
|
||||
}
|
||||
@@ -9,16 +9,20 @@
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-ai-chat</artifactId>
|
||||
<artifactId>service-ai-cli</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai-core</artifactId>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
||||
<groupId>cn.hutool</groupId>
|
||||
<artifactId>hutool-all</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.freemarker</groupId>
|
||||
<artifactId>freemarker</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -38,4 +42,5 @@
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
@SpringBootApplication
|
||||
public class Generator implements ApplicationRunner {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(Generator.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) throws Exception {
|
||||
new LlamaSwapTool().generate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.extra.template.Template;
|
||||
import cn.hutool.extra.template.TemplateConfig;
|
||||
import cn.hutool.extra.template.TemplateEngine;
|
||||
import cn.hutool.extra.template.TemplateUtil;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
public abstract class GeneratorTool {
|
||||
private final TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
|
||||
|
||||
protected void generateTemplate(String templatePath, Map<?, ?> data, String targetScriptPath) {
|
||||
Template template = engine.getTemplate(templatePath);
|
||||
String script = template.render(data);
|
||||
FileUtil.del(targetScriptPath);
|
||||
FileUtil.writeString(script, targetScriptPath, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public abstract void generate() throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package com.lanyuanxiaoyao.service.ai.cli;
|
||||
|
||||
import cn.hutool.core.collection.ListUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250612
|
||||
*/
|
||||
public class LlamaSwapTool extends GeneratorTool {
|
||||
private static final String API_KEY = "*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4";
|
||||
|
||||
public static String displayName(String name) {
|
||||
return name.replaceAll("\\s+", "_")
|
||||
.replaceAll("\\.", "_")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void generate() {
|
||||
generateTemplate(
|
||||
"llama-swap.ftl",
|
||||
Map.of(
|
||||
"models", List.of(
|
||||
llamaCppEmbeddingCmd("BGE/bge-m3-q4km", "bge-m3-Q4_K_M.gguf", 20),
|
||||
vllmEmbeddingCmd("BGE/bge-m3", "bge-m3", 5),
|
||||
llamaCppRerankerCmd("BGE/beg-reranker-v2-q4km", "bge-reranker-v2-m3-Q4_K_M.gguf", 20),
|
||||
vllmRerankerCmd("BGE/beg-reranker-v2", "bge-reranker-v2-m3", 5),
|
||||
|
||||
vllmCmd("Qwen3/qwen3-0.6b", "Qwen3-0.6B", 5, true),
|
||||
vllmCmd("Qwen3/qwen3-1.7b", "Qwen3-1.7B", 5, true),
|
||||
vllmCmd("Qwen3/qwen3-4b", "Qwen3-4B", 8, true),
|
||||
llamaCppCmd("Qwen3/qwen3-4b-q4km", "Qwen3-4B-Q4_K_M.gguf", 35),
|
||||
llamaCppCmd("Qwen3/qwen3-8b-q4km", "Qwen3-8B-Q4_K_M.gguf", 35),
|
||||
|
||||
vllmEmbeddingCmd("Qwen3/qwen3-embedding-0.6b", "Qwen3-Embedding-0.6B", 5),
|
||||
vllmEmbeddingCmd("Qwen3/qwen3-embedding-4b", "Qwen3-Embedding-4B", 8),
|
||||
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-4b-q4km", "Qwen3-Embedding-4B-Q4_K_M.gguf", 35),
|
||||
llamaCppEmbeddingCmd("Qwen3/qwen3-embedding-8b-q4km", "Qwen3-Embedding-8B-Q4_K_M.gguf", 35),
|
||||
|
||||
// 0.9.1 vllm还未支持
|
||||
// vllmRerankerCmd("Qwen3/qwen3-reranker-0.6b", "Qwen3-Reranker-0.6B", 5),
|
||||
// vllmRerankerCmd("Qwen3/qwen3-reranker-4b", "Qwen3-Reranker-4B", 8),
|
||||
|
||||
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b", "Qwen2.5-VL-7B-Instruct-BF16.gguf", 35),
|
||||
llamaCppVisualCmd("Qwen2.5/qwen2.5-vl-7b-q4km", "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", 35),
|
||||
vllmCmd("Qwen2.5/qwen2.5-vl-3b-instruct", "Qwen2.5-VL-3B-Instruct", 8, false),
|
||||
vllmCmd("Qwen2.5/qwen2.5-vl-7b-instruct", "Qwen2.5-VL-7B-Instruct", 8, false),
|
||||
|
||||
llamaCppVisualCmd("MiniCPM/minicpm-o-2.6-7.6b-q4km", "MiniCPM-o-2_6-7.6B-Q4_K_M.gguf", 35),
|
||||
vllmCmd("MiniCPM/minicpm-o-2.6-7.6b", "MiniCPM-o-2_6", 10, false)
|
||||
)
|
||||
),
|
||||
"config.yaml"
|
||||
);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppEmbeddingCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, true, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppRerankerCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, true, false);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppVisualCmd(String name, String model, Integer thread) {
|
||||
return llamaCppCmd(name, model, thread, false, false, true);
|
||||
}
|
||||
|
||||
private DockerCmd llamaCppCmd(String name, String model, Integer thread, Boolean isEmbedding, Boolean isReranker, Boolean isVisual) {
|
||||
List<String> arguments = ListUtil.list(
|
||||
false,
|
||||
StrUtil.format("-m /models/{}", model),
|
||||
"--port ${PORT}",
|
||||
StrUtil.format("--api-key {}", API_KEY),
|
||||
"-c 0",
|
||||
"-b 4096",
|
||||
StrUtil.format("-t {}", thread),
|
||||
"-np 5",
|
||||
"--log-disable",
|
||||
"--no-webui"
|
||||
);
|
||||
if (isEmbedding) {
|
||||
arguments.add("--embedding");
|
||||
arguments.add("-ub 8192");
|
||||
arguments.add("--pooling mean");
|
||||
} else if (isReranker) {
|
||||
arguments.add("--reranking");
|
||||
} else if (isVisual) {
|
||||
arguments.add(StrUtil.format("--mmproj /models/{}.mmproj", model));
|
||||
} else {
|
||||
arguments.add("--jinja");
|
||||
}
|
||||
return new DockerCmd(
|
||||
"ghcr.io/ggml-org/llama.cpp:server",
|
||||
name,
|
||||
model,
|
||||
StrUtil.format("http://llamacpp-{}:${PORT}", displayName(model)),
|
||||
List.of(StrUtil.format("--name llamacpp-{}", displayName(model))),
|
||||
arguments
|
||||
);
|
||||
}
|
||||
|
||||
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isReasonable) {
|
||||
return vllmCmd(name, model, cache, false, false, isReasonable);
|
||||
}
|
||||
|
||||
private DockerCmd vllmEmbeddingCmd(String name, String model, Integer cache) {
|
||||
return vllmCmd(name, model, cache, true, false, false);
|
||||
}
|
||||
|
||||
private DockerCmd vllmRerankerCmd(String name, String model, Integer cache) {
|
||||
return vllmCmd(name, model, cache, false, true, false);
|
||||
}
|
||||
|
||||
private DockerCmd vllmVisualCmd(String name, String model, Integer cache, Boolean isReasonable) {
|
||||
return vllmCmd(name, model, cache, false, false, isReasonable);
|
||||
}
|
||||
|
||||
private DockerCmd vllmCmd(String name, String model, Integer cache, Boolean isEmbedding, Boolean isReranker, Boolean isReasonable) {
|
||||
List<String> arguments = ListUtil.list(
|
||||
false,
|
||||
StrUtil.format("--model /models/{}", model),
|
||||
StrUtil.format("--served-model-name {}", name),
|
||||
"--port ${PORT}",
|
||||
StrUtil.format("--api-key {}", API_KEY),
|
||||
"--disable-log-requests",
|
||||
"--uvicorn-log-level error",
|
||||
"--trust-remote-code"
|
||||
);
|
||||
if (isEmbedding) {
|
||||
arguments.add("--task embedding");
|
||||
} else if (isReranker) {
|
||||
arguments.add("--task score");
|
||||
} else if (isReasonable) {
|
||||
arguments.add("--enable-auto-tool-choice");
|
||||
arguments.add("--tool-call-parser hermes");
|
||||
arguments.add("--enable-reasoning");
|
||||
arguments.add("--reasoning-parser deepseek_r1");
|
||||
}
|
||||
return new DockerCmd(
|
||||
"vllm-server-cpu:0.8.5.post1",
|
||||
name,
|
||||
model,
|
||||
StrUtil.format("http://vllm-{}:${PORT}", displayName(model)),
|
||||
List.of(
|
||||
StrUtil.format("--name vllm-{}", displayName(model)),
|
||||
"--privileged=true",
|
||||
"--shm-size=4g",
|
||||
StrUtil.format("-e VLLM_CPU_KVCACHE_SPACE={}", cache)
|
||||
),
|
||||
arguments
|
||||
);
|
||||
}
|
||||
|
||||
public static class DockerCmd {
|
||||
private String image;
|
||||
private String name;
|
||||
private String model;
|
||||
private String proxy;
|
||||
private List<String> options = ListUtil.list(
|
||||
false,
|
||||
"--rm",
|
||||
"--network llama",
|
||||
"-v /data/models:/models"
|
||||
);
|
||||
private List<String> arguments = ListUtil.list(false);
|
||||
|
||||
public DockerCmd(String image, String name, String model, String proxy, List<String> options, List<String> arguments) {
|
||||
this.image = image;
|
||||
this.name = name;
|
||||
this.model = model;
|
||||
this.proxy = proxy;
|
||||
this.options.addAll(options);
|
||||
this.arguments.addAll(arguments);
|
||||
}
|
||||
|
||||
public String getImage() {
|
||||
return image;
|
||||
}
|
||||
|
||||
public void setImage(String image) {
|
||||
this.image = image;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public String getProxy() {
|
||||
return proxy;
|
||||
}
|
||||
|
||||
public void setProxy(String proxy) {
|
||||
this.proxy = proxy;
|
||||
}
|
||||
|
||||
public List<String> getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public void setOptions(List<String> options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public List<String> getArguments() {
|
||||
return arguments;
|
||||
}
|
||||
|
||||
public void setArguments(List<String> arguments) {
|
||||
this.arguments = arguments;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DockerCmd{" +
|
||||
"image='" + image + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", model='" + model + '\'' +
|
||||
", proxy='" + proxy + '\'' +
|
||||
", options=" + options +
|
||||
", arguments=" + arguments +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
healthCheckTimeout: 600
|
||||
logLevel: warn
|
||||
models:
|
||||
<#list models as model>
|
||||
"${model.name}":
|
||||
proxy: ${model.proxy}
|
||||
ttl: 86400
|
||||
cmd: |
|
||||
docker run
|
||||
<#list model.options as option>
|
||||
${option}
|
||||
</#list>
|
||||
${model.image}
|
||||
<#list model.arguments as arg>
|
||||
${arg}
|
||||
</#list>
|
||||
</#list>
|
||||
groups:
|
||||
"persistent":
|
||||
swap: false
|
||||
exclusive: false
|
||||
members:
|
||||
<#list models as model>
|
||||
- "${model.name}"
|
||||
</#list>
|
||||
@@ -17,22 +17,30 @@ import org.springframework.web.reactive.function.client.WebClient;
|
||||
*/
|
||||
@Configuration
|
||||
public class WebClientConfiguration {
|
||||
private HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public RestClient.Builder restClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
return generateRestClientBuilder();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public WebClient.Builder webClientBuilder() {
|
||||
return generateWebClientBuilder();
|
||||
}
|
||||
|
||||
private static HttpClient httpClient() {
|
||||
return HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static RestClient.Builder generateRestClientBuilder() {
|
||||
return RestClient.builder()
|
||||
.requestFactory(new JdkClientHttpRequestFactory(httpClient()));
|
||||
}
|
||||
|
||||
public static WebClient.Builder generateWebClientBuilder() {
|
||||
return WebClient.builder()
|
||||
.clientConnector(new JdkClientHttpConnector(httpClient()));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.lanyuanxiaoyao.service.ai.core.entity.amis;
|
||||
|
||||
/**
|
||||
* Crud 响应
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-07-06
|
||||
*/
|
||||
public class AmisCrudResponse extends AmisMapResponse {
|
||||
public void setData(Iterable<?> list) {
|
||||
getData().put("items", list);
|
||||
}
|
||||
|
||||
public void setTotal(Long total) {
|
||||
getData().put("total", total);
|
||||
}
|
||||
|
||||
public void setTotal(Integer total) {
|
||||
setTotal(total.longValue());
|
||||
}
|
||||
|
||||
public void setData(Iterable<?> list, Long total) {
|
||||
setData(list);
|
||||
setTotal(total);
|
||||
}
|
||||
|
||||
public void setData(Iterable<?> list, Integer total) {
|
||||
setData(list, total.longValue());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.core.entity.amis;
|
||||
|
||||
/**
|
||||
* Crud 响应
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-07-06
|
||||
*/
|
||||
public class AmisDetailResponse extends AmisMapResponse {
|
||||
public void setDetail(Object detail) {
|
||||
getData().put("detail", detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.lanyuanxiaoyao.service.ai.core.entity.amis;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Map 响应
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-07-06
|
||||
*/
|
||||
public class AmisMapResponse extends AmisResponse<Map<String, Object>> {
|
||||
public AmisMapResponse() {
|
||||
setData(new HashMap<>());
|
||||
}
|
||||
|
||||
public AmisMapResponse setData(String key, Object value) {
|
||||
getData().put(key, value);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.lanyuanxiaoyao.service.ai.core.entity.amis;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Amis 组件结构化返回值
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2022-09-21
|
||||
*/
|
||||
public class AmisResponse<T> {
|
||||
private static final int SUCCESS_STATUS = 0;
|
||||
private static final int ERROR_STATUS = 500;
|
||||
private static final String SUCCESS_MESSAGE = "OK";
|
||||
private static final String ERROR_MESSAGE = "ERROR";
|
||||
private Integer status;
|
||||
private String message;
|
||||
private T data;
|
||||
|
||||
public static AmisResponse<Object> responseError() {
|
||||
return responseError(ERROR_MESSAGE);
|
||||
}
|
||||
|
||||
public static AmisResponse<Object> responseError(String message) {
|
||||
AmisResponse<Object> response = new AmisResponse<>();
|
||||
response.setStatus(ERROR_STATUS);
|
||||
response.setMessage(message);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisResponse<Object> responseSuccess() {
|
||||
AmisResponse<Object> response = new AmisResponse<>();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(SUCCESS_MESSAGE);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisResponse<Object> responseSuccess(String message) {
|
||||
AmisResponse<Object> response = new AmisResponse<>();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(message);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static <E> AmisResponse<E> responseSuccess(String message, E data) {
|
||||
AmisResponse<E> response = new AmisResponse<>();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(message);
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static <E> AmisResponse<E> responseSuccess(E data) {
|
||||
AmisResponse<E> response = new AmisResponse<>();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(SUCCESS_MESSAGE);
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisMapResponse responseMapData() {
|
||||
AmisMapResponse response = new AmisMapResponse();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(SUCCESS_MESSAGE);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisMapResponse responseMapData(Map<String, Object> data) {
|
||||
AmisMapResponse response = responseMapData();
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisMapResponse responseMapData(String key, Object value) {
|
||||
AmisMapResponse response = responseMapData();
|
||||
response.setData(key, value);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisCrudResponse responseCrudData(Iterable<?> data) {
|
||||
AmisCrudResponse response = new AmisCrudResponse();
|
||||
response.setStatus(SUCCESS_STATUS);
|
||||
response.setMessage(SUCCESS_MESSAGE);
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisCrudResponse responseCrudData(Iterable<?> data, Integer total) {
|
||||
AmisCrudResponse response = responseCrudData(data);
|
||||
response.setTotal(total);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisCrudResponse responseCrudData(Iterable<?> data, Long total) {
|
||||
AmisCrudResponse response = responseCrudData(data);
|
||||
response.setTotal(total);
|
||||
return response;
|
||||
}
|
||||
|
||||
public static AmisDetailResponse responseDetailData(Object detail) {
|
||||
AmisDetailResponse response = new AmisDetailResponse();
|
||||
response.setDetail(detail);
|
||||
return response;
|
||||
}
|
||||
|
||||
public Integer getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(Integer status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public T getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(T data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AmisResponse{" +
|
||||
"status=" + status +
|
||||
", message='" + message + '\'' +
|
||||
", data=" + data +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.security.config.Customizer;
|
||||
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
||||
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
||||
@@ -12,9 +13,6 @@ import org.springframework.security.core.userdetails.User;
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.security.provisioning.InMemoryUserDetailsManager;
|
||||
import org.springframework.security.web.SecurityFilterChain;
|
||||
import org.springframework.web.cors.CorsConfiguration;
|
||||
import org.springframework.web.cors.CorsConfigurationSource;
|
||||
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
@@ -23,29 +21,22 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
|
||||
@Configuration
|
||||
@EnableWebSecurity
|
||||
public class SecurityConfig {
|
||||
private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class);
|
||||
|
||||
@Bean
|
||||
public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception {
|
||||
return http.authorizeHttpRequests(registry -> registry.anyRequest().authenticated())
|
||||
return http.authorizeHttpRequests(
|
||||
registry -> registry
|
||||
.requestMatchers(HttpMethod.OPTIONS, "/**")
|
||||
.permitAll()
|
||||
.anyRequest()
|
||||
.authenticated()
|
||||
)
|
||||
.httpBasic(Customizer.withDefaults())
|
||||
.cors(AbstractHttpConfigurer::disable)
|
||||
.csrf(AbstractHttpConfigurer::disable)
|
||||
.cors(configurer -> configurer.configurationSource(corsConfigurationSource()))
|
||||
.formLogin(AbstractHttpConfigurer::disable)
|
||||
.build();
|
||||
}
|
||||
|
||||
private CorsConfigurationSource corsConfigurationSource() {
|
||||
CorsConfiguration configuration = new CorsConfiguration();
|
||||
configuration.setAllowCredentials(true);
|
||||
configuration.addAllowedHeader("*");
|
||||
configuration.addAllowedMethod("*");
|
||||
configuration.addAllowedOriginPattern("*");
|
||||
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
|
||||
source.registerCorsConfiguration("/**", configuration);
|
||||
return source;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public InMemoryUserDetailsManager userDetailsService(SecurityProperties securityProperties) {
|
||||
UserDetails user = User.builder()
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-ai-knowledge</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai-core</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-vector-store-qdrant</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-markdown-document-reader</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.yomahub</groupId>
|
||||
<artifactId>liteflow-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -1,29 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge;
|
||||
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
public class KnowledgeApplication implements ApplicationRunner {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(KnowledgeApplication.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.PointVO;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.reader.TextLineReader;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.service.KnowledgeService;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.reader.TextReader;
|
||||
import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
|
||||
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("knowledge")
|
||||
public class KnowledgeController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeController.class);
|
||||
|
||||
private final KnowledgeService knowledgeService;
|
||||
private final QdrantClient client;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
public KnowledgeController(KnowledgeService knowledgeService, VectorStore vectorStore, EmbeddingModel embeddingModel) {
|
||||
this.knowledgeService = knowledgeService;
|
||||
client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
|
||||
@PostMapping("add")
|
||||
public void add(
|
||||
@RequestParam("name") String name,
|
||||
@RequestParam("strategy") String strategy
|
||||
) throws ExecutionException, InterruptedException {
|
||||
knowledgeService.add(name, strategy);
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public ImmutableList<KnowledgeVO> list() {
|
||||
return knowledgeService.list();
|
||||
}
|
||||
|
||||
@GetMapping("list_points")
|
||||
public ImmutableList<PointVO> listPoints(@RequestParam("name") String name) throws ExecutionException, InterruptedException {
|
||||
Points.ScrollResponse response = client.scrollAsync(
|
||||
Points.ScrollPoints.newBuilder()
|
||||
.setCollectionName(name)
|
||||
// .setLimit(2)
|
||||
.setWithPayload(Points.WithPayloadSelector.newBuilder().setEnable(true).build())
|
||||
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(false).build())
|
||||
.build()
|
||||
)
|
||||
.get();
|
||||
return response.getResultList()
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.collect(point -> {
|
||||
PointVO vo = new PointVO();
|
||||
vo.setId(point.getId().getUuid());
|
||||
vo.setText(point.getPayloadMap().get("doc_content").getStringValue());
|
||||
return vo;
|
||||
})
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public void delete(@RequestParam("name") String name) throws ExecutionException, InterruptedException {
|
||||
knowledgeService.remove(name);
|
||||
}
|
||||
|
||||
@PostMapping("preview_text")
|
||||
public ImmutableList<PointVO> previewText(
|
||||
@RequestParam("name") String name,
|
||||
@RequestParam(value = "mode", defaultValue = "normal") String mode,
|
||||
@RequestParam(value = "type", defaultValue = "text") String type,
|
||||
@RequestParam("content") String content
|
||||
) {
|
||||
TextReader reader = new TextLineReader(new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)));
|
||||
return reader.get()
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.collect(doc -> {
|
||||
PointVO vo = new PointVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@PostMapping(value = "process_text", consumes = "text/plain;charset=utf-8")
|
||||
public void processText(
|
||||
@RequestParam("name") String name,
|
||||
@RequestBody String text
|
||||
) {
|
||||
VectorStore source = QdrantVectorStore.builder(client, embeddingModel)
|
||||
.collectionName(name)
|
||||
.initializeSchema(true)
|
||||
.build();
|
||||
MarkdownDocumentReader reader = new MarkdownDocumentReader(
|
||||
new ByteArrayResource(text.getBytes(StandardCharsets.UTF_8)),
|
||||
MarkdownDocumentReaderConfig.builder()
|
||||
.withHorizontalRuleCreateDocument(true)
|
||||
.withIncludeCodeBlock(false)
|
||||
.withIncludeBlockquote(false)
|
||||
.build()
|
||||
);
|
||||
source.add(reader.get());
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
public class Knowledge {
|
||||
private Long id;
|
||||
private Long vectorSourceId;
|
||||
private String name;
|
||||
private String strategy;
|
||||
|
||||
public Long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(Long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public Long getVectorSourceId() {
|
||||
return vectorSourceId;
|
||||
}
|
||||
|
||||
public void setVectorSourceId(Long vectorSourceId) {
|
||||
this.vectorSourceId = vectorSourceId;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStrategy() {
|
||||
return strategy;
|
||||
}
|
||||
|
||||
public void setStrategy(String strategy) {
|
||||
this.strategy = strategy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Knowledge{" +
|
||||
"id=" + id +
|
||||
", vectorSourceId=" + vectorSourceId +
|
||||
", name='" + name + '\'' +
|
||||
", strategy='" + strategy + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class KnowledgeVO {
|
||||
private String name;
|
||||
private String strategy;
|
||||
private Long size;
|
||||
private Long points;
|
||||
private Long segments;
|
||||
private String status;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStrategy() {
|
||||
return strategy;
|
||||
}
|
||||
|
||||
public void setStrategy(String strategy) {
|
||||
this.strategy = strategy;
|
||||
}
|
||||
|
||||
public Long getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
public void setSize(Long size) {
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public Long getPoints() {
|
||||
return points;
|
||||
}
|
||||
|
||||
public void setPoints(Long points) {
|
||||
this.points = points;
|
||||
}
|
||||
|
||||
public Long getSegments() {
|
||||
return segments;
|
||||
}
|
||||
|
||||
public void setSegments(Long segments) {
|
||||
this.segments = segments;
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public void setStatus(String status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CollectionVO{" +
|
||||
"name='" + name + '\'' +
|
||||
", strategy='" + strategy + '\'' +
|
||||
", size=" + size +
|
||||
", points=" + points +
|
||||
", segments=" + segments +
|
||||
", status='" + status + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
public class PointVO {
|
||||
private String id;
|
||||
private String text;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getText() {
|
||||
return text;
|
||||
}
|
||||
|
||||
public void setText(String text) {
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "PointVO{" +
|
||||
"id='" + id + '\'' +
|
||||
", text='" + text + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.reader;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.reader.TextReader;
|
||||
import org.springframework.core.io.Resource;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
public class TextLineReader extends TextReader {
|
||||
public TextLineReader(Resource resource) {
|
||||
super(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> get() {
|
||||
return super.get()
|
||||
.stream()
|
||||
.flatMap(doc -> {
|
||||
String text = doc.getText();
|
||||
if (StrUtil.isBlank(text)) {
|
||||
return Stream.of(doc);
|
||||
}
|
||||
return Stream.of(text.split("\n\n"))
|
||||
.filter(StrUtil::isNotBlank)
|
||||
.map(line -> new Document(line, doc.getMetadata()));
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class EmbeddingService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmbeddingService.class);
|
||||
|
||||
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeGroupService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeGroupService.class);
|
||||
private static final String GROUP_TABLE_NAME = "service_ai_group";
|
||||
|
||||
private final JdbcTemplate template;
|
||||
|
||||
public KnowledgeGroupService(JdbcTemplate template) {
|
||||
this.template = template;
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void add(Long knowledgeId, String name) {
|
||||
template.update(
|
||||
SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name")
|
||||
.values()
|
||||
.addValue("?", "?", "?")
|
||||
.precompileSql(),
|
||||
IdUtil.getSnowflakeNextId(),
|
||||
knowledgeId,
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(Long groupId) {
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
groupId
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void removeByKnowledgeId(Long knowledgeId) {
|
||||
template.update(
|
||||
SqlBuilder.delete(GROUP_TABLE_NAME)
|
||||
.whereEq("knowledge_id", "?")
|
||||
.precompileSql(),
|
||||
knowledgeId
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
package com.lanyuanxiaoyao.service.ai.knowledge.service;
|
||||
|
||||
import club.kingon.sql.builder.SqlBuilder;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeService.class);
|
||||
private static final String KNOWLEDGE_TABLE_NAME = "service_ai_knowledge";
|
||||
|
||||
private final JdbcTemplate template;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
private final QdrantClient client;
|
||||
private final KnowledgeGroupService knowledgeGroupService;
|
||||
|
||||
public KnowledgeService(JdbcTemplate template, EmbeddingModel embeddingModel, VectorStore vectorStore, KnowledgeGroupService knowledgeGroupService) {
|
||||
this.template = template;
|
||||
this.embeddingModel = embeddingModel;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.knowledgeGroupService = knowledgeGroupService;
|
||||
}
|
||||
|
||||
public Knowledge get(Long id) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
Knowledge.class,
|
||||
id
|
||||
);
|
||||
}
|
||||
|
||||
public Knowledge get(String name) {
|
||||
return template.queryForObject(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("name", "?")
|
||||
.precompileSql(),
|
||||
Knowledge.class,
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void add(String name, String strategy) throws ExecutionException, InterruptedException {
|
||||
Integer count = template.queryForObject(
|
||||
SqlBuilder.select("count(*)")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("name", "?")
|
||||
.precompileSql(),
|
||||
Integer.class,
|
||||
name
|
||||
);
|
||||
if (count > 0) {
|
||||
throw new RuntimeException("名称已存在");
|
||||
}
|
||||
|
||||
long id = IdUtil.getSnowflakeNextId();
|
||||
long vectorSourceId = IdUtil.getSnowflakeNextId();
|
||||
template.update(
|
||||
SqlBuilder.insertInto(KNOWLEDGE_TABLE_NAME, "id", "vector_source_id", "name", "strategy")
|
||||
.values()
|
||||
.addValue("?", "?", "?", "?")
|
||||
.precompileSql(),
|
||||
id,
|
||||
vectorSourceId,
|
||||
name,
|
||||
strategy
|
||||
);
|
||||
client.createCollectionAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.valueOf(strategy))
|
||||
.setSize(embeddingModel.dimensions())
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
|
||||
public ImmutableList<KnowledgeVO> list() {
|
||||
return template.query(
|
||||
SqlBuilder.select("id", "vector_source_id", "name", "strategy")
|
||||
.from(KNOWLEDGE_TABLE_NAME)
|
||||
.build(),
|
||||
(rs, index) -> {
|
||||
Knowledge knowledge = new Knowledge();
|
||||
knowledge.setId(rs.getLong(1));
|
||||
knowledge.setVectorSourceId(rs.getLong(2));
|
||||
knowledge.setName(rs.getString(3));
|
||||
knowledge.setStrategy(rs.getString(4));
|
||||
return knowledge;
|
||||
}
|
||||
)
|
||||
.stream()
|
||||
.map(knowledge -> {
|
||||
try {
|
||||
Collections.CollectionInfo info = client.getCollectionInfoAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
KnowledgeVO vo = new KnowledgeVO();
|
||||
vo.setName(knowledge.getName());
|
||||
vo.setPoints(info.getPointsCount());
|
||||
vo.setSegments(info.getSegmentsCount());
|
||||
vo.setStatus(info.getStatus().name());
|
||||
Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
|
||||
vo.setStrategy(vectorParams.getDistance().name());
|
||||
vo.setSize(vectorParams.getSize());
|
||||
return vo;
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void remove(String name) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = get(name);
|
||||
if (ObjectUtil.isNull(knowledge)) {
|
||||
throw new RuntimeException(StrUtil.format("{} 不存在"));
|
||||
}
|
||||
template.update(
|
||||
SqlBuilder.delete(KNOWLEDGE_TABLE_NAME)
|
||||
.whereEq("id", "?")
|
||||
.precompileSql(),
|
||||
knowledge.getId()
|
||||
);
|
||||
knowledgeGroupService.removeByKnowledgeId(knowledge.getId());
|
||||
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
spring:
|
||||
application:
|
||||
name: service-ai-knowledge
|
||||
profiles:
|
||||
include: common,metrics,forest
|
||||
cloud:
|
||||
zookeeper:
|
||||
enabled: true
|
||||
connect-string: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
|
||||
discovery:
|
||||
enabled: ${spring.cloud.zookeeper.enabled}
|
||||
root: /hudi-services
|
||||
instance-id: ${spring.application.name}-127.0.0.1-${random.uuid}-20250514
|
||||
metadata:
|
||||
discovery: zookeeper
|
||||
ip: 127.0.0.1
|
||||
hostname: localhost
|
||||
hostname_full: localhost
|
||||
start_time: 20250514112750
|
||||
datasource:
|
||||
url: jdbc:mysql://localhost:3307/ai?useSSL=false
|
||||
username: test
|
||||
password: test
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
security:
|
||||
meta:
|
||||
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
|
||||
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
|
||||
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
|
||||
ai:
|
||||
openai:
|
||||
base-url: http://132.121.206.65:10086
|
||||
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
|
||||
chat:
|
||||
options:
|
||||
model: 'Qwen3-1.7'
|
||||
embedding:
|
||||
options:
|
||||
model: 'Bge-m3'
|
||||
vectorstore:
|
||||
qdrant:
|
||||
api-key: lanyuanxiaoyao
|
||||
jasypt:
|
||||
encryptor:
|
||||
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
|
||||
server:
|
||||
port: 8080
|
||||
liteflow:
|
||||
rule-source: config/flow.xml
|
||||
print-banner: false
|
||||
check-node-exists: false
|
||||
@@ -1,14 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<flow>
|
||||
<chain name="embedding">
|
||||
SER(
|
||||
embedding_start,
|
||||
SWITCH(embedding_mode_switch).TO(
|
||||
normal_embedding,
|
||||
llm_embedding,
|
||||
qa_embedding
|
||||
),
|
||||
embedding_finish
|
||||
);
|
||||
</chain>
|
||||
</flow>
|
||||
@@ -1,34 +0,0 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %clr(%5p) %clr([${HOSTNAME}]){yellow} %clr([%t]){magenta} %clr(%logger{40}){cyan} #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Console"/>
|
||||
<!-- <appender-ref ref="RollingFile"/>-->
|
||||
</root>
|
||||
</configuration>
|
||||
130
service-ai/service-ai-web/pom.xml
Normal file
130
service-ai/service-ai-web/pom.xml
Normal file
@@ -0,0 +1,130 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>service-ai-web</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lanyuanxiaoyao</groupId>
|
||||
<artifactId>service-ai-core</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jetty</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-vector-store-qdrant</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-markdown-document-reader</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-jpa</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.blinkfox</groupId>
|
||||
<artifactId>fenix-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.yomahub</groupId>
|
||||
<artifactId>liteflow-spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-tika-document-reader</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-pdf-document-reader</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.noear</groupId>
|
||||
<artifactId>solon-ai</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.noear</groupId>
|
||||
<artifactId>solon-ai-dialect-openai</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<annotationProcessorPaths>
|
||||
<path>
|
||||
<groupId>org.mapstruct</groupId>
|
||||
<artifactId>mapstruct-processor</artifactId>
|
||||
<version>${mapstruct.version}</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok-mapstruct-binding</artifactId>
|
||||
<version>0.2.0</version>
|
||||
</path>
|
||||
<path>
|
||||
<groupId>org.hibernate</groupId>
|
||||
<artifactId>hibernate-jpamodelgen</artifactId>
|
||||
<version>6.6.8.Final</version>
|
||||
</path>
|
||||
</annotationProcessorPaths>
|
||||
<compilerArgs>
|
||||
<arg>-Amapstruct.defaultComponentModel=spring</arg>
|
||||
<arg>-Amapstruct.defaultInjectionStrategy=constructor</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web;
|
||||
|
||||
import com.blinkfox.fenix.EnableFenix;
|
||||
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@SpringBootApplication(scanBasePackages = "com.lanyuanxiaoyao.service")
|
||||
@EnableDiscoveryClient
|
||||
@EnableConfigurationProperties
|
||||
@EnableEncryptableProperties
|
||||
@EnableRetry
|
||||
@EnableScheduling
|
||||
@EnableFenix
|
||||
@EnableJpaAuditing
|
||||
public class WebApplication implements ApplicationRunner, ApplicationContextAware {
|
||||
private static ApplicationContext context;
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(WebApplication.class, args);
|
||||
}
|
||||
|
||||
public static <T> T getBean(Class<T> clazz) {
|
||||
return context.getBean(clazz);
|
||||
}
|
||||
|
||||
public static <T> T getBean(String name, Class<T> clazz) {
|
||||
return context.getBean(name, clazz);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
WebApplication.context = context;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface DetailController<DETAIL_ITEM> {
|
||||
String DETAIL = "/detail/{id}";
|
||||
|
||||
AmisResponse<DETAIL_ITEM> detail(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface ListController {
|
||||
String LIST = "/list";
|
||||
|
||||
AmisCrudResponse list() throws Exception;
|
||||
|
||||
AmisCrudResponse list(Query query) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface RemoveController {
|
||||
String REMOVE = "/remove/{id}";
|
||||
|
||||
AmisResponse<Object> remove(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SaveController<SAVE_ITEM> {
|
||||
String SAVE = "/save";
|
||||
|
||||
AmisResponse<Long> save(SAVE_ITEM item) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> extends SaveController<SAVE_ITEM>, ListController, DetailController<DETAIL_ITEM>, RemoveController {
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-26
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class SimpleControllerSupport<ENTITY extends SimpleEntity, SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> implements SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> {
|
||||
protected final SimpleServiceSupport<ENTITY> service;
|
||||
|
||||
public SimpleControllerSupport(SimpleServiceSupport<ENTITY> service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
@PostMapping(SAVE)
|
||||
@Override
|
||||
public AmisResponse<Long> save(@RequestBody SAVE_ITEM item) throws Exception {
|
||||
SaveItemMapper<ENTITY, SAVE_ITEM> mapper = saveItemMapper();
|
||||
return AmisResponse.responseSuccess(service.save(mapper.from(item)));
|
||||
}
|
||||
|
||||
@GetMapping(LIST)
|
||||
@Override
|
||||
public AmisCrudResponse list() throws Exception {
|
||||
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
|
||||
return AmisCrudResponse.responseCrudData(
|
||||
service.list()
|
||||
.collect(entity -> {
|
||||
try {
|
||||
return mapper.from(entity);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@PostMapping(LIST)
|
||||
@Override
|
||||
public AmisCrudResponse list(@RequestBody Query query) throws Exception {
|
||||
if (ObjectUtil.isNull(query)) {
|
||||
return AmisCrudResponse.responseCrudData(List.of(), 0);
|
||||
}
|
||||
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
|
||||
Page<ENTITY> result = service.list(query);
|
||||
return AmisCrudResponse.responseCrudData(
|
||||
result.get()
|
||||
.map(entity -> {
|
||||
try {
|
||||
return mapper.from(entity);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.toList(),
|
||||
result.getTotalElements()
|
||||
);
|
||||
}
|
||||
|
||||
@GetMapping(DETAIL)
|
||||
@Override
|
||||
public AmisResponse<DETAIL_ITEM> detail(@PathVariable("id") Long id) throws Exception {
|
||||
DetailItemMapper<ENTITY, DETAIL_ITEM> mapper = detailItemMapper();
|
||||
return AmisResponse.responseSuccess(mapper.from(service.detailOrThrow(id)));
|
||||
}
|
||||
|
||||
@GetMapping(REMOVE)
|
||||
@Override
|
||||
public AmisResponse<Object> remove(@PathVariable("id") Long id) throws Exception {
|
||||
service.remove(id);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
|
||||
protected abstract SaveItemMapper<ENTITY, SAVE_ITEM> saveItemMapper();
|
||||
|
||||
protected abstract ListItemMapper<ENTITY, LIST_ITEM> listItemMapper();
|
||||
|
||||
protected abstract DetailItemMapper<ENTITY, DETAIL_ITEM> detailItemMapper();
|
||||
|
||||
public interface SaveItemMapper<ENTITY, SAVE_ITEM> {
|
||||
ENTITY from(SAVE_ITEM item) throws Exception;
|
||||
}
|
||||
|
||||
public interface ListItemMapper<ENTITY, LIST_ITEM> {
|
||||
LIST_ITEM from(ENTITY entity) throws Exception;
|
||||
}
|
||||
|
||||
public interface DetailItemMapper<ENTITY, DETAIL_ITEM> {
|
||||
DETAIL_ITEM from(ENTITY entity) throws Exception;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.controller.query;
|
||||
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.map.ImmutableMap;
|
||||
|
||||
/**
|
||||
* 统一查询
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-12-03
|
||||
*/
|
||||
@Data
|
||||
public class Query {
|
||||
private Queryable query;
|
||||
private ImmutableList<Sortable> sort;
|
||||
private Pageable page;
|
||||
|
||||
@Data
|
||||
public static class Queryable {
|
||||
private ImmutableList<String> nullEqual;
|
||||
private ImmutableList<String> notNullEqual;
|
||||
private ImmutableList<String> empty;
|
||||
private ImmutableList<String> notEmpty;
|
||||
private ImmutableMap<String, ?> equal;
|
||||
private ImmutableMap<String, ?> notEqual;
|
||||
private ImmutableMap<String, String> like;
|
||||
private ImmutableMap<String, String> notLike;
|
||||
private ImmutableMap<String, ?> great;
|
||||
private ImmutableMap<String, ?> less;
|
||||
private ImmutableMap<String, ?> greatEqual;
|
||||
private ImmutableMap<String, ?> lessEqual;
|
||||
private ImmutableMap<String, ImmutableList<?>> in;
|
||||
private ImmutableMap<String, ImmutableList<?>> notIn;
|
||||
private ImmutableMap<String, Between> between;
|
||||
private ImmutableMap<String, Between> notBetween;
|
||||
|
||||
@Data
|
||||
public static class Between {
|
||||
private String start;
|
||||
private String end;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Sortable {
|
||||
private String column;
|
||||
private Direction direction;
|
||||
|
||||
public enum Direction {
|
||||
ASC,
|
||||
DESC,
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Pageable {
|
||||
private Integer index;
|
||||
private Integer size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.Id;
|
||||
import jakarta.persistence.MappedSuperclass;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.GenericGenerator;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* 实体类公共字段
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-20
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@MappedSuperclass
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
public class IdOnlyEntity {
|
||||
@Id
|
||||
@GeneratedValue(generator = "snowflake")
|
||||
@GenericGenerator(name = "snowflake", strategy = "com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator")
|
||||
private Long id;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.MappedSuperclass;
|
||||
import java.time.LocalDateTime;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.springframework.data.annotation.CreatedDate;
|
||||
import org.springframework.data.annotation.LastModifiedDate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* 实体类公共字段
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-20
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString(callSuper = true)
|
||||
@MappedSuperclass
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
public class SimpleEntity extends IdOnlyEntity {
|
||||
@CreatedDate
|
||||
private LocalDateTime createdTime;
|
||||
@LastModifiedDate
|
||||
private LocalDateTime modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.entity;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 创建对象使用的接收类
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-26
|
||||
*/
|
||||
@Data
|
||||
public class SimpleItem {
|
||||
private Long id;
|
||||
private LocalDateTime createdTime;
|
||||
private LocalDateTime modifiedTime;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.repository;
|
||||
|
||||
import com.blinkfox.fenix.jpa.FenixJpaRepository;
|
||||
import com.blinkfox.fenix.specification.FenixJpaSpecificationExecutor;
|
||||
import org.springframework.data.repository.NoRepositoryBean;
|
||||
import org.springframework.data.repository.query.QueryByExampleExecutor;
|
||||
|
||||
/**
|
||||
* 整合一下
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@NoRepositoryBean
|
||||
public interface SimpleRepository<E, ID> extends FenixJpaRepository<E, ID>, FenixJpaSpecificationExecutor<E>, QueryByExampleExecutor<E> {
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.service;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import java.util.Optional;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-28
|
||||
*/
|
||||
public interface SimpleService<ENTITY extends SimpleEntity> {
|
||||
Long save(ENTITY entity) throws Exception;
|
||||
|
||||
Long count() throws Exception;
|
||||
|
||||
ImmutableList<ENTITY> list() throws Exception;
|
||||
|
||||
ImmutableList<ENTITY> list(ImmutableSet<Long> ids) throws Exception;
|
||||
|
||||
Page<ENTITY> list(Query query) throws Exception;
|
||||
|
||||
Optional<ENTITY> detailOptional(Long id) throws Exception;
|
||||
|
||||
ENTITY detail(Long id) throws Exception;
|
||||
|
||||
ENTITY detailOrThrow(Long id) throws Exception;
|
||||
|
||||
ENTITY detailOrNull(Long id) throws Exception;
|
||||
|
||||
void remove(Long id) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.base.service;
|
||||
|
||||
import cn.hutool.core.bean.BeanUtil;
|
||||
import cn.hutool.core.bean.copier.CopyOptions;
|
||||
import cn.hutool.core.util.EnumUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import jakarta.persistence.criteria.CriteriaBuilder;
|
||||
import jakarta.persistence.criteria.CriteriaQuery;
|
||||
import jakarta.persistence.criteria.Path;
|
||||
import jakarta.persistence.criteria.Predicate;
|
||||
import jakarta.persistence.criteria.Root;
|
||||
import jakarta.transaction.Transactional;
|
||||
import java.util.Optional;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.list.MutableList;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.data.domain.Sort;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implements SimpleService<ENTITY> {
|
||||
private static final Integer DEFAULT_PAGE_INDEX = 1;
|
||||
private static final Integer DEFAULT_PAGE_SIZE = 10;
|
||||
protected final SimpleRepository<ENTITY, Long> repository;
|
||||
|
||||
public SimpleServiceSupport(SimpleRepository<ENTITY, Long> repository) {
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
@Transactional(rollbackOn = Throwable.class)
|
||||
@Override
|
||||
public Long save(ENTITY entity) {
|
||||
if (ObjectUtil.isNotNull(entity.getId())) {
|
||||
Long id = entity.getId();
|
||||
ENTITY targetEntity = repository.findById(entity.getId()).orElseThrow(() -> new IdNotFoundException(id));
|
||||
BeanUtil.copyProperties(
|
||||
entity,
|
||||
targetEntity,
|
||||
CopyOptions.create()
|
||||
.setIgnoreProperties(
|
||||
IdOnlyEntity_.ID,
|
||||
SimpleEntity_.CREATED_TIME,
|
||||
SimpleEntity_.MODIFIED_TIME
|
||||
)
|
||||
);
|
||||
entity = targetEntity;
|
||||
}
|
||||
entity = repository.save(entity);
|
||||
return entity.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long count() {
|
||||
return repository.count(
|
||||
(root, query, builder) ->
|
||||
builder.and(
|
||||
listPredicate(root, query, builder)
|
||||
.reject(ObjectUtil::isNull)
|
||||
.toArray(new Predicate[]{})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<ENTITY> list() {
|
||||
return Lists.immutable.ofAll(repository.findAll(
|
||||
(root, query, builder) ->
|
||||
builder.and(
|
||||
listPredicate(root, query, builder)
|
||||
.reject(ObjectUtil::isNull)
|
||||
.toArray(new Predicate[]{})
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableList<ENTITY> list(ImmutableSet<Long> ids) {
|
||||
return Lists.immutable.ofAll(repository.findAll(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.add(builder.in(root.get("id")).value(ids));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
private <Y> Path<Y> column(Root<ENTITY> root, String column) {
|
||||
String[] columns = StrUtil.splitToArray(column, "/");
|
||||
Path<Y> path = root.get(columns[0]);
|
||||
for (int i = 1; i < columns.length; i++) {
|
||||
path = path.get(columns[i]);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private <Y> Object value(Path<Y> column, Object value) {
|
||||
Class<?> javaType = column.getJavaType();
|
||||
if (EnumUtil.isEnum(javaType)) {
|
||||
return EnumUtil.fromString((Class<Enum>) javaType, (String) value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected ImmutableList<Predicate> queryPredicates(Query.Queryable queryable, Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
|
||||
MutableList<Predicate> predicates = Lists.mutable.empty();
|
||||
if (ObjectUtil.isEmpty(queryable)) {
|
||||
return predicates.toImmutable();
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNullEqual())) {
|
||||
queryable.getNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotNullEqual())) {
|
||||
queryable.getNotNullEqual().forEach(column -> predicates.add(builder.isNull(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getEmpty())) {
|
||||
queryable.getEmpty().forEach(column -> predicates.add(builder.isEmpty(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotEmpty())) {
|
||||
queryable.getNotEmpty().forEach(column -> predicates.add(builder.isNotEmpty(column(root, column))));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getEqual())) {
|
||||
queryable.getEqual().forEachKeyValue((column, value) -> {
|
||||
Path<Object> path = column(root, column);
|
||||
predicates.add(builder.equal(path, value(path, value)));
|
||||
});
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotEqual())) {
|
||||
queryable.getEqual().forEachKeyValue((column, value) -> {
|
||||
Path<Object> path = column(root, column);
|
||||
predicates.add(builder.notEqual(path, value(path, value)));
|
||||
});
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLike())) {
|
||||
queryable.getLike().forEachKeyValue((column, value) -> predicates.add(builder.like(column(root, column), value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotLike())) {
|
||||
queryable.getNotLike().forEachKeyValue((column, value) -> predicates.add(builder.notLike(column(root, column), value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getGreat())) {
|
||||
queryable.getGreat().forEachKeyValue((column, value) -> predicates.add(builder.greaterThan(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLess())) {
|
||||
queryable.getLess().forEachKeyValue((column, value) -> predicates.add(builder.lessThan(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getGreatEqual())) {
|
||||
queryable.getGreatEqual().forEachKeyValue((column, value) -> predicates.add(builder.greaterThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getLessEqual())) {
|
||||
queryable.getLessEqual().forEachKeyValue((column, value) -> predicates.add(builder.lessThanOrEqualTo(column(root, column), (Comparable<Object>) value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getIn())) {
|
||||
queryable.getIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value)));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotIn())) {
|
||||
queryable.getNotIn().forEachKeyValue((column, value) -> predicates.add(builder.in(column(root, column)).value(value).not()));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getBetween())) {
|
||||
queryable.getBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
|
||||
}
|
||||
if (ObjectUtil.isNotEmpty(queryable.getNotBetween())) {
|
||||
queryable.getNotBetween().forEachKeyValue((column, value) -> predicates.add(builder.between(column(root, column), value.getStart(), value.getEnd())));
|
||||
}
|
||||
return predicates.toImmutable();
|
||||
}
|
||||
|
||||
protected ImmutableList<Predicate> listPredicate(Root<ENTITY> root, CriteriaQuery<?> query, CriteriaBuilder builder) {
|
||||
return Lists.immutable.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<ENTITY> list(Query listQuery) {
|
||||
PageRequest pageRequest = PageRequest.of(DEFAULT_PAGE_INDEX - 1, DEFAULT_PAGE_SIZE, Sort.by(SimpleEntity_.CREATED_TIME).descending());
|
||||
if (ObjectUtil.isNotNull(listQuery.getPage())) {
|
||||
pageRequest = PageRequest.of(
|
||||
ObjectUtil.defaultIfNull(listQuery.getPage().getIndex(), DEFAULT_PAGE_INDEX) - 1,
|
||||
ObjectUtil.defaultIfNull(listQuery.getPage().getSize(), DEFAULT_PAGE_SIZE),
|
||||
Sort.by(SimpleEntity_.CREATED_TIME).descending()
|
||||
);
|
||||
}
|
||||
return repository.findAll(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.addAllIterable(queryPredicates(listQuery.getQuery(), root, query, builder));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
},
|
||||
pageRequest
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ENTITY> detailOptional(Long id) {
|
||||
if (ObjectUtil.isNull(id)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return repository.findOne(
|
||||
(root, query, builder) -> {
|
||||
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
|
||||
predicates.add(builder.equal(root.get(IdOnlyEntity_.id), id));
|
||||
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detail(Long id) {
|
||||
return detailOrNull(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detailOrThrow(Long id) {
|
||||
return detailOptional(id).orElseThrow(() -> new IdNotFoundException(id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ENTITY detailOrNull(Long id) {
|
||||
return detailOptional(id).orElse(null);
|
||||
}
|
||||
|
||||
@Transactional(rollbackOn = Throwable.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
if (ObjectUtil.isNotNull(id)) {
|
||||
repository.deleteById(id);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class IdNotFoundException extends RuntimeException {
|
||||
public IdNotFoundException() {
|
||||
super("资源不存在");
|
||||
}
|
||||
|
||||
public IdNotFoundException(Long id) {
|
||||
super(StrUtil.format("ID为{}的资源不存在", id));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "file-store")
|
||||
public class FileStoreProperties {
|
||||
private String downloadPrefix;
|
||||
private String uploadPath;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.time.Duration;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.noear.solon.ai.reranking.RerankingModel;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatModel;
|
||||
import org.springframework.ai.deepseek.DeepSeekChatOptions;
|
||||
import org.springframework.ai.deepseek.api.DeepSeekApi;
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiChatModel;
|
||||
import org.springframework.ai.openai.OpenAiChatOptions;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
@Slf4j
|
||||
@Configuration
|
||||
public class LlmConfiguration {
|
||||
@Bean("chat")
|
||||
public ChatClient.Builder chatClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getChat(), "chat properties is null");
|
||||
DeepSeekApi.Builder apiBuilder = DeepSeekApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getChat().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getChat().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getChat().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getChat().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
DeepSeekChatModel.builder()
|
||||
.deepSeekApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
DeepSeekChatOptions.builder()
|
||||
.model(llmProperties.getChat().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean("visual")
|
||||
public ChatClient.Builder visualClientBuilder(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getVisual(), "visual properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getVisual().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getVisual().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getVisual().getEndpoint())) {
|
||||
apiBuilder.completionsPath(llmProperties.getVisual().getEndpoint());
|
||||
}
|
||||
return ChatClient.builder(
|
||||
OpenAiChatModel.builder()
|
||||
.openAiApi(apiBuilder.build())
|
||||
.defaultOptions(
|
||||
OpenAiChatOptions.builder()
|
||||
.model(llmProperties.getVisual().getModel())
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EmbeddingModel embeddingModel(LlmProperties llmProperties, WebClient.Builder webClientBuilder, RestClient.Builder restClientBuilder) {
|
||||
Assert.notNull(llmProperties.getEmbedding(), "embedding properties is null");
|
||||
OpenAiApi.Builder apiBuilder = OpenAiApi.builder()
|
||||
.baseUrl(StrUtil.firstNonBlank(llmProperties.getEmbedding().getBaseUrl(), llmProperties.getBaseUrl()))
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getEmbedding().getApiKey(), llmProperties.getApiKey()))
|
||||
.webClientBuilder(webClientBuilder)
|
||||
.restClientBuilder(restClientBuilder);
|
||||
if (StrUtil.isNotBlank(llmProperties.getEmbedding().getEndpoint())) {
|
||||
apiBuilder.embeddingsPath(llmProperties.getEmbedding().getEndpoint());
|
||||
}
|
||||
return new OpenAiEmbeddingModel(
|
||||
apiBuilder.build(),
|
||||
MetadataMode.EMBED,
|
||||
OpenAiEmbeddingOptions.builder()
|
||||
.model(llmProperties.getEmbedding().getModel())
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RerankingModel rerankingModel(LlmProperties llmProperties) {
|
||||
Assert.notNull(llmProperties.getReranker(), "reranker properties is null");
|
||||
String url = llmProperties.getBaseUrl();
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getBaseUrl())) {
|
||||
url = llmProperties.getReranker().getBaseUrl();
|
||||
}
|
||||
if (StrUtil.isNotBlank(llmProperties.getReranker().getEndpoint())) {
|
||||
url += llmProperties.getReranker().getEndpoint();
|
||||
} else {
|
||||
url += "/v1/rerank";
|
||||
}
|
||||
return RerankingModel.of(url)
|
||||
.apiKey(StrUtil.firstNonBlank(llmProperties.getReranker().getApiKey(), llmProperties.getApiKey()))
|
||||
.model(llmProperties.getReranker().getModel())
|
||||
.timeout(Duration.ofMinutes(10))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Data
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "spring.llm")
|
||||
public class LlmProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private ChatProperties chat;
|
||||
private ChatProperties visual;
|
||||
private ChatProperties embedding;
|
||||
private ChatProperties reranker;
|
||||
|
||||
@Data
|
||||
public static class ChatProperties {
|
||||
private String baseUrl;
|
||||
private String apiKey;
|
||||
private String model;
|
||||
private String endpoint;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
|
||||
public interface Prompts {
|
||||
String hudiBase = """
|
||||
Hudi数据同步服务
|
||||
实现从源端数据库(TeleDB、TelePG)到Hudi表的数据实时同步,并通过Hudi-Hive插件将Hudi表封为Hive外表(即目标端),供外部系统进行SQL查询。
|
||||
数据同步任务:基于Flink开发,运行在Hadoop Yarn集群上,该任务常驻运行在集群上,每个任务负责一张或多张表的据同步,同步任务从指定的Pulsar队列中实时读取数据,经过数据转换和业务处理后,写入Hudi表。
|
||||
数据压缩任务:基于Flink开发,运行在Hadoop Yarn集群上,该任务根据指定的调度策略周期性定时启动,每个任务将张指定的Hudi表中的log数据压缩为parquet数据,当压缩完成时,上一次压缩到当前压缩之间被写入的数据才能被外部统查询。
|
||||
服务管理平台:使用Spring Cloud框架开发的微服务集群,用于管理同步任务和压缩任务的运行、停止和状态监控等息,提供友好的前端交互网页。
|
||||
源端(上游):类似TeleDB、TelePG这些提供原始数据的数据库。
|
||||
目标端(下游):数据同步到Hudi表后提供给外部系统查询用的Hive表。
|
||||
TeleDB:中国电信自研分布式MySQL集群组件,逻辑上的一张表在物理上被水平分片(Sharding)为多个“set”表存储在个MySQL节点,支持弹性扩容和容灾。
|
||||
TelePG:中国电信自研分布式PostgreSQL集群组件,其架构和功能特性与TeleDB高度相似。
|
||||
逻辑表:一张逻辑表对应一张TeleDB或TelePG上的业务表;逻辑表中包含广东所有地市的业务数据,数据量大的表通常按地市进行分片(分区)。
|
||||
Hudi表:逻辑表数据经同步任务处理后实际存储的位置,根据逻辑表的数据量,数据量大的会根据地市分为A、B表,B也被称为大表,通常包含广深东佛(广州、深圳、东莞、佛山)四个大区的数据,A表包含广东除了广深东佛外的其他市,特大表,如acct_item,会按一个地市对应一个Hudi表;Hudi表统一配置为MOR表,使用Bucket Index索引。
|
||||
Hive表:通过Hudi-Hive插件创建的Hive外表,作为下游系统的唯一查询入口,该表逻辑上对应源端的一张逻辑表,它Hudi服务内部可能存在的多个物理Hudi表(如A表、B表或地市分表)聚合封装成一个逻辑视图,透明地对外提供完整的辑表数据查询。
|
||||
重点表:根据业务系统的要求,对于有的表及时性和数据完整性有更高的要求,这些表被称为重点表,在tb_app_collect_table_info表中的tags字段,包含“FOCUS”字符的是重点表。
|
||||
Flink 任务:即数据同步任务,根据逻辑表的数据量通常有如下规则:
|
||||
大数据量:采用 1逻辑表:1 Flink任务 模式。该Flink任务内聚合处理该逻辑表对应的所有Hudi表(如A表 + B表或多地市表)的同步子任务。
|
||||
小数据量:采用 N逻辑表:1 Flink任务 模式。一个Flink任务内聚合处理多张低数据量逻辑表对应的所有Hudi表同步自务。
|
||||
Pulsar队列(消息队列):源端TeleDB逻辑表增量日志(包含新增-I、更新-U、删除-D、DDL操作类型)由Canal同步组件实时写入Pulsar队列。TelePG逻辑表增量日志由PGSync组件以相同逻辑同步到对应队列。
|
||||
压缩调度服务:service-scheduler(单实例):按策略将压缩任务放入预调度队列,定时将预调度任务转移至各集群对应的压缩任务队列;service-launcher(与集群一一对应):定时轮询对应集群的压缩任务队列,发现任务即调度执行;service-queue:提供队列机制;scheduler与launcher均基于集群资源状态动态调节任务产生与执行速率,利用队列缓冲,避免资源超限。
|
||||
压缩调度:压缩任务耗时长、资源大。为平衡效率与资源,调度服务通常从凌晨2点起,每3小时调度一次全部Hudi表的压缩任务。
|
||||
跨天调度:为确保关键表数据在0点后及时可用(此时Hive最新数据常未达0点),0点至2点间,对重点表进行更高频压缩调度。此高频率调度持续直至目标表被标记为“已跨天”
|
||||
跨天判断:按照先后次序1. 源端同步组件(Canal/PGSync)判断源表数据是否跨天。若跨天,向队列写入跨天消息;2.同步任务接收跨天消息,在tb_app_collect_table_version表插入记录;3.独立程序判断Hudi表数据是否已跨天。若跨天,更新tb_app_collect_table_version对应记录状态。此时逻辑表标记为“已跨天”;一张表必须先接收到跨天标记,然后才能完成跨天。
|
||||
集群:指Hadoop Yarn集群,同步任务仅在b12运行,压缩任务主要在b12运行,部分重点表在b1或a4运行,调度服务根据资源动态在多个集群间分配压缩任务;其中b12集群使用default队列,b1集群使用datalake队列,a4集群使用ten_iap.datalake队列。
|
||||
""";
|
||||
|
||||
String hudiDatabase = """
|
||||
Hudi数据同步服务使用的数据表详情如下
|
||||
hudi_collect_build_b12.tb_app_collect_table_info表:记录同步任务配置信息
|
||||
id:主键
|
||||
alias:表别名,同样可以唯一标识该记录
|
||||
flink_job_id:tb_app_flink_job_config表主键,对应的Flink任务
|
||||
hudi_job_id:hudi:tb_app_hudi_job_config表主键,对应Hudi表的配置
|
||||
sync_yarn_job_id:tb_app_yarn_job_config表主键,同步任务的yarn资源配置
|
||||
compaction_yarn_job_id:tb_app_yarn_job_config表主键,压缩任务的yarn资源配置
|
||||
src_db:源端数据库名称
|
||||
src_type:源端数据库类型,取值有teledb、telepg
|
||||
src_schema:源端数据库schema名称
|
||||
src_table:源端表名
|
||||
src_pulsar_addr:pulsar地址
|
||||
src_topic:pulsar队列名称
|
||||
tgt_hdfs_path:Hudi表对应的hdfs路径
|
||||
status:逻辑删除状态(y为正常,n为删除)
|
||||
filter_field:过滤字段,用于指定源端消息中的某个字段,进行消息过滤,如B表,同步任务会使用CITY_ID字段,并过滤出该字段为广深东佛编码的消息
|
||||
filter_values:过滤值
|
||||
filter_type:过滤类型,INCLUDE表示包含指定过滤值的记录保留,EXCLUDE表示包含指定过滤值的记录丢弃
|
||||
bucket_number:Hudi bucket index的参数值
|
||||
partition_field:Hudi表分区字段,通常使用CITY_ID
|
||||
priority:优先级,整数,数字越大优先级越高,优先级高,该表对应在压缩调度队列中会排在更靠前的位置
|
||||
hive_db:目标端hive库名称
|
||||
hive_table:目标端hive表名
|
||||
tags:标签,使用英文逗号分隔,用于标识表的特殊属性
|
||||
|
||||
hudi_collect_build_b12.tb_app_collect_table_version表:记录跨天版本
|
||||
flink_job_id:tb_app_flink_job_config表主键,对应的Flink任务
|
||||
alias:表别名 tb_app_collect_table_info表alias字段,对应唯一同步任务
|
||||
version:版本,格式为yyyyMMdd,如2025年6月6日跨天版本,则该值为20250605
|
||||
op_ts:操作时间,接收到跨天标记的时间
|
||||
create_time:记录创建时间
|
||||
update_time:记录创建时间
|
||||
scheduled:是否已跨天,1为已跨天,0为未跨天
|
||||
|
||||
hudi_collect_build_b12.tb_app_flink_job_config表:Flink 任务配置
|
||||
id:主键
|
||||
name:Flink任务名称
|
||||
status:逻辑删除状态(y为正常,n为删除)
|
||||
application_id:flink任务对应的yarn任务的application id
|
||||
|
||||
hudi_collect_build_b12.tb_app_hudi_job_config表:Hudi表原生配置
|
||||
id:主键
|
||||
source_tasks:读取Pulsar消息的Flink算子并行度
|
||||
write_tasks:写Hudi表的Flink算子并行度
|
||||
keep_file_version:保留数据文件版本
|
||||
keep_commit_version:保留时间线版本
|
||||
|
||||
hudi_collect_build_b12.tb_app_hudi_sync_state表:同步、压缩任务运行状态
|
||||
id:主键,由flink_job_id和alias使用短横线连接而成,格式为:flink_job_id-alias
|
||||
message_id:最新消费到的Pulsar消息的message id
|
||||
source_start_time:同步任务启动时间
|
||||
source_receive_time:同步任务最新接收到消息队列消息的时间
|
||||
source_checkpoint_time:同步任务对应Flink任务最近一次执行checkpoint的时间,这个时间每15分钟更新一次,可以用来判断flink任务是否还在运行
|
||||
source_publish_time:同步任务接收到的最新一条消息队列的消息被发布到队列中的时间
|
||||
source_op_time:同步任务接收到的最新一条消息队列的消息在源端产生的时间
|
||||
source_application_id:同步任务对应Flink任务对应的yarn任务的application id
|
||||
source_cluster:同步任务运行的yarn集群
|
||||
compaction_start_time:最近一次压缩任务启动时间
|
||||
compaction_finish_time:最近一次压缩任务完成时间
|
||||
compaction_application_id:压缩任务对应的Flink任务对应的yarn任务的application id
|
||||
compaction_cluster:压缩任务运行所在的yarn集群
|
||||
|
||||
hudi_collect_build_b12.tb_app_yarn_job_config表:同步、压缩任务对应的yarn任务资源配置
|
||||
id:主键
|
||||
job_manager_memory:Job Manager内存(MB)
|
||||
task_manager_memory:Task Manager内存(MB)
|
||||
""";
|
||||
|
||||
String hudi = StrUtil.format(
|
||||
"""
|
||||
{}
|
||||
|
||||
{}
|
||||
""",
|
||||
hudiBase,
|
||||
hudiDatabase
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.configuration;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.hibernate.engine.spi.SharedSessionContractImplementor;
|
||||
import org.hibernate.id.IdentifierGenerator;
|
||||
|
||||
/**
|
||||
* 使用雪花算法作为ID生成器
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-14
|
||||
*/
|
||||
@Slf4j
|
||||
public class SnowflakeIdGenerator implements IdentifierGenerator {
|
||||
|
||||
@Override
|
||||
public Serializable generate(SharedSessionContractImplementor session, Object object) {
|
||||
try {
|
||||
return Snowflake.next();
|
||||
} catch (Exception e) {
|
||||
log.error("Generate snowflake id failed", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Snowflake {
|
||||
/**
|
||||
* 起始的时间戳
|
||||
*/
|
||||
private final static long START_TIMESTAMP = 1;
|
||||
|
||||
/**
|
||||
* 序列号占用的位数
|
||||
*/
|
||||
private final static long SEQUENCE_BIT = 11;
|
||||
|
||||
/**
|
||||
* 序列号最大值
|
||||
*/
|
||||
private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
|
||||
|
||||
/**
|
||||
* 时间戳值向左位移
|
||||
*/
|
||||
private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
|
||||
|
||||
/**
|
||||
* 序列号
|
||||
*/
|
||||
private static long sequence = 0;
|
||||
/**
|
||||
* 上一次时间戳
|
||||
*/
|
||||
private static long lastTimestamp = -1;
|
||||
|
||||
public static synchronized long next() {
|
||||
long currentTimestamp = nowTimestamp();
|
||||
if (currentTimestamp < lastTimestamp) {
|
||||
throw new RuntimeException("Clock have moved backwards.");
|
||||
}
|
||||
|
||||
if (currentTimestamp == lastTimestamp) {
|
||||
// 相同毫秒内, 序列号自增
|
||||
sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
|
||||
// 同一毫秒的序列数已经达到最大
|
||||
if (sequence == 0) {
|
||||
currentTimestamp = nextTimestamp();
|
||||
}
|
||||
} else {
|
||||
// 不同毫秒内, 序列号置为0
|
||||
sequence = 0;
|
||||
}
|
||||
|
||||
lastTimestamp = currentTimestamp;
|
||||
return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
|
||||
}
|
||||
|
||||
private static long nextTimestamp() {
|
||||
long milli = nowTimestamp();
|
||||
while (milli <= lastTimestamp) {
|
||||
milli = nowTimestamp();
|
||||
}
|
||||
return milli;
|
||||
}
|
||||
|
||||
private static long nowTimestamp() {
|
||||
return Instant.now().toEpochMilli();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.io.IoUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.core.util.URLUtil;
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.FileStoreProperties;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.FileChannel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
/**
|
||||
* 文件上传接口
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/upload")
|
||||
public class DataFileController {
|
||||
private final FileStoreProperties fileStoreProperties;
|
||||
private final DataFileService dataFileService;
|
||||
private final String uploadFolderPath;
|
||||
private final String cacheFolderPath;
|
||||
private final String sliceFolderPath;
|
||||
|
||||
public DataFileController(FileStoreProperties fileStoreProperties, DataFileService dataFileService) {
|
||||
this.fileStoreProperties = fileStoreProperties;
|
||||
this.dataFileService = dataFileService;
|
||||
|
||||
this.uploadFolderPath = fileStoreProperties.getUploadPath();
|
||||
this.cacheFolderPath = StrUtil.format("{}/cache", uploadFolderPath);
|
||||
this.sliceFolderPath = StrUtil.format("{}/slice", uploadFolderPath);
|
||||
}
|
||||
|
||||
@PostMapping("")
|
||||
public AmisResponse<FinishResponse> upload(@RequestParam("file") MultipartFile file) throws IOException {
|
||||
String filename = file.getOriginalFilename();
|
||||
Long id = dataFileService.initialDataFile(filename);
|
||||
String url = StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), id);
|
||||
byte[] bytes = file.getBytes();
|
||||
String originMd5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
|
||||
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, originMd5));
|
||||
if (targetFile.exists()) {
|
||||
dataFileService.updateDataFile(id, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), originMd5, file.getContentType());
|
||||
return AmisResponse.responseSuccess(new FinishResponse(id, filename, String.valueOf(id), url));
|
||||
}
|
||||
File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, id));
|
||||
cacheFile = FileUtil.writeBytes(bytes, cacheFile);
|
||||
String targetMd5 = SecureUtil.md5(cacheFile);
|
||||
if (!StrUtil.equals(originMd5, targetMd5)) {
|
||||
throw new RuntimeException("文件上传失败,校验不匹配");
|
||||
}
|
||||
FileUtil.move(cacheFile, targetFile, true);
|
||||
dataFileService.updateDataFile(id, FileUtil.getAbsolutePath(targetFile), FileUtil.size(targetFile), targetMd5, file.getContentType());
|
||||
return AmisResponse.responseSuccess(new FinishResponse(id, filename, String.valueOf(id), url));
|
||||
}
|
||||
|
||||
@GetMapping("/download/{id}")
|
||||
public void download(@PathVariable("id") Long id, HttpServletResponse response) throws IOException {
|
||||
DataFile dataFile = dataFileService.downloadFile(id);
|
||||
File targetFile = new File(dataFile.getPath());
|
||||
response.setHeader("Content-Type", dataFile.getType());
|
||||
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
|
||||
response.setHeader("Content-Disposition", StrUtil.format("attachment; filename={}", URLUtil.encodeAll(dataFile.getFilename())));
|
||||
IoUtil.copy(new FileInputStream(targetFile), response.getOutputStream());
|
||||
}
|
||||
|
||||
@PostMapping("/start")
|
||||
public AmisResponse<StartResponse> start(@RequestBody StartRequest request) {
|
||||
Long id = dataFileService.initialDataFile(request.filename);
|
||||
return AmisResponse.responseSuccess(new StartResponse(id.toString()));
|
||||
}
|
||||
|
||||
@PostMapping("/slice")
|
||||
public AmisResponse<SliceResponse> slice(
|
||||
@RequestParam("uploadId")
|
||||
Long uploadId,
|
||||
@RequestParam("partNumber")
|
||||
Integer sequence,
|
||||
@RequestParam("partSize")
|
||||
Long size,
|
||||
@RequestParam("file")
|
||||
MultipartFile file
|
||||
) throws IOException {
|
||||
byte[] bytes = file.getBytes();
|
||||
String md5 = SecureUtil.md5(new ByteArrayInputStream(bytes));
|
||||
String targetFilename = StrUtil.format("{}-{}", sequence, md5);
|
||||
String targetFilePath = sliceFilePath(uploadId, targetFilename);
|
||||
FileUtil.mkParentDirs(targetFilePath);
|
||||
FileUtil.writeBytes(bytes, targetFilePath);
|
||||
return AmisResponse.responseSuccess(new SliceResponse(targetFilename));
|
||||
}
|
||||
|
||||
private String sliceFilePath(Long uploadId, String sliceFilename) {
|
||||
return StrUtil.format("{}/{}/{}", sliceFolderPath, uploadId, sliceFilename);
|
||||
}
|
||||
|
||||
@PostMapping("finish")
|
||||
public AmisResponse<FinishResponse> finish(@RequestBody FinishRequest request) {
|
||||
if (request.partList.anySatisfy(part -> !FileUtil.exist(sliceFilePath(request.uploadId, part.eTag)))) {
|
||||
throw new RuntimeException("文件校验失败,请重新上传");
|
||||
}
|
||||
try {
|
||||
File cacheFile = new File(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId));
|
||||
FileUtil.mkParentDirs(cacheFile);
|
||||
if (cacheFile.createNewFile()) {
|
||||
try (FileOutputStream fos = new FileOutputStream(cacheFile)) {
|
||||
try (FileChannel fosChannel = fos.getChannel()) {
|
||||
for (FinishRequest.Part part : request.partList) {
|
||||
File sliceFile = new File(sliceFilePath(request.uploadId, part.eTag));
|
||||
try (FileInputStream fis = new FileInputStream(sliceFile)) {
|
||||
try (FileChannel fisChannel = fis.getChannel()) {
|
||||
fisChannel.transferTo(0, fisChannel.size(), fosChannel);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String md5 = SecureUtil.md5(cacheFile);
|
||||
File targetFile = new File(StrUtil.format("{}/{}", uploadFolderPath, md5));
|
||||
if (!targetFile.exists()) {
|
||||
FileUtil.move(cacheFile, targetFile, true);
|
||||
}
|
||||
String absolutePath = FileUtil.getAbsolutePath(targetFile);
|
||||
dataFileService.updateDataFile(
|
||||
request.uploadId,
|
||||
absolutePath,
|
||||
FileUtil.size(targetFile),
|
||||
SecureUtil.md5(targetFile),
|
||||
FileUtil.getMimeType(absolutePath)
|
||||
);
|
||||
return AmisResponse.responseSuccess(new FinishResponse(
|
||||
request.uploadId,
|
||||
request.filename,
|
||||
request.uploadId.toString(),
|
||||
StrUtil.format("{}/upload/download/{}", fileStoreProperties.getDownloadPrefix(), request.uploadId)
|
||||
));
|
||||
} else {
|
||||
throw new RuntimeException("合并文件失败");
|
||||
}
|
||||
} catch (Throwable throwable) {
|
||||
throw new RuntimeException(throwable);
|
||||
} finally {
|
||||
FileUtil.del(StrUtil.format("{}/{}", cacheFolderPath, request.uploadId));
|
||||
FileUtil.del(StrUtil.format("{}/{}", sliceFolderPath, request.uploadId));
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class StartRequest {
|
||||
private String name;
|
||||
private String filename;
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class StartResponse {
|
||||
private String uploadId;
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class SliceResponse {
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class FinishRequest {
|
||||
private String filename;
|
||||
private Long uploadId;
|
||||
private ImmutableList<Part> partList;
|
||||
|
||||
@Data
|
||||
public static final class Part {
|
||||
private Integer partNumber;
|
||||
@JsonProperty("eTag")
|
||||
private String eTag;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static final class FinishResponse {
|
||||
private Long id;
|
||||
private String filename;
|
||||
private String value;
|
||||
private String url;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.chat;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.Prompts;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.MessageVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.ChartTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.TableTool;
|
||||
import com.lanyuanxiaoyao.service.ai.web.tools.YarnTool;
|
||||
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
/**
|
||||
* 聊天
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250514
|
||||
*/
|
||||
@Controller
|
||||
@RequestMapping("chat")
|
||||
public class ChatController {
|
||||
private static final Logger logger = LoggerFactory.getLogger(ChatController.class);
|
||||
private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
private static final String ROLE_ASSISTANT = "assistant";
|
||||
private static final String ROLE_USER = "user";
|
||||
|
||||
private final ChatClient chatClient;
|
||||
|
||||
public ChatController(@Qualifier("chat") ChatClient.Builder builder) {
|
||||
this.chatClient = builder.build();
|
||||
}
|
||||
|
||||
private ChatClient.ChatClientRequestSpec buildRequest(ImmutableList<MessageVO> messages) {
|
||||
ChatClient.ChatClientRequestSpec spec = chatClient.prompt()
|
||||
.system(
|
||||
StrUtil.format("""
|
||||
你是一名专业的AI运维助手,专职负责“Hudi数据同步服务”的平台运维工作。你的核心职责是:
|
||||
1.友好解答:积极、专业地解答用户(通常是平台管理员或用户)关于该平台运维工作的疑问。
|
||||
2.知识驱动:在解答时,应尽可能通过各种方式(知识库、上下文、外部工具等)全面获取准确知识和数据来支持回答。
|
||||
3.诚实守界:
|
||||
对于无法通过已有知识或数据确认的问题,必须明确告知用户你无法解答,切勿捏造信息或提供不确定的答案。
|
||||
对于与该Hudi数据同步服务平台运维工作无关的问题,需委婉拒绝用户,并明确说明超出你的职责和能力范围。
|
||||
|
||||
对话语言:中文
|
||||
|
||||
{}
|
||||
|
||||
当前时间为:{}
|
||||
|
||||
""",
|
||||
Prompts.hudi,
|
||||
LocalDateTime.now().format(formatter)
|
||||
))
|
||||
.messages(
|
||||
messages
|
||||
.collect(message -> StrUtil.equals(message.getRole(), ROLE_ASSISTANT)
|
||||
? new AssistantMessage(message.getContent())
|
||||
: new UserMessage(message.getContent()))
|
||||
.collect(message -> (Message) message)
|
||||
.toList()
|
||||
);
|
||||
spec.tools(
|
||||
new TableTool(),
|
||||
new YarnTool(),
|
||||
new ChartTool()
|
||||
);
|
||||
return spec;
|
||||
}
|
||||
|
||||
@PostMapping("sync")
|
||||
@ResponseBody
|
||||
public MessageVO chatSync(
|
||||
@RequestBody ImmutableList<MessageVO> messages
|
||||
) {
|
||||
ChatResponse response = buildRequest(messages)
|
||||
.call()
|
||||
.chatResponse();
|
||||
return toMessage(response);
|
||||
}
|
||||
|
||||
@PostMapping("async")
|
||||
public SseEmitter chatAsync(
|
||||
@RequestBody ImmutableList<MessageVO> messages,
|
||||
HttpServletResponse httpResponse
|
||||
) {
|
||||
httpResponse.setHeader("X-Accel-Buffering", "no");
|
||||
|
||||
SseEmitter emitter = new SseEmitter(20 * 60 * 1000L);
|
||||
ExecutorProvider.EXECUTORS.submit(() -> {
|
||||
buildRequest(messages)
|
||||
.stream()
|
||||
.chatResponse()
|
||||
.subscribe(
|
||||
response -> {
|
||||
try {
|
||||
emitter.send(
|
||||
SseEmitter.event()
|
||||
.data(toMessage(response))
|
||||
.reconnectTime(5 * 1000L)
|
||||
.build()
|
||||
);
|
||||
} catch (IOException e) {
|
||||
emitter.completeWithError(e);
|
||||
}
|
||||
},
|
||||
emitter::completeWithError,
|
||||
emitter::complete
|
||||
);
|
||||
});
|
||||
emitter.onTimeout(() -> emitter.completeWithError(new TimeoutException("SseEmitter Timeout")));
|
||||
return emitter;
|
||||
}
|
||||
|
||||
private MessageVO toMessage(ChatResponse response) {
|
||||
AssistantMessage message = Optional.ofNullable(response)
|
||||
.map(ChatResponse::getResult)
|
||||
.map(Generation::getOutput)
|
||||
.orElseThrow(() -> new RuntimeException("ChatResponse is null"));
|
||||
MessageVO vo = new MessageVO();
|
||||
vo.setRole(ROLE_ASSISTANT);
|
||||
vo.setContent(message.getText());
|
||||
if (message instanceof DeepSeekAssistantMessage deepseekMessage) {
|
||||
vo.setReason(deepseekMessage.getReasoningContent());
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.feedback;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Sets;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("feedback")
|
||||
public class FeedbackController extends SimpleControllerSupport<Feedback, FeedbackController.SaveItem, FeedbackController.ListItem, FeedbackController.ListItem> {
|
||||
private final FeedbackService feedbackService;
|
||||
private final DataFileService dataFileService;
|
||||
|
||||
public FeedbackController(FeedbackService feedbackService, DataFileService dataFileService) {
|
||||
super(feedbackService);
|
||||
this.feedbackService = feedbackService;
|
||||
this.dataFileService = dataFileService;
|
||||
}
|
||||
|
||||
@GetMapping("reanalysis/{id}")
|
||||
public void reanalysis(@PathVariable("id") Long id) {
|
||||
feedbackService.reanalysis(id);
|
||||
}
|
||||
|
||||
@PostMapping("conclude")
|
||||
public void conclude(@RequestBody ConcludeItem item) {
|
||||
feedbackService.updateConclusion(item.getId(), item.getConclusion());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Feedback, SaveItem> saveItemMapper() {
|
||||
return item -> {
|
||||
Feedback feedback = new Feedback();
|
||||
feedback.setSource(item.getSource());
|
||||
feedback.setPictures(dataFileService.downloadFile(item.getPictures()).toSet());
|
||||
return feedback;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Feedback, ListItem> listItemMapper() {
|
||||
return Mappers.getMapper(ListItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Feedback, ListItem> detailItemMapper() {
|
||||
ListItemMapper<Feedback, ListItem> mapper = listItemMapper();
|
||||
return mapper::from;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String source;
|
||||
private ImmutableSet<Long> pictures;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class ConcludeItem {
|
||||
private Long id;
|
||||
private String conclusion;
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static class ListItem extends SimpleItem {
|
||||
private String source;
|
||||
private ImmutableSet<Long> pictures;
|
||||
private String analysis;
|
||||
private String conclusion;
|
||||
private Feedback.Status status;
|
||||
|
||||
@org.mapstruct.Mapper(
|
||||
imports = {
|
||||
IdOnlyEntity.class,
|
||||
Sets.class
|
||||
}
|
||||
)
|
||||
public interface Mapper extends ListItemMapper<Feedback, ListItem> {
|
||||
@Mapping(target = "pictures", expression = "java(Sets.immutable.ofAll(feedback.getPictures()).collect(IdOnlyEntity::getId))")
|
||||
@Override
|
||||
ListItem from(Feedback feedback);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge/group")
|
||||
public class GroupController extends SimpleControllerSupport<Group, GroupController.SaveItem, GroupController.ListItem, GroupController.ListItem> {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
|
||||
public GroupController(GroupService groupService, KnowledgeBaseService knowledgeBaseService) {
|
||||
super(groupService);
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Group, SaveItem> saveItemMapper() {
|
||||
return item -> {
|
||||
Group group = new Group();
|
||||
group.setName(item.getName());
|
||||
group.setKnowledge(knowledgeBaseService.detailOrThrow(item.getKnowledgeId()));
|
||||
return group;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Group, ListItem> listItemMapper() {
|
||||
return Mappers.getMapper(ListItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Group, ListItem> detailItemMapper() {
|
||||
ListItemMapper<Group, ListItem> mapper = listItemMapper();
|
||||
return mapper::from;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String name;
|
||||
private Long knowledgeId;
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static final class ListItem extends SimpleItem {
|
||||
private String name;
|
||||
private Group.Status status;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends ListItemMapper<Group, ListItem> {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisMapResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.EmbeddingService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.mapstruct.Mapping;
|
||||
import org.mapstruct.factory.Mappers;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250515
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge")
|
||||
public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge, KnowledgeBaseController.SaveItem, KnowledgeBaseController.ListItem, KnowledgeBaseController.ListItem> {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final EmbeddingService embeddingService;
|
||||
|
||||
public KnowledgeBaseController(KnowledgeBaseService knowledgeBaseService, EmbeddingService embeddingService) {
|
||||
super(knowledgeBaseService);
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.embeddingService = embeddingService;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SaveItemMapper<Knowledge, SaveItem> saveItemMapper() {
|
||||
return Mappers.getMapper(SaveItem.Mapper.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ListItemMapper<Knowledge, ListItem> listItemMapper() {
|
||||
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
|
||||
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected DetailItemMapper<Knowledge, ListItem> detailItemMapper() {
|
||||
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
|
||||
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
|
||||
}
|
||||
|
||||
@GetMapping("{id}/name")
|
||||
public AmisMapResponse name(@PathVariable("id") Long id) {
|
||||
return AmisResponse.responseMapData()
|
||||
.setData("name", knowledgeBaseService.getName(id));
|
||||
}
|
||||
|
||||
@PostMapping("update_description")
|
||||
public void updateDescription(
|
||||
@RequestParam("id") Long id,
|
||||
@RequestParam("description") String description
|
||||
) throws Exception {
|
||||
knowledgeBaseService.updateDescription(id, description);
|
||||
}
|
||||
|
||||
@PostMapping("preview_text")
|
||||
public AmisResponse<?> previewText(@RequestBody SubmitItem item) {
|
||||
if (StrUtil.equals("text", item.getType())) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(item.getMode(), item.getContent())
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else if (StrUtil.equals("file", item.getType())) {
|
||||
return AmisResponse.responseCrudData(
|
||||
embeddingService.preview(item.getMode(), item.getFiles())
|
||||
.collect(doc -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(doc.getId());
|
||||
vo.setText(doc.getText());
|
||||
return vo;
|
||||
})
|
||||
);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + item.getType());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text")
|
||||
public void submitText(@RequestBody SubmitItem item) {
|
||||
if (StrUtil.equals("text", item.getMode())) {
|
||||
embeddingService.submit(item.getId(), item.getMode(), item.getContent());
|
||||
} else if (StrUtil.equals("file", item.getType())) {
|
||||
embeddingService.submit(item.getId(), item.getMode(), item.getFiles());
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported type: " + item.getType());
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("submit_text_directly")
|
||||
public void submitDirectly(@RequestBody SubmitDirectlyItem item) {
|
||||
embeddingService.submitDirectly(item.getId(), item.getName(), Lists.immutable.ofAll(StrUtil.split(item.getContent(), item.getSplitKey())));
|
||||
}
|
||||
|
||||
@PostMapping("query")
|
||||
public ImmutableList<String> query(@RequestBody QueryItem item) throws ExecutionException, InterruptedException, IOException {
|
||||
return knowledgeBaseService.query(item.getId(), item.getText(), item.getLimit(), item.getThreshold());
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SaveItem {
|
||||
private String name;
|
||||
private String strategy;
|
||||
private String description;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends SaveItemMapper<Knowledge, SaveItem> {
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public static final class ListItem extends SimpleItem {
|
||||
private Long vectorSourceId;
|
||||
private String name;
|
||||
private String description;
|
||||
private String strategy;
|
||||
private Long size;
|
||||
private Long points;
|
||||
private Long segments;
|
||||
private String status;
|
||||
|
||||
@org.mapstruct.Mapper
|
||||
public interface Mapper extends ListItemMapper<Knowledge, ListItem> {
|
||||
@Mapping(source = "info.config.params.vectorsConfig.params.distance", target = "strategy")
|
||||
@Mapping(source = "info.config.params.vectorsConfig.params.size", target = "size")
|
||||
@Mapping(source = "info.pointsCount", target = "points")
|
||||
@Mapping(source = "info.segmentsCount", target = "segments")
|
||||
@Mapping(source = "info.status", target = "status")
|
||||
ListItem from(Knowledge knowledge, Collections.CollectionInfo info);
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SubmitItem {
|
||||
private Long id;
|
||||
private String mode;
|
||||
private String type;
|
||||
private String content;
|
||||
private ImmutableList<Long> files;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class SubmitDirectlyItem {
|
||||
private Long id;
|
||||
private String name;
|
||||
private String splitKey = "\n\n";
|
||||
private String content;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static final class QueryItem {
|
||||
private Long id;
|
||||
private Integer limit = 5;
|
||||
private Double threshold = 0.6;
|
||||
private String text;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.SegmentService;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("knowledge/segment")
|
||||
public class SegmentController {
|
||||
private final SegmentService segmentService;
|
||||
|
||||
public SegmentController(SegmentService segmentService) {
|
||||
this.segmentService = segmentService;
|
||||
}
|
||||
|
||||
@GetMapping("list")
|
||||
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId, @RequestParam("group_id") Long groupId) throws ExecutionException, InterruptedException {
|
||||
return AmisResponse.responseCrudData(segmentService.list(knowledgeId, groupId));
|
||||
}
|
||||
|
||||
@GetMapping("delete")
|
||||
public AmisResponse<?> delete(@RequestParam("knowledge_id") Long knowledgeId, @RequestParam("segment_id") Long segmentId) throws ExecutionException, InterruptedException {
|
||||
segmentService.remove(knowledgeId, segmentId);
|
||||
return AmisResponse.responseSuccess();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
|
||||
/**
|
||||
* 上传文件
|
||||
*
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2024-11-21
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_file")
|
||||
@NoArgsConstructor
|
||||
public class DataFile extends SimpleEntity {
|
||||
private String filename;
|
||||
private Long size;
|
||||
private String md5;
|
||||
private String path;
|
||||
private String type;
|
||||
|
||||
public DataFile(String filename) {
|
||||
this.filename = filename;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.ConstraintMode;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.ForeignKey;
|
||||
import jakarta.persistence.JoinTable;
|
||||
import jakarta.persistence.NamedAttributeNode;
|
||||
import jakarta.persistence.NamedEntityGraph;
|
||||
import jakarta.persistence.OneToMany;
|
||||
import jakarta.persistence.Table;
|
||||
import java.util.Set;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_feedback")
|
||||
@NamedEntityGraph(name = "feedback.detail", attributeNodes = {
|
||||
@NamedAttributeNode("pictures")
|
||||
})
|
||||
public class Feedback extends SimpleEntity {
|
||||
@Column(nullable = false, columnDefinition = "longtext")
|
||||
private String source;
|
||||
@OneToMany(fetch = FetchType.EAGER)
|
||||
@JoinTable(catalog = Constants.DATABASE_NAME, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
|
||||
@ToString.Exclude
|
||||
private Set<DataFile> pictures;
|
||||
@Column(columnDefinition = "longtext")
|
||||
private String analysis;
|
||||
@Column(columnDefinition = "longtext")
|
||||
private String conclusion;
|
||||
@Column(nullable = false)
|
||||
private Status status = Status.ANALYSIS_PROCESSING;
|
||||
|
||||
public enum Status {
|
||||
ANALYSIS_PROCESSING,
|
||||
ANALYSIS_SUCCESS,
|
||||
FINISHED,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.ConstraintMode;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.ForeignKey;
|
||||
import jakarta.persistence.JoinColumn;
|
||||
import jakarta.persistence.ManyToOne;
|
||||
import jakarta.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_group")
|
||||
public class Group extends SimpleEntity {
|
||||
@Column(nullable = false)
|
||||
private String name;
|
||||
@Column(nullable = false)
|
||||
private Status status = Status.RUNNING;
|
||||
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(nullable = false, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
|
||||
@ToString.Exclude
|
||||
private Knowledge knowledge;
|
||||
|
||||
public enum Status {
|
||||
RUNNING,
|
||||
FINISHED,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import jakarta.persistence.CascadeType;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.EntityListeners;
|
||||
import jakarta.persistence.FetchType;
|
||||
import jakarta.persistence.OneToMany;
|
||||
import jakarta.persistence.Table;
|
||||
import java.util.Set;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@Entity
|
||||
@DynamicUpdate
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_knowledge")
|
||||
public class Knowledge extends SimpleEntity {
|
||||
@Column(nullable = false)
|
||||
private Long vectorSourceId;
|
||||
@Column(nullable = false)
|
||||
private String name;
|
||||
@Column(nullable = false, columnDefinition = "longtext")
|
||||
private String description;
|
||||
@Column(nullable = false)
|
||||
private Strategy strategy = Strategy.Cosine;
|
||||
|
||||
@OneToMany(fetch = FetchType.LAZY, mappedBy = "knowledge", cascade = CascadeType.ALL)
|
||||
@ToString.Exclude
|
||||
private Set<Group> groups;
|
||||
|
||||
public enum Strategy {
|
||||
Cosine,
|
||||
Euclid,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.context;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.factory.Maps;
|
||||
import org.springframework.ai.document.Document;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class EmbeddingContext {
|
||||
private Long vectorSourceId;
|
||||
private Long groupId;
|
||||
private Config config;
|
||||
private String content;
|
||||
private String file;
|
||||
private String fileFormat;
|
||||
@Builder.Default
|
||||
private List<Document> documents = Lists.mutable.empty();
|
||||
@Builder.Default
|
||||
private Map<String, Object> metadata = Maps.mutable.empty();
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static final class Config {
|
||||
@Builder.Default
|
||||
private SplitStrategy splitStrategy = SplitStrategy.NORMAL;
|
||||
|
||||
public enum SplitStrategy {
|
||||
NORMAL, LLM, QA
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.context;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250616
|
||||
*/
|
||||
@Data
|
||||
public class FeedbackContext {
|
||||
private Feedback feedback;
|
||||
private String optimizedSource;
|
||||
private List<String> pictureDescriptions = Lists.mutable.empty();
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class MessageVO {
|
||||
private String role;
|
||||
private String content;
|
||||
private String reason;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.entity.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250516
|
||||
*/
|
||||
@Data
|
||||
public class SegmentVO {
|
||||
private String id;
|
||||
private String text;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface DataFileRepository extends SimpleRepository<DataFile, Long> {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.data.jpa.repository.EntityGraph;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@SuppressWarnings("NullableProblems")
|
||||
@Repository
|
||||
public interface FeedbackRepository extends SimpleRepository<Feedback, Long> {
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
List<Feedback> findAll(Specification<Feedback> specification);
|
||||
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
List<Feedback> findAll(Specification<Feedback> specification, Sort sort);
|
||||
|
||||
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
|
||||
@Override
|
||||
Optional<Feedback> findOne(Specification<Feedback> specification);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface GroupRepository extends SimpleRepository<Group, Long> {
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.repository;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface KnowledgeRepository extends SimpleRepository<Knowledge, Long> {
|
||||
Boolean existsKnowledgeByName(String name);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.DataFileRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.eclipse.collections.api.factory.Sets;
|
||||
import org.eclipse.collections.api.set.ImmutableSet;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250527
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DataFileService {
|
||||
private final DataFileRepository dataFileRepository;
|
||||
|
||||
public DataFileService(DataFileRepository dataFileRepository) {
|
||||
this.dataFileRepository = dataFileRepository;
|
||||
}
|
||||
|
||||
public DataFile downloadFile(Long id) {
|
||||
return dataFileRepository.findById(id).orElseThrow(() -> new RuntimeException(StrUtil.format("Datafile not exists: {}", id)));
|
||||
}
|
||||
|
||||
public ImmutableSet<DataFile> downloadFile(ImmutableSet<Long> ids) {
|
||||
return Sets.immutable.ofAll(dataFileRepository.findAllById(ids));
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long initialDataFile(String filename) {
|
||||
DataFile dataFile = dataFileRepository.save(new DataFile(filename));
|
||||
return dataFile.getId();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDataFile(Long id, String path, Long size, String md5, String type) {
|
||||
DataFile dataFile = downloadFile(id);
|
||||
dataFile.setPath(path);
|
||||
dataFile.setSize(size);
|
||||
dataFile.setMd5(md5);
|
||||
dataFile.setType(type);
|
||||
dataFileRepository.save(dataFile);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Pair;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
|
||||
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
|
||||
import com.yomahub.liteflow.core.FlowExecutor;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class EmbeddingService {
|
||||
private final DataFileService dataFileService;
|
||||
private final FlowExecutor executor;
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final GroupService groupService;
|
||||
private final ExecutorService executors = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
|
||||
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
|
||||
public EmbeddingService(DataFileService dataFileService, FlowExecutor executor, KnowledgeBaseService knowledgeBaseService, GroupService groupService) {
|
||||
this.dataFileService = dataFileService;
|
||||
this.executor = executor;
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.groupService = groupService;
|
||||
}
|
||||
|
||||
public ImmutableList<Document> preview(String mode, String content) {
|
||||
if (content.length() > 2000) {
|
||||
content = content.substring(0, 2000);
|
||||
}
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.content(content)
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_preview", null, context);
|
||||
return Lists.immutable.ofAll(context.getDocuments());
|
||||
}
|
||||
|
||||
public ImmutableList<Document> preview(String mode, ImmutableList<Long> ids) {
|
||||
DataFile dataFile = dataFileService.downloadFile(ids.get(0));
|
||||
String content = FileUtil.readString(dataFile.getPath(), StandardCharsets.UTF_8);
|
||||
return preview(mode, content);
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, String content) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
Group group = new Group();
|
||||
group.setName(StrUtil.format("文本-{}", IdUtil.nanoId(10)));
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.content(content)
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_submit", null, context);
|
||||
groupService.finish(groupId);
|
||||
});
|
||||
}
|
||||
|
||||
public void submit(Long id, String mode, ImmutableList<Long> ids) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
List<Pair<Long, DataFile>> dataFiles = Lists.mutable.empty();
|
||||
for (Long fileId : ids) {
|
||||
DataFile dataFile = dataFileService.downloadFile(fileId);
|
||||
Group group = new Group();
|
||||
group.setName(dataFile.getFilename());
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
dataFiles.add(Pair.of(groupId, dataFile));
|
||||
}
|
||||
for (Pair<Long, DataFile> pair : dataFiles) {
|
||||
Long groupId = pair.getKey();
|
||||
DataFile dataFile = pair.getValue();
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.file(dataFile.getPath())
|
||||
.fileFormat(dataFile.getFilename())
|
||||
.config(EmbeddingContext.Config.builder()
|
||||
.splitStrategy(EmbeddingContext.Config.SplitStrategy.valueOf(mode))
|
||||
.build())
|
||||
.build();
|
||||
executor.execute2Resp("embedding_submit", null, context);
|
||||
groupService.finish(groupId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void submitDirectly(Long id, String name, ImmutableList<String> contents) {
|
||||
executors.submit(() -> {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
|
||||
String groupName = name;
|
||||
if (StrUtil.isBlank(groupName)) {
|
||||
groupName = StrUtil.format("外部-{}", IdUtil.nanoId(10));
|
||||
}
|
||||
Group group = new Group();
|
||||
group.setName(groupName);
|
||||
group.setKnowledge(knowledge);
|
||||
Long groupId = groupService.save(group);
|
||||
EmbeddingContext context = EmbeddingContext.builder()
|
||||
.vectorSourceId(knowledge.getVectorSourceId())
|
||||
.groupId(groupId)
|
||||
.build();
|
||||
context.setDocuments(
|
||||
contents
|
||||
.collect(StrUtil::trim)
|
||||
.collect(content ->
|
||||
Document.builder()
|
||||
.text(content)
|
||||
.metadata(new HashMap<>())
|
||||
.build()
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
executor.execute2Resp("embedding_submit_directly", null, context);
|
||||
groupService.finish(groupId);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.feedback;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback_;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.FeedbackRepository;
|
||||
import com.yomahub.liteflow.core.FlowExecutor;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class FeedbackService extends SimpleServiceSupport<Feedback> {
|
||||
private final FlowExecutor executor;
|
||||
|
||||
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
|
||||
public FeedbackService(FeedbackRepository feedbackRepository, FlowExecutor executor) {
|
||||
super(feedbackRepository);
|
||||
this.executor = executor;
|
||||
}
|
||||
|
||||
@Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
|
||||
public void analysis() {
|
||||
List<Feedback> feedbacks = repository.findAll(
|
||||
builder -> builder
|
||||
.andEquals(Feedback_.STATUS, Feedback.Status.ANALYSIS_PROCESSING)
|
||||
.build()
|
||||
);
|
||||
for (Feedback feedback : feedbacks) {
|
||||
FeedbackContext context = new FeedbackContext();
|
||||
context.setFeedback(feedback);
|
||||
executor.execute2Resp("feedback_analysis", null, context);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateAnalysis(Long id, String analysis) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setAnalysis(analysis);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateConclusion(Long id, String conclusion) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setConclusion(conclusion);
|
||||
feedback.setStatus(Feedback.Status.FINISHED);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateStatus(Long id, Feedback.Status status) {
|
||||
Feedback feedback = detailOrThrow(id);
|
||||
feedback.setStatus(status);
|
||||
repository.save(feedback);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void reanalysis(Long id) {
|
||||
updateStatus(id, Feedback.Status.ANALYSIS_PROCESSING);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.GroupRepository;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import lombok.SneakyThrows;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class GroupService extends SimpleServiceSupport<Group> {
|
||||
private final QdrantClient client;
|
||||
|
||||
public GroupService(GroupRepository groupRepository, VectorStore vectorStore) {
|
||||
super(groupRepository);
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void finish(Long id) {
|
||||
Group group = detailOrThrow(id);
|
||||
group.setStatus(Group.Status.FINISHED);
|
||||
save(group);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
Group group = detailOrThrow(id);
|
||||
Knowledge knowledge = group.getKnowledge();
|
||||
client.deleteAsync(
|
||||
String.valueOf(knowledge.getVectorSourceId()),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
|
||||
.build()
|
||||
).get();
|
||||
super.remove(id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
|
||||
import com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.repository.KnowledgeRepository;
|
||||
import com.lanyuanxiaoyao.service.common.Constants;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import lombok.SneakyThrows;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250522
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
|
||||
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
|
||||
private final KnowledgeRepository knowledgeRepository;
|
||||
private final EmbeddingModel model;
|
||||
private final QdrantClient client;
|
||||
|
||||
public KnowledgeBaseService(KnowledgeRepository knowledgeRepository, EmbeddingModel model, VectorStore vectorStore) {
|
||||
super(knowledgeRepository);
|
||||
this.knowledgeRepository = knowledgeRepository;
|
||||
this.model = model;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public Long save(Knowledge entity) {
|
||||
if (knowledgeRepository.existsKnowledgeByName(entity.getName())) {
|
||||
throw new RuntimeException("名称已存在");
|
||||
}
|
||||
|
||||
Long vectorSourceId = SnowflakeIdGenerator.Snowflake.next();
|
||||
client.createCollectionAsync(
|
||||
String.valueOf(vectorSourceId),
|
||||
Collections.VectorParams.newBuilder()
|
||||
.setDistance(Collections.Distance.valueOf(entity.getStrategy().name()))
|
||||
.setSize(model.dimensions())
|
||||
.build()
|
||||
).get();
|
||||
|
||||
entity.setVectorSourceId(vectorSourceId);
|
||||
return super.save(entity);
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateDescription(Long id, String description) throws Exception {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
knowledge.setDescription(description);
|
||||
save(knowledge);
|
||||
}
|
||||
|
||||
public String getName(Long id) {
|
||||
return detailOrThrow(id).getName();
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Override
|
||||
public void remove(Long id) {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
for (Group group : knowledge.getGroups()) {
|
||||
client.deleteAsync(
|
||||
String.valueOf(knowledge.getVectorSourceId()),
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
|
||||
.build()
|
||||
).get();
|
||||
}
|
||||
super.remove(id);
|
||||
}
|
||||
|
||||
public ImmutableList<String> query(
|
||||
Long id,
|
||||
String text,
|
||||
Integer limit,
|
||||
Double threshold
|
||||
) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = detailOrThrow(id);
|
||||
Boolean exists = client.collectionExistsAsync(String.valueOf(knowledge.getVectorSourceId())).get();
|
||||
if (!exists) {
|
||||
throw new RuntimeException(StrUtil.format("{} not exists", id));
|
||||
}
|
||||
VectorStore vs = QdrantVectorStore.builder(client, model)
|
||||
.collectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.initializeSchema(false)
|
||||
.build();
|
||||
List<Document> documents = vs.similaritySearch(
|
||||
SearchRequest.builder()
|
||||
.query(text)
|
||||
.topK(limit)
|
||||
.similarityThreshold(threshold)
|
||||
.build()
|
||||
);
|
||||
return Lists.immutable.ofAll(documents)
|
||||
.collect(Document::getText);
|
||||
}
|
||||
|
||||
public Collections.CollectionInfo collectionInfo(Long vectorSourceId) throws ExecutionException, InterruptedException {
|
||||
return client.getCollectionInfoAsync(String.valueOf(vectorSourceId)).get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
|
||||
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.vo.SegmentVO;
|
||||
import io.qdrant.client.ConditionFactory;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Points;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.stream.Collectors;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250528
|
||||
*/
|
||||
@Service
|
||||
public class SegmentService {
|
||||
private final KnowledgeBaseService knowledgeBaseService;
|
||||
private final QdrantClient client;
|
||||
|
||||
public SegmentService(KnowledgeBaseService knowledgeBaseService, VectorStore vectorStore) {
|
||||
this.knowledgeBaseService = knowledgeBaseService;
|
||||
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
}
|
||||
|
||||
public ImmutableList<SegmentVO> list(Long knowledgeId, Long groupId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
|
||||
Points.ScrollResponse response = client.scrollAsync(
|
||||
Points.ScrollPoints.newBuilder()
|
||||
.setCollectionName(String.valueOf(knowledge.getVectorSourceId()))
|
||||
.setWithPayload(Points.WithPayloadSelector.newBuilder().setEnable(true).build())
|
||||
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(false).build())
|
||||
.setFilter(
|
||||
Points.Filter.newBuilder()
|
||||
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.get();
|
||||
return response.getResultList()
|
||||
.stream()
|
||||
.collect(Collectors.toCollection(Lists.mutable::empty))
|
||||
.collect(point -> {
|
||||
SegmentVO vo = new SegmentVO();
|
||||
vo.setId(point.getId().getUuid());
|
||||
vo.setText(point.getPayloadMap().get("doc_content").getStringValue());
|
||||
return vo;
|
||||
})
|
||||
.toImmutable();
|
||||
}
|
||||
|
||||
public void remove(Long knowledgeId, Long segmentId) throws ExecutionException, InterruptedException {
|
||||
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
|
||||
client.deletePayloadAsync(
|
||||
String.valueOf(knowledgeId),
|
||||
List.of(String.valueOf(segmentId)),
|
||||
null,
|
||||
null,
|
||||
null
|
||||
).get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package com.lanyuanxiaoyao.service.ai.web.service.node;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.lang.Assert;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext;
|
||||
import com.yomahub.liteflow.annotation.LiteflowComponent;
|
||||
import com.yomahub.liteflow.annotation.LiteflowMethod;
|
||||
import com.yomahub.liteflow.core.NodeComponent;
|
||||
import com.yomahub.liteflow.enums.LiteFlowMethodEnum;
|
||||
import com.yomahub.liteflow.enums.NodeTypeEnum;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.document.DocumentReader;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.reader.ExtractedTextFormatter;
|
||||
import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
|
||||
import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
|
||||
import org.springframework.ai.reader.tika.TikaDocumentReader;
|
||||
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.core.io.PathResource;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @version 20250523
|
||||
*/
|
||||
@Slf4j
|
||||
@LiteflowComponent
|
||||
public class EmbeddingNodes {
|
||||
private final ChatClient.Builder chatClientBuilder;
|
||||
private final QdrantClient qdrantClient;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
public EmbeddingNodes(@Qualifier("chat") ChatClient.Builder builder, VectorStore vectorStore, EmbeddingModel embeddingModel) {
|
||||
this.chatClientBuilder = builder;
|
||||
this.qdrantClient = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_BOOLEAN, nodeId = "embedding_check_if_file_needed", nodeName = "判断是否需要读取文件", nodeType = NodeTypeEnum.BOOLEAN)
|
||||
public boolean checkIfFileReadNeeded(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
Assert.notNull(context, "EmbeddingContext is null");
|
||||
if (StrUtil.isNotBlank(context.getFile())) {
|
||||
Assert.isTrue(FileUtil.exist(context.getFile()), "File [{}] not exist", context.getFile());
|
||||
return true;
|
||||
}
|
||||
Assert.notBlank(context.getContent(), "Contents is empty");
|
||||
return false;
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_SWITCH, nodeId = "file_reader_switch", nodeName = "判断文件格式", nodeType = NodeTypeEnum.SWITCH)
|
||||
public String fileReaderSwitch(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
String extName = FileUtil.extName(context.getFileFormat());
|
||||
return switch (extName.toLowerCase()) {
|
||||
case "txt", "md", "markdown" -> "txt_file_reader";
|
||||
case "pdf" -> "pdf_file_reader";
|
||||
case "doc", "docx", "xls", "xlsx", "ppt", "pptx", "html", "xml", "wps", "et", "dpt" -> "any_file_reader";
|
||||
default -> throw new IllegalStateException("Unsupported ext: " + extName);
|
||||
};
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "txt_file_reader", nodeName = "读取文本文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void txtFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.setContent(FileUtil.readString(context.getFile(), Charset.defaultCharset()));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "pdf_file_reader", nodeName = "读取pdf文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void pdfFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
PagePdfDocumentReader reader = new PagePdfDocumentReader(
|
||||
new PathResource(context.getFile()),
|
||||
PdfDocumentReaderConfig.builder()
|
||||
.withPageTopMargin(0)
|
||||
.withPageExtractedTextFormatter(ExtractedTextFormatter.builder()
|
||||
.withNumberOfTopTextLinesToDelete(0)
|
||||
.build())
|
||||
.build());
|
||||
context.setContent(readBySpringAiReader(reader));
|
||||
}
|
||||
|
||||
/**
|
||||
* <a href="https://tika.apache.org/3.1.0/formats.html#Database_formats">Tika支持的格式</a>
|
||||
*/
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "any_file_reader", nodeName = "使用Tika尝试读取文件", nodeType = NodeTypeEnum.COMMON)
|
||||
public void anyFileReader(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.setContent(readBySpringAiReader(new TikaDocumentReader(new PathResource(context.getFile()))));
|
||||
}
|
||||
|
||||
private String readBySpringAiReader(DocumentReader reader) {
|
||||
return reader.get()
|
||||
.stream()
|
||||
.map(Document::getText)
|
||||
.collect(Collectors.joining("\n"))
|
||||
.trim();
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS_SWITCH, nodeId = "split_switch", nodeName = "判断使用什么分段方法", nodeType = NodeTypeEnum.SWITCH)
|
||||
public String splitSwitch(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
return switch (context.getConfig().getSplitStrategy()) {
|
||||
case NORMAL -> "normal_split";
|
||||
case LLM -> "llm_split";
|
||||
case QA -> "qa_split";
|
||||
};
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "normal_split", nodeName = "使用普通分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void normalSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
TokenTextSplitter splitter = new TokenTextSplitter(200, 100, 5, 200, true);
|
||||
Document document = Document.builder()
|
||||
.text(context.getContent())
|
||||
.build();
|
||||
context.setDocuments(splitter.split(document));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "llm_split", nodeName = "使用大模型分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void llmSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
context.getDocuments().addAll(llmSplit(
|
||||
"""
|
||||
请你将用户输入的文本进行语义切分,生成用于知识库检索的文本段。
|
||||
每个文本段要尽可能多地覆盖用户输入文本的各方面知识和细节,包括但不限于主题、概念、关键信息等。对于关键的数字、理论、细节等,要严格遵循原文,不能进行任何虚构和捏造不存在的知识,确保输出内容准确、真实且全面。
|
||||
输出格式为纯文本段,分段之间使用“---”作为分割,方便后续使用代码进行切分。
|
||||
输出文本避免添加markdown格式,保持文本格式紧凑。
|
||||
切分过程中,要注重保持文本的完整性和逻辑性,确保每个文本段都能独立地表达出清晰、准确的信息,以便更好地进行知识库检索。
|
||||
""",
|
||||
context.getContent(),
|
||||
context.getMetadata()
|
||||
));
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "qa_split", nodeName = "使用Q/A格式分段", nodeType = NodeTypeEnum.COMMON)
|
||||
public void qaSplit(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
// language=TEXT
|
||||
context.getDocuments().addAll(llmSplit(
|
||||
"""
|
||||
对用户输入的文本,生成多组高质量的问答对。请遵循以下指南:
|
||||
|
||||
1. 问题部分:
|
||||
为同一个主题创建尽可能多的不同表述的问题,确保问题的多样性。
|
||||
每个问题应考虑用户可能的多种问法,例如:
|
||||
直接询问(如“什么是...?”)
|
||||
请求确认(如“是否可以说...?”)
|
||||
寻求解释(如“请解释一下...的含义。”)
|
||||
假设性问题(如“如果...会怎样?”)
|
||||
例子请求(如“能否举个例子说明...?”)
|
||||
问题应涵盖文本中的关键信息、主要概念和细节,确保不遗漏重要内容。
|
||||
2. 答案部分:
|
||||
提供一个全面、信息丰富的答案,涵盖问题的所有可能角度,确保逻辑连贯。
|
||||
答案应直接基于给定文本,确保准确性和一致性。
|
||||
包含相关的细节,如日期、名称、职位等具体信息,必要时提供背景信息以增强理解。
|
||||
3. 格式:
|
||||
使用"问:"标记问题的开始,问题文本应在一个段落内完成。
|
||||
使用"答:"标记答案的开始,答案应清晰分段,便于阅读。
|
||||
问答对之间用“---”分隔,以提高可读性。
|
||||
4. 内容要求:
|
||||
确保问答对紧密围绕文本主题,避免偏离主题。
|
||||
避免添加文本中未提及的信息,确保信息的真实性。
|
||||
一个问题搭配一个答案,避免一组问答对中同时涉及多个问题。
|
||||
如果文本信息不足以回答某个方面,可以在答案中说明 "根据给定信息无法确定",并尽量提供相关的上下文。
|
||||
除了问答对本身,避免输出任何与问答对无关的提示性、引导性、解释性的文本。
|
||||
|
||||
格式样例:
|
||||
问:苹果通常是什么颜色的?
|
||||
答:红色。
|
||||
---
|
||||
问:苹果长在树上还是地上?
|
||||
答:苹果长在树上。
|
||||
""",
|
||||
context.getContent(),
|
||||
context.getMetadata()
|
||||
));
|
||||
}
|
||||
|
||||
private List<Document> llmSplit(String prompt, String content, Map<String, Object> metadata) {
|
||||
ChatClient client = chatClientBuilder.build();
|
||||
String response = client.prompt()
|
||||
.system(prompt)
|
||||
.user(content)
|
||||
.call()
|
||||
.content();
|
||||
Assert.notBlank(response, "LLM response is empty");
|
||||
// noinspection DataFlowIssue
|
||||
return Arrays.stream(StrUtil.trim(response).split("---"))
|
||||
.map(text -> text.replaceAll("(?!^.+) +$", ""))
|
||||
.map(StrUtil::trim)
|
||||
.map(text -> Document.builder()
|
||||
.text(text)
|
||||
.metadata(Optional.ofNullable(metadata).orElse(new HashMap<>()))
|
||||
.build())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "import_vector_source", nodeName = "导入向量库", nodeType = NodeTypeEnum.COMMON)
|
||||
public void importVectorSource(NodeComponent node) {
|
||||
EmbeddingContext context = node.getContextBean(EmbeddingContext.class);
|
||||
Assert.notNull(context.getVectorSourceId(), "VectorSourceId is null");
|
||||
Assert.notNull(context.getGroupId(), "GroupId is null");
|
||||
if (ObjectUtil.isNotEmpty(context.getDocuments())) {
|
||||
VectorStore vs = QdrantVectorStore.builder(qdrantClient, embeddingModel)
|
||||
.collectionName(String.valueOf(context.getVectorSourceId()))
|
||||
.build();
|
||||
for (Document document : context.getDocuments()) {
|
||||
Map<String, Object> metadata = document.getMetadata();
|
||||
if (StrUtil.isNotBlank(context.getFileFormat()))
|
||||
metadata.put("filename", context.getFileFormat());
|
||||
if (StrUtil.isNotBlank(context.getFile()))
|
||||
metadata.put("filepath", context.getFile());
|
||||
metadata.put("group_id", String.valueOf(context.getGroupId()));
|
||||
metadata.put("vector_source_id", String.valueOf(context.getVectorSourceId()));
|
||||
}
|
||||
vs.add(context.getDocuments());
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user