lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(knowledge): 完成知识库部署相关脚本调整

Browse Source
This commit is contained in:
v-zhangjc9
2025-05-28 18:46:20 +08:00
parent e6e24dff27
commit 8c9cb6f21d
11 changed files with 67 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/WebConfiguration.java Normal file
View File

@@ -0,0 +1,24 @@
package com.lanyuanxiaoyao.service.ai.core.configuration;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
/**
* web 配置
*
* @author lanyuanxiaoyao
* @date 2023-04-21
*/
@Configuration
public class WebConfiguration implements WebMvcConfigurer {
@Override
public void addCorsMappings(CorsRegistry registry) {
// 避免跨域影响调试
registry.addMapping("/**")
.allowedOriginPatterns("*")
.allowCredentials(true)
.allowedMethods("*")
.maxAge(3600);
}
}

1
service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/DataFileController.java
View File

@@ -90,7 +90,6 @@ public class DataFileController {
@PostMapping("/start")
public AmisResponse<StartResponse> start(@RequestBody StartRequest request) {
log.info("Request: {}", request);
Long id = dataFileService.initialDataFile(request.filename);
return AmisResponse.responseSuccess(new StartResponse(id.toString()));
}

38
service-ai/service-ai-knowledge/src/main/resources/application.yml
View File

@@ -2,31 +2,7 @@ spring:
application:
name: service-ai-knowledge
profiles:
include: common,metrics,forest
cloud:
zookeeper:
enabled: true
connect-string: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
discovery:
enabled: ${spring.cloud.zookeeper.enabled}
root: /hudi-services
instance-id: ${spring.application.name}-127.0.0.1-${random.uuid}-20250514
metadata:
discovery: zookeeper
ip: 127.0.0.1
hostname: localhost
hostname_full: localhost
start_time: 20250514112750
datasource:
url: jdbc:mysql://localhost:3307/ai?useSSL=false
username: test
password: test
driver-class-name: com.mysql.cj.jdbc.Driver
security:
meta:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
include: random-port,common,discovery,metrics,forest
ai:
openai:
base-url: http://132.121.206.65:10086
@@ -39,16 +15,10 @@ spring:
model: 'Bge-m3-vllm'
vectorstore:
qdrant:
api-key: lanyuanxiaoyao
jasypt:
encryptor:
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
server:
port: 8080
host: 132.121.206.65
port: 29463
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
liteflow:
rule-source: config/flow.xml
print-banner: false
check-node-exists: false
knowledge:
download-prefix: "http://localhost:8080"
upload-path: /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-ai/temp

2
service-ai/service-ai-knowledge/src/main/resources/logback-spring.xml
View File

@@ -29,6 +29,6 @@
<root level="INFO">
<appender-ref ref="Console"/>
<!-- <appender-ref ref="RollingFile"/>-->
<appender-ref ref="RollingFile"/>
</root>
</configuration>

13
service-cli/service-cli-runner/src/main/resources/application.yml
View File

@@ -168,3 +168,16 @@ deploy:
source-jar: service-ai-chat-1.0.0-SNAPSHOT.jar
jdk: "jdk17"
replicas: 1
service-ai-knowledge:
order: 6
groups:
- "ai"
source-jar: service-ai-knowledge-1.0.0-SNAPSHOT.jar
jdk: "jdk17"
replicas: 1
arguments:
"[knowledge.upload-path]": ${deploy.runtime.data-path}/knowledge
"[spring.datasource.url]": ${deploy.runtime.database.config.url}
"[spring.datasource.username]": ${deploy.runtime.database.config.username}
"[spring.datasource.password]": ${deploy.runtime.database.config.password}
"[spring.datasource.driver-class-name]": ${deploy.runtime.database.config.driver-class-name}

2
service-configuration/src/test/java/com/lanyuanxiaoyao/service/configuration/TestSecurityDecrypt.java
View File

@@ -21,6 +21,6 @@ public class TestSecurityDecrypt {
config.setStringOutputType("base64");
encryptor.setConfig(config);
System.out.println(encryptor.decrypt("GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu"));
System.out.println(encryptor.encrypt("jdHyKdp9qxNqCK3c"));
}
}

35
service-web/client/src/pages/ai/knowledge/DataDetail.tsx
View File

@@ -1,6 +1,6 @@
import React from 'react'
import {useNavigate, useParams} from 'react-router'
import {amisRender, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx'
import {amisRender, commonInfo, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx'
const statusMapping = [
mappingItem('解析中', 'RUNNING', 'bg-warning'),
@@ -24,16 +24,7 @@ const DataDetail: React.FC = () => {
{
type: 'service',
className: 'inline',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/knowledge/name',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
id: knowledge_id,
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/knowledge/name?id=${knowledge_id}`,
body: {
type: 'tpl',
tpl: '${name}',
@@ -47,16 +38,7 @@ const DataDetail: React.FC = () => {
body: [
{
type: 'crud',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/group/list',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
knowledge_id,
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/group/list?knowledge_id=${knowledge_id}`,
...crudCommonOptions(),
headerToolbar: [
'reload',
@@ -128,16 +110,7 @@ const DataDetail: React.FC = () => {
level: 'link',
size: 'sm',
actionType: 'ajax',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/group/delete',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
id: '${id}',
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/group/delete?id=\${id}`,
confirmText: '确认删除',
confirmTitle: '删除',
},

55
service-web/client/src/pages/ai/knowledge/DataImport.tsx
View File

@@ -1,7 +1,7 @@
import React from 'react'
import {useParams} from 'react-router'
import styled from 'styled-components'
import {amisRender} from '../../../util/amis.tsx'
import {amisRender, commonInfo} from '../../../util/amis.tsx'
const ImportDataDiv = styled.div`
.antd-EditorControl {
@@ -23,16 +23,7 @@ const DataImport: React.FC = () => {
{
type: 'service',
className: 'inline',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/knowledge/name',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
id: knowledge_id,
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/knowledge/name?id=${knowledge_id}`,
body: {
type: 'tpl',
tpl: '${name}',
@@ -112,35 +103,17 @@ const DataImport: React.FC = () => {
autoUpload: false,
drag: true,
multiple: true,
useChunk: true,
accept: '*',
// 5MB 5242880
// 100MB 104857600
// 500MB 524288000
// 1GB 1073741824
maxSize: '',
maxLength: 0,
startChunkApi: {
method: 'post',
url: 'http://127.0.0.1:8080/upload/start',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
chunkApi: {
method: 'post',
url: 'http://127.0.0.1:8080/upload/slice',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
finishChunkApi: {
method: 'post',
url: 'http://127.0.0.1:8080/upload/finish',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
maxSize: 104857600,
receiver: `${commonInfo.baseAiKnowledgeUrl}/upload`
// useChunk: true,
// startChunkApi: `post:${commonInfo.baseAiKnowledgeUrl}/upload/start`,
// chunkApi: `post:${commonInfo.baseAiKnowledgeUrl}/upload/slice`,
// finishChunkApi: `post:${commonInfo.baseAiKnowledgeUrl}/upload/finish`,
},
{
className: 'text-right',
@@ -148,7 +121,7 @@ const DataImport: React.FC = () => {
buttons: [
{
type: 'reset',
label: '重置'
label: '重置',
},
{
type: 'submit',
@@ -157,10 +130,7 @@ const DataImport: React.FC = () => {
level: 'secondary',
api: {
method: 'post',
url: 'http://127.0.0.1:8080/knowledge/preview_text',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
url: `${commonInfo.baseAiKnowledgeUrl}/knowledge/preview_text`,
dataType: 'form',
data: {
mode: '${mode|default:undefined}',
@@ -178,10 +148,7 @@ const DataImport: React.FC = () => {
actionType: 'ajax',
api: {
method: 'post',
url: 'http://127.0.0.1:8080/knowledge/submit_text',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
url: `${commonInfo.baseAiKnowledgeUrl}/knowledge/submit_text`,
dataType: 'form',
data: {
id: knowledge_id,

31
service-web/client/src/pages/ai/knowledge/DataSegment.tsx
View File

@@ -1,6 +1,6 @@
import React from 'react'
import {useParams} from 'react-router'
import {amisRender, crudCommonOptions} from '../../../util/amis.tsx'
import {amisRender, commonInfo, crudCommonOptions} from '../../../util/amis.tsx'
const DataDetail: React.FC = () => {
const {knowledge_id, group_id} = useParams()
@@ -18,16 +18,7 @@ const DataDetail: React.FC = () => {
{
type: 'service',
className: 'inline',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/knowledge/name',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
id: knowledge_id,
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/knowledge/name?id=${knowledge_id}`,
body: {
type: 'tpl',
tpl: '${name}',
@@ -41,17 +32,7 @@ const DataDetail: React.FC = () => {
body: [
{
type: 'crud',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/segment/list',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
knowledge_id,
group_id,
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/segment/list?knowledge_id=${knowledge_id}&group_id=${group_id}`,
...crudCommonOptions(),
headerToolbar: [
'reload',
@@ -110,12 +91,6 @@ const DataDetail: React.FC = () => {
level: 'link',
size: 'sm',
actionType: 'ajax',
api: {
method: 'get',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
confirmText: '删除后无法恢复，确认删除该记录？',
confirmTitle: '删除',
},

16
service-web/client/src/pages/ai/knowledge/Knowledge.tsx
View File

@@ -1,6 +1,6 @@
import React from 'react'
import {useNavigate} from 'react-router'
import {amisRender, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx'
import {amisRender, commonInfo, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx'
const strategyMapping = [
mappingItem('文本', 'Cosine'),
@@ -25,12 +25,7 @@ const Knowledge: React.FC = () => {
body: [
{
type: 'crud',
api: {
url: 'http://127.0.0.1:8080/knowledge/list',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
api: `${commonInfo.baseAiKnowledgeUrl}/knowledge/list`,
...crudCommonOptions(),
headerToolbar: [
'reload',
@@ -47,11 +42,8 @@ const Knowledge: React.FC = () => {
body: {
type: 'form',
api: {
url: 'http://127.0.0.1:8080/knowledge/add',
url: `${commonInfo.baseAiKnowledgeUrl}/knowledge/add`,
dataType: 'form',
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
},
body: [
{
@@ -156,7 +148,7 @@ const Knowledge: React.FC = () => {
actionType: 'ajax',
api: {
method: 'get',
url: 'http://127.0.0.1:8080/knowledge/delete',
url: `${commonInfo.baseAiKnowledgeUrl}/knowledge/delete`,
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},

2
service-web/client/src/util/amis.tsx
View File

@@ -8,6 +8,8 @@ import axios from 'axios'
export const commonInfo = {
baseUrl: 'http://132.126.207.130:35690/hudi_services/service_web',
baseAiChatUrl: 'http://132.126.207.130:35690/hudi_services/ai_chat',
baseAiKnowledgeUrl: 'http://132.126.207.130:35690/hudi_services/ai_knowledge',
// baseUrl: '/hudi_services/service_web',
clusters: {
// hudi同步运行集群和yarn队列名称