diff --git a/service-ai/pom.xml b/service-ai/pom.xml
index f165b52..b689311 100644
--- a/service-ai/pom.xml
+++ b/service-ai/pom.xml
@@ -131,6 +131,11 @@
hutool-all
${hutool.version}
+
+ com.yomahub
+ liteflow-spring-boot-starter
+ 2.13.2
+
diff --git a/service-ai/service-ai-knowledge/pom.xml b/service-ai/service-ai-knowledge/pom.xml
index 702f182..098c03a 100644
--- a/service-ai/service-ai-knowledge/pom.xml
+++ b/service-ai/service-ai-knowledge/pom.xml
@@ -28,12 +28,24 @@
org.springframework.ai
- spring-ai-qdrant-store
+ spring-ai-starter-vector-store-qdrant
org.springframework.ai
spring-ai-markdown-document-reader
+
+ org.springframework.boot
+ spring-boot-starter-jdbc
+
+
+ com.mysql
+ mysql-connector-j
+
+
+ com.yomahub
+ liteflow-spring-boot-starter
+
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/KnowledgeController.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/KnowledgeController.java
index 7ece6a9..0cca981 100644
--- a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/KnowledgeController.java
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/KnowledgeController.java
@@ -1,9 +1,10 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
-import com.lanyuanxiaoyao.service.ai.knowledge.entity.CollectionVO;
-import com.lanyuanxiaoyao.service.ai.knowledge.entity.PointVO;
+import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
+import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.PointVO;
+import com.lanyuanxiaoyao.service.ai.knowledge.reader.TextLineReader;
+import com.lanyuanxiaoyao.service.ai.knowledge.service.KnowledgeService;
import io.qdrant.client.QdrantClient;
-import io.qdrant.client.grpc.Collections;
import io.qdrant.client.grpc.Points;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutionException;
@@ -13,6 +14,7 @@ import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.reader.TextReader;
import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
import org.springframework.ai.vectorstore.VectorStore;
@@ -34,10 +36,12 @@ import org.springframework.web.bind.annotation.RestController;
public class KnowledgeController {
private static final Logger logger = LoggerFactory.getLogger(KnowledgeController.class);
+ private final KnowledgeService knowledgeService;
private final QdrantClient client;
private final EmbeddingModel embeddingModel;
- public KnowledgeController(VectorStore vectorStore, EmbeddingModel embeddingModel) {
+ public KnowledgeController(KnowledgeService knowledgeService, VectorStore vectorStore, EmbeddingModel embeddingModel) {
+ this.knowledgeService = knowledgeService;
client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
this.embeddingModel = embeddingModel;
}
@@ -47,39 +51,12 @@ public class KnowledgeController {
@RequestParam("name") String name,
@RequestParam("strategy") String strategy
) throws ExecutionException, InterruptedException {
- logger.info("Enter method: add[name, strategy]. name:{},strategy:{}", name, strategy);
- client.createCollectionAsync(
- name,
- Collections.VectorParams.newBuilder()
- .setDistance(Collections.Distance.valueOf(strategy))
- .setSize(embeddingModel.dimensions())
- .build()
- ).get();
+ knowledgeService.add(name, strategy);
}
@GetMapping("list")
- public ImmutableList list() throws ExecutionException, InterruptedException {
- return client.listCollectionsAsync()
- .get()
- .stream()
- .collect(Collectors.toCollection(Lists.mutable::empty))
- .collect(name -> {
- try {
- Collections.CollectionInfo info = client.getCollectionInfoAsync(name).get();
- CollectionVO vo = new CollectionVO();
- vo.setName(name);
- vo.setPoints(info.getPointsCount());
- vo.setSegments(info.getSegmentsCount());
- vo.setStatus(info.getStatus().name());
- Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
- vo.setStrategy(vectorParams.getDistance().name());
- vo.setSize(vectorParams.getSize());
- return vo;
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- })
- .toImmutable();
+ public ImmutableList list() {
+ return knowledgeService.list();
}
@GetMapping("list_points")
@@ -107,16 +84,27 @@ public class KnowledgeController {
@GetMapping("delete")
public void delete(@RequestParam("name") String name) throws ExecutionException, InterruptedException {
- client.deleteCollectionAsync(name).get();
+ knowledgeService.remove(name);
}
- @PostMapping(value = "preview_text", consumes = "text/plain;charset=utf-8")
- public ImmutableList previewText(
+ @PostMapping("preview_text")
+ public ImmutableList previewText(
@RequestParam("name") String name,
@RequestParam(value = "mode", defaultValue = "normal") String mode,
- @RequestBody String text
+ @RequestParam(value = "type", defaultValue = "text") String type,
+ @RequestParam("content") String content
) {
- return Lists.immutable.empty();
+ TextReader reader = new TextLineReader(new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)));
+ return reader.get()
+ .stream()
+ .collect(Collectors.toCollection(Lists.mutable::empty))
+ .collect(doc -> {
+ PointVO vo = new PointVO();
+ vo.setId(doc.getId());
+ vo.setText(doc.getText());
+ return vo;
+ })
+ .toImmutable();
}
@PostMapping(value = "process_text", consumes = "text/plain;charset=utf-8")
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/Knowledge.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/Knowledge.java
new file mode 100644
index 0000000..016a92c
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/Knowledge.java
@@ -0,0 +1,54 @@
+package com.lanyuanxiaoyao.service.ai.knowledge.entity;
+
+/**
+ * @author lanyuanxiaoyao
+ * @version 20250522
+ */
+public class Knowledge {
+ private Long id;
+ private Long vectorSourceId;
+ private String name;
+ private String strategy;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Long getVectorSourceId() {
+ return vectorSourceId;
+ }
+
+ public void setVectorSourceId(Long vectorSourceId) {
+ this.vectorSourceId = vectorSourceId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getStrategy() {
+ return strategy;
+ }
+
+ public void setStrategy(String strategy) {
+ this.strategy = strategy;
+ }
+
+ @Override
+ public String toString() {
+ return "Knowledge{" +
+ "id=" + id +
+ ", vectorSourceId=" + vectorSourceId +
+ ", name='" + name + '\'' +
+ ", strategy='" + strategy + '\'' +
+ '}';
+ }
+}
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/CollectionVO.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/KnowledgeVO.java
similarity index 93%
rename from service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/CollectionVO.java
rename to service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/KnowledgeVO.java
index 5301b50..1e26302 100644
--- a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/CollectionVO.java
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/KnowledgeVO.java
@@ -1,10 +1,10 @@
-package com.lanyuanxiaoyao.service.ai.knowledge.entity;
+package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
-public class CollectionVO {
+public class KnowledgeVO {
private String name;
private String strategy;
private Long size;
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/PointVO.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/PointVO.java
similarity index 89%
rename from service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/PointVO.java
rename to service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/PointVO.java
index a8ff85e..0cf40a2 100644
--- a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/PointVO.java
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/vo/PointVO.java
@@ -1,4 +1,4 @@
-package com.lanyuanxiaoyao.service.ai.knowledge.entity;
+package com.lanyuanxiaoyao.service.ai.knowledge.entity.vo;
/**
* @author lanyuanxiaoyao
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/reader/TextLineReader.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/reader/TextLineReader.java
new file mode 100644
index 0000000..ceb1c28
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/reader/TextLineReader.java
@@ -0,0 +1,34 @@
+package com.lanyuanxiaoyao.service.ai.knowledge.reader;
+
+import cn.hutool.core.util.StrUtil;
+import java.util.List;
+import java.util.stream.Stream;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.reader.TextReader;
+import org.springframework.core.io.Resource;
+
+/**
+ * @author lanyuanxiaoyao
+ * @version 20250522
+ */
+public class TextLineReader extends TextReader {
+ public TextLineReader(Resource resource) {
+ super(resource);
+ }
+
+ @Override
+ public List get() {
+ return super.get()
+ .stream()
+ .flatMap(doc -> {
+ String text = doc.getText();
+ if (StrUtil.isBlank(text)) {
+ return Stream.of(doc);
+ }
+ return Stream.of(text.split("\n\n"))
+ .filter(StrUtil::isNotBlank)
+ .map(line -> new Document(line, doc.getMetadata()));
+ })
+ .toList();
+ }
+}
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/EmbeddingService.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/EmbeddingService.java
new file mode 100644
index 0000000..f85466f
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/EmbeddingService.java
@@ -0,0 +1,16 @@
+package com.lanyuanxiaoyao.service.ai.knowledge.service;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author lanyuanxiaoyao
+ * @version 20250522
+ */
+@Service
+public class EmbeddingService {
+ private static final Logger logger = LoggerFactory.getLogger(EmbeddingService.class);
+
+
+}
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeGroupService.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeGroupService.java
new file mode 100644
index 0000000..b9c72f7
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeGroupService.java
@@ -0,0 +1,58 @@
+package com.lanyuanxiaoyao.service.ai.knowledge.service;
+
+import club.kingon.sql.builder.SqlBuilder;
+import cn.hutool.core.util.IdUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * @author lanyuanxiaoyao
+ * @version 20250522
+ */
+@Service
+public class KnowledgeGroupService {
+ private static final Logger logger = LoggerFactory.getLogger(KnowledgeGroupService.class);
+ private static final String GROUP_TABLE_NAME = "service_ai_group";
+
+ private final JdbcTemplate template;
+
+ public KnowledgeGroupService(JdbcTemplate template) {
+ this.template = template;
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void add(Long knowledgeId, String name) {
+ template.update(
+ SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name")
+ .values()
+ .addValue("?", "?", "?")
+ .precompileSql(),
+ IdUtil.getSnowflakeNextId(),
+ knowledgeId,
+ name
+ );
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void remove(Long groupId) {
+ template.update(
+ SqlBuilder.delete(GROUP_TABLE_NAME)
+ .whereEq("id", "?")
+ .precompileSql(),
+ groupId
+ );
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void removeByKnowledgeId(Long knowledgeId) {
+ template.update(
+ SqlBuilder.delete(GROUP_TABLE_NAME)
+ .whereEq("knowledge_id", "?")
+ .precompileSql(),
+ knowledgeId
+ );
+ }
+}
diff --git a/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeService.java b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeService.java
new file mode 100644
index 0000000..f4709b2
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/service/KnowledgeService.java
@@ -0,0 +1,151 @@
+package com.lanyuanxiaoyao.service.ai.knowledge.service;
+
+import club.kingon.sql.builder.SqlBuilder;
+import cn.hutool.core.util.IdUtil;
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.ai.knowledge.entity.Knowledge;
+import com.lanyuanxiaoyao.service.ai.knowledge.entity.vo.KnowledgeVO;
+import io.qdrant.client.QdrantClient;
+import io.qdrant.client.grpc.Collections;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.list.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * @author lanyuanxiaoyao
+ * @version 20250522
+ */
+@Service
+public class KnowledgeService {
+ private static final Logger logger = LoggerFactory.getLogger(KnowledgeService.class);
+ private static final String KNOWLEDGE_TABLE_NAME = "service_ai_knowledge";
+
+ private final JdbcTemplate template;
+ private final EmbeddingModel embeddingModel;
+ private final QdrantClient client;
+ private final KnowledgeGroupService knowledgeGroupService;
+
+ public KnowledgeService(JdbcTemplate template, EmbeddingModel embeddingModel, VectorStore vectorStore, KnowledgeGroupService knowledgeGroupService) {
+ this.template = template;
+ this.embeddingModel = embeddingModel;
+ this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
+ this.knowledgeGroupService = knowledgeGroupService;
+ }
+
+ public Knowledge get(Long id) {
+ return template.queryForObject(
+ SqlBuilder.select("id", "vector_source_id", "name", "strategy")
+ .from(KNOWLEDGE_TABLE_NAME)
+ .whereEq("id", "?")
+ .precompileSql(),
+ Knowledge.class,
+ id
+ );
+ }
+
+ public Knowledge get(String name) {
+ return template.queryForObject(
+ SqlBuilder.select("id", "vector_source_id", "name", "strategy")
+ .from(KNOWLEDGE_TABLE_NAME)
+ .whereEq("name", "?")
+ .precompileSql(),
+ Knowledge.class,
+ name
+ );
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void add(String name, String strategy) throws ExecutionException, InterruptedException {
+ Integer count = template.queryForObject(
+ SqlBuilder.select("count(*)")
+ .from(KNOWLEDGE_TABLE_NAME)
+ .whereEq("name", "?")
+ .precompileSql(),
+ Integer.class,
+ name
+ );
+ if (count > 0) {
+ throw new RuntimeException("名称已存在");
+ }
+
+ long id = IdUtil.getSnowflakeNextId();
+ long vectorSourceId = IdUtil.getSnowflakeNextId();
+ template.update(
+ SqlBuilder.insertInto(KNOWLEDGE_TABLE_NAME, "id", "vector_source_id", "name", "strategy")
+ .values()
+ .addValue("?", "?", "?", "?")
+ .precompileSql(),
+ id,
+ vectorSourceId,
+ name,
+ strategy
+ );
+ client.createCollectionAsync(
+ String.valueOf(vectorSourceId),
+ Collections.VectorParams.newBuilder()
+ .setDistance(Collections.Distance.valueOf(strategy))
+ .setSize(embeddingModel.dimensions())
+ .build()
+ ).get();
+ }
+
+ public ImmutableList list() {
+ return template.query(
+ SqlBuilder.select("id", "vector_source_id", "name", "strategy")
+ .from(KNOWLEDGE_TABLE_NAME)
+ .build(),
+ (rs, index) -> {
+ Knowledge knowledge = new Knowledge();
+ knowledge.setId(rs.getLong(1));
+ knowledge.setVectorSourceId(rs.getLong(2));
+ knowledge.setName(rs.getString(3));
+ knowledge.setStrategy(rs.getString(4));
+ return knowledge;
+ }
+ )
+ .stream()
+ .map(knowledge -> {
+ try {
+ Collections.CollectionInfo info = client.getCollectionInfoAsync(String.valueOf(knowledge.getVectorSourceId())).get();
+ KnowledgeVO vo = new KnowledgeVO();
+ vo.setName(knowledge.getName());
+ vo.setPoints(info.getPointsCount());
+ vo.setSegments(info.getSegmentsCount());
+ vo.setStatus(info.getStatus().name());
+ Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
+ vo.setStrategy(vectorParams.getDistance().name());
+ vo.setSize(vectorParams.getSize());
+ return vo;
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .collect(Collectors.toCollection(Lists.mutable::empty))
+ .toImmutable();
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void remove(String name) throws ExecutionException, InterruptedException {
+ Knowledge knowledge = get(name);
+ if (ObjectUtil.isNull(knowledge)) {
+ throw new RuntimeException(StrUtil.format("{} 不存在"));
+ }
+ template.update(
+ SqlBuilder.delete(KNOWLEDGE_TABLE_NAME)
+ .whereEq("id", "?")
+ .precompileSql(),
+ knowledge.getId()
+ );
+ knowledgeGroupService.removeByKnowledgeId(knowledge.getId());
+ client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
+ }
+}
\ No newline at end of file
diff --git a/service-ai/service-ai-knowledge/src/main/resources/application.yml b/service-ai/service-ai-knowledge/src/main/resources/application.yml
index b390e70..52b26d9 100644
--- a/service-ai/service-ai-knowledge/src/main/resources/application.yml
+++ b/service-ai/service-ai-knowledge/src/main/resources/application.yml
@@ -17,6 +17,11 @@ spring:
hostname: localhost
hostname_full: localhost
start_time: 20250514112750
+ datasource:
+ url: jdbc:mysql://localhost:3307/ai?useSSL=false
+ username: test
+ password: test
+ driver-class-name: com.mysql.cj.jdbc.Driver
security:
meta:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
@@ -39,4 +44,8 @@ jasypt:
encryptor:
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
server:
- port: 8080
\ No newline at end of file
+ port: 8080
+liteflow:
+ rule-source: config/flow.xml
+ print-banner: false
+ check-node-exists: false
diff --git a/service-ai/service-ai-knowledge/src/main/resources/config/flow.xml b/service-ai/service-ai-knowledge/src/main/resources/config/flow.xml
new file mode 100644
index 0000000..9b93803
--- /dev/null
+++ b/service-ai/service-ai-knowledge/src/main/resources/config/flow.xml
@@ -0,0 +1,14 @@
+
+
+
+ SER(
+ embedding_start,
+ SWITCH(embedding_mode_switch).TO(
+ normal_embedding,
+ llm_embedding,
+ qa_embedding
+ ),
+ embedding_finish
+ );
+
+
\ No newline at end of file