lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): 完善知识库接口

Browse Source
This commit is contained in:
v-zhangjc9
2025-05-16 19:00:55 +08:00
parent be976290b6
commit 5d49c82190
22 changed files with 800 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/KnowledgeApplication.java
View File

@@ -1,19 +1,13 @@
package com.lanyuanxiaoyao.service.ai.knowledge;
import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties;
import jakarta.annotation.Resource;
import java.net.MalformedURLException;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.model.Media;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.core.io.FileUrlResource;
import org.springframework.retry.annotation.EnableRetry;
import org.springframework.util.MimeTypeUtils;
/**
* @author lanyuanxiaoyao
@@ -25,35 +19,11 @@ import org.springframework.util.MimeTypeUtils;
@EnableEncryptableProperties
@EnableRetry
public class KnowledgeApplication implements ApplicationRunner {
@Resource
private ChatClient.Builder builder;
public static void main(String[] args) {
SpringApplication.run(KnowledgeApplication.class, args);
}
@Override
public void run(ApplicationArguments args) throws Exception {
ChatClient client = builder.build();
String content = client.prompt()
.user(
prompt -> {
try {
prompt
.text("如实描述图片中的内容，不要加入自己的思考以及与图片内容无关的任何文本")
.media(
Media.builder()
.mimeType(MimeTypeUtils.IMAGE_PNG)
.data(new FileUrlResource("/Users/lanyuanxiaoyao/Pictures/数据使用合同签订.png"))
.build()
);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
)
.call()
.content();
System.out.println(content);
public void run(ApplicationArguments args) {
}
}

125
service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/controller/KnowledgeController.java
View File

@@ -1,9 +1,28 @@
package com.lanyuanxiaoyao.service.ai.knowledge.controller;
import jakarta.annotation.PostConstruct;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.CollectionVO;
import com.lanyuanxiaoyao.service.ai.knowledge.entity.PointVO;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.grpc.Collections;
import io.qdrant.client.grpc.Points;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
@@ -15,10 +34,108 @@ import org.springframework.web.bind.annotation.RestController;
public class KnowledgeController {
private static final Logger logger = LoggerFactory.getLogger(KnowledgeController.class);
public KnowledgeController() {
private final QdrantClient client;
private final EmbeddingModel embeddingModel;
public KnowledgeController(VectorStore vectorStore, EmbeddingModel embeddingModel) {
client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
this.embeddingModel = embeddingModel;
}
@PostConstruct
public void initial() {
@PostMapping("add")
public void add(
@RequestParam("name") String name,
@RequestParam("strategy") String strategy
) throws ExecutionException, InterruptedException {
logger.info("Enter method: add[name, strategy]. name:{},strategy:{}", name, strategy);
client.createCollectionAsync(
name,
Collections.VectorParams.newBuilder()
.setDistance(Collections.Distance.valueOf(strategy))
.setSize(embeddingModel.dimensions())
.build()
).get();
}
@GetMapping("list")
public ImmutableList<CollectionVO> list() throws ExecutionException, InterruptedException {
return client.listCollectionsAsync()
.get()
.stream()
.collect(Collectors.toCollection(Lists.mutable::empty))
.collect(name -> {
try {
Collections.CollectionInfo info = client.getCollectionInfoAsync(name).get();
CollectionVO vo = new CollectionVO();
vo.setName(name);
vo.setPoints(info.getPointsCount());
vo.setSegments(info.getSegmentsCount());
vo.setStatus(info.getStatus().name());
Collections.VectorParams vectorParams = info.getConfig().getParams().getVectorsConfig().getParams();
vo.setStrategy(vectorParams.getDistance().name());
vo.setSize(vectorParams.getSize());
return vo;
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
})
.toImmutable();
}
@GetMapping("list_points")
public ImmutableList<PointVO> listPoints(@RequestParam("name") String name) throws ExecutionException, InterruptedException {
Points.ScrollResponse response = client.scrollAsync(
Points.ScrollPoints.newBuilder()
.setCollectionName(name)
// .setLimit(2)
.setWithPayload(Points.WithPayloadSelector.newBuilder().setEnable(true).build())
.setWithVectors(Points.WithVectorsSelector.newBuilder().setEnable(false).build())
.build()
)
.get();
return response.getResultList()
.stream()
.collect(Collectors.toCollection(Lists.mutable::empty))
.collect(point -> {
PointVO vo = new PointVO();
vo.setId(point.getId().getUuid());
vo.setText(point.getPayloadMap().get("doc_content").getStringValue());
return vo;
})
.toImmutable();
}
@GetMapping("delete")
public void delete(@RequestParam("name") String name) throws ExecutionException, InterruptedException {
client.deleteCollectionAsync(name).get();
}
@PostMapping(value = "preview_text", consumes = "text/plain;charset=utf-8")
public ImmutableList<String> previewText(
@RequestParam("name") String name,
@RequestParam(value = "mode", defaultValue = "normal") String mode,
@RequestBody String text
) {
return Lists.immutable.empty();
}
@PostMapping(value = "process_text", consumes = "text/plain;charset=utf-8")
public void processText(
@RequestParam("name") String name,
@RequestBody String text
) {
VectorStore source = QdrantVectorStore.builder(client, embeddingModel)
.collectionName(name)
.initializeSchema(true)
.build();
MarkdownDocumentReader reader = new MarkdownDocumentReader(
new ByteArrayResource(text.getBytes(StandardCharsets.UTF_8)),
MarkdownDocumentReaderConfig.builder()
.withHorizontalRuleCreateDocument(true)
.withIncludeCodeBlock(false)
.withIncludeBlockquote(false)
.build()
);
source.add(reader.get());
}
}

74
service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/CollectionVO.java Normal file
View File

@@ -0,0 +1,74 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
public class CollectionVO {
private String name;
private String strategy;
private Long size;
private Long points;
private Long segments;
private String status;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStrategy() {
return strategy;
}
public void setStrategy(String strategy) {
this.strategy = strategy;
}
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
}
public Long getPoints() {
return points;
}
public void setPoints(Long points) {
this.points = points;
}
public Long getSegments() {
return segments;
}
public void setSegments(Long segments) {
this.segments = segments;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
@Override
public String toString() {
return "CollectionVO{" +
"name='" + name + '\'' +
", strategy='" + strategy + '\'' +
", size=" + size +
", points=" + points +
", segments=" + segments +
", status='" + status + '\'' +
'}';
}
}

34
service-ai/service-ai-knowledge/src/main/java/com/lanyuanxiaoyao/service/ai/knowledge/entity/PointVO.java Normal file
View File

@@ -0,0 +1,34 @@
package com.lanyuanxiaoyao.service.ai.knowledge.entity;
/**
* @author lanyuanxiaoyao
* @version 20250516
*/
public class PointVO {
private String id;
private String text;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getText() {
return text;
}
public void setText(String text) {
this.text = text;
}
@Override
public String toString() {
return "PointVO{" +
"id='" + id + '\'' +
", text='" + text + '\'' +
'}';
}
}

13
service-ai/service-ai-knowledge/src/main/resources/application.yml
View File

@@ -24,18 +24,17 @@ spring:
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
ai:
openai:
base-url: http://localhost:3000
api-key: '*XMySqV%>hR&v>>g*NwCs3tpQ5FVMFEF2VHVTj<MYQd$&@$sY7CgqNyea4giJi4'
base-url: http://132.121.206.65:10086
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
chat:
options:
model: '/models/InternVL3-1B-Instruct-Q8_0.gguf'
model: 'Qwen3-1.7'
embedding:
options:
model: 'Bge-m3'
# vectorstore:
# qdrant:
# api-key: lanyuanxiaoyao
# initialize-schema: true
vectorstore:
qdrant:
api-key: lanyuanxiaoyao
jasypt:
encryptor:
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'