lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai-web): 完成JPA存储适配

Browse Source
This commit is contained in:
v-zhangjc9
2025-06-23 16:53:34 +08:00
parent e48d7e8649
commit 5b3c27ea48
32 changed files with 458 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
service-ai/database/service_ai_feedback.sql
View File

@@ -1,11 +1,18 @@
CREATE TABLE `service_ai_feedback`
(
`id` bigint NOT NULL,
`source` longtext NOT NULL,
`conclusion` longtext,
`id` bigint NOT NULL,
`created_time` datetime(6) DEFAULT NULL,
`modified_time` datetime(6) DEFAULT NULL,
`analysis` longtext,
`pictures` longtext,
`status` varchar(50) NOT NULL DEFAULT 'ANALYSIS_PROCESSING',
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) DEFAULT CHARSET = utf8mb4;
`conclusion` longtext,
`source` longtext NOT NULL,
`status` tinyint NOT NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARSET = utf8mb4;
CREATE TABLE `service_ai_feedback_pictures`
(
`feedback_id` bigint NOT NULL,
`pictures_id` bigint NOT NULL,
PRIMARY KEY (`feedback_id`, `pictures_id`)
) DEFAULT CHARSET = utf8mb4;

17
service-ai/database/service_ai_file.sql
View File

@@ -1,11 +1,12 @@
CREATE TABLE `service_ai_file`
(
`id` bigint NOT NULL,
`filename` varchar(500) DEFAULT NULL,
`size` bigint DEFAULT NULL,
`md5` varchar(100) DEFAULT NULL,
`path` varchar(500) DEFAULT NULL,
`type` varchar(50) DEFAULT NULL,
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`id` bigint NOT NULL,
`created_time` datetime(6) DEFAULT NULL,
`modified_time` datetime(6) DEFAULT NULL,
`filename` varchar(255) DEFAULT NULL,
`md5` varchar(255) DEFAULT NULL,
`path` varchar(255) DEFAULT NULL,
`size` bigint DEFAULT NULL,
`type` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARSET = utf8mb4;
) DEFAULT CHARSET = utf8mb4;

11
service-ai/database/service_ai_group.sql
View File

@@ -1,9 +1,10 @@
CREATE TABLE `service_ai_group`
(
`id` bigint NOT NULL,
`created_time` datetime(6) DEFAULT NULL,
`modified_time` datetime(6) DEFAULT NULL,
`name` varchar(255) NOT NULL,
`status` tinyint NOT NULL,
`knowledge_id` bigint NOT NULL,
`name` varchar(100) NOT NULL,
`status` varchar(10) NOT NULL DEFAULT 'RUNNING',
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) DEFAULT CHARSET = utf8mb4;
PRIMARY KEY (`id`)
) DEFAULT CHARSET=utf8mb4;

12
service-ai/database/service_ai_knowledge.sql
View File

@@ -1,11 +1,11 @@
CREATE TABLE `service_ai_knowledge`
(
`id` bigint NOT NULL,
`vector_source_id` varchar(100) NOT NULL,
`name` varchar(100) NOT NULL,
`created_time` datetime(6) DEFAULT NULL,
`modified_time` datetime(6) DEFAULT NULL,
`description` longtext NOT NULL,
`strategy` varchar(10) NOT NULL,
`created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`name` varchar(255) NOT NULL,
`strategy` tinyint NOT NULL,
`vector_source_id` bigint NOT NULL,
PRIMARY KEY (`id`)
) DEFAULT CHARSET = utf8mb4;
) DEFAULT CHARSET = utf8mb4;

5
service-ai/pom.xml
View File

@@ -110,6 +110,11 @@
<artifactId>jasypt-spring-boot-starter</artifactId>
<version>3.0.5</version>
</dependency>
<dependency>
<groupId>com.blinkfox</groupId>
<artifactId>fenix-spring-boot-starter</artifactId>
<version>3.0.0</version>
</dependency>
<!-- 日志相关 -->
<dependency>

74
service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/SnowflakeId.java
View File

@@ -1,74 +0,0 @@
package com.lanyuanxiaoyao.service.ai.core.configuration;
import java.time.Instant;
/**
* 使用雪花算法作为ID生成器
*
* @author lanyuanxiaoyao
* @date 2024-11-14
*/
public class SnowflakeId {
/**
* 起始的时间戳
*/
private final static long START_TIMESTAMP = 1;
/**
* 序列号占用的位数
*/
private final static long SEQUENCE_BIT = 11;
/**
* 序列号最大值
*/
private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
/**
* 时间戳值向左位移
*/
private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
/**
* 序列号
*/
private static long sequence = 0;
/**
* 上一次时间戳
*/
private static long lastTimestamp = -1;
public static synchronized long next() {
long currentTimestamp = nowTimestamp();
if (currentTimestamp < lastTimestamp) {
throw new RuntimeException("Clock have moved backwards.");
}
if (currentTimestamp == lastTimestamp) {
// 相同毫秒内, 序列号自增
sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
// 同一毫秒的序列数已经达到最大
if (sequence == 0) {
currentTimestamp = nextTimestamp();
}
} else {
// 不同毫秒内, 序列号置为0
sequence = 0;
}
lastTimestamp = currentTimestamp;
return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
}
private static long nextTimestamp() {
long milli = nowTimestamp();
while (milli <= lastTimestamp) {
milli = nowTimestamp();
}
return milli;
}
private static long nowTimestamp() {
return Instant.now().toEpochMilli();
}
}

1
service-ai/service-ai-web/pom.xml
View File

@@ -53,7 +53,6 @@
<dependency>
<groupId>com.blinkfox</groupId>
<artifactId>fenix-spring-boot-starter</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>com.mysql</groupId>

9
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/ListController.java
View File

@@ -1,17 +1,16 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
import org.eclipse.collections.api.list.ImmutableList;
/**
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface ListController<LIST_ITEM> {
public interface ListController {
String LIST = "/list";
AmisResponse<ImmutableList<LIST_ITEM>> list() throws Exception;
AmisCrudResponse list() throws Exception;
AmisResponse<ImmutableList<LIST_ITEM>> list(Query query) throws Exception;
AmisCrudResponse list(Query query) throws Exception;
}

2
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleController.java
View File

@@ -4,5 +4,5 @@ package com.lanyuanxiaoyao.service.ai.web.base.controller;
* @author lanyuanxiaoyao
* @date 2024-11-28
*/
public interface SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> extends SaveController<SAVE_ITEM>, ListController<LIST_ITEM>, DetailController<DETAIL_ITEM>, RemoveController {
public interface SimpleController<SAVE_ITEM, LIST_ITEM, DETAIL_ITEM> extends SaveController<SAVE_ITEM>, ListController, DetailController<DETAIL_ITEM>, RemoveController {
}

52
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleControllerSupport.java
View File

@@ -1,13 +1,14 @@
package com.lanyuanxiaoyao.service.ai.web.base.controller;
import cn.hutool.core.util.ObjectUtil;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.springframework.data.domain.Page;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@@ -34,43 +35,52 @@ public abstract class SimpleControllerSupport<ENTITY extends SimpleEntity, SAVE_
@GetMapping(LIST)
@Override
public AmisResponse<ImmutableList<LIST_ITEM>> list() throws Exception {
public AmisCrudResponse list() throws Exception {
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
return AmisResponse.responseSuccess(service.list().collect(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
}));
return AmisCrudResponse.responseCrudData(
service.list()
.collect(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
);
}
@PostMapping(LIST)
@Override
public AmisResponse<ImmutableList<LIST_ITEM>> list(@RequestBody Query query) throws Exception {
public AmisCrudResponse list(@RequestBody Query query) throws Exception {
if (ObjectUtil.isNull(query)) {
return AmisResponse.responseSuccess(Lists.immutable.empty());
return AmisCrudResponse.responseCrudData(List.of(), 0);
}
ListItemMapper<ENTITY, LIST_ITEM> mapper = listItemMapper();
return AmisResponse.responseSuccess(service.list(query).collect(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
}));
Page<ENTITY> result = service.list(query);
return AmisCrudResponse.responseCrudData(
result.get()
.map(entity -> {
try {
return mapper.from(entity);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
.toList(),
result.getTotalElements()
);
}
@GetMapping(DETAIL)
@Override
public AmisResponse<DETAIL_ITEM> detail(@PathVariable Long id) throws Exception {
public AmisResponse<DETAIL_ITEM> detail(@PathVariable("id") Long id) throws Exception {
DetailItemMapper<ENTITY, DETAIL_ITEM> mapper = detailItemMapper();
return AmisResponse.responseSuccess(mapper.from(service.detailOrThrow(id)));
}
@GetMapping(REMOVE)
@Override
public AmisResponse<Object> remove(@PathVariable Long id) throws Exception {
public AmisResponse<Object> remove(@PathVariable("id") Long id) throws Exception {
service.remove(id);
return AmisResponse.responseSuccess();
}

3
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleService.java
View File

@@ -5,6 +5,7 @@ import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import java.util.Optional;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.set.ImmutableSet;
import org.springframework.data.domain.Page;
/**
* @author lanyuanxiaoyao
@@ -19,7 +20,7 @@ public interface SimpleService<ENTITY extends SimpleEntity> {
ImmutableList<ENTITY> list(ImmutableSet<Long> ids) throws Exception;
ImmutableList<ENTITY> list(Query query) throws Exception;
Page<ENTITY> list(Query query) throws Exception;
Optional<ENTITY> detailOptional(Long id) throws Exception;

30
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleServiceSupport.java
View File

@@ -22,6 +22,8 @@ import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.set.ImmutableSet;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
/**
@@ -30,6 +32,8 @@ import org.springframework.data.domain.Sort;
*/
@Slf4j
public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implements SimpleService<ENTITY> {
private static final Integer DEFAULT_PAGE_INDEX = 1;
private static final Integer DEFAULT_PAGE_SIZE = 10;
protected final SimpleRepository<ENTITY, Long> repository;
public SimpleServiceSupport(SimpleRepository<ENTITY, Long> repository) {
@@ -38,7 +42,7 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
@Transactional(rollbackOn = Throwable.class)
@Override
public Long save(ENTITY entity) throws Exception {
public Long save(ENTITY entity) {
if (ObjectUtil.isNotNull(entity.getId())) {
Long id = entity.getId();
ENTITY targetEntity = repository.findById(entity.getId()).orElseThrow(() -> new IdNotFoundException(id));
@@ -59,7 +63,7 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
}
@Override
public Long count() throws Exception {
public Long count() {
return repository.count(
(root, query, builder) ->
builder.and(
@@ -71,7 +75,7 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
}
@Override
public ImmutableList<ENTITY> list() throws Exception {
public ImmutableList<ENTITY> list() {
return Lists.immutable.ofAll(repository.findAll(
(root, query, builder) ->
builder.and(
@@ -83,7 +87,7 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
}
@Override
public ImmutableList<ENTITY> list(ImmutableSet<Long> ids) throws Exception {
public ImmutableList<ENTITY> list(ImmutableSet<Long> ids) {
return Lists.immutable.ofAll(repository.findAll(
(root, query, builder) -> {
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
@@ -179,15 +183,23 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
}
@Override
public ImmutableList<ENTITY> list(Query listQuery) throws Exception {
return Lists.immutable.ofAll(repository.findAll(
public Page<ENTITY> list(Query listQuery) {
PageRequest pageRequest = PageRequest.of(DEFAULT_PAGE_INDEX - 1, DEFAULT_PAGE_SIZE, Sort.by(SimpleEntity_.CREATED_TIME).descending());
if (ObjectUtil.isNotNull(listQuery.getPage())) {
pageRequest = PageRequest.of(
ObjectUtil.defaultIfNull(listQuery.getPage().getIndex(), DEFAULT_PAGE_INDEX) - 1,
ObjectUtil.defaultIfNull(listQuery.getPage().getSize(), DEFAULT_PAGE_SIZE),
Sort.by(SimpleEntity_.CREATED_TIME).descending()
);
}
return repository.findAll(
(root, query, builder) -> {
MutableList<Predicate> predicates = Lists.mutable.ofAll(listPredicate(root, query, builder));
predicates.addAllIterable(queryPredicates(listQuery.getQuery(), root, query, builder));
return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()]));
},
Sort.by(SimpleEntity_.CREATED_TIME).descending()
));
pageRequest
);
}
@Override
@@ -221,7 +233,7 @@ public abstract class SimpleServiceSupport<ENTITY extends SimpleEntity> implemen
@Transactional(rollbackOn = Throwable.class)
@Override
public void remove(Long id) throws Exception {
public void remove(Long id) {
if (ObjectUtil.isNotNull(id)) {
repository.deleteById(id);
}

35
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java
View File

@@ -4,6 +4,7 @@ import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport
import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
import com.lanyuanxiaoyao.service.ai.web.service.DataFileService;
import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -22,12 +23,14 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("feedback")
public class FeedbackController extends SimpleControllerSupport<Feedback, FeedbackController.SaveItem, FeedbackController.ListItem, FeedbackController.DetailItem> {
public class FeedbackController extends SimpleControllerSupport<Feedback, FeedbackController.SaveItem, FeedbackController.ListItem, FeedbackController.ListItem> {
private final FeedbackService feedbackService;
private final DataFileService dataFileService;
public FeedbackController(FeedbackService feedbackService) {
public FeedbackController(FeedbackService feedbackService, DataFileService dataFileService) {
super(feedbackService);
this.feedbackService = feedbackService;
this.dataFileService = dataFileService;
}
@GetMapping("reanalysis/{id}")
@@ -42,7 +45,12 @@ public class FeedbackController extends SimpleControllerSupport<Feedback, Feedba
@Override
protected SaveItemMapper<Feedback, SaveItem> saveItemMapper() {
return null;
return item -> {
Feedback feedback = new Feedback();
feedback.setSource(item.getSource());
feedback.setPictures(dataFileService.downloadFile(item.getPictures()).toSet());
return feedback;
};
}
@Override
@@ -51,8 +59,9 @@ public class FeedbackController extends SimpleControllerSupport<Feedback, Feedba
}
@Override
protected DetailItemMapper<Feedback, DetailItem> detailItemMapper() {
return Mappers.getMapper(DetailItem.Mapper.class);
protected DetailItemMapper<Feedback, ListItem> detailItemMapper() {
ListItemMapper<Feedback, ListItem> mapper = listItemMapper();
return mapper::from;
}
@Data
@@ -88,20 +97,4 @@ public class FeedbackController extends SimpleControllerSupport<Feedback, Feedba
ListItem from(Feedback feedback);
}
}
@Data
@EqualsAndHashCode(callSuper = true)
public static final class DetailItem extends ListItem {
@org.mapstruct.Mapper(
imports = {
IdOnlyEntity.class,
Sets.class
}
)
public interface Mapper extends DetailItemMapper<Feedback, DetailItem> {
@Mapping(target = "pictures", expression = "java(Sets.immutable.ofAll(feedback.getPictures()).collect(IdOnlyEntity::getId))")
@Override
DetailItem from(Feedback feedback);
}
}
}

61
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/GroupController.java
View File

@@ -1,12 +1,15 @@
package com.lanyuanxiaoyao.service.ai.web.controller.knowledge;
import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse;
import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem;
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
import java.util.concurrent.ExecutionException;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.mapstruct.factory.Mappers;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
@@ -16,21 +19,49 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("knowledge/group")
public class GroupController {
private final GroupService groupService;
public class GroupController extends SimpleControllerSupport<Group, GroupController.SaveItem, GroupController.ListItem, GroupController.ListItem> {
private final KnowledgeBaseService knowledgeBaseService;
public GroupController(GroupService groupService) {
this.groupService = groupService;
public GroupController(GroupService groupService, KnowledgeBaseService knowledgeBaseService) {
super(groupService);
this.knowledgeBaseService = knowledgeBaseService;
}
@GetMapping("list")
public AmisResponse<?> list(@RequestParam("knowledge_id") Long knowledgeId) {
return AmisResponse.responseCrudData(groupService.list(knowledgeId));
@Override
protected SaveItemMapper<Group, SaveItem> saveItemMapper() {
return item -> {
Group group = new Group();
group.setName(item.getName());
group.setKnowledge(knowledgeBaseService.detailOrThrow(item.getKnowledgeId()));
return group;
};
}
@GetMapping("delete")
public AmisResponse<?> delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException {
groupService.remove(id);
return AmisResponse.responseSuccess();
@Override
protected ListItemMapper<Group, ListItem> listItemMapper() {
return Mappers.getMapper(ListItem.Mapper.class);
}
@Override
protected DetailItemMapper<Group, ListItem> detailItemMapper() {
ListItemMapper<Group, ListItem> mapper = listItemMapper();
return mapper::from;
}
@Data
public static final class SaveItem {
private String name;
private Long knowledgeId;
}
@Data
@EqualsAndHashCode(callSuper = true)
public static final class ListItem extends SimpleItem {
private String name;
private Group.Status status;
@org.mapstruct.Mapper
public interface Mapper extends ListItemMapper<Group, ListItem> {
}
}
}

126
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/KnowledgeBaseController.java
View File

@@ -34,7 +34,7 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
@RestController
@RequestMapping("knowledge")
public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge, KnowledgeBaseController.SaveItem, KnowledgeBaseController.ListItem, KnowledgeBaseController.DetailItem> {
public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge, KnowledgeBaseController.SaveItem, KnowledgeBaseController.ListItem, KnowledgeBaseController.ListItem> {
private final KnowledgeBaseService knowledgeBaseService;
private final EmbeddingService embeddingService;
@@ -44,20 +44,6 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
this.embeddingService = embeddingService;
}
@PostMapping("update_description")
public void updateDescription(
@RequestParam("id") Long id,
@RequestParam("description") String description
) throws Exception {
knowledgeBaseService.updateDescription(id, description);
}
@GetMapping("name/{id}")
public AmisMapResponse name(@PathVariable("id") Long id) {
return AmisResponse.responseMapData()
.setData("name", knowledgeBaseService.getName(id));
}
@Override
protected SaveItemMapper<Knowledge, SaveItem> saveItemMapper() {
return Mappers.getMapper(SaveItem.Mapper.class);
@@ -66,25 +52,34 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
@Override
protected ListItemMapper<Knowledge, ListItem> listItemMapper() {
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
return knowledge -> mapper.from(knowledge, knowledgeBaseService.getCollectionInfoById(knowledge.getVectorSourceId()));
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
}
@Override
protected DetailItemMapper<Knowledge, DetailItem> detailItemMapper() {
DetailItem.Mapper mapper = Mappers.getMapper(DetailItem.Mapper.class);
return knowledge -> mapper.from(knowledge, knowledgeBaseService.getCollectionInfoById(knowledge.getVectorSourceId()));
protected DetailItemMapper<Knowledge, ListItem> detailItemMapper() {
ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class);
return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId()));
}
@GetMapping("{id}/name")
public AmisMapResponse name(@PathVariable("id") Long id) {
return AmisResponse.responseMapData()
.setData("name", knowledgeBaseService.getName(id));
}
@PostMapping("update_description")
public void updateDescription(
@RequestParam("id") Long id,
@RequestParam("description") String description
) throws Exception {
knowledgeBaseService.updateDescription(id, description);
}
@PostMapping("preview_text")
public AmisResponse<?> previewText(
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
@RequestParam(value = "type", defaultValue = "text") String type,
@RequestParam(value = "content", required = false) String content,
@RequestParam(value = "files", required = false) String files
) {
if (StrUtil.equals("text", type)) {
public AmisResponse<?> previewText(@RequestBody SubmitItem item) {
if (StrUtil.equals("text", item.getType())) {
return AmisResponse.responseCrudData(
embeddingService.preview(mode, content)
embeddingService.preview(item.getMode(), item.getContent())
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
@@ -92,9 +87,9 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
return vo;
})
);
} else if (StrUtil.equals("file", type)) {
} else if (StrUtil.equals("file", item.getType())) {
return AmisResponse.responseCrudData(
embeddingService.preview(mode, Lists.immutable.of(files.split(",")))
embeddingService.preview(item.getMode(), item.getFiles())
.collect(doc -> {
SegmentVO vo = new SegmentVO();
vo.setId(doc.getId());
@@ -103,45 +98,29 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
})
);
} else {
throw new IllegalArgumentException("Unsupported type: " + type);
throw new IllegalArgumentException("Unsupported type: " + item.getType());
}
}
@PostMapping("submit_text")
public void submitText(
@RequestParam("id") Long id,
@RequestParam(value = "mode", defaultValue = "NORMAL") String mode,
@RequestParam(value = "type", defaultValue = "text") String type,
@RequestParam(value = "content", required = false) String content,
@RequestParam(value = "files", required = false) String files
) {
if (StrUtil.equals("text", type)) {
embeddingService.submit(id, mode, content);
} else if (StrUtil.equals("file", type)) {
embeddingService.submit(id, mode, Lists.immutable.of(files.split(",")));
public void submitText(@RequestBody SubmitItem item) {
if (StrUtil.equals("text", item.getMode())) {
embeddingService.submit(item.getId(), item.getMode(), item.getContent());
} else if (StrUtil.equals("file", item.getType())) {
embeddingService.submit(item.getId(), item.getMode(), item.getFiles());
} else {
throw new IllegalArgumentException("Unsupported type: " + type);
throw new IllegalArgumentException("Unsupported type: " + item.getType());
}
}
@PostMapping("submit_text_directly")
public void submitDirectly(
@RequestParam("id") Long id,
@RequestParam(value = "name", required = false) String name,
@RequestParam(value = "split_key", defaultValue = "\n\n") String splitKey,
@RequestBody String content
) {
embeddingService.submitDirectly(id, name, Lists.immutable.of(content.split(splitKey)));
public void submitDirectly(@RequestBody SubmitDirectlyItem item) {
embeddingService.submitDirectly(item.getId(), item.getName(), Lists.immutable.ofAll(StrUtil.split(item.getContent(), item.getSplitKey())));
}
@PostMapping("query")
public ImmutableList<String> query(
@RequestParam("id") Long id,
@RequestParam(value = "limit", defaultValue = "5") Integer limit,
@RequestParam(value = "threshold", defaultValue = "0.6") Double threshold,
@RequestBody String text
) throws ExecutionException, InterruptedException, IOException {
return knowledgeBaseService.query(id, text, limit, threshold);
public ImmutableList<String> query(@RequestBody QueryItem item) throws ExecutionException, InterruptedException, IOException {
return knowledgeBaseService.query(item.getId(), item.getText(), item.getLimit(), item.getThreshold());
}
@Data
@@ -157,7 +136,7 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
@Data
@EqualsAndHashCode(callSuper = true)
public static class ListItem extends SimpleItem {
public static final class ListItem extends SimpleItem {
private Long vectorSourceId;
private String name;
private String description;
@@ -179,16 +158,27 @@ public class KnowledgeBaseController extends SimpleControllerSupport<Knowledge,
}
@Data
@EqualsAndHashCode(callSuper = true)
public static final class DetailItem extends ListItem {
@org.mapstruct.Mapper
public interface Mapper extends DetailItemMapper<Knowledge, ListItem> {
@Mapping(source = "info.config.params.vectorsConfig.params.distance", target = "strategy")
@Mapping(source = "info.config.params.vectorsConfig.params.size", target = "size")
@Mapping(source = "info.pointsCount", target = "points")
@Mapping(source = "info.segmentsCount", target = "segments")
@Mapping(source = "info.status", target = "status")
DetailItem from(Knowledge knowledge, Collections.CollectionInfo info);
}
public static final class SubmitItem {
private Long id;
private String mode;
private String type;
private String content;
private ImmutableList<Long> files;
}
@Data
public static final class SubmitDirectlyItem {
private Long id;
private String name;
private String splitKey = "\n\n";
private String content;
}
@Data
public static final class QueryItem {
private Long id;
private Integer limit = 5;
private Double threshold = 0.6;
private String text;
}
}

3
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/DataFile.java
View File

@@ -1,6 +1,7 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import lombok.Getter;
@@ -20,7 +21,7 @@ import org.hibernate.annotations.DynamicUpdate;
@ToString
@Entity
@DynamicUpdate
@Table(name = "service_ai_file")
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_file")
@NoArgsConstructor
public class DataFile extends SimpleEntity {
private String filename;

17
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java
View File

@@ -1,12 +1,16 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.ConstraintMode;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.JoinTable;
import jakarta.persistence.NamedAttributeNode;
import jakarta.persistence.NamedEntityGraph;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.util.Set;
@@ -22,15 +26,22 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(name = "service_ai_feedback")
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_feedback")
@NamedEntityGraph(name = "feedback.detail", attributeNodes = {
@NamedAttributeNode("pictures")
})
public class Feedback extends SimpleEntity {
@Column(nullable = false, columnDefinition = "longtext")
private String source;
@OneToMany(fetch = FetchType.LAZY)
@JoinTable(foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
@OneToMany(fetch = FetchType.EAGER)
@JoinTable(catalog = Constants.DATABASE_NAME, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
@ToString.Exclude
private Set<DataFile> pictures;
@Column(columnDefinition = "longtext")
private String analysis;
@Column(columnDefinition = "longtext")
private String conclusion;
@Column(nullable = false)
private Status status = Status.ANALYSIS_PROCESSING;
public enum Status {

13
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Group.java
View File

@@ -1,6 +1,8 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.Column;
import jakarta.persistence.ConstraintMode;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
@@ -25,13 +27,20 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(name = "service_ai_group")
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_group")
public class Group extends SimpleEntity {
@Column(nullable = false)
private String name;
private String status;
@Column(nullable = false)
private Status status = Status.RUNNING;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
@ToString.Exclude
private Knowledge knowledge;
public enum Status {
RUNNING,
FINISHED,
}
}

15
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Knowledge.java
View File

@@ -1,7 +1,9 @@
package com.lanyuanxiaoyao.service.ai.web.entity;
import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity;
import com.lanyuanxiaoyao.service.common.Constants;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.FetchType;
@@ -24,14 +26,23 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@DynamicUpdate
@EntityListeners(AuditingEntityListener.class)
@Table(name = "service_ai_knowledge")
@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_knowledge")
public class Knowledge extends SimpleEntity {
@Column(nullable = false)
private Long vectorSourceId;
@Column(nullable = false)
private String name;
@Column(nullable = false, columnDefinition = "longtext")
private String description;
private String strategy;
@Column(nullable = false)
private Strategy strategy = Strategy.Cosine;
@OneToMany(fetch = FetchType.LAZY, mappedBy = "knowledge", cascade = CascadeType.ALL)
@ToString.Exclude
private Set<Group> groups;
public enum Strategy {
Cosine,
Euclid,
}
}

17
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/repository/FeedbackRepository.java
View File

@@ -2,8 +2,25 @@ package com.lanyuanxiaoyao.service.ai.web.repository;
import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository;
import com.lanyuanxiaoyao.service.ai.web.entity.Feedback;
import java.util.List;
import java.util.Optional;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.data.jpa.repository.EntityGraph;
import org.springframework.stereotype.Repository;
@SuppressWarnings("NullableProblems")
@Repository
public interface FeedbackRepository extends SimpleRepository<Feedback, Long> {
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
@Override
List<Feedback> findAll(Specification<Feedback> specification);
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
@Override
List<Feedback> findAll(Specification<Feedback> specification, Sort sort);
@EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH)
@Override
Optional<Feedback> findOne(Specification<Feedback> specification);
}

32
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/EmbeddingService.java
View File

@@ -5,6 +5,7 @@ import cn.hutool.core.lang.Pair;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.entity.DataFile;
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext;
import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService;
@@ -54,16 +55,19 @@ public class EmbeddingService {
return Lists.immutable.ofAll(context.getDocuments());
}
public ImmutableList<Document> preview(String mode, ImmutableList<String> ids) {
DataFile dataFile = dataFileService.downloadFile(Long.parseLong(ids.get(0)));
public ImmutableList<Document> preview(String mode, ImmutableList<Long> ids) {
DataFile dataFile = dataFileService.downloadFile(ids.get(0));
String content = FileUtil.readString(dataFile.getPath(), StandardCharsets.UTF_8);
return preview(mode, content);
}
public void submit(Long id, String mode, String content) {
executors.submit(() -> {
Knowledge knowledge = knowledgeBaseService.get(id);
Long groupId = groupService.add(knowledge.getId(), StrUtil.format("文本-{}", IdUtil.nanoId(10)));
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
Group group = new Group();
group.setName(StrUtil.format("文本-{}", IdUtil.nanoId(10)));
group.setKnowledge(knowledge);
Long groupId = groupService.save(group);
EmbeddingContext context = EmbeddingContext.builder()
.vectorSourceId(knowledge.getVectorSourceId())
.groupId(groupId)
@@ -77,13 +81,16 @@ public class EmbeddingService {
});
}
public void submit(Long id, String mode, ImmutableList<String> ids) {
public void submit(Long id, String mode, ImmutableList<Long> ids) {
executors.submit(() -> {
Knowledge knowledge = knowledgeBaseService.get(id);
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
List<Pair<Long, DataFile>> dataFiles = Lists.mutable.empty();
for (String fileId : ids) {
DataFile dataFile = dataFileService.downloadFile(Long.parseLong(fileId));
Long groupId = groupService.add(id, dataFile.getFilename());
for (Long fileId : ids) {
DataFile dataFile = dataFileService.downloadFile(fileId);
Group group = new Group();
group.setName(dataFile.getFilename());
group.setKnowledge(knowledge);
Long groupId = groupService.save(group);
dataFiles.add(Pair.of(groupId, dataFile));
}
for (Pair<Long, DataFile> pair : dataFiles) {
@@ -106,12 +113,15 @@ public class EmbeddingService {
public void submitDirectly(Long id, String name, ImmutableList<String> contents) {
executors.submit(() -> {
Knowledge knowledge = knowledgeBaseService.get(id);
Knowledge knowledge = knowledgeBaseService.detailOrThrow(id);
String groupName = name;
if (StrUtil.isBlank(groupName)) {
groupName = StrUtil.format("外部-{}", IdUtil.nanoId(10));
}
Long groupId = groupService.add(knowledge.getId(), groupName);
Group group = new Group();
group.setName(groupName);
group.setKnowledge(knowledge);
Long groupId = groupService.save(group);
EmbeddingContext context = EmbeddingContext.builder()
.vectorSourceId(knowledge.getVectorSourceId())
.groupId(groupId)

4
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java
View File

@@ -7,7 +7,9 @@ import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext;
import com.lanyuanxiaoyao.service.ai.web.repository.FeedbackRepository;
import com.yomahub.liteflow.core.FlowExecutor;
import java.util.List;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -22,7 +24,7 @@ public class FeedbackService extends SimpleServiceSupport<Feedback> {
this.executor = executor;
}
// @Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
@Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
public void analysis() {
List<Feedback> feedbacks = repository.findAll(
builder -> builder

120
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/GroupService.java
View File

@@ -1,21 +1,14 @@
package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.entry.Alias;
import club.kingon.sql.builder.entry.Column;
import com.lanyuanxiaoyao.service.ai.core.configuration.SnowflakeId;
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
import com.lanyuanxiaoyao.service.ai.web.repository.GroupRepository;
import io.qdrant.client.ConditionFactory;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.grpc.Points;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import lombok.SneakyThrows;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -24,107 +17,34 @@ import org.springframework.transaction.annotation.Transactional;
* @version 20250522
*/
@Service
public class GroupService {
public static final String GROUP_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_group";
private static final RowMapper<Group> groupMapper = (rs, row) -> {
Group vo = new Group();
vo.setId(rs.getLong(1));
vo.setName(rs.getString(2));
vo.setStatus(rs.getString(3));
return vo;
};
private final JdbcTemplate template;
public class GroupService extends SimpleServiceSupport<Group> {
private final QdrantClient client;
public GroupService(JdbcTemplate template, VectorStore vectorStore) {
this.template = template;
public GroupService(GroupRepository groupRepository, VectorStore vectorStore) {
super(groupRepository);
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
}
public Group get(Long id) {
return template.queryForObject(
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
.from(GROUP_TABLE_NAME)
.whereEq("id", id)
.orderByDesc("created_time")
.build(),
groupMapper
);
}
@Transactional(rollbackFor = Exception.class)
public Long add(Long knowledgeId, String name) {
long id = SnowflakeId.next();
template.update(
SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name", "status")
.values()
.addValue("?", "?", "?", "?")
.precompileSql(),
id,
knowledgeId,
name,
"RUNNING"
);
return id;
}
public ImmutableList<Group> list(Long knowledgeId) {
return template.query(
SqlBuilder.select("id", "name", "status", "created_time", "modified_time")
.from(GROUP_TABLE_NAME)
.whereEq("knowledge_id", knowledgeId)
.orderByDesc("created_time")
.build(),
groupMapper
)
.stream()
.collect(Collectors.toCollection(Lists.mutable::empty))
.toImmutable();
public void finish(Long id) {
Group group = detailOrThrow(id);
group.setStatus(Group.Status.FINISHED);
save(group);
}
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
public void finish(Long groupId) {
template.update(
SqlBuilder.update(GROUP_TABLE_NAME)
.set("status", "FINISHED")
.whereEq("id", groupId)
.build()
);
}
@Transactional(rollbackFor = Exception.class)
public void remove(Long groupId) throws ExecutionException, InterruptedException {
Long vectorSourceId = template.queryForObject(
SqlBuilder.select("k.vector_source_id")
.from(Alias.of(GROUP_TABLE_NAME, "g"), Alias.of(KnowledgeBaseService.KNOWLEDGE_TABLE_NAME, "k"))
.whereEq("g.knowledge_id", Column.as("k.id"))
.andEq("g.id", groupId)
.precompileSql(),
Long.class,
groupId
);
@Override
public void remove(Long id) {
Group group = detailOrThrow(id);
Knowledge knowledge = group.getKnowledge();
client.deleteAsync(
String.valueOf(vectorSourceId),
String.valueOf(knowledge.getVectorSourceId()),
Points.Filter.newBuilder()
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(vectorSourceId)))
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId)))
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
.build()
).get();
template.update(
SqlBuilder.delete(GROUP_TABLE_NAME)
.whereEq("id", groupId)
.build()
);
}
@Transactional(rollbackFor = Exception.class)
public void removeByKnowledgeId(Long knowledgeId) {
template.update(
SqlBuilder.delete(GROUP_TABLE_NAME)
.whereEq("knowledge_id", "?")
.precompileSql(),
knowledgeId
);
super.remove(id);
}
}

34
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/KnowledgeBaseService.java
View File

@@ -3,13 +3,17 @@ package com.lanyuanxiaoyao.service.ai.web.service.knowledge;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport;
import com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator;
import com.lanyuanxiaoyao.service.ai.web.entity.Group;
import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge;
import com.lanyuanxiaoyao.service.ai.web.repository.KnowledgeRepository;
import com.lanyuanxiaoyao.service.common.Constants;
import io.qdrant.client.ConditionFactory;
import io.qdrant.client.QdrantClient;
import io.qdrant.client.grpc.Collections;
import io.qdrant.client.grpc.Points;
import java.util.List;
import java.util.concurrent.ExecutionException;
import lombok.SneakyThrows;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.springframework.ai.document.Document;
@@ -17,7 +21,6 @@ import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -28,16 +31,6 @@ import org.springframework.transaction.annotation.Transactional;
@Service
public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge";
public static final String[] KNOWLEDGE_COLUMNS = new String[]{"id", "vector_source_id", "name", "description", "strategy", "created_time", "modified_time"};
private static final RowMapper<Knowledge> knowledgeMapper = (rs, row) -> {
Knowledge knowledge = new Knowledge();
knowledge.setId(rs.getLong(1));
knowledge.setVectorSourceId(rs.getLong(2));
knowledge.setName(rs.getString(3));
knowledge.setDescription(rs.getString(4));
knowledge.setStrategy(rs.getString(5));
return knowledge;
};
private final KnowledgeRepository knowledgeRepository;
private final EmbeddingModel model;
private final QdrantClient client;
@@ -49,8 +42,9 @@ public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
}
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
public Long save(Knowledge entity) throws Exception {
public Long save(Knowledge entity) {
if (knowledgeRepository.existsKnowledgeByName(entity.getName())) {
throw new RuntimeException("名称已存在");
}
@@ -59,7 +53,7 @@ public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
client.createCollectionAsync(
String.valueOf(vectorSourceId),
Collections.VectorParams.newBuilder()
.setDistance(Collections.Distance.valueOf(entity.getStrategy()))
.setDistance(Collections.Distance.valueOf(entity.getStrategy().name()))
.setSize(model.dimensions())
.build()
).get();
@@ -79,11 +73,21 @@ public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
return detailOrThrow(id).getName();
}
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
@Override
public void remove(Long id) throws Exception {
public void remove(Long id) {
Knowledge knowledge = detailOrThrow(id);
client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get();
for (Group group : knowledge.getGroups()) {
client.deleteAsync(
String.valueOf(knowledge.getVectorSourceId()),
Points.Filter.newBuilder()
.addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId())))
.addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId())))
.build()
).get();
}
super.remove(id);
}
@@ -113,7 +117,7 @@ public class KnowledgeBaseService extends SimpleServiceSupport<Knowledge> {
.collect(Document::getText);
}
public Collections.CollectionInfo getCollectionInfoById(Long vectorSourceId) throws ExecutionException, InterruptedException {
public Collections.CollectionInfo collectionInfo(Long vectorSourceId) throws ExecutionException, InterruptedException {
return client.getCollectionInfoAsync(String.valueOf(vectorSourceId)).get();
}
}

6
service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/SegmentService.java
View File

@@ -27,8 +27,8 @@ public class SegmentService {
this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow();
}
public ImmutableList<SegmentVO> list(Long id, Long groupId) throws ExecutionException, InterruptedException {
Knowledge knowledge = knowledgeBaseService.get(id);
public ImmutableList<SegmentVO> list(Long knowledgeId, Long groupId) throws ExecutionException, InterruptedException {
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
Points.ScrollResponse response = client.scrollAsync(
Points.ScrollPoints.newBuilder()
.setCollectionName(String.valueOf(knowledge.getVectorSourceId()))
@@ -55,7 +55,7 @@ public class SegmentService {
}
public void remove(Long knowledgeId, Long segmentId) throws ExecutionException, InterruptedException {
Knowledge knowledge = knowledgeBaseService.get(knowledgeId);
Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId);
client.deletePayloadAsync(
String.valueOf(knowledgeId),
List.of(String.valueOf(segmentId)),

41
service-ai/service-ai-web/src/main/resources/application.yml
View File

@@ -1,12 +1,11 @@
server:
compression:
enabled: true
port: 8080
spring:
application:
name: service-ai-web
profiles:
include: common,forest
include: random-port,common,discovery,metrics,forest
mvc:
async:
request-timeout: 3600000
@@ -22,47 +21,27 @@ spring:
ai:
vectorstore:
qdrant:
host: 132.121.206.65
port: 29463
api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==)
host: 192.168.100.140
port: 6334
llm:
base-url: https://api.siliconflow.cn
api-key: sk-xrguybusoqndpqvgzgvllddzgjamksuecyqdaygdwnrnqfwo
base-url: http://132.121.206.65:10086
api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R)
chat:
base-url: ${spring.llm.base-url}/v1
model: 'Qwen/Qwen3-8B'
model: 'Qwen3/qwen3-1.7b'
visual:
base-url: https://open.bigmodel.cn/api/paas/v4
endpoint: /chat/completions
model: 'glm-4v-flash'
model: 'Qwen2.5/qwen2.5-vl-7b-q4km'
embedding:
model: 'BAAI/bge-m3'
model: 'Qwen3/qwen3-embedding-4b'
reranker:
model: 'BAAI/bge-reranker-v2-m3'
cloud:
discovery:
enabled: false
zookeeper:
enabled: false
datasource:
url: jdbc:mysql://192.168.100.140:3306/hudi_collect_build_b12?useSSL=false&allowPublicKeyRetrieval=true
username: root
password: rootless
driver-class-name: com.mysql.cj.jdbc.Driver
security:
meta:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
model: 'BGE/beg-reranker-v2'
jpa:
show-sql: true
generate-ddl: true
generate-ddl: false
liteflow:
rule-source: liteflow.xml
print-banner: false
check-node-exists: false
jasypt:
encryptor:
password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.'
fenix:
print-banner: false

2
service-ai/service-ai-web/src/main/resources/logback-spring.xml
View File

@@ -25,7 +25,7 @@
</appender>
<logger name="com.zaxxer.hikari" level="ERROR"/>
<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="WARN"/>
<logger name="org.hibernate.SQL" level="DEBUG"/>
<root level="INFO">
<appender-ref ref="Console"/>

20
service-web/client/src/pages/ai/knowledge/DataDetail.tsx
View File

@@ -24,7 +24,7 @@ const DataDetail: React.FC = () => {
{
type: 'service',
className: 'inline',
api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`,
api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`,
body: {
type: 'tpl',
tpl: '${name}',
@@ -38,7 +38,21 @@ const DataDetail: React.FC = () => {
body: [
{
type: 'crud',
api: `${commonInfo.baseAiUrl}/knowledge/group/list?knowledge_id=${knowledge_id}`,
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/group/list`,
data: {
query: {
equal: {
'knowledge/id': knowledge_id,
}
},
page: {
index: '${page}',
size: '${perPage}',
}
}
},
...crudCommonOptions(),
headerToolbar: [
'reload',
@@ -146,7 +160,7 @@ const DataDetail: React.FC = () => {
level: 'link',
size: 'sm',
actionType: 'ajax',
api: `get:${commonInfo.baseAiUrl}/knowledge/group/delete?id=\${id}`,
api: `get:${commonInfo.baseAiUrl}/knowledge/group/remove/\${id}`,
confirmText: '确认删除',
confirmTitle: '删除',
},

7
service-web/client/src/pages/ai/knowledge/DataImport.tsx
View File

@@ -23,7 +23,7 @@ const DataImport: React.FC = () => {
{
type: 'service',
className: 'inline',
api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`,
api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`,
body: {
type: 'tpl',
tpl: '${name}',
@@ -42,6 +42,7 @@ const DataImport: React.FC = () => {
body: [
{
id: 'a5219cc7-72dd-4199-9eeb-61305fe41075',
debug: commonInfo.debug,
type: 'form',
wrapWithPanel: false,
actions: [],
@@ -103,6 +104,8 @@ const DataImport: React.FC = () => {
autoUpload: false,
drag: true,
multiple: true,
joinValues: false,
extractValue: true,
accept: '*',
// 5MB 5242880
// 100MB 104857600
@@ -131,7 +134,6 @@ const DataImport: React.FC = () => {
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/preview_text`,
dataType: 'form',
data: {
mode: '${mode|default:undefined}',
type: '${type|default:undefined}',
@@ -149,7 +151,6 @@ const DataImport: React.FC = () => {
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/submit_text`,
dataType: 'form',
data: {
id: knowledge_id,
mode: '${mode|default:undefined}',

2
service-web/client/src/pages/ai/knowledge/DataSegment.tsx
View File

@@ -18,7 +18,7 @@ const DataDetail: React.FC = () => {
{
type: 'service',
className: 'inline',
api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`,
api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`,
body: {
type: 'tpl',
tpl: '${name}',

149
service-web/client/src/pages/ai/knowledge/Knowledge.tsx
View File

@@ -1,6 +1,13 @@
import React from 'react'
import {useNavigate} from 'react-router'
import {amisRender, commonInfo, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx'
import {
amisRender,
commonInfo,
crudCommonOptions,
mappingField,
mappingItem,
paginationTemplate,
} from '../../../util/amis.tsx'
const strategyMapping = [
mappingItem('文本', 'Cosine'),
@@ -25,62 +32,74 @@ const Knowledge: React.FC = () => {
body: [
{
type: 'crud',
api: `${commonInfo.baseAiUrl}/knowledge/list`,
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/list`,
data: {
page: {
index: '${page}',
size: '${perPage}',
}
}
},
...crudCommonOptions(),
headerToolbar: [
'reload',
{
type: 'action',
label: '',
icon: 'fa fa-plus',
tooltip: '新增',
tooltipPlacement: 'top',
actionType: 'dialog',
dialog: {
title: '新增知识库',
size: 'md',
body: {
type: 'form',
api: {
url: `${commonInfo.baseAiUrl}/knowledge/add`,
dataType: 'form',
...paginationTemplate(
10,
5,
[
{
type: 'action',
label: '',
icon: 'fa fa-plus',
tooltip: '新增',
tooltipPlacement: 'top',
actionType: 'dialog',
dialog: {
title: '新增知识库',
size: 'md',
body: {
type: 'form',
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/save`,
},
body: [
{
type: 'input-text',
name: 'name',
label: '名称',
required: true,
},
{
type: 'textarea',
name: 'description',
label: '描述',
required: true,
},
{
type: 'select',
name: 'strategy',
label: '类型',
value: 'Cosine',
required: true,
options: [
{
label: '文本',
value: 'Cosine',
},
{
label: '图片',
value: 'Euclid',
disabled: true,
},
],
},
],
},
body: [
{
type: 'input-text',
name: 'name',
label: '名称',
required: true,
},
{
type: 'textarea',
name: 'description',
label: '描述',
required: true,
},
{
type: 'select',
name: 'strategy',
label: '类型',
value: 'Cosine',
required: true,
options: [
{
label: '文本',
value: 'Cosine',
},
{
label: '图片',
value: 'Euclid',
disabled: true,
},
],
},
],
},
},
},
],
],
),
columns: [
{
name: 'name',
@@ -128,13 +147,12 @@ const Knowledge: React.FC = () => {
api: {
method: 'post',
url: `${commonInfo.baseAiUrl}/knowledge/update_description`,
dataType: 'form',
},
mode: 'normal',
body: [
{
type: 'hidden',
name: "id",
name: 'id',
// value: '${id}',
},
{
@@ -142,10 +160,10 @@ const Knowledge: React.FC = () => {
name: 'description',
label: '描述',
required: true,
}
]
}
}
},
],
},
},
},
{
type: 'action',
@@ -192,17 +210,8 @@ const Knowledge: React.FC = () => {
level: 'link',
size: 'sm',
actionType: 'ajax',
api: {
method: 'get',
url: `${commonInfo.baseAiUrl}/knowledge/delete`,
headers: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
},
data: {
id: '${id}',
},
},
confirmText: '确认删除',
api: `get:${commonInfo.baseAiUrl}/knowledge/remove/\${id}`,
confirmText: '确认删除知识库：${name}',
confirmTitle: '删除',
},
],

8
service-web/client/src/util/amis.tsx
View File

@@ -10,8 +10,8 @@ import {isEqual} from 'licia'
export const commonInfo = {
debug: isEqual(import.meta.env.MODE, 'development'),
baseUrl: 'http://132.126.207.130:35690/hudi_services/service_web',
// baseAiUrl: 'http://132.126.207.130:35690/hudi_services/service_ai_web',
baseAiUrl: 'http://localhost:8080',
baseAiUrl: 'http://132.126.207.130:35690/hudi_services/service_ai_web',
// baseAiUrl: 'http://localhost:8080',
authorizationHeaders: {
'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==',
'Content-Type': 'application/json',
@@ -311,16 +311,18 @@ export function paginationCommonOptions(perPage = true, maxButtons = 5) {
return option
}
export function paginationTemplate(perPage = 20, maxButtons = 5) {
export function paginationTemplate(perPage = 20, maxButtons = 5, extraHeaders: Schema[] = [], extraFooters: Schema[] = []) {
return {
perPage: perPage,
headerToolbar: [
'reload',
paginationCommonOptions(true, maxButtons),
...extraHeaders,
],
footerToolbar: [
'statistics',
paginationCommonOptions(true, maxButtons),
...extraFooters,
],
}
}