From 5b3c27ea48df318e423f1525613bb64045afa333 Mon Sep 17 00:00:00 2001 From: v-zhangjc9 Date: Mon, 23 Jun 2025 16:53:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(ai-web):=20=E5=AE=8C=E6=88=90JPA=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- service-ai/database/service_ai_feedback.sql | 23 ++- service-ai/database/service_ai_file.sql | 17 +- service-ai/database/service_ai_group.sql | 11 +- service-ai/database/service_ai_knowledge.sql | 12 +- service-ai/pom.xml | 5 + .../ai/core/configuration/SnowflakeId.java | 74 --------- service-ai/service-ai-web/pom.xml | 1 - .../web/base/controller/ListController.java | 9 +- .../web/base/controller/SimpleController.java | 2 +- .../controller/SimpleControllerSupport.java | 52 +++--- .../ai/web/base/service/SimpleService.java | 3 +- .../base/service/SimpleServiceSupport.java | 30 ++-- .../feedback/FeedbackController.java | 35 ++-- .../controller/knowledge/GroupController.java | 61 +++++-- .../knowledge/KnowledgeBaseController.java | 126 +++++++-------- .../service/ai/web/entity/DataFile.java | 3 +- .../service/ai/web/entity/Feedback.java | 17 +- .../service/ai/web/entity/Group.java | 13 +- .../service/ai/web/entity/Knowledge.java | 15 +- .../ai/web/repository/FeedbackRepository.java | 17 ++ .../ai/web/service/EmbeddingService.java | 32 ++-- .../web/service/feedback/FeedbackService.java | 4 +- .../web/service/knowledge/GroupService.java | 120 +++----------- .../knowledge/KnowledgeBaseService.java | 34 ++-- .../web/service/knowledge/SegmentService.java | 6 +- .../src/main/resources/application.yml | 41 ++--- .../src/main/resources/logback-spring.xml | 2 +- .../src/pages/ai/knowledge/DataDetail.tsx | 20 ++- .../src/pages/ai/knowledge/DataImport.tsx | 7 +- .../src/pages/ai/knowledge/DataSegment.tsx | 2 +- .../src/pages/ai/knowledge/Knowledge.tsx | 149 ++++++++++-------- service-web/client/src/util/amis.tsx | 8 +- 32 files changed, 458 insertions(+), 493 deletions(-) delete mode 100644 service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/SnowflakeId.java diff --git a/service-ai/database/service_ai_feedback.sql b/service-ai/database/service_ai_feedback.sql index f9ce3ba..8baf92a 100644 --- a/service-ai/database/service_ai_feedback.sql +++ b/service-ai/database/service_ai_feedback.sql @@ -1,11 +1,18 @@ CREATE TABLE `service_ai_feedback` ( - `id` bigint NOT NULL, - `source` longtext NOT NULL, - `conclusion` longtext, + `id` bigint NOT NULL, + `created_time` datetime(6) DEFAULT NULL, + `modified_time` datetime(6) DEFAULT NULL, `analysis` longtext, - `pictures` longtext, - `status` varchar(50) NOT NULL DEFAULT 'ANALYSIS_PROCESSING', - `created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) DEFAULT CHARSET = utf8mb4; \ No newline at end of file + `conclusion` longtext, + `source` longtext NOT NULL, + `status` tinyint NOT NULL, + PRIMARY KEY (`id`) +) DEFAULT CHARSET = utf8mb4; + +CREATE TABLE `service_ai_feedback_pictures` +( + `feedback_id` bigint NOT NULL, + `pictures_id` bigint NOT NULL, + PRIMARY KEY (`feedback_id`, `pictures_id`) +) DEFAULT CHARSET = utf8mb4; diff --git a/service-ai/database/service_ai_file.sql b/service-ai/database/service_ai_file.sql index 9ee9588..d0bb88f 100644 --- a/service-ai/database/service_ai_file.sql +++ b/service-ai/database/service_ai_file.sql @@ -1,11 +1,12 @@ CREATE TABLE `service_ai_file` ( - `id` bigint NOT NULL, - `filename` varchar(500) DEFAULT NULL, - `size` bigint DEFAULT NULL, - `md5` varchar(100) DEFAULT NULL, - `path` varchar(500) DEFAULT NULL, - `type` varchar(50) DEFAULT NULL, - `created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `id` bigint NOT NULL, + `created_time` datetime(6) DEFAULT NULL, + `modified_time` datetime(6) DEFAULT NULL, + `filename` varchar(255) DEFAULT NULL, + `md5` varchar(255) DEFAULT NULL, + `path` varchar(255) DEFAULT NULL, + `size` bigint DEFAULT NULL, + `type` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) -) DEFAULT CHARSET = utf8mb4; \ No newline at end of file +) DEFAULT CHARSET = utf8mb4; diff --git a/service-ai/database/service_ai_group.sql b/service-ai/database/service_ai_group.sql index b5c2333..eafead9 100644 --- a/service-ai/database/service_ai_group.sql +++ b/service-ai/database/service_ai_group.sql @@ -1,9 +1,10 @@ CREATE TABLE `service_ai_group` ( `id` bigint NOT NULL, + `created_time` datetime(6) DEFAULT NULL, + `modified_time` datetime(6) DEFAULT NULL, + `name` varchar(255) NOT NULL, + `status` tinyint NOT NULL, `knowledge_id` bigint NOT NULL, - `name` varchar(100) NOT NULL, - `status` varchar(10) NOT NULL DEFAULT 'RUNNING', - `created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) DEFAULT CHARSET = utf8mb4; \ No newline at end of file + PRIMARY KEY (`id`) +) DEFAULT CHARSET=utf8mb4; diff --git a/service-ai/database/service_ai_knowledge.sql b/service-ai/database/service_ai_knowledge.sql index 25f7b0a..90f30f5 100644 --- a/service-ai/database/service_ai_knowledge.sql +++ b/service-ai/database/service_ai_knowledge.sql @@ -1,11 +1,11 @@ CREATE TABLE `service_ai_knowledge` ( `id` bigint NOT NULL, - `vector_source_id` varchar(100) NOT NULL, - `name` varchar(100) NOT NULL, + `created_time` datetime(6) DEFAULT NULL, + `modified_time` datetime(6) DEFAULT NULL, `description` longtext NOT NULL, - `strategy` varchar(10) NOT NULL, - `created_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `name` varchar(255) NOT NULL, + `strategy` tinyint NOT NULL, + `vector_source_id` bigint NOT NULL, PRIMARY KEY (`id`) -) DEFAULT CHARSET = utf8mb4; \ No newline at end of file +) DEFAULT CHARSET = utf8mb4; diff --git a/service-ai/pom.xml b/service-ai/pom.xml index 9763d44..472629d 100644 --- a/service-ai/pom.xml +++ b/service-ai/pom.xml @@ -110,6 +110,11 @@ jasypt-spring-boot-starter 3.0.5 + + com.blinkfox + fenix-spring-boot-starter + 3.0.0 + diff --git a/service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/SnowflakeId.java b/service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/SnowflakeId.java deleted file mode 100644 index 59da9ee..0000000 --- a/service-ai/service-ai-core/src/main/java/com/lanyuanxiaoyao/service/ai/core/configuration/SnowflakeId.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.lanyuanxiaoyao.service.ai.core.configuration; - -import java.time.Instant; - -/** - * 使用雪花算法作为ID生成器 - * - * @author lanyuanxiaoyao - * @date 2024-11-14 - */ -public class SnowflakeId { - /** - * 起始的时间戳 - */ - private final static long START_TIMESTAMP = 1; - - /** - * 序列号占用的位数 - */ - private final static long SEQUENCE_BIT = 11; - - /** - * 序列号最大值 - */ - private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT); - - /** - * 时间戳值向左位移 - */ - private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT; - - /** - * 序列号 - */ - private static long sequence = 0; - /** - * 上一次时间戳 - */ - private static long lastTimestamp = -1; - - public static synchronized long next() { - long currentTimestamp = nowTimestamp(); - if (currentTimestamp < lastTimestamp) { - throw new RuntimeException("Clock have moved backwards."); - } - - if (currentTimestamp == lastTimestamp) { - // 相同毫秒内, 序列号自增 - sequence = (sequence + 1) & MAX_SEQUENCE_BIT; - // 同一毫秒的序列数已经达到最大 - if (sequence == 0) { - currentTimestamp = nextTimestamp(); - } - } else { - // 不同毫秒内, 序列号置为0 - sequence = 0; - } - - lastTimestamp = currentTimestamp; - return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence; - } - - private static long nextTimestamp() { - long milli = nowTimestamp(); - while (milli <= lastTimestamp) { - milli = nowTimestamp(); - } - return milli; - } - - private static long nowTimestamp() { - return Instant.now().toEpochMilli(); - } -} diff --git a/service-ai/service-ai-web/pom.xml b/service-ai/service-ai-web/pom.xml index 4e650c6..fa93ea7 100644 --- a/service-ai/service-ai-web/pom.xml +++ b/service-ai/service-ai-web/pom.xml @@ -53,7 +53,6 @@ com.blinkfox fenix-spring-boot-starter - 3.0.0 com.mysql diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/ListController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/ListController.java index 02dd5b2..91ad12e 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/ListController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/ListController.java @@ -1,17 +1,16 @@ package com.lanyuanxiaoyao.service.ai.web.base.controller; -import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse; +import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse; import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query; -import org.eclipse.collections.api.list.ImmutableList; /** * @author lanyuanxiaoyao * @date 2024-11-28 */ -public interface ListController { +public interface ListController { String LIST = "/list"; - AmisResponse> list() throws Exception; + AmisCrudResponse list() throws Exception; - AmisResponse> list(Query query) throws Exception; + AmisCrudResponse list(Query query) throws Exception; } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleController.java index 93713c0..36e77e4 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleController.java @@ -4,5 +4,5 @@ package com.lanyuanxiaoyao.service.ai.web.base.controller; * @author lanyuanxiaoyao * @date 2024-11-28 */ -public interface SimpleController extends SaveController, ListController, DetailController, RemoveController { +public interface SimpleController extends SaveController, ListController, DetailController, RemoveController { } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleControllerSupport.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleControllerSupport.java index bd9197f..5f077b1 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleControllerSupport.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/controller/SimpleControllerSupport.java @@ -1,13 +1,14 @@ package com.lanyuanxiaoyao.service.ai.web.base.controller; import cn.hutool.core.util.ObjectUtil; +import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisCrudResponse; import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse; import com.lanyuanxiaoyao.service.ai.web.base.controller.query.Query; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport; +import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.api.list.ImmutableList; +import org.springframework.data.domain.Page; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -34,43 +35,52 @@ public abstract class SimpleControllerSupport> list() throws Exception { + public AmisCrudResponse list() throws Exception { ListItemMapper mapper = listItemMapper(); - return AmisResponse.responseSuccess(service.list().collect(entity -> { - try { - return mapper.from(entity); - } catch (Exception e) { - throw new RuntimeException(e); - } - })); + return AmisCrudResponse.responseCrudData( + service.list() + .collect(entity -> { + try { + return mapper.from(entity); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + ); } @PostMapping(LIST) @Override - public AmisResponse> list(@RequestBody Query query) throws Exception { + public AmisCrudResponse list(@RequestBody Query query) throws Exception { if (ObjectUtil.isNull(query)) { - return AmisResponse.responseSuccess(Lists.immutable.empty()); + return AmisCrudResponse.responseCrudData(List.of(), 0); } ListItemMapper mapper = listItemMapper(); - return AmisResponse.responseSuccess(service.list(query).collect(entity -> { - try { - return mapper.from(entity); - } catch (Exception e) { - throw new RuntimeException(e); - } - })); + Page result = service.list(query); + return AmisCrudResponse.responseCrudData( + result.get() + .map(entity -> { + try { + return mapper.from(entity); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .toList(), + result.getTotalElements() + ); } @GetMapping(DETAIL) @Override - public AmisResponse detail(@PathVariable Long id) throws Exception { + public AmisResponse detail(@PathVariable("id") Long id) throws Exception { DetailItemMapper mapper = detailItemMapper(); return AmisResponse.responseSuccess(mapper.from(service.detailOrThrow(id))); } @GetMapping(REMOVE) @Override - public AmisResponse remove(@PathVariable Long id) throws Exception { + public AmisResponse remove(@PathVariable("id") Long id) throws Exception { service.remove(id); return AmisResponse.responseSuccess(); } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleService.java index 44bdef2..47e8279 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleService.java @@ -5,6 +5,7 @@ import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; import java.util.Optional; import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.set.ImmutableSet; +import org.springframework.data.domain.Page; /** * @author lanyuanxiaoyao @@ -19,7 +20,7 @@ public interface SimpleService { ImmutableList list(ImmutableSet ids) throws Exception; - ImmutableList list(Query query) throws Exception; + Page list(Query query) throws Exception; Optional detailOptional(Long id) throws Exception; diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleServiceSupport.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleServiceSupport.java index 0ab5181..6c4e687 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleServiceSupport.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/base/service/SimpleServiceSupport.java @@ -22,6 +22,8 @@ import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.set.ImmutableSet; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; /** @@ -30,6 +32,8 @@ import org.springframework.data.domain.Sort; */ @Slf4j public abstract class SimpleServiceSupport implements SimpleService { + private static final Integer DEFAULT_PAGE_INDEX = 1; + private static final Integer DEFAULT_PAGE_SIZE = 10; protected final SimpleRepository repository; public SimpleServiceSupport(SimpleRepository repository) { @@ -38,7 +42,7 @@ public abstract class SimpleServiceSupport implemen @Transactional(rollbackOn = Throwable.class) @Override - public Long save(ENTITY entity) throws Exception { + public Long save(ENTITY entity) { if (ObjectUtil.isNotNull(entity.getId())) { Long id = entity.getId(); ENTITY targetEntity = repository.findById(entity.getId()).orElseThrow(() -> new IdNotFoundException(id)); @@ -59,7 +63,7 @@ public abstract class SimpleServiceSupport implemen } @Override - public Long count() throws Exception { + public Long count() { return repository.count( (root, query, builder) -> builder.and( @@ -71,7 +75,7 @@ public abstract class SimpleServiceSupport implemen } @Override - public ImmutableList list() throws Exception { + public ImmutableList list() { return Lists.immutable.ofAll(repository.findAll( (root, query, builder) -> builder.and( @@ -83,7 +87,7 @@ public abstract class SimpleServiceSupport implemen } @Override - public ImmutableList list(ImmutableSet ids) throws Exception { + public ImmutableList list(ImmutableSet ids) { return Lists.immutable.ofAll(repository.findAll( (root, query, builder) -> { MutableList predicates = Lists.mutable.ofAll(listPredicate(root, query, builder)); @@ -179,15 +183,23 @@ public abstract class SimpleServiceSupport implemen } @Override - public ImmutableList list(Query listQuery) throws Exception { - return Lists.immutable.ofAll(repository.findAll( + public Page list(Query listQuery) { + PageRequest pageRequest = PageRequest.of(DEFAULT_PAGE_INDEX - 1, DEFAULT_PAGE_SIZE, Sort.by(SimpleEntity_.CREATED_TIME).descending()); + if (ObjectUtil.isNotNull(listQuery.getPage())) { + pageRequest = PageRequest.of( + ObjectUtil.defaultIfNull(listQuery.getPage().getIndex(), DEFAULT_PAGE_INDEX) - 1, + ObjectUtil.defaultIfNull(listQuery.getPage().getSize(), DEFAULT_PAGE_SIZE), + Sort.by(SimpleEntity_.CREATED_TIME).descending() + ); + } + return repository.findAll( (root, query, builder) -> { MutableList predicates = Lists.mutable.ofAll(listPredicate(root, query, builder)); predicates.addAllIterable(queryPredicates(listQuery.getQuery(), root, query, builder)); return builder.and(predicates.reject(ObjectUtil::isNull).toArray(new Predicate[predicates.size()])); }, - Sort.by(SimpleEntity_.CREATED_TIME).descending() - )); + pageRequest + ); } @Override @@ -221,7 +233,7 @@ public abstract class SimpleServiceSupport implemen @Transactional(rollbackOn = Throwable.class) @Override - public void remove(Long id) throws Exception { + public void remove(Long id) { if (ObjectUtil.isNotNull(id)) { repository.deleteById(id); } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java index bac96c1..90e1803 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/feedback/FeedbackController.java @@ -4,6 +4,7 @@ import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport import com.lanyuanxiaoyao.service.ai.web.base.entity.IdOnlyEntity; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem; import com.lanyuanxiaoyao.service.ai.web.entity.Feedback; +import com.lanyuanxiaoyao.service.ai.web.service.DataFileService; import com.lanyuanxiaoyao.service.ai.web.service.feedback.FeedbackService; import lombok.Data; import lombok.EqualsAndHashCode; @@ -22,12 +23,14 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController @RequestMapping("feedback") -public class FeedbackController extends SimpleControllerSupport { +public class FeedbackController extends SimpleControllerSupport { private final FeedbackService feedbackService; + private final DataFileService dataFileService; - public FeedbackController(FeedbackService feedbackService) { + public FeedbackController(FeedbackService feedbackService, DataFileService dataFileService) { super(feedbackService); this.feedbackService = feedbackService; + this.dataFileService = dataFileService; } @GetMapping("reanalysis/{id}") @@ -42,7 +45,12 @@ public class FeedbackController extends SimpleControllerSupport saveItemMapper() { - return null; + return item -> { + Feedback feedback = new Feedback(); + feedback.setSource(item.getSource()); + feedback.setPictures(dataFileService.downloadFile(item.getPictures()).toSet()); + return feedback; + }; } @Override @@ -51,8 +59,9 @@ public class FeedbackController extends SimpleControllerSupport detailItemMapper() { - return Mappers.getMapper(DetailItem.Mapper.class); + protected DetailItemMapper detailItemMapper() { + ListItemMapper mapper = listItemMapper(); + return mapper::from; } @Data @@ -88,20 +97,4 @@ public class FeedbackController extends SimpleControllerSupport { - @Mapping(target = "pictures", expression = "java(Sets.immutable.ofAll(feedback.getPictures()).collect(IdOnlyEntity::getId))") - @Override - DetailItem from(Feedback feedback); - } - } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/GroupController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/GroupController.java index 78ab472..e02ccb1 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/GroupController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/GroupController.java @@ -1,12 +1,15 @@ package com.lanyuanxiaoyao.service.ai.web.controller.knowledge; -import com.lanyuanxiaoyao.service.ai.core.entity.amis.AmisResponse; +import com.lanyuanxiaoyao.service.ai.web.base.controller.SimpleControllerSupport; +import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleItem; +import com.lanyuanxiaoyao.service.ai.web.entity.Group; import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService; -import java.util.concurrent.ExecutionException; +import com.lanyuanxiaoyao.service.ai.web.service.knowledge.KnowledgeBaseService; +import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.GetMapping; +import org.mapstruct.factory.Mappers; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** @@ -16,21 +19,49 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController @RequestMapping("knowledge/group") -public class GroupController { - private final GroupService groupService; +public class GroupController extends SimpleControllerSupport { + private final KnowledgeBaseService knowledgeBaseService; - public GroupController(GroupService groupService) { - this.groupService = groupService; + public GroupController(GroupService groupService, KnowledgeBaseService knowledgeBaseService) { + super(groupService); + this.knowledgeBaseService = knowledgeBaseService; } - @GetMapping("list") - public AmisResponse list(@RequestParam("knowledge_id") Long knowledgeId) { - return AmisResponse.responseCrudData(groupService.list(knowledgeId)); + @Override + protected SaveItemMapper saveItemMapper() { + return item -> { + Group group = new Group(); + group.setName(item.getName()); + group.setKnowledge(knowledgeBaseService.detailOrThrow(item.getKnowledgeId())); + return group; + }; } - @GetMapping("delete") - public AmisResponse delete(@RequestParam("id") Long id) throws ExecutionException, InterruptedException { - groupService.remove(id); - return AmisResponse.responseSuccess(); + @Override + protected ListItemMapper listItemMapper() { + return Mappers.getMapper(ListItem.Mapper.class); + } + + @Override + protected DetailItemMapper detailItemMapper() { + ListItemMapper mapper = listItemMapper(); + return mapper::from; + } + + @Data + public static final class SaveItem { + private String name; + private Long knowledgeId; + } + + @Data + @EqualsAndHashCode(callSuper = true) + public static final class ListItem extends SimpleItem { + private String name; + private Group.Status status; + + @org.mapstruct.Mapper + public interface Mapper extends ListItemMapper { + } } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/KnowledgeBaseController.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/KnowledgeBaseController.java index af7f910..26e2c5d 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/KnowledgeBaseController.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/controller/knowledge/KnowledgeBaseController.java @@ -34,7 +34,7 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController @RequestMapping("knowledge") -public class KnowledgeBaseController extends SimpleControllerSupport { +public class KnowledgeBaseController extends SimpleControllerSupport { private final KnowledgeBaseService knowledgeBaseService; private final EmbeddingService embeddingService; @@ -44,20 +44,6 @@ public class KnowledgeBaseController extends SimpleControllerSupport saveItemMapper() { return Mappers.getMapper(SaveItem.Mapper.class); @@ -66,25 +52,34 @@ public class KnowledgeBaseController extends SimpleControllerSupport listItemMapper() { ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class); - return knowledge -> mapper.from(knowledge, knowledgeBaseService.getCollectionInfoById(knowledge.getVectorSourceId())); + return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId())); } @Override - protected DetailItemMapper detailItemMapper() { - DetailItem.Mapper mapper = Mappers.getMapper(DetailItem.Mapper.class); - return knowledge -> mapper.from(knowledge, knowledgeBaseService.getCollectionInfoById(knowledge.getVectorSourceId())); + protected DetailItemMapper detailItemMapper() { + ListItem.Mapper mapper = Mappers.getMapper(ListItem.Mapper.class); + return knowledge -> mapper.from(knowledge, knowledgeBaseService.collectionInfo(knowledge.getVectorSourceId())); + } + + @GetMapping("{id}/name") + public AmisMapResponse name(@PathVariable("id") Long id) { + return AmisResponse.responseMapData() + .setData("name", knowledgeBaseService.getName(id)); + } + + @PostMapping("update_description") + public void updateDescription( + @RequestParam("id") Long id, + @RequestParam("description") String description + ) throws Exception { + knowledgeBaseService.updateDescription(id, description); } @PostMapping("preview_text") - public AmisResponse previewText( - @RequestParam(value = "mode", defaultValue = "NORMAL") String mode, - @RequestParam(value = "type", defaultValue = "text") String type, - @RequestParam(value = "content", required = false) String content, - @RequestParam(value = "files", required = false) String files - ) { - if (StrUtil.equals("text", type)) { + public AmisResponse previewText(@RequestBody SubmitItem item) { + if (StrUtil.equals("text", item.getType())) { return AmisResponse.responseCrudData( - embeddingService.preview(mode, content) + embeddingService.preview(item.getMode(), item.getContent()) .collect(doc -> { SegmentVO vo = new SegmentVO(); vo.setId(doc.getId()); @@ -92,9 +87,9 @@ public class KnowledgeBaseController extends SimpleControllerSupport { SegmentVO vo = new SegmentVO(); vo.setId(doc.getId()); @@ -103,45 +98,29 @@ public class KnowledgeBaseController extends SimpleControllerSupport query( - @RequestParam("id") Long id, - @RequestParam(value = "limit", defaultValue = "5") Integer limit, - @RequestParam(value = "threshold", defaultValue = "0.6") Double threshold, - @RequestBody String text - ) throws ExecutionException, InterruptedException, IOException { - return knowledgeBaseService.query(id, text, limit, threshold); + public ImmutableList query(@RequestBody QueryItem item) throws ExecutionException, InterruptedException, IOException { + return knowledgeBaseService.query(item.getId(), item.getText(), item.getLimit(), item.getThreshold()); } @Data @@ -157,7 +136,7 @@ public class KnowledgeBaseController extends SimpleControllerSupport { - @Mapping(source = "info.config.params.vectorsConfig.params.distance", target = "strategy") - @Mapping(source = "info.config.params.vectorsConfig.params.size", target = "size") - @Mapping(source = "info.pointsCount", target = "points") - @Mapping(source = "info.segmentsCount", target = "segments") - @Mapping(source = "info.status", target = "status") - DetailItem from(Knowledge knowledge, Collections.CollectionInfo info); - } + public static final class SubmitItem { + private Long id; + private String mode; + private String type; + private String content; + private ImmutableList files; + } + + @Data + public static final class SubmitDirectlyItem { + private Long id; + private String name; + private String splitKey = "\n\n"; + private String content; + } + + @Data + public static final class QueryItem { + private Long id; + private Integer limit = 5; + private Double threshold = 0.6; + private String text; } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/DataFile.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/DataFile.java index a82e7b1..0b9f4ad 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/DataFile.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/DataFile.java @@ -1,6 +1,7 @@ package com.lanyuanxiaoyao.service.ai.web.entity; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; +import com.lanyuanxiaoyao.service.common.Constants; import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Getter; @@ -20,7 +21,7 @@ import org.hibernate.annotations.DynamicUpdate; @ToString @Entity @DynamicUpdate -@Table(name = "service_ai_file") +@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_file") @NoArgsConstructor public class DataFile extends SimpleEntity { private String filename; diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java index df583bd..bc50bcb 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Feedback.java @@ -1,12 +1,16 @@ package com.lanyuanxiaoyao.service.ai.web.entity; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; +import com.lanyuanxiaoyao.service.common.Constants; +import jakarta.persistence.Column; import jakarta.persistence.ConstraintMode; import jakarta.persistence.Entity; import jakarta.persistence.EntityListeners; import jakarta.persistence.FetchType; import jakarta.persistence.ForeignKey; import jakarta.persistence.JoinTable; +import jakarta.persistence.NamedAttributeNode; +import jakarta.persistence.NamedEntityGraph; import jakarta.persistence.OneToMany; import jakarta.persistence.Table; import java.util.Set; @@ -22,15 +26,22 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; @Entity @DynamicUpdate @EntityListeners(AuditingEntityListener.class) -@Table(name = "service_ai_feedback") +@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_feedback") +@NamedEntityGraph(name = "feedback.detail", attributeNodes = { + @NamedAttributeNode("pictures") +}) public class Feedback extends SimpleEntity { + @Column(nullable = false, columnDefinition = "longtext") private String source; - @OneToMany(fetch = FetchType.LAZY) - @JoinTable(foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) + @OneToMany(fetch = FetchType.EAGER) + @JoinTable(catalog = Constants.DATABASE_NAME, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT), inverseForeignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) @ToString.Exclude private Set pictures; + @Column(columnDefinition = "longtext") private String analysis; + @Column(columnDefinition = "longtext") private String conclusion; + @Column(nullable = false) private Status status = Status.ANALYSIS_PROCESSING; public enum Status { diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Group.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Group.java index 26a0211..828bcd5 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Group.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Group.java @@ -1,6 +1,8 @@ package com.lanyuanxiaoyao.service.ai.web.entity; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; +import com.lanyuanxiaoyao.service.common.Constants; +import jakarta.persistence.Column; import jakarta.persistence.ConstraintMode; import jakarta.persistence.Entity; import jakarta.persistence.EntityListeners; @@ -25,13 +27,20 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; @Entity @DynamicUpdate @EntityListeners(AuditingEntityListener.class) -@Table(name = "service_ai_group") +@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_group") public class Group extends SimpleEntity { + @Column(nullable = false) private String name; - private String status; + @Column(nullable = false) + private Status status = Status.RUNNING; @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(nullable = false, foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) @ToString.Exclude private Knowledge knowledge; + + public enum Status { + RUNNING, + FINISHED, + } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Knowledge.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Knowledge.java index 65638e2..6dd04fc 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Knowledge.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/entity/Knowledge.java @@ -1,7 +1,9 @@ package com.lanyuanxiaoyao.service.ai.web.entity; import com.lanyuanxiaoyao.service.ai.web.base.entity.SimpleEntity; +import com.lanyuanxiaoyao.service.common.Constants; import jakarta.persistence.CascadeType; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.EntityListeners; import jakarta.persistence.FetchType; @@ -24,14 +26,23 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; @Entity @DynamicUpdate @EntityListeners(AuditingEntityListener.class) -@Table(name = "service_ai_knowledge") +@Table(catalog = Constants.DATABASE_NAME, name = "service_ai_knowledge") public class Knowledge extends SimpleEntity { + @Column(nullable = false) private Long vectorSourceId; + @Column(nullable = false) private String name; + @Column(nullable = false, columnDefinition = "longtext") private String description; - private String strategy; + @Column(nullable = false) + private Strategy strategy = Strategy.Cosine; @OneToMany(fetch = FetchType.LAZY, mappedBy = "knowledge", cascade = CascadeType.ALL) @ToString.Exclude private Set groups; + + public enum Strategy { + Cosine, + Euclid, + } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/repository/FeedbackRepository.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/repository/FeedbackRepository.java index a81535a..4d6335e 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/repository/FeedbackRepository.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/repository/FeedbackRepository.java @@ -2,8 +2,25 @@ package com.lanyuanxiaoyao.service.ai.web.repository; import com.lanyuanxiaoyao.service.ai.web.base.repository.SimpleRepository; import com.lanyuanxiaoyao.service.ai.web.entity.Feedback; +import java.util.List; +import java.util.Optional; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.data.jpa.repository.EntityGraph; import org.springframework.stereotype.Repository; +@SuppressWarnings("NullableProblems") @Repository public interface FeedbackRepository extends SimpleRepository { + @EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH) + @Override + List findAll(Specification specification); + + @EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH) + @Override + List findAll(Specification specification, Sort sort); + + @EntityGraph(value = "feedback.detail", type = EntityGraph.EntityGraphType.FETCH) + @Override + Optional findOne(Specification specification); } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/EmbeddingService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/EmbeddingService.java index 5d89356..3d7b493 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/EmbeddingService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/EmbeddingService.java @@ -5,6 +5,7 @@ import cn.hutool.core.lang.Pair; import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.StrUtil; import com.lanyuanxiaoyao.service.ai.web.entity.DataFile; +import com.lanyuanxiaoyao.service.ai.web.entity.Group; import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge; import com.lanyuanxiaoyao.service.ai.web.entity.context.EmbeddingContext; import com.lanyuanxiaoyao.service.ai.web.service.knowledge.GroupService; @@ -54,16 +55,19 @@ public class EmbeddingService { return Lists.immutable.ofAll(context.getDocuments()); } - public ImmutableList preview(String mode, ImmutableList ids) { - DataFile dataFile = dataFileService.downloadFile(Long.parseLong(ids.get(0))); + public ImmutableList preview(String mode, ImmutableList ids) { + DataFile dataFile = dataFileService.downloadFile(ids.get(0)); String content = FileUtil.readString(dataFile.getPath(), StandardCharsets.UTF_8); return preview(mode, content); } public void submit(Long id, String mode, String content) { executors.submit(() -> { - Knowledge knowledge = knowledgeBaseService.get(id); - Long groupId = groupService.add(knowledge.getId(), StrUtil.format("文本-{}", IdUtil.nanoId(10))); + Knowledge knowledge = knowledgeBaseService.detailOrThrow(id); + Group group = new Group(); + group.setName(StrUtil.format("文本-{}", IdUtil.nanoId(10))); + group.setKnowledge(knowledge); + Long groupId = groupService.save(group); EmbeddingContext context = EmbeddingContext.builder() .vectorSourceId(knowledge.getVectorSourceId()) .groupId(groupId) @@ -77,13 +81,16 @@ public class EmbeddingService { }); } - public void submit(Long id, String mode, ImmutableList ids) { + public void submit(Long id, String mode, ImmutableList ids) { executors.submit(() -> { - Knowledge knowledge = knowledgeBaseService.get(id); + Knowledge knowledge = knowledgeBaseService.detailOrThrow(id); List> dataFiles = Lists.mutable.empty(); - for (String fileId : ids) { - DataFile dataFile = dataFileService.downloadFile(Long.parseLong(fileId)); - Long groupId = groupService.add(id, dataFile.getFilename()); + for (Long fileId : ids) { + DataFile dataFile = dataFileService.downloadFile(fileId); + Group group = new Group(); + group.setName(dataFile.getFilename()); + group.setKnowledge(knowledge); + Long groupId = groupService.save(group); dataFiles.add(Pair.of(groupId, dataFile)); } for (Pair pair : dataFiles) { @@ -106,12 +113,15 @@ public class EmbeddingService { public void submitDirectly(Long id, String name, ImmutableList contents) { executors.submit(() -> { - Knowledge knowledge = knowledgeBaseService.get(id); + Knowledge knowledge = knowledgeBaseService.detailOrThrow(id); String groupName = name; if (StrUtil.isBlank(groupName)) { groupName = StrUtil.format("外部-{}", IdUtil.nanoId(10)); } - Long groupId = groupService.add(knowledge.getId(), groupName); + Group group = new Group(); + group.setName(groupName); + group.setKnowledge(knowledge); + Long groupId = groupService.save(group); EmbeddingContext context = EmbeddingContext.builder() .vectorSourceId(knowledge.getVectorSourceId()) .groupId(groupId) diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java index b3ed3cd..e1e7d8b 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/feedback/FeedbackService.java @@ -7,7 +7,9 @@ import com.lanyuanxiaoyao.service.ai.web.entity.context.FeedbackContext; import com.lanyuanxiaoyao.service.ai.web.repository.FeedbackRepository; import com.yomahub.liteflow.core.FlowExecutor; import java.util.List; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -22,7 +24,7 @@ public class FeedbackService extends SimpleServiceSupport { this.executor = executor; } - // @Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES) + @Scheduled(initialDelay = 1, fixedDelay = 1, timeUnit = TimeUnit.MINUTES) public void analysis() { List feedbacks = repository.findAll( builder -> builder diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/GroupService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/GroupService.java index f12bb3a..195c33f 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/GroupService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/GroupService.java @@ -1,21 +1,14 @@ package com.lanyuanxiaoyao.service.ai.web.service.knowledge; -import club.kingon.sql.builder.SqlBuilder; -import club.kingon.sql.builder.entry.Alias; -import club.kingon.sql.builder.entry.Column; -import com.lanyuanxiaoyao.service.ai.core.configuration.SnowflakeId; +import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport; import com.lanyuanxiaoyao.service.ai.web.entity.Group; -import com.lanyuanxiaoyao.service.common.Constants; +import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge; +import com.lanyuanxiaoyao.service.ai.web.repository.GroupRepository; import io.qdrant.client.ConditionFactory; import io.qdrant.client.QdrantClient; import io.qdrant.client.grpc.Points; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.api.list.ImmutableList; +import lombok.SneakyThrows; import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -24,107 +17,34 @@ import org.springframework.transaction.annotation.Transactional; * @version 20250522 */ @Service -public class GroupService { - public static final String GROUP_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_group"; - private static final RowMapper groupMapper = (rs, row) -> { - Group vo = new Group(); - vo.setId(rs.getLong(1)); - vo.setName(rs.getString(2)); - vo.setStatus(rs.getString(3)); - return vo; - }; - - private final JdbcTemplate template; +public class GroupService extends SimpleServiceSupport { private final QdrantClient client; - public GroupService(JdbcTemplate template, VectorStore vectorStore) { - this.template = template; + public GroupService(GroupRepository groupRepository, VectorStore vectorStore) { + super(groupRepository); this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow(); } - public Group get(Long id) { - return template.queryForObject( - SqlBuilder.select("id", "name", "status", "created_time", "modified_time") - .from(GROUP_TABLE_NAME) - .whereEq("id", id) - .orderByDesc("created_time") - .build(), - groupMapper - ); - } - @Transactional(rollbackFor = Exception.class) - public Long add(Long knowledgeId, String name) { - long id = SnowflakeId.next(); - template.update( - SqlBuilder.insertInto(GROUP_TABLE_NAME, "id", "knowledge_id", "name", "status") - .values() - .addValue("?", "?", "?", "?") - .precompileSql(), - id, - knowledgeId, - name, - "RUNNING" - ); - return id; - } - - public ImmutableList list(Long knowledgeId) { - return template.query( - SqlBuilder.select("id", "name", "status", "created_time", "modified_time") - .from(GROUP_TABLE_NAME) - .whereEq("knowledge_id", knowledgeId) - .orderByDesc("created_time") - .build(), - groupMapper - ) - .stream() - .collect(Collectors.toCollection(Lists.mutable::empty)) - .toImmutable(); + public void finish(Long id) { + Group group = detailOrThrow(id); + group.setStatus(Group.Status.FINISHED); + save(group); } + @SneakyThrows @Transactional(rollbackFor = Exception.class) - public void finish(Long groupId) { - template.update( - SqlBuilder.update(GROUP_TABLE_NAME) - .set("status", "FINISHED") - .whereEq("id", groupId) - .build() - ); - } - - @Transactional(rollbackFor = Exception.class) - public void remove(Long groupId) throws ExecutionException, InterruptedException { - Long vectorSourceId = template.queryForObject( - SqlBuilder.select("k.vector_source_id") - .from(Alias.of(GROUP_TABLE_NAME, "g"), Alias.of(KnowledgeBaseService.KNOWLEDGE_TABLE_NAME, "k")) - .whereEq("g.knowledge_id", Column.as("k.id")) - .andEq("g.id", groupId) - .precompileSql(), - Long.class, - groupId - ); + @Override + public void remove(Long id) { + Group group = detailOrThrow(id); + Knowledge knowledge = group.getKnowledge(); client.deleteAsync( - String.valueOf(vectorSourceId), + String.valueOf(knowledge.getVectorSourceId()), Points.Filter.newBuilder() - .addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(vectorSourceId))) - .addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(groupId))) + .addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId()))) + .addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId()))) .build() ).get(); - template.update( - SqlBuilder.delete(GROUP_TABLE_NAME) - .whereEq("id", groupId) - .build() - ); - } - - @Transactional(rollbackFor = Exception.class) - public void removeByKnowledgeId(Long knowledgeId) { - template.update( - SqlBuilder.delete(GROUP_TABLE_NAME) - .whereEq("knowledge_id", "?") - .precompileSql(), - knowledgeId - ); + super.remove(id); } } diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/KnowledgeBaseService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/KnowledgeBaseService.java index 766641a..329a0d3 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/KnowledgeBaseService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/KnowledgeBaseService.java @@ -3,13 +3,17 @@ package com.lanyuanxiaoyao.service.ai.web.service.knowledge; import cn.hutool.core.util.StrUtil; import com.lanyuanxiaoyao.service.ai.web.base.service.SimpleServiceSupport; import com.lanyuanxiaoyao.service.ai.web.configuration.SnowflakeIdGenerator; +import com.lanyuanxiaoyao.service.ai.web.entity.Group; import com.lanyuanxiaoyao.service.ai.web.entity.Knowledge; import com.lanyuanxiaoyao.service.ai.web.repository.KnowledgeRepository; import com.lanyuanxiaoyao.service.common.Constants; +import io.qdrant.client.ConditionFactory; import io.qdrant.client.QdrantClient; import io.qdrant.client.grpc.Collections; +import io.qdrant.client.grpc.Points; import java.util.List; import java.util.concurrent.ExecutionException; +import lombok.SneakyThrows; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ImmutableList; import org.springframework.ai.document.Document; @@ -17,7 +21,6 @@ import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore; -import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -28,16 +31,6 @@ import org.springframework.transaction.annotation.Transactional; @Service public class KnowledgeBaseService extends SimpleServiceSupport { public static final String KNOWLEDGE_TABLE_NAME = Constants.DATABASE_NAME + ".service_ai_knowledge"; - public static final String[] KNOWLEDGE_COLUMNS = new String[]{"id", "vector_source_id", "name", "description", "strategy", "created_time", "modified_time"}; - private static final RowMapper knowledgeMapper = (rs, row) -> { - Knowledge knowledge = new Knowledge(); - knowledge.setId(rs.getLong(1)); - knowledge.setVectorSourceId(rs.getLong(2)); - knowledge.setName(rs.getString(3)); - knowledge.setDescription(rs.getString(4)); - knowledge.setStrategy(rs.getString(5)); - return knowledge; - }; private final KnowledgeRepository knowledgeRepository; private final EmbeddingModel model; private final QdrantClient client; @@ -49,8 +42,9 @@ public class KnowledgeBaseService extends SimpleServiceSupport { this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow(); } + @SneakyThrows @Transactional(rollbackFor = Exception.class) - public Long save(Knowledge entity) throws Exception { + public Long save(Knowledge entity) { if (knowledgeRepository.existsKnowledgeByName(entity.getName())) { throw new RuntimeException("名称已存在"); } @@ -59,7 +53,7 @@ public class KnowledgeBaseService extends SimpleServiceSupport { client.createCollectionAsync( String.valueOf(vectorSourceId), Collections.VectorParams.newBuilder() - .setDistance(Collections.Distance.valueOf(entity.getStrategy())) + .setDistance(Collections.Distance.valueOf(entity.getStrategy().name())) .setSize(model.dimensions()) .build() ).get(); @@ -79,11 +73,21 @@ public class KnowledgeBaseService extends SimpleServiceSupport { return detailOrThrow(id).getName(); } + @SneakyThrows @Transactional(rollbackFor = Exception.class) @Override - public void remove(Long id) throws Exception { + public void remove(Long id) { Knowledge knowledge = detailOrThrow(id); client.deleteCollectionAsync(String.valueOf(knowledge.getVectorSourceId())).get(); + for (Group group : knowledge.getGroups()) { + client.deleteAsync( + String.valueOf(knowledge.getVectorSourceId()), + Points.Filter.newBuilder() + .addMust(ConditionFactory.matchKeyword("vector_source_id", String.valueOf(knowledge.getVectorSourceId()))) + .addMust(ConditionFactory.matchKeyword("group_id", String.valueOf(group.getId()))) + .build() + ).get(); + } super.remove(id); } @@ -113,7 +117,7 @@ public class KnowledgeBaseService extends SimpleServiceSupport { .collect(Document::getText); } - public Collections.CollectionInfo getCollectionInfoById(Long vectorSourceId) throws ExecutionException, InterruptedException { + public Collections.CollectionInfo collectionInfo(Long vectorSourceId) throws ExecutionException, InterruptedException { return client.getCollectionInfoAsync(String.valueOf(vectorSourceId)).get(); } } \ No newline at end of file diff --git a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/SegmentService.java b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/SegmentService.java index a5f9e77..61d63e3 100644 --- a/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/SegmentService.java +++ b/service-ai/service-ai-web/src/main/java/com/lanyuanxiaoyao/service/ai/web/service/knowledge/SegmentService.java @@ -27,8 +27,8 @@ public class SegmentService { this.client = (QdrantClient) vectorStore.getNativeClient().orElseThrow(); } - public ImmutableList list(Long id, Long groupId) throws ExecutionException, InterruptedException { - Knowledge knowledge = knowledgeBaseService.get(id); + public ImmutableList list(Long knowledgeId, Long groupId) throws ExecutionException, InterruptedException { + Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId); Points.ScrollResponse response = client.scrollAsync( Points.ScrollPoints.newBuilder() .setCollectionName(String.valueOf(knowledge.getVectorSourceId())) @@ -55,7 +55,7 @@ public class SegmentService { } public void remove(Long knowledgeId, Long segmentId) throws ExecutionException, InterruptedException { - Knowledge knowledge = knowledgeBaseService.get(knowledgeId); + Knowledge knowledge = knowledgeBaseService.detailOrThrow(knowledgeId); client.deletePayloadAsync( String.valueOf(knowledgeId), List.of(String.valueOf(segmentId)), diff --git a/service-ai/service-ai-web/src/main/resources/application.yml b/service-ai/service-ai-web/src/main/resources/application.yml index 34bb207..cef88da 100644 --- a/service-ai/service-ai-web/src/main/resources/application.yml +++ b/service-ai/service-ai-web/src/main/resources/application.yml @@ -1,12 +1,11 @@ server: compression: enabled: true - port: 8080 spring: application: name: service-ai-web profiles: - include: common,forest + include: random-port,common,discovery,metrics,forest mvc: async: request-timeout: 3600000 @@ -22,47 +21,27 @@ spring: ai: vectorstore: qdrant: + host: 132.121.206.65 + port: 29463 api-key: ENC(0/0UkIKeAvyV17yNqSU3v04wmm8CdWKe4BYSSJa2FuBtK12TcZRJPdwk+ZpYnpISv+KmVTUrrmFBzAYrDR3ysA==) - host: 192.168.100.140 - port: 6334 llm: - base-url: https://api.siliconflow.cn - api-key: sk-xrguybusoqndpqvgzgvllddzgjamksuecyqdaygdwnrnqfwo + base-url: http://132.121.206.65:10086 + api-key: ENC(K+Hff9QGC+fcyi510VIDd9CaeK/IN5WBJ9rlkUsHEdDgIidW+stHHJlsK0lLPUXXREha+ToQZqqDXJrqSE+GUKCXklFhelD8bRHFXBIeP/ZzT2cxhzgKUXgjw3S0Qw2R) chat: base-url: ${spring.llm.base-url}/v1 - model: 'Qwen/Qwen3-8B' + model: 'Qwen3/qwen3-1.7b' visual: - base-url: https://open.bigmodel.cn/api/paas/v4 - endpoint: /chat/completions - model: 'glm-4v-flash' + model: 'Qwen2.5/qwen2.5-vl-7b-q4km' embedding: - model: 'BAAI/bge-m3' + model: 'Qwen3/qwen3-embedding-4b' reranker: - model: 'BAAI/bge-reranker-v2-m3' - cloud: - discovery: - enabled: false - zookeeper: - enabled: false - datasource: - url: jdbc:mysql://192.168.100.140:3306/hudi_collect_build_b12?useSSL=false&allowPublicKeyRetrieval=true - username: root - password: rootless - driver-class-name: com.mysql.cj.jdbc.Driver - security: - meta: - authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu) - username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==) - darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=) + model: 'BGE/beg-reranker-v2' jpa: show-sql: true - generate-ddl: true + generate-ddl: false liteflow: rule-source: liteflow.xml print-banner: false check-node-exists: false -jasypt: - encryptor: - password: 'r#(R,P"Dp^A47>WSn:Wn].gs/+"v:q_Q*An~zF*g-@j@jtSTv5H/,S-3:R?r9R}.' fenix: print-banner: false diff --git a/service-ai/service-ai-web/src/main/resources/logback-spring.xml b/service-ai/service-ai-web/src/main/resources/logback-spring.xml index d686c9c..957e01d 100644 --- a/service-ai/service-ai-web/src/main/resources/logback-spring.xml +++ b/service-ai/service-ai-web/src/main/resources/logback-spring.xml @@ -25,7 +25,7 @@ - + diff --git a/service-web/client/src/pages/ai/knowledge/DataDetail.tsx b/service-web/client/src/pages/ai/knowledge/DataDetail.tsx index bc9041c..6b61c80 100644 --- a/service-web/client/src/pages/ai/knowledge/DataDetail.tsx +++ b/service-web/client/src/pages/ai/knowledge/DataDetail.tsx @@ -24,7 +24,7 @@ const DataDetail: React.FC = () => { { type: 'service', className: 'inline', - api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`, + api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`, body: { type: 'tpl', tpl: '${name}', @@ -38,7 +38,21 @@ const DataDetail: React.FC = () => { body: [ { type: 'crud', - api: `${commonInfo.baseAiUrl}/knowledge/group/list?knowledge_id=${knowledge_id}`, + api: { + method: 'post', + url: `${commonInfo.baseAiUrl}/knowledge/group/list`, + data: { + query: { + equal: { + 'knowledge/id': knowledge_id, + } + }, + page: { + index: '${page}', + size: '${perPage}', + } + } + }, ...crudCommonOptions(), headerToolbar: [ 'reload', @@ -146,7 +160,7 @@ const DataDetail: React.FC = () => { level: 'link', size: 'sm', actionType: 'ajax', - api: `get:${commonInfo.baseAiUrl}/knowledge/group/delete?id=\${id}`, + api: `get:${commonInfo.baseAiUrl}/knowledge/group/remove/\${id}`, confirmText: '确认删除', confirmTitle: '删除', }, diff --git a/service-web/client/src/pages/ai/knowledge/DataImport.tsx b/service-web/client/src/pages/ai/knowledge/DataImport.tsx index 7f2d8cf..c55df64 100644 --- a/service-web/client/src/pages/ai/knowledge/DataImport.tsx +++ b/service-web/client/src/pages/ai/knowledge/DataImport.tsx @@ -23,7 +23,7 @@ const DataImport: React.FC = () => { { type: 'service', className: 'inline', - api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`, + api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`, body: { type: 'tpl', tpl: '${name}', @@ -42,6 +42,7 @@ const DataImport: React.FC = () => { body: [ { id: 'a5219cc7-72dd-4199-9eeb-61305fe41075', + debug: commonInfo.debug, type: 'form', wrapWithPanel: false, actions: [], @@ -103,6 +104,8 @@ const DataImport: React.FC = () => { autoUpload: false, drag: true, multiple: true, + joinValues: false, + extractValue: true, accept: '*', // 5MB 5242880 // 100MB 104857600 @@ -131,7 +134,6 @@ const DataImport: React.FC = () => { api: { method: 'post', url: `${commonInfo.baseAiUrl}/knowledge/preview_text`, - dataType: 'form', data: { mode: '${mode|default:undefined}', type: '${type|default:undefined}', @@ -149,7 +151,6 @@ const DataImport: React.FC = () => { api: { method: 'post', url: `${commonInfo.baseAiUrl}/knowledge/submit_text`, - dataType: 'form', data: { id: knowledge_id, mode: '${mode|default:undefined}', diff --git a/service-web/client/src/pages/ai/knowledge/DataSegment.tsx b/service-web/client/src/pages/ai/knowledge/DataSegment.tsx index 2d3c664..542e097 100644 --- a/service-web/client/src/pages/ai/knowledge/DataSegment.tsx +++ b/service-web/client/src/pages/ai/knowledge/DataSegment.tsx @@ -18,7 +18,7 @@ const DataDetail: React.FC = () => { { type: 'service', className: 'inline', - api: `${commonInfo.baseAiUrl}/knowledge/name?id=${knowledge_id}`, + api: `${commonInfo.baseAiUrl}/knowledge/${knowledge_id}/name`, body: { type: 'tpl', tpl: '${name}', diff --git a/service-web/client/src/pages/ai/knowledge/Knowledge.tsx b/service-web/client/src/pages/ai/knowledge/Knowledge.tsx index 234bfb3..c6b9bbd 100644 --- a/service-web/client/src/pages/ai/knowledge/Knowledge.tsx +++ b/service-web/client/src/pages/ai/knowledge/Knowledge.tsx @@ -1,6 +1,13 @@ import React from 'react' import {useNavigate} from 'react-router' -import {amisRender, commonInfo, crudCommonOptions, mappingField, mappingItem} from '../../../util/amis.tsx' +import { + amisRender, + commonInfo, + crudCommonOptions, + mappingField, + mappingItem, + paginationTemplate, +} from '../../../util/amis.tsx' const strategyMapping = [ mappingItem('文本', 'Cosine'), @@ -25,62 +32,74 @@ const Knowledge: React.FC = () => { body: [ { type: 'crud', - api: `${commonInfo.baseAiUrl}/knowledge/list`, + api: { + method: 'post', + url: `${commonInfo.baseAiUrl}/knowledge/list`, + data: { + page: { + index: '${page}', + size: '${perPage}', + } + } + }, ...crudCommonOptions(), - headerToolbar: [ - 'reload', - { - type: 'action', - label: '', - icon: 'fa fa-plus', - tooltip: '新增', - tooltipPlacement: 'top', - actionType: 'dialog', - dialog: { - title: '新增知识库', - size: 'md', - body: { - type: 'form', - api: { - url: `${commonInfo.baseAiUrl}/knowledge/add`, - dataType: 'form', + ...paginationTemplate( + 10, + 5, + [ + { + type: 'action', + label: '', + icon: 'fa fa-plus', + tooltip: '新增', + tooltipPlacement: 'top', + actionType: 'dialog', + dialog: { + title: '新增知识库', + size: 'md', + body: { + type: 'form', + api: { + method: 'post', + url: `${commonInfo.baseAiUrl}/knowledge/save`, + }, + body: [ + { + type: 'input-text', + name: 'name', + label: '名称', + required: true, + }, + { + type: 'textarea', + name: 'description', + label: '描述', + required: true, + }, + { + type: 'select', + name: 'strategy', + label: '类型', + value: 'Cosine', + required: true, + options: [ + { + label: '文本', + value: 'Cosine', + }, + { + label: '图片', + value: 'Euclid', + disabled: true, + }, + ], + }, + ], }, - body: [ - { - type: 'input-text', - name: 'name', - label: '名称', - required: true, - }, - { - type: 'textarea', - name: 'description', - label: '描述', - required: true, - }, - { - type: 'select', - name: 'strategy', - label: '类型', - value: 'Cosine', - required: true, - options: [ - { - label: '文本', - value: 'Cosine', - }, - { - label: '图片', - value: 'Euclid', - disabled: true, - }, - ], - }, - ], }, }, - }, - ], + ], + ), columns: [ { name: 'name', @@ -128,13 +147,12 @@ const Knowledge: React.FC = () => { api: { method: 'post', url: `${commonInfo.baseAiUrl}/knowledge/update_description`, - dataType: 'form', }, mode: 'normal', body: [ { type: 'hidden', - name: "id", + name: 'id', // value: '${id}', }, { @@ -142,10 +160,10 @@ const Knowledge: React.FC = () => { name: 'description', label: '描述', required: true, - } - ] - } - } + }, + ], + }, + }, }, { type: 'action', @@ -192,17 +210,8 @@ const Knowledge: React.FC = () => { level: 'link', size: 'sm', actionType: 'ajax', - api: { - method: 'get', - url: `${commonInfo.baseAiUrl}/knowledge/delete`, - headers: { - 'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==', - }, - data: { - id: '${id}', - }, - }, - confirmText: '确认删除', + api: `get:${commonInfo.baseAiUrl}/knowledge/remove/\${id}`, + confirmText: '确认删除知识库：${name}', confirmTitle: '删除', }, ], diff --git a/service-web/client/src/util/amis.tsx b/service-web/client/src/util/amis.tsx index 9c4b3b9..5698b8e 100644 --- a/service-web/client/src/util/amis.tsx +++ b/service-web/client/src/util/amis.tsx @@ -10,8 +10,8 @@ import {isEqual} from 'licia' export const commonInfo = { debug: isEqual(import.meta.env.MODE, 'development'), baseUrl: 'http://132.126.207.130:35690/hudi_services/service_web', - // baseAiUrl: 'http://132.126.207.130:35690/hudi_services/service_ai_web', - baseAiUrl: 'http://localhost:8080', + baseAiUrl: 'http://132.126.207.130:35690/hudi_services/service_ai_web', + // baseAiUrl: 'http://localhost:8080', authorizationHeaders: { 'Authorization': 'Basic QXhoRWJzY3dzSkRiWU1IMjpjWXhnM2I0UHRXb1ZENVNqRmF5V3h0blNWc2p6UnNnNA==', 'Content-Type': 'application/json', @@ -311,16 +311,18 @@ export function paginationCommonOptions(perPage = true, maxButtons = 5) { return option } -export function paginationTemplate(perPage = 20, maxButtons = 5) { +export function paginationTemplate(perPage = 20, maxButtons = 5, extraHeaders: Schema[] = [], extraFooters: Schema[] = []) { return { perPage: perPage, headerToolbar: [ 'reload', paginationCommonOptions(true, maxButtons), + ...extraHeaders, ], footerToolbar: [ 'statistics', paginationCommonOptions(true, maxButtons), + ...extraFooters, ], } }