重构 jeesite-ai 代码目录，新增 parent-ai，重命名 cms-ai 为 ai-cms
This commit is contained in:
thinkgem
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Copyright (c) 2013-Now http://jeesite.com All rights reserved.
|
||||
* No deletion without permission, or be held responsible to law.
|
||||
*/
|
||||
package com.jeesite.modules.ai.cms.config;
|
||||
|
||||
import com.jeesite.common.config.Global;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
|
||||
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
||||
|
||||
/**
|
||||
* MVC 异步任务池定义
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@Configuration
|
||||
public class AiCmsWebMvcConfig implements WebMvcConfigurer {
|
||||
|
||||
@Override
|
||||
public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
|
||||
configurer.setTaskExecutor(webMvcAsyncTaskExecutor());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ThreadPoolTaskExecutor webMvcAsyncTaskExecutor() {
|
||||
ThreadPoolTaskExecutor bean = new ThreadPoolTaskExecutor();
|
||||
bean.setCorePoolSize(Global.getPropertyToInteger("web.taskPool.corePoolSize", "8"));
|
||||
bean.setMaxPoolSize(Global.getPropertyToInteger("web.taskPool.maxPoolSize", "20"));
|
||||
bean.setKeepAliveSeconds(Global.getPropertyToInteger("web.taskPool.keepAliveSeconds", "60"));
|
||||
bean.setQueueCapacity(Global.getPropertyToInteger("web.taskPool.queueCapacity", String.valueOf(Integer.MAX_VALUE)));
|
||||
bean.setThreadNamePrefix("web-async-");
|
||||
return bean;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Copyright (c) 2013-Now http://jeesite.com All rights reserved.
|
||||
* No deletion without permission, or be held responsible to law.
|
||||
*/
|
||||
package com.jeesite.modules.ai.cms.service;
|
||||
|
||||
import com.jeesite.common.collect.ListUtils;
|
||||
import com.jeesite.common.collect.MapUtils;
|
||||
import com.jeesite.common.config.Global;
|
||||
import com.jeesite.common.io.IOUtils;
|
||||
import com.jeesite.common.lang.StringUtils;
|
||||
import com.jeesite.common.lang.TimeUtils;
|
||||
import com.jeesite.common.utils.PageUtils;
|
||||
import com.jeesite.common.web.http.HttpClientUtils;
|
||||
import com.jeesite.common.web.http.ServletUtils;
|
||||
import com.jeesite.modules.cms.entity.Article;
|
||||
import com.jeesite.modules.cms.service.ArticleVectorStore;
|
||||
import com.jeesite.modules.cms.utils.CmsUtils;
|
||||
import com.vladsch.flexmark.html.renderer.LinkType;
|
||||
import com.vladsch.flexmark.html.renderer.ResolvedLink;
|
||||
import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter;
|
||||
import com.vladsch.flexmark.html2md.converter.HtmlLinkResolver;
|
||||
import com.vladsch.flexmark.html2md.converter.HtmlLinkResolverFactory;
|
||||
import com.vladsch.flexmark.html2md.converter.HtmlNodeConverterContext;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.apache.tika.Tika;
|
||||
import org.apache.tika.config.TikaConfig;
|
||||
import org.apache.tika.exception.TikaException;
|
||||
import org.apache.tika.io.TikaInputStream;
|
||||
import org.apache.tika.metadata.Metadata;
|
||||
import org.apache.tika.mime.MediaType;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* CMS 文章向量库存储
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@Service
|
||||
public class ArticleVectorStoreImpl implements ArticleVectorStore {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
private final VectorStore vectorStore;
|
||||
|
||||
public ArticleVectorStoreImpl(ObjectProvider<VectorStore> vectorStore) {
|
||||
this.vectorStore = vectorStore.getIfAvailable();
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存文章到向量库
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@Override
|
||||
public void save(Article article) {
|
||||
if (vectorStore == null) return;
|
||||
Map<String, Object> metadata = MapUtils.newHashMap();
|
||||
metadata.put("id", article.getId());
|
||||
metadata.put("siteCode", article.getCategory().getSite().getSiteCode());
|
||||
metadata.put("categoryCode", article.getCategory().getCategoryCode());
|
||||
metadata.put("categoryName", article.getCategory().getCategoryName());
|
||||
metadata.put("title", article.getTitle());
|
||||
metadata.put("href", article.getHref());
|
||||
metadata.put("keywords", article.getKeywords());
|
||||
metadata.put("description", article.getDescription());
|
||||
metadata.put("url", article.getUrl());
|
||||
metadata.put("status", article.getStatus());
|
||||
metadata.put("createBy", article.getCreateBy());
|
||||
metadata.put("createDate", article.getCreateDate());
|
||||
metadata.put("updateBy", article.getUpdateBy());
|
||||
metadata.put("updateDate", article.getUpdateDate());
|
||||
List<String> attachmentList = ListUtils.newArrayList();
|
||||
String content = article.getTitle() + ", " + article.getKeywords() + ", "
|
||||
+ article.getDescription() + ", " + FlexmarkHtmlConverter.builder()
|
||||
.linkResolverFactory(getHtmlLinkResolverFactory(attachmentList)).build()
|
||||
.convert(article.getArticleData().getContent())
|
||||
+ ", attachment: " + attachmentList;
|
||||
List<Document> documents = List.of(new Document(article.getId(), content, metadata));
|
||||
List<Document> splitDocuments = new TokenTextSplitter().apply(documents);
|
||||
this.delete(article); // 删除原数据
|
||||
ListUtils.pageList(splitDocuments, 10, params -> {
|
||||
vectorStore.add((List<Document>)params[0]); // 增加新数据
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析文章中的连接并提取内容
|
||||
* @author ThinkGem
|
||||
*/
|
||||
private @NotNull HtmlLinkResolverFactory getHtmlLinkResolverFactory(List<String> attachmentList) {
|
||||
HttpServletRequest request = ServletUtils.getRequest();
|
||||
return new HtmlLinkResolverFactory() {
|
||||
@Override
|
||||
public @NotNull Set<Class<?>> getAfterDependents() {
|
||||
return Set.of();
|
||||
}
|
||||
@Override
|
||||
public @NotNull Set<Class<?>> getBeforeDependents() {
|
||||
return Set.of();
|
||||
}
|
||||
@Override
|
||||
public boolean affectsGlobalScope() {
|
||||
return false;
|
||||
}
|
||||
@Override
|
||||
public HtmlLinkResolver apply(HtmlNodeConverterContext htmlNodeConverterContext) {
|
||||
return (node, context, resolvedLink) -> {
|
||||
if ("a".equalsIgnoreCase(node.nodeName())) {
|
||||
String href = node.attributes().get("href"); String url = href;
|
||||
if (StringUtils.contains(url, "://")) {
|
||||
// 只提取系统允许跳转的附件内容，外部网站内容不进行提取，shiro.allowRedirects 参数设置范围
|
||||
if (ServletUtils.isAllowRedirects(request, url)) {
|
||||
try (InputStream is = HttpClientUtils.getInputStream(url, null)) {
|
||||
if (is != null) {
|
||||
String text = getDocumentText(is);
|
||||
attachmentList.add(url + text);
|
||||
}
|
||||
} catch (IOException | TikaException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
String ctxPath = Global.getCtxPath();
|
||||
if (StringUtils.isNotBlank(ctxPath) && StringUtils.startsWith(url, ctxPath)){
|
||||
url = url.substring(ctxPath.length());
|
||||
}
|
||||
try (InputStream is = IOUtils.getFileInputStream(Global.getUserfilesBaseDir(url))){
|
||||
if (is != null) {
|
||||
String text = getDocumentText(is);
|
||||
attachmentList.add(url + text);
|
||||
}
|
||||
} catch (IOException | TikaException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return new ResolvedLink(LinkType.LINK, href);
|
||||
}
|
||||
return resolvedLink;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* 获取文章附件中的内容
|
||||
* @author ThinkGem
|
||||
*/
|
||||
private static @NotNull String getDocumentText(InputStream is) throws IOException, TikaException {
|
||||
TikaConfig config = TikaConfig.getDefaultConfig();
|
||||
Tika tika = new Tika(config);
|
||||
Metadata metadata = new Metadata();
|
||||
TikaInputStream stream = TikaInputStream.get(is);
|
||||
MediaType mimetype = tika.getDetector().detect(stream, metadata);
|
||||
if (mimetype != null && StringUtils.equals(mimetype.getType(), "text")) {
|
||||
String text = IOUtils.toString(stream, StandardCharsets.UTF_8);
|
||||
if (StringUtils.isNotBlank(text)) {
|
||||
return FlexmarkHtmlConverter.builder().build().convert(text);
|
||||
} else {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
String content = tika.parseToString(stream, metadata);
|
||||
return content.lines()
|
||||
.map(String::strip).filter(line -> !line.isEmpty())
|
||||
.reduce((a, b) -> a + System.lineSeparator() + b)
|
||||
.orElse(StringUtils.EMPTY);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除向量库文章
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@Override
|
||||
public void delete(Article article) {
|
||||
if (vectorStore == null) return;
|
||||
if (StringUtils.isNotBlank(article.getId())) {
|
||||
vectorStore.delete(new FilterExpressionBuilder().eq("id", article.getId()).build());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 重建向量库文章
|
||||
* @author ThinkGem
|
||||
*/
|
||||
public String rebuild(Article article) {
|
||||
if (vectorStore == null) return null;
|
||||
logger.debug("开始重建向量库。 siteCode: {}, categoryCode: {}",
|
||||
article.getCategory().getSite().getSiteCode(),
|
||||
article.getCategory().getCategoryCode());
|
||||
long start = System.currentTimeMillis();
|
||||
try{
|
||||
article.setIsQueryArticleData(true); // 查询文章内容
|
||||
PageUtils.findList(article, null, e -> {
|
||||
List<Article> list = CmsUtils.getArticleService().findList((Article) e);
|
||||
if (!list.isEmpty()) {
|
||||
list.forEach(this::save);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}catch(Exception ex){
|
||||
logger.error("重建向量库失败", ex);
|
||||
return "重建向量库失败：" + ex.getMessage();
|
||||
}
|
||||
String message = "重建向量库完成！ 用时" + TimeUtils.formatTime(System.currentTimeMillis() - start) + "。";
|
||||
logger.debug(message);
|
||||
return message;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Copyright (c) 2013-Now http://jeesite.com All rights reserved.
|
||||
* No deletion without permission, or be held responsible to law.
|
||||
*/
|
||||
package com.jeesite.modules.ai.cms.service;
|
||||
|
||||
import com.jeesite.common.cache.CacheUtils;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.springframework.ai.chat.memory.ChatMemoryRepository;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* AI 对话消息存储
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@Service
|
||||
public class CacheChatMemoryRepository implements ChatMemoryRepository {
|
||||
|
||||
private static final String CMS_CHAT_MSG_CACHE = "cmsChatMsgCache";
|
||||
|
||||
@Override
|
||||
public @NotNull List<String> findConversationIds() {
|
||||
return CacheUtils.getCache(CMS_CHAT_MSG_CACHE).keys().stream().map(Object::toString).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public @NotNull List<Message> findByConversationId(@NotNull String conversationId) {
|
||||
return CacheUtils.computeIfAbsent(CMS_CHAT_MSG_CACHE, conversationId, k -> List.of());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveAll(@NotNull String conversationId, @NotNull List<Message> messages) {
|
||||
CacheUtils.put(CMS_CHAT_MSG_CACHE, conversationId, messages);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByConversationId(@NotNull String conversationId) {
|
||||
CacheUtils.remove(CMS_CHAT_MSG_CACHE, conversationId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Copyright (c) 2013-Now http://jeesite.com All rights reserved.
|
||||
* No deletion without permission, or be held responsible to law.
|
||||
*/
|
||||
package com.jeesite.modules.ai.cms.web;
|
||||
|
||||
import com.jeesite.common.config.Global;
|
||||
import com.jeesite.common.web.BaseController;
|
||||
import com.jeesite.modules.ai.cms.service.AiCmsChatService;
|
||||
import com.jeesite.modules.sys.entity.Area;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* AI 聊天控制器类
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("${adminPath}/cms/chat")
|
||||
public class CmsAiChatController extends BaseController {
|
||||
|
||||
private final AiCmsChatService aiCmsChatService;
|
||||
|
||||
public CmsAiChatController(AiCmsChatService aiCmsChatService) {
|
||||
this.aiCmsChatService = aiCmsChatService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取聊天对话消息
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@RequestMapping("/message")
|
||||
public List<Message> message(String id) {
|
||||
return aiCmsChatService.getChatMessage(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聊天对话列表
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@RequestMapping("/list")
|
||||
public Collection<Map<String, Object>> list() {
|
||||
return aiCmsChatService.getChatCacheMap().values().stream()
|
||||
.sorted(Comparator.comparing(map -> (String) map.get("id"),
|
||||
Comparator.reverseOrder())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 新建或更新聊天对话
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@RequestMapping("/save")
|
||||
public String save(String id, String title) {
|
||||
Map<String, Object> map = aiCmsChatService.saveChatConversation(id, title);
|
||||
return renderResult(Global.TRUE, "保存成功", map);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除聊天对话
|
||||
* @author ThinkGem
|
||||
*/
|
||||
@RequestMapping("/delete")
|
||||
public String delete(@RequestParam String id) {
|
||||
aiCmsChatService.deleteChatConversation(id);
|
||||
return renderResult(Global.TRUE, "删除成功", id);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聊天对话，流输出
|
||||
* @author ThinkGem
|
||||
* http://127.0.0.1:8980/js/a/cms/chat/stream?id=1&message=你好
|
||||
*/
|
||||
@RequestMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||
public Flux<ChatResponse> stream(@RequestParam String id, @RequestParam String message, HttpServletRequest request) {
|
||||
return aiCmsChatService.chatStream(id, message, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聊天对话，文本输出
|
||||
* @author ThinkGem
|
||||
* http://127.0.0.1:8980/js/a/cms/chat/text?message=你好
|
||||
*/
|
||||
@RequestMapping(value = "/text")
|
||||
public String text(@RequestParam String message) {
|
||||
return aiCmsChatService.chatText(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聊天对话，结构化输出 JSON
|
||||
* @author ThinkGem
|
||||
* http://127.0.0.1:8980/js/a/cms/chat/json?message=张三
|
||||
* http://127.0.0.1:8980/js/a/cms/chat/json?message=打开客厅的灯
|
||||
*/
|
||||
@RequestMapping(value = "/json")
|
||||
public Map<String, Object> json(@RequestParam String message) {
|
||||
return aiCmsChatService.chatJson(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聊天对话，结构化输出 Entity
|
||||
* @author ThinkGem
|
||||
* http://127.0.0.1:8980/js/a/cms/chat/entity?message=北京
|
||||
*/
|
||||
@RequestMapping(value = "/entity")
|
||||
public List<Area> entity(@RequestParam String message) {
|
||||
return aiCmsChatService.chatArea(message);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
|
||||
## 重要提示（Tip）：
|
||||
|
||||
## 请勿在该配置文件中添加其它任何配置（添加也不会生效）。
|
||||
## 该文件，仅仅是为了让 jeesite-ai-cms.yml 文件，
|
||||
## 在 IDEA 中有一个自动完成及帮助提示，并无其它用意。
|
||||
## 参数配置请在 jeesite-ai-cms.yml 文件中添加。
|
||||
|
||||
spring:
|
||||
config:
|
||||
import:
|
||||
- classpath:config/jeesite-ai-cms.yml
|
||||
@@ -0,0 +1,5 @@
|
||||
5.11.1
|
||||
5.12.0
|
||||
5.12.1
|
||||
5.13.0
|
||||
5.13.1
|
||||
15
modules/ai/ai-cms/src/test/resources/application.yml
Normal file
15
modules/ai/ai-cms/src/test/resources/application.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
|
||||
# 数据库连接
|
||||
jdbc:
|
||||
|
||||
# Mysql 数据库配置
|
||||
type: mysql
|
||||
driver: com.mysql.cj.jdbc.Driver
|
||||
url: jdbc:mysql://127.0.0.1:3306/jeesite_v5?useSSL=false&allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=CONVERT_TO_NULL&serverTimezone=Asia/Shanghai
|
||||
username: root
|
||||
password: 123456
|
||||
testSql: SELECT 1
|
||||
|
||||
# 日志配置
|
||||
logging:
|
||||
config: classpath:logback-test.xml
|
||||
20
modules/ai/ai-cms/src/test/resources/logback-test.xml
Normal file
20
modules/ai/ai-cms/src/test/resources/logback-test.xml
Normal file
@@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration debug="false" scan="false">
|
||||
|
||||
<!-- Logger level setting -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml" />
|
||||
<include resource="config/logger-default.xml" />
|
||||
|
||||
<!-- Console log output -->
|
||||
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} %clr(%-5p) %clr([%-39logger{39}]){cyan} - %m%n%wEx</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- Level: FATAL 0 ERROR 3 WARN 4 INFO 6 DEBUG 7 -->
|
||||
<root level="WARN">
|
||||
<appender-ref ref="console" />
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
Reference in New Issue
Block a user