lyd/ai-rag-knowledge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: lyd:26011801-rag-git
Branches Tags
lyd:master lyd:26011802-open-ai lyd:26011801-rag-git lyd:26011401-rag-api lyd:26011301-rag lyd:25090701-stream-ui lyd:25060702-stream-api lyd:25060701-init-prodject
..
compare: lyd:25090701-stream-ui
Branches Tags
lyd:26011802-open-ai lyd:master lyd:26011801-rag-git lyd:26011401-rag-api lyd:26011301-rag lyd:25090701-stream-ui lyd:25060702-stream-api lyd:25060701-init-prodject

1 Commits
26011801-r ... 25090701-s

Author SHA1 Message Date
liyongde
86bf82b3e6 feat: 流式应答ui 2025-09-07 11:42:01 +08:00
16 changed files with 26 additions and 1813 deletions
Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -35,6 +35,4 @@ build/
.vscode/
### Mac OS ###
.DS_Store
/ai-rag-app/cloned-repo/
/.idea/
.DS_Store

36
ai-rag-api/src/main/java/com/storm/dev/api/IRAGService.java
View File

@@ -1,36 +0,0 @@
package com.storm.dev.api;
import com.storm.dev.api.response.Response;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.multipart.MultipartFile;
import reactor.core.publisher.Flux;
import java.util.List;
/**
* @author: lyd
* @date: 2026/1/14 23:41
*/
public interface IRAGService {
/**
* 获取标签列表
*
* @return
*/
Response<List<String>> queryRagTagList();
/**
* 上传知识库
*
* @param ragTag
* @param files
* @return
*/
Response<String> uploadFile(String ragTag, List<MultipartFile> files);
ChatResponse generateStreamRag(String model, String ragTag, String message);
Response<String> analyzeGitRepository(String repoUrl, String userName, String token) throws Exception;
}

20
ai-rag-api/src/main/java/com/storm/dev/api/response/Response.java
View File

@@ -1,20 +0,0 @@
package com.storm.dev.api.response;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Response<T> implements Serializable {
private String code;
private String info;
private T data;
}

18
ai-rag-app/pom.xml
View File

@@ -29,16 +29,16 @@
<!-- <artifactId>spring-ai-openai-spring-boot-starter</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-tika-document-reader</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.ai</groupId>-->
<!-- <artifactId>spring-ai-tika-document-reader</artifactId>-->
<!-- </dependency>-->
<!-- 处理知识库：向量库 -->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-pgvector-store-spring-boot-starter</artifactId>
</dependency>
<!-- &lt;!&ndash; 处理知识库：向量库 &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.ai</groupId>-->
<!-- <artifactId>spring-ai-pgvector-store-spring-boot-starter</artifactId>-->
<!-- </dependency>-->
<!-- 使用ollama的api -->
<dependency>

26
ai-rag-app/src/main/java/com/storm/dev/config/OllamaConfig.java
View File

@@ -1,16 +1,10 @@
package com.storm.dev.config;
import org.springframework.ai.ollama.OllamaChatClient;
import org.springframework.ai.ollama.OllamaEmbeddingClient;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.PgVectorStore;
import org.springframework.ai.vectorstore.SimpleVectorStore;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
/**
* 注入OllamaApi、OllamaChatClient对象
@@ -30,24 +24,4 @@ public class OllamaConfig {
return new OllamaChatClient(ollamaApi);
}
@Bean
public TokenTextSplitter tokenTextSplitter() {
return new TokenTextSplitter();
}
@Bean
public SimpleVectorStore simpleVectorStore(OllamaApi ollamaApi) {
OllamaEmbeddingClient embeddingClient = new OllamaEmbeddingClient(ollamaApi);
embeddingClient.withDefaultOptions(OllamaOptions.create().withModel("nomic-embed-text"));
return new SimpleVectorStore(embeddingClient);
}
@Bean
public PgVectorStore pgVectorStore(OllamaApi ollamaApi, JdbcTemplate jdbcTemplate) {
OllamaEmbeddingClient embeddingClient = new OllamaEmbeddingClient(ollamaApi);
embeddingClient.withDefaultOptions(OllamaOptions.create().withModel("nomic-embed-text"));
return new PgVectorStore(jdbcTemplate, embeddingClient);
}
}

17
ai-rag-app/src/main/resources/application-dev.yml
View File

@@ -2,25 +2,16 @@ server:
port: 8090
spring:
datasource:
driver-class-name: org.postgresql.Driver
username: postgres
password: postgres
url: jdbc:postgresql://192.168.109.134:15432/ai-rag-knowledge
type: com.zaxxer.hikari.HikariDataSource
ai:
ollama:
base-url: http://192.168.109.134:11434
embedding:
options:
num-batch: 512
model: nomic-embed-text
base-url: http://117.72.202.142:11434
# Redis
redis:
sdk:
config:
host: 127.0.0.1
port: 6379
host: 117.72.202.142
port: 16379
pool-size: 10
min-idle-size: 5
idle-timeout: 30000

86
ai-rag-app/src/test/java/com/storm/dev/text/GitTest.java
View File

@@ -1,86 +0,0 @@
package com.storm.dev.text;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.ai.document.Document;
import org.springframework.ai.ollama.OllamaChatClient;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.PgVectorStore;
import org.springframework.ai.vectorstore.SimpleVectorStore;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.PathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
/**
* @author: lyd
* @date: 2026/1/18 14:55
*/
@Slf4j
@RunWith(SpringRunner.class)
@SpringBootTest
public class GitTest {
@Resource
private OllamaChatClient ollamaChatClient;
@Resource
private TokenTextSplitter tokenTextSplitter;
@Resource
private SimpleVectorStore simpleVectorStore;
@Resource
private PgVectorStore pgVectorStore;
public final String LOCALPATH = "./cloned-repo";
@Test
public void test() throws Exception {
String repoUrl = "https://gitee.com/liyongde/java-trial.git";
String username = "liyongde";
String password = "a1c280a3bfe97eb5a53f7f04a01e7fca";
log.info("克隆路径：" + new File(LOCALPATH).getAbsolutePath());
FileUtils.deleteDirectory(new File(LOCALPATH));
Git git = Git.cloneRepository()
.setURI(repoUrl)
.setDirectory(new File(LOCALPATH))
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
.call();
git.close();
}
@Test
public void test_file() throws IOException {
Files.walkFileTree(Path.of(LOCALPATH), new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
log.info("文件路径:{}", file.toString());
PathResource resource = new PathResource(file);
TikaDocumentReader reader = new TikaDocumentReader(resource);
List<Document> documents = reader.get();
List<Document> documentSplitterList = tokenTextSplitter.apply(documents);
documents.forEach(doc -> doc.getMetadata().put("knowledge", "java-trial"));
documentSplitterList.forEach(doc -> doc.getMetadata().put("knowledge", "java-trial"));
pgVectorStore.accept(documentSplitterList);
return super.visitFile(file, attrs);
}
});
}
}

92
ai-rag-app/src/test/java/com/storm/dev/text/RAGApiTest.java
View File

@@ -1,92 +0,0 @@
package com.storm.dev.text;
import com.alibaba.fastjson.JSON;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.ai.document.Document;
import org.springframework.ai.ollama.OllamaChatClient;
import org.springframework.ai.ollama.api.OllamaApi;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.PgVectorStore;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.SimpleVectorStore;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import reactor.core.publisher.Flux;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @author: lyd
* @date: 2026/1/14 21:47
*/
@Slf4j
@RunWith(SpringRunner.class)
@SpringBootTest
public class RAGApiTest {
@Resource
private OllamaChatClient ollamaChatClient;
@Resource
private TokenTextSplitter tokenTextSplitter;
@Resource
private SimpleVectorStore simpleVectorStore;
@Resource
private PgVectorStore pgVectorStore;
@Test
public void upload() {
// 上传
TikaDocumentReader reader = new TikaDocumentReader("./data/file.text");
List<Document> documents = reader.get();
List<Document> documentSplitterList = tokenTextSplitter.apply(documents);
// 打标
documents.forEach(document -> document.getMetadata().put("knowledge", "德德"));
documentSplitterList.forEach(document -> document.getMetadata().put("knowledge", "德德"));
pgVectorStore.accept(documentSplitterList);
log.info("上传完成!");
}
@Test
public void chat() {
// 构建提问
String message = "拆装出库的操作流程是什么？";
// 构建推理模板
String SYSTEM_PROMPT = """
Use the information from the DOCUMENTS section to provide accurate answers but act as if you knew this information innately.
If unsure, simply state that you don't know.
Another thing you need to note is that your reply must be in Chinese!
DOCUMENTS:
{documents}
""";
// 读取向量库信息
SearchRequest request = SearchRequest.query(message).withTopK(5).withFilterExpression("knowledge == '富士迈泰国项目软件方案'");
// 相似性搜索
List<Document> documents = pgVectorStore.similaritySearch(request);
String documentsCollectors = documents.stream().map(Document::getContent).collect(Collectors.joining());
// 推理：RAG
Message ragMessage = new SystemPromptTemplate(SYSTEM_PROMPT).createMessage(Map.of("documents", documentsCollectors));
ArrayList<Message> messages = new ArrayList<>();
messages.add(new UserMessage(message));
messages.add(ragMessage);
// 提问
// ChatResponse chatResponse = ollamaChatClient.call(new Prompt(messages, OllamaOptions.create().withModel("deepseek-r1:7b")));
Flux<ChatResponse> stream = ollamaChatClient.stream(new Prompt(messages, OllamaOptions.create().withModel("deepseek-r1:7b")));
log.info("测试结果:{}", JSON.toJSONString(stream));
}
}

1
ai-rag-app/src/test/resources/data/file.text
View File

@@ -1 +0,0 @@
李永德，1999年12月31日出生，福建泉州人。

16
ai-rag-trigger/pom.xml
View File

@@ -27,14 +27,14 @@
<!-- <groupId>org.springframework.ai</groupId>-->
<!-- <artifactId>spring-ai-openai-spring-boot-starter</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-tika-document-reader</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-pgvector-store</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.ai</groupId>-->
<!-- <artifactId>spring-ai-tika-document-reader</artifactId>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.ai</groupId>-->
<!-- <artifactId>spring-ai-pgvector-store</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-ollama</artifactId>

4
ai-rag-trigger/src/main/java/com/storm/dev/trigger/http/OllamaController.java
View File

@@ -22,7 +22,7 @@ public class OllamaController implements IAiService {
private OllamaChatClient chatClient;
/**
* http://localhost:8090/api/v1/ollama/generate?model=deepseek-r1:7b&message=1+1
* http://localhost:8090/api/v1/ollama/generate?model=deepseek-r1:1.5b&message=1+1
*/
@GetMapping("generate")
@Override
@@ -31,7 +31,7 @@ public class OllamaController implements IAiService {
}
/**
* http://localhost:8090/api/v1/ollama/generate_stream?model=deepseek-r1:7b&message=hi
* http://localhost:8090/api/v1/ollama/generate_stream?model=deepseek-r1:1.5b&message=hi
*/
@GetMapping("generate_stream")
@Override

187
ai-rag-trigger/src/main/java/com/storm/dev/trigger/http/RAGController.java
View File

@@ -1,187 +0,0 @@
package com.storm.dev.trigger.http;
import com.alibaba.fastjson.JSON;
import com.storm.dev.api.IRAGService;
import com.storm.dev.api.response.Response;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.redisson.api.RList;
import org.redisson.api.RedissonClient;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.ai.document.Document;
import org.springframework.ai.ollama.OllamaChatClient;
import org.springframework.ai.ollama.api.OllamaOptions;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.PgVectorStore;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.SimpleVectorStore;
import org.springframework.core.io.PathResource;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import reactor.core.publisher.Flux;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @author: lyd
* @date: 2026/1/14 23:43
*/
@Slf4j
@RestController()
@CrossOrigin("*")
@RequestMapping("/api/v1/rag/")
public class RAGController implements IRAGService {
@Resource
private RedissonClient redissonClient;
@Resource
private OllamaChatClient ollamaChatClient;
@Resource
private TokenTextSplitter tokenTextSplitter;
@Resource
private SimpleVectorStore simpleVectorStore;
@Resource
private PgVectorStore pgVectorStore;
@Override
@RequestMapping(value = "query_rag_tag_list", method = RequestMethod.GET)
public Response<List<String>> queryRagTagList() {
RList<String> ragTag = redissonClient.getList("ragTag");
return Response.<List<String>>builder()
.code("0000")
.info("调用成功")
.data(ragTag)
.build();
}
@Override
@RequestMapping(value = "file/upload", method = RequestMethod.POST, headers = "content-type=multipart/form-data")
public Response<String> uploadFile(@RequestParam String ragTag, @RequestParam("file") List<MultipartFile> files) {
log.info("上传知识库开始 {}", ragTag);
for (MultipartFile file : files) {
// 上传
TikaDocumentReader reader = new TikaDocumentReader(file.getResource());
List<Document> documents = reader.get();
List<Document> documentSplitterList = tokenTextSplitter.apply(documents);
// 打标
documents.forEach(document -> document.getMetadata().put("knowledge", ragTag));
documentSplitterList.forEach(document -> document.getMetadata().put("knowledge", ragTag));
pgVectorStore.accept(documentSplitterList);
// 可以用MySQL存储
RList<String> elements = redissonClient.getList("ragTag");
if (!elements.contains(ragTag)){
elements.add(ragTag);
}
log.info("上传完成!");
}
return Response.<String>builder().code("0000").info("调用成功").build();
}
@Override
@RequestMapping(value = "generate_stream_rag", method = RequestMethod.GET)
public ChatResponse generateStreamRag(@RequestParam String model, @RequestParam String ragTag, @RequestParam String message) {
log.info("用户选择模型：{}，知识库：{}，提问问题：{}", model, ragTag, message);
// 构建推理模板
String SYSTEM_PROMPT = """
Use the information from the DOCUMENTS section to provide accurate answers but act as if you knew this information innately.
If unsure, simply state that you don't know.
Another thing you need to note is that your reply must be in Chinese!
DOCUMENTS:
{documents}
""";
// 读取向量库信息
SearchRequest request = SearchRequest.query(message).withTopK(5).withFilterExpression("knowledge == '" + ragTag + "'");
// 相似性搜索
List<Document> documents = pgVectorStore.similaritySearch(request);
String documentsCollectors = documents.stream().map(Document::getContent).collect(Collectors.joining());
// 推理：RAG
Message ragMessage = new SystemPromptTemplate(SYSTEM_PROMPT).createMessage(Map.of("documents", documentsCollectors));
ArrayList<Message> messages = new ArrayList<>();
messages.add(new UserMessage(message));
messages.add(ragMessage);
// 提问
// Flux<ChatResponse> chatResponse = ollamaChatClient.stream(new Prompt(messages, OllamaOptions.create().withModel(model)));
ChatResponse call = ollamaChatClient.call(new Prompt(messages, OllamaOptions.create().withModel(model)));
log.info("测试结果:{}", call);
return call;
}
@RequestMapping(value = "analyze_git_repository", method = RequestMethod.POST)
@Override
public Response<String> analyzeGitRepository(@RequestParam String repoUrl, @RequestParam String userName, @RequestParam String token) throws Exception {
String localPath = "./git-cloned-repo";
String repoProjectName = extractProjectName(repoUrl);
log.info("克隆路径：{}", new File(localPath).getAbsolutePath());
FileUtils.deleteDirectory(new File(localPath));
Git git = Git.cloneRepository()
.setURI(repoUrl)
.setDirectory(new File(localPath))
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(userName, token))
.call();
Files.walkFileTree(Paths.get(localPath), new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
log.info("{} 遍历解析路径，上传知识库:{}", repoProjectName, file.getFileName());
try {
TikaDocumentReader reader = new TikaDocumentReader(new PathResource(file));
List<Document> documents = reader.get();
List<Document> documentSplitterList = tokenTextSplitter.apply(documents);
documents.forEach(doc -> doc.getMetadata().put("knowledge", repoProjectName));
documentSplitterList.forEach(doc -> doc.getMetadata().put("knowledge", repoProjectName));
pgVectorStore.accept(documentSplitterList);
} catch (Exception e) {
log.error("遍历解析路径，上传知识库失败:{}", file.getFileName());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
log.info("Failed to access file: {} - {}", file.toString(), exc.getMessage());
return FileVisitResult.CONTINUE;
}
});
FileUtils.deleteDirectory(new File(localPath));
RList<String> elements = redissonClient.getList("ragTag");
if (!elements.contains(repoProjectName)) {
elements.add(repoProjectName);
}
git.close();
log.info("遍历解析路径，上传完成:{}", repoUrl);
return Response.<String>builder().code("0000").info("调用成功").build();
}
private String extractProjectName(String repoUrl) {
String[] parts = repoUrl.split("/");
String projectNameWithGit = parts[parts.length - 1];
return projectNameWithGit.replace(".git", "");
}
}

43
docs/docker-compose-environment-aliyun2.yml
View File

@@ -1,43 +0,0 @@
# docker-compose -f docker-compose-environment-aliyun.yml up -d
version: '3'
services:
# 对话模型
# ollama pull deepseek-r1:1.5b
# 运行模型
# ollama run deepseek-r1:1.5b
# 联网模型
# ollama pull nomic-embed-text
ollama:
image: registry.cn-hangzhou.aliyuncs.com/xfg-studio/ollama:0.5.10
container_name: ollama
restart: unless-stopped
ports:
- "11434:11434"
vector_db:
image: registry.cn-hangzhou.aliyuncs.com/xfg-studio/pgvector:v0.5.0
container_name: vector_db
restart: always
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- POSTGRES_DB=ai-rag-knowledge
- PGPASSWORD=postgres
volumes:
- ./pgvector/sql/init.sql:/docker-entrypoint-initdb.d/init.sql
logging:
options:
max-size: 10m
max-file: "3"
ports:
- '15432:5432'
healthcheck:
test: "pg_isready -U postgres -d ai-rag-knowledge"
interval: 2s
timeout: 20s
retries: 10
networks:
- my-network
networks:
my-network:
driver: bridge

2
docs/nginx/html/ai-case-04.html
View File

@@ -69,7 +69,7 @@
addMessage('<span class="animate-pulse">▍</span>');
// 构建API URL
const apiUrl = `http://localhost:8090/api/v1/ollama/generate_stream?model=deepseek-r1:7b&message=${encodeURIComponent(message)}`;
const apiUrl = `http://localhost:8090/api/v1/ollama/generate_stream?model=deepseek-r1:1.5b&message=${encodeURIComponent(message)}`;
// 使用EventSource接收流式响应
const eventSource = new EventSource(apiUrl);

2
docs/nginx/html/index.html
View File

@@ -56,7 +56,7 @@
<script>
const API_BASE = 'http://localhost:8090/api/v1/ollama/generate_stream';
const MODEL = 'deepseek-r1:7b';
const MODEL = 'deepseek-r1:1.5b';
const chatContainer = document.getElementById('chatContainer');
const messageInput = document.getElementById('messageInput');
const sendButton = document.getElementById('sendButton');

1285
docs/nginx/html/rag-ai.html
View File

File diff suppressed because it is too large Load Diff