feat: rag-git的api

2026-01-18 16:59:11 +08:00
parent e042e548f9
commit 24d189f945
4 changed files with 162 additions and 1 deletions
--- a/ai-rag-app/src/test/java/com/storm/dev/text/GitTest.java
+++ b/ai-rag-app/src/test/java/com/storm/dev/text/GitTest.java
@@ -0,0 +1,86 @@
+package com.storm.dev.text;
+
+import jakarta.annotation.Resource;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.ollama.OllamaChatClient;
+import org.springframework.ai.reader.tika.TikaDocumentReader;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+import org.springframework.ai.vectorstore.PgVectorStore;
+import org.springframework.ai.vectorstore.SimpleVectorStore;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.PathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+
+/**
+ * @author: lyd
+ * @date: 2026/1/18 14:55
+ */
+@Slf4j
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class GitTest {
+    @Resource
+    private OllamaChatClient ollamaChatClient;
+    @Resource
+    private TokenTextSplitter tokenTextSplitter;
+    @Resource
+    private SimpleVectorStore simpleVectorStore;
+    @Resource
+    private PgVectorStore pgVectorStore;
+
+    public final String LOCALPATH = "./cloned-repo";
+    @Test
+    public void test() throws Exception {
+        String repoUrl = "https://gitee.com/liyongde/java-trial.git";
+        String username = "liyongde";
+        String password = "a1c280a3bfe97eb5a53f7f04a01e7fca";
+
+
+        log.info("克隆路径：" + new File(LOCALPATH).getAbsolutePath());
+
+        FileUtils.deleteDirectory(new File(LOCALPATH));
+
+        Git git = Git.cloneRepository()
+                .setURI(repoUrl)
+                .setDirectory(new File(LOCALPATH))
+                .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
+                .call();
+
+        git.close();
+    }
+
+    @Test
+    public void test_file() throws IOException {
+        Files.walkFileTree(Path.of(LOCALPATH), new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                log.info("文件路径:{}", file.toString());
+                PathResource resource = new PathResource(file);
+                TikaDocumentReader reader = new TikaDocumentReader(resource);
+
+                List<Document> documents = reader.get();
+                List<Document> documentSplitterList = tokenTextSplitter.apply(documents);
+                documents.forEach(doc -> doc.getMetadata().put("knowledge", "java-trial"));
+                documentSplitterList.forEach(doc -> doc.getMetadata().put("knowledge", "java-trial"));
+
+                pgVectorStore.accept(documentSplitterList);
+                return super.visitFile(file, attrs);
+            }
+        });
+    }
+}