From 79e439a1047b957494a3507bfa317ae0e16d73a9 Mon Sep 17 00:00:00 2001 From: liyongde <1419499670@qq.com> Date: Wed, 30 Jul 2025 17:32:50 +0800 Subject: [PATCH] =?UTF-8?q?opt:=20lucene=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lms/nladmin-system/nlsso-server/pom.xml | 3 +- .../nl/config/lucene/LogMessageConstant.java | 83 +++++++-- .../org/nl/config/lucene/LuceneAppender.java | 42 +++-- .../java/org/nl/config/lucene/Searcher.java | 164 ++++++++++++++++++ .../lucence/LuceneLogController.java | 49 ++---- .../wms/system_manage/enums/TagNameEnum.java | 23 +++ .../service/logserver/LuceneLogService.java | 22 ++- .../logserver/LuceneLogServiceImpl.java | 155 +++-------------- .../main/resources/config/application-dev.yml | 10 +- .../src/main/resources/logback-spring.xml | 96 ++++++---- lms/nladmin-ui/src/views/lucene/index.vue | 15 +- 11 files changed, 404 insertions(+), 258 deletions(-) create mode 100644 lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/Searcher.java create mode 100644 lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/enums/TagNameEnum.java diff --git a/lms/nladmin-system/nlsso-server/pom.xml b/lms/nladmin-system/nlsso-server/pom.xml index b1d5f4e..0350f23 100644 --- a/lms/nladmin-system/nlsso-server/pom.xml +++ b/lms/nladmin-system/nlsso-server/pom.xml @@ -291,9 +291,10 @@ org.quartz-scheduler quartz + com.yomahub - tlog-core + tlog-all-spring-boot-starter 1.5.0 diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LogMessageConstant.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LogMessageConstant.java index 14ddcca..7c42750 100644 --- a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LogMessageConstant.java +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LogMessageConstant.java @@ -6,41 +6,86 @@ package org.nl.config.lucene; * @Date: 2023/8/25 */ public class LogMessageConstant { - /** */ - public final static String SORT_NAME = "time"; - /** 级别 */ + /** + * 标签 + */ + public final static String FIELD_LABEL = "label"; + /** + * 时间 + */ + public final static String FIELD_SORT_NAME = "time"; + /** + * IP + */ + public final static String FIELD_IP = "ip"; + /** + * 级别 + */ public final static String FIELD_LEVEL = "level"; - /** 时间 */ + /** + * 时间 + */ public final static String FIELD_TIMESTAMP = "timestamp"; - /** 类的限定名 */ + /** + * 类的限定名 + */ public final static String FIELD_CLASS_NAME = "logger"; - /** 线程名 */ + /** + * 线程名 + */ public final static String FIELD_THREAD = "thread"; - /** 日志内容 */ + /** + * 日志内容 + */ public final static String FIELD_MESSAGE = "message"; + /** + * tlogTraceId + */ public final static String FIELD_TRACEID = "tlogTraceId"; // 定义颜色值 - /** 文本颜色：黑色 */ + /** + * 文本颜色：黑色 + */ public final static String COLOR_BLACK = "\u001B[30m"; - /** 文本颜色：红色 */ + /** + * 文本颜色：红色 + */ public final static String COLOR_RED = "\u001B[31m"; - /** 文本颜色：绿色 */ + /** + * 文本颜色：绿色 + */ public final static String COLOR_GREEN = "\u001B[32m"; - /** 文本颜色：黄色 */ + /** + * 文本颜色：黄色 + */ public final static String COLOR_YELLOW = "\u001B[33m"; - /** 文本颜色：蓝色 */ + /** + * 文本颜色：蓝色 + */ public final static String COLOR_BLUE = "\u001B[34m"; - /** 文本颜色：品红色 */ + /** + * 文本颜色：品红色 + */ public final static String COLOR_MAGENTA = "\u001B[35m"; - /** 文本颜色：青色 */ + /** + * 文本颜色：青色 + */ public final static String COLOR_CYAN = "\u001B[36m"; - /** 文本颜色：白色 */ + /** + * 文本颜色：白色 + */ public final static String COLOR_WHITE = "\u001B[37m"; - /** 文本颜色重置 */ + /** + * 文本颜色重置 + */ public final static String COLOR_RESET = "\u001B[0m"; - /** 背景颜色：黄色 */ + /** + * 背景颜色：黄色 + */ public final static String BACKGROUND_YELLOW = "\u001B[43m"; - /** 索引路径 */ - public final static String INDEX_DIR = "E:\\lucene\\index"; + /** + * 索引路径 + */ + public final static String INDEX_DIR = "D:\\software\\lucene\\index"; } diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LuceneAppender.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LuceneAppender.java index a4c204a..6cfd6d4 100644 --- a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LuceneAppender.java +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/LuceneAppender.java @@ -8,6 +8,8 @@ package org.nl.config.lucene; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.AppenderBase; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; import com.alibaba.ttl.TransmittableThreadLocal; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.analysis.Analyzer; @@ -20,6 +22,8 @@ import org.nl.common.utils.YmlConfigFileUtil; import org.wltea.analyzer.lucene.IKAnalyzer; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -58,26 +62,36 @@ public class LuceneAppender extends AppenderBase { @Override protected void append(ILoggingEvent event) { + String message = event.getFormattedMessage(); + Map mdcPropertyMap = event.getMDCPropertyMap(); Document doc = new Document(); - for (Property property : this.properties.getProperties()) { - LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context); - String encode = encoder.encode(event); - doc.add(new StringField(property.getName(), encode, Field.Store.YES)); + long timeStamp = event.getTimeStamp(); + // 获取本机的IP地址 + String ipAddress = "-"; + try { + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); } - Map map = event.getMDCPropertyMap(); - if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){ - doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES)); - }else { - doc.add(new StringField("traceId"," ", Field.Store.YES)); + String formattedDateTime = DateUtil.format(new java.util.Date(timeStamp), "yyyy-MM-dd HH:mm:ss.SSS"); + doc.add(new LongPoint(LogMessageConstant.FIELD_SORT_NAME, timeStamp)); + doc.add(new NumericDocValuesField(LogMessageConstant.FIELD_SORT_NAME, timeStamp)); + doc.add(new StringField(LogMessageConstant.FIELD_LEVEL, event.getLevel().toString(), Field.Store.YES)); + doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, formattedDateTime, Field.Store.YES)); + doc.add(new StoredField(LogMessageConstant.FIELD_CLASS_NAME, event.getLoggerName())); + doc.add(new StoredField(LogMessageConstant.FIELD_IP, ipAddress)); + doc.add(new StoredField(LogMessageConstant.FIELD_THREAD, event.getThreadName())); + if (ObjectUtil.isNotEmpty(mdcPropertyMap) && ObjectUtil.isNotEmpty(mdcPropertyMap.get(LogMessageConstant.FIELD_TRACEID))) { + String traceId = mdcPropertyMap.get(LogMessageConstant.FIELD_TRACEID); + doc.add(new StringField(LogMessageConstant.FIELD_TRACEID, traceId, Field.Store.YES)); + doc.add(new StringField(LogMessageConstant.FIELD_LABEL, ObjectUtil.isNotEmpty(mdcPropertyMap.get("tag_name")) + ? mdcPropertyMap.get("tag_name") : "-", Field.Store.YES)); } - - doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES)); - doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES)); - doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp())); + doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, message, Field.Store.YES)); try { indexWriter.addDocument(doc); indexWriter.commit(); - } catch (Exception e) { + } catch (IOException e) { e.printStackTrace(); } } diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/Searcher.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/Searcher.java new file mode 100644 index 0000000..f9f0352 --- /dev/null +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/config/lucene/Searcher.java @@ -0,0 +1,164 @@ +package org.nl.config.lucene; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.io.IOException; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.*; + +/** + * lucene查询器 + */ +@Slf4j +public class Searcher { + + public static Map search(String indexDir, JSONObject whereJson) throws Exception { + //获取要查询的路径，也就是索引所在的位置 + Directory dir = FSDirectory.open(Paths.get(indexDir)); + IndexReader reader = DirectoryReader.open(dir); + //构建IndexSearcher + IndexSearcher searcher = new IndexSearcher(reader); + //标准分词器，会自动去掉空格啊，is a the等单词 + Analyzer analyzer = new IKAnalyzer(true); + + // 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数： + // 每页条数 + int pageSize = Integer.parseInt(whereJson.get("size").toString()); + // 当前页码 + int pageNum = Integer.parseInt(whereJson.get("page").toString()) - 1; + // 当前页的起始条数 + int start = pageNum * pageSize; + // 当前页的结束条数（不能包含） + int end = start + pageSize; + // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 + Sort sort = new Sort(new SortField(LogMessageConstant.FIELD_SORT_NAME, SortField.Type.LONG, true)); + + TopDocs docs = null; + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + Calendar calendar = Calendar.getInstance(); + calendar.set(1970, 0, 1); + //时间范围查询 + JSONArray createTime = whereJson.getJSONArray("createTime"); + String startDate = DateUtil.format(calendar.getTime(), "yyyy-MM-dd HH:mm:ss.SSS"); + String endDate = DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS"); + + if (createTime != null) { + startDate = createTime.getString(0); + endDate = createTime.getString(1); + } + // 字段之间的与或非关系，MUST表示and，MUST_NOT表示not，SHOULD表示or，有几个fields就必须有几个clauses + TermRangeQuery termRangeQuery = new TermRangeQuery("timestamp", new BytesRef(startDate), + new BytesRef(endDate), true, true); + booleanQueryBuilder.add(termRangeQuery, BooleanClause.Occur.MUST); + if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_MESSAGE))) { + //查询解析器 + QueryParser queryParser = new QueryParser("message", analyzer); + Query query = queryParser.parse("message:" + whereJson.getString("message") + "~"); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_TRACEID))) { + //查询解析器 + TermQuery termQuery = new TermQuery(new Term(LogMessageConstant.FIELD_TRACEID, + whereJson.getString(LogMessageConstant.FIELD_TRACEID).trim())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_LABEL))) { + //查询解析器 + TermQuery termQuery = new TermQuery(new Term(LogMessageConstant.FIELD_LABEL, + whereJson.getString(LogMessageConstant.FIELD_LABEL).trim())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_LEVEL))) { + //查询解析器 + TermQuery termQuery = new TermQuery(new Term(LogMessageConstant.FIELD_LEVEL, + whereJson.get(LogMessageConstant.FIELD_LEVEL).toString())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + // 使用实体接收 + List list = new ArrayList<>(); + TopFieldCollector collector = TopFieldCollector.create(sort, 20000, 0); + searcher.search(booleanQueryBuilder.build(), collector); + docs = collector.topDocs(pageNum*pageSize, pageSize); + ScoreDoc[] scoreDocs = docs.scoreDocs; + int totalSize = collector.getTotalHits(); + + for (ScoreDoc scoreDoc : scoreDocs) { + Document doc = reader.document(scoreDoc.doc); + String logInfo = LogMessageConstant.COLOR_RED + doc.get(LogMessageConstant.FIELD_TIMESTAMP) + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_BLUE + doc.get(LogMessageConstant.FIELD_IP) + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_GREEN + "[" + doc.get(LogMessageConstant.FIELD_THREAD) + "]" + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_BLACK + doc.get(LogMessageConstant.FIELD_LEVEL) + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_MAGENTA + doc.get(LogMessageConstant.FIELD_CLASS_NAME) + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_GREEN + "<" + doc.get(LogMessageConstant.FIELD_TRACEID) + ">" + + LogMessageConstant.COLOR_RESET + " - " + + LogMessageConstant.COLOR_BLACK + highlightKeyword(doc.get(LogMessageConstant.FIELD_MESSAGE), whereJson.getString("message")); + list.add(logInfo); + } + reader.close(); + JSONObject jo = new JSONObject(); + jo.put("content", list); + jo.put("totalElements", totalSize); + return jo; + } + + public static String highlightKeyword(String text, String keyword) { + if (ObjectUtil.isEmpty(keyword)) { + return text; + } + + int startIndex = text.indexOf(keyword); + if (startIndex != -1) { + int endIndex = startIndex + keyword.length(); + String beforeKeyword = text.substring(0, startIndex); + String afterKeyword = text.substring(endIndex); + String highlightedKeyword = LogMessageConstant.BACKGROUND_YELLOW + keyword + LogMessageConstant.COLOR_RESET + + LogMessageConstant.COLOR_BLACK; + return beforeKeyword + highlightedKeyword + afterKeyword; + } else { + return text; + } + } + + public static void main(String[] args) throws IOException, ParseException { + // 获取当前时间 + LocalDateTime now = LocalDateTime.now(); + // 减去七天 + LocalDateTime sevenDaysAgo = now.minus(7, ChronoUnit.DAYS); + // 转换为 Date 类型 + Date sevenDaysAgoDate = Date.from(sevenDaysAgo.atZone(ZoneId.systemDefault()).toInstant()); + // 获取时间戳 + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"); + Date date = dateFormat.parse(String.valueOf(sevenDaysAgo)); + long timestamp = date.getTime(); + System.out.println(now); + System.out.println(sevenDaysAgo); + System.out.println(sevenDaysAgoDate); + System.out.println(timestamp); + } +} diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/lucence/LuceneLogController.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/lucence/LuceneLogController.java index 54cc93b..b1db846 100644 --- a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/lucence/LuceneLogController.java +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/lucence/LuceneLogController.java @@ -1,6 +1,8 @@ package org.nl.wms.system_manage.controller.lucence; import cn.dev33.satoken.annotation.SaIgnore; +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.nl.common.logging.annotation.Log; import org.nl.wms.system_manage.service.logserver.LogQuery; @@ -16,9 +18,9 @@ import org.springframework.web.bind.annotation.*; * @date 2023年01月29日 18:55 * @desc desc */ - @RestController -@RequestMapping("/api/esLog") +@RequiredArgsConstructor +@RequestMapping("/api/lucene") @Slf4j public class LuceneLogController { @@ -26,44 +28,13 @@ public class LuceneLogController { private LuceneLogService luceneLogService; - @GetMapping("/labels/{type}") - //("获取标签") - public ResponseEntity labelsValues(@PathVariable String type) { - return new ResponseEntity<>(luceneLogService.getLabelsValues(type), HttpStatus.OK); + @PostMapping("/getAll") + public ResponseEntity get(@RequestBody JSONObject whereJson) { + return new ResponseEntity<>(luceneLogService.getAll(whereJson), HttpStatus.OK); } - @PostMapping("/query") - //("日志查询") - public ResponseEntity queryAll(@RequestBody LogQuery query) { - return new ResponseEntity<>(luceneLogService.query(query), HttpStatus.OK); + @PostMapping("/getTagName") + public ResponseEntity getTagName() { + return new ResponseEntity<>(luceneLogService.getTagName(), HttpStatus.OK); } - @DeleteMapping("/clearLogs") - //("清空日志") - public ResponseEntity clearLogs(@RequestBody LogQuery query) { - luceneLogService.clearLogs(query); - return new ResponseEntity<>(HttpStatus.OK); - } - - @GetMapping("/thread") - @SaIgnore - @Log("线程追踪测试") - //("清空日志") - public ResponseEntity thread(@RequestBody LogQuery query) { - log.info("线程链路测试"+Thread.currentThread().getName()); - log.error("线程链路测试"+Thread.currentThread().getName()); - luceneLogService.syncdemo(); - Thread thread = new Thread(()->{ - try { - Thread.sleep(5000); - log.info("线程链路测试"+Thread.currentThread().getName()); - log.error("线程链路测试"+Thread.currentThread().getName()); - log.info("线程链路测试"+Thread.currentThread().getName()); - }catch (Exception ex){ - ex.printStackTrace(); - } - }); - thread.start(); - return new ResponseEntity<>(HttpStatus.OK); - } - } diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/enums/TagNameEnum.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/enums/TagNameEnum.java new file mode 100644 index 0000000..6c8b0b1 --- /dev/null +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/enums/TagNameEnum.java @@ -0,0 +1,23 @@ +package org.nl.wms.system_manage.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @Author: lyd + * @Description: 日志标签枚举 + * @Date: 2023/12/28 + */ +@Getter +@AllArgsConstructor +public enum TagNameEnum { + /** + * LMS系统 + */ + LMS("LMS系统"), + /** + * 标记符号 + */ + MARK_SYMBOL("-"); + private final String tag; +} diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogService.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogService.java index 1ec06a9..d857ac7 100644 --- a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogService.java +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogService.java @@ -3,6 +3,9 @@ package org.nl.wms.system_manage.service.logserver; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import java.util.List; +import java.util.Map; + /** * @author ldjun * @version 1.0 @@ -12,21 +15,16 @@ import com.alibaba.fastjson.JSONObject; public interface LuceneLogService { /** - * 清空日志 + * 获取数据分页 + * + * @param whereJson 条件 + * @return Map */ - void clearLogs(LogQuery query); - /** - * 获取labels和values树 - * @return - */ - JSONArray getLabelsValues(String type); + Map getAll(JSONObject whereJson); /** - * 日志查询 - * @param logQuery + * 获取枚举的标签名称 * @return */ - JSONObject query(LogQuery logQuery); - - void syncdemo(); + List getTagName(); } diff --git a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogServiceImpl.java b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogServiceImpl.java index 709d4dc..b19d1ce 100644 --- a/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogServiceImpl.java +++ b/lms/nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/logserver/LuceneLogServiceImpl.java @@ -17,156 +17,47 @@ import org.apache.lucene.search.*; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.BytesRef; import org.nl.common.exception.BadRequestException; +import org.nl.config.language.LangProcess; import org.nl.config.lucene.LuceneAppender; +import org.nl.config.lucene.Searcher; +import org.nl.wms.system_manage.enums.TagNameEnum; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wltea.analyzer.lucene.IKAnalyzer; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; @Service @Slf4j public class LuceneLogServiceImpl implements LuceneLogService { - - static String[] INFO_LEVEL = new String[]{"DEBUG","INFO","WARN","ERROR"}; - + /** + * 日志索引目录 + */ @Value("${lucene.index.path}") - private String indexUrl; + private String luceneUrl; @Override - public void clearLogs(LogQuery query) { + public Map getAll(JSONObject whereJson) { + JSONObject jo = new JSONObject(); try { - IndexWriter indexWriter = LuceneAppender.indexWriter; - if (indexWriter != null){ - BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - DateTime offset = DateUtil.offset(new Date(), DateField.HOUR_OF_DAY, -10); - TermRangeQuery termRangeQuery = new TermRangeQuery( - "requestTime",null, - new BytesRef(DateUtil.format(offset, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); - booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); - indexWriter.deleteDocuments(termRangeQuery); - indexWriter.commit(); - } - }catch (Exception ex){ - throw new BadRequestException("删除失败："+ex.getMessage()); + JSONObject jsonObject = (JSONObject) Searcher.search(luceneUrl, whereJson); + JSONArray array = jsonObject.getJSONArray("content"); + int totalElements = Integer.parseInt(jsonObject.get("totalElements").toString()); + jo.put("content", array); + jo.put("totalElements", totalElements); + } catch (Exception e) { + log.error("索引查询为空", e); + throw new NullPointerException(LangProcess.msg("error_NullPoint")); } + + return jo; } @Override - public JSONArray getLabelsValues(String type) { - JSONArray result = new JSONArray(); - for (String v : INFO_LEVEL) { - JSONObject item = new JSONObject(); - item.put("label", v); - item.put("value", v); - result.add(item); - } - return result; - } - - - @Override - public JSONObject query(LogQuery logQuery) { - JSONObject res = new JSONObject(); - PageResult page = new PageResult(); - page.setPage(logQuery.getPage()); - page.setPageSize(logQuery.getSize()); - try { - FSDirectory directory = FSDirectory.open(Paths.get(indexUrl)); - DirectoryReader open = DirectoryReader.open(directory); - IndexSearcher searcher = new IndexSearcher(open); - // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 - BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - //时间范围查询 - Date startDate = logQuery.getStartTime(); - Date endDate = logQuery.getEndTime(); - - if (startDate == null){ - Calendar calendar=Calendar.getInstance(); - calendar.set(1970, 0, 1); - startDate = calendar.getTime(); } - if (endDate == null){ endDate = new DateTime(); } - TermRangeQuery termRangeQuery = new TermRangeQuery( - "requestTime", - new BytesRef(DateUtil.format(startDate, "yyyy-MM-dd HH:mm:ss.SSS")), - new BytesRef(DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); - booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); - - // 字段之间的与或非关系，MUST表示and，MUST_NOT表示not，SHOULD表示or，有几个fields就必须有几个clauses - if (ObjectUtil.isNotEmpty(logQuery.getTraceId())){ - TermQuery termQuery = new TermQuery(new Term("traceId", logQuery.getTraceId())); - booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); - } - if (ObjectUtil.isNotEmpty(logQuery.getLogLevel())){ - TermQuery termQuery = new TermQuery(new Term("logLevel", logQuery.getLogLevel())); - booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); - } - if (ObjectUtil.isNotEmpty(logQuery.getSystem())) { - TermQuery termQuery = new TermQuery(new Term("system", logQuery.getSystem())); - booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); - } - if (ObjectUtil.isNotEmpty(logQuery.getRequestMethod())) { - TermQuery termQuery = new TermQuery(new Term("requestMethod", logQuery.getRequestMethod())); - booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); - } - if (Boolean.TRUE.equals(logQuery.getIsRequest())) { - Term traceid = new Term("traceId"); - TermQuery termQuery = new TermQuery(traceid); - booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST_NOT); - } - if (ObjectUtil.isNotEmpty(logQuery.getMessage())){ - //查询解析器 - QueryParser queryParser = new QueryParser("message", new IKAnalyzer(true)); - Query query = queryParser.parse(logQuery.getMessage()); - booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); - } - - TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("time", SortField.Type.LONG,true)), 20000, 0); - searcher.search(booleanQueryBuilder.build(), collector); - TopDocs topDocs = collector.topDocs((logQuery.getPage()-1)*logQuery.getSize(), logQuery.getSize()); - int totalSize = collector.getTotalHits(); - ScoreDoc[] scoreDocs = topDocs.scoreDocs; - - List list = new ArrayList<>(); - for (ScoreDoc scoreDoc : scoreDocs) { - Document doc = open.document(scoreDoc.doc); - LogRepositoryDTO dto = new LogRepositoryDTO(); - dto.setSystem(doc.get("system")); - dto.setTraceId(doc.get("traceId")); - dto.setLogger(doc.get("logger")); - dto.setLogLevel(doc.get("logLevel")); - dto.setMessage(doc.get("message")); - dto.setRequestIp(doc.get("requestIp")); - dto.setRequestMethod(doc.get("requestMethod")); - dto.setRequestTime(doc.get("requestTime")); - dto.setThread(doc.get("thread")); - list.add(dto); - } - open.close(); - directory.close(); - page.addAll(list); - page.setTotal(scoreDocs.length); - res.put("total", totalSize); - }catch (Exception ex){ - ex.printStackTrace(); - } - res.put("page",page); - return res; - - } - - - @Override - @Async - public void syncdemo() { - log.info("Async线程链路测试"+Thread.currentThread().getName()); - log.error("Async线程链路测试"+Thread.currentThread().getName()); - log.info("Async线程链路测试"+Thread.currentThread().getName()); - log.info("线程链路测试"+Thread.currentThread().getName()); + public List getTagName() { + return Arrays.stream(TagNameEnum.values()).map(TagNameEnum::getTag).collect(Collectors.toList()); } } diff --git a/lms/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml b/lms/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml index 1d8373e..8806b17 100644 --- a/lms/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml +++ b/lms/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml @@ -9,10 +9,11 @@ spring: druid: db-type: com.alibaba.druid.pool.DruidDataSource driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:wms_standardv1}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false - # url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:wms_oulun}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false + url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:zzzw_lms}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false +# url: jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:zzzw_lms}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false username: ${DB_USER:root} - password: ${DB_PWD:123456} + password: ${DB_PWD:P@ssw0rd.} +# password: ${DB_PWD:12356} # 初始连接数 initial-size: 15 # 最小连接数 @@ -134,6 +135,3 @@ sa-token: token-prefix: is-read-cookie: false is-print: false -lucene: - index: - path: D:\lms\lucene\index diff --git a/lms/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml b/lms/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml index 516ab71..5e6935b 100644 --- a/lms/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml +++ b/lms/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml @@ -61,44 +61,33 @@ - - - system - lms - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - + + + INFO + ACCEPT + NEXT + + + + WARN + ACCEPT + NEXT + + + + ERROR + ACCEPT + DENY + - + + 512 - + @@ -138,7 +127,48 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lms/nladmin-ui/src/views/lucene/index.vue b/lms/nladmin-ui/src/views/lucene/index.vue index d08b9e0..e0fc4d8 100644 --- a/lms/nladmin-ui/src/views/lucene/index.vue +++ b/lms/nladmin-ui/src/views/lucene/index.vue @@ -62,6 +62,15 @@ :end-placeholder="$t('Log.end_time')" /> + + + {{ $t('common.Query') }} @@ -73,7 +82,8 @@
-
+
+
@@ -126,7 +136,8 @@ export default { size: 100, total: 0, createTime: '' - } + }, + compressedView: true } }, created() {