diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java deleted file mode 100644 index ba9aeab0..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.nl.config.lucene; -/** - * @author ldjun - * @version 1.0 - * @date 2023年08月24日 13:00 - * @desc desc - */ - -import ch.qos.logback.classic.AsyncAppender; -import ch.qos.logback.classic.spi.ILoggingEvent; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.MDC; - -import java.util.Map; - -public class AsyncLuceneAppender extends AsyncAppender { - - - @Override - protected void append(ILoggingEvent event) { - String traceId = LuceneAppender.traceIdTL.get(); - if (StringUtils.isNotEmpty(traceId)){ - MDC.put("traceId",traceId); - Map mdcPropertyMap = event.getMDCPropertyMap(); - if (mdcPropertyMap.getClass().getName().contains("SynchronizedMap")){ - mdcPropertyMap.put("traceId",traceId); - } - MDC.clear(); - } - super.append(event); - } -} diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java deleted file mode 100644 index 14ddcca0..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.nl.config.lucene; - -/** - * @Author: lyd - * @Description: 定义lucene相关常量 - * @Date: 2023/8/25 - */ -public class LogMessageConstant { - /** */ - public final static String SORT_NAME = "time"; - /** 级别 */ - public final static String FIELD_LEVEL = "level"; - /** 时间 */ - public final static String FIELD_TIMESTAMP = "timestamp"; - /** 类的限定名 */ - public final static String FIELD_CLASS_NAME = "logger"; - /** 线程名 */ - public final static String FIELD_THREAD = "thread"; - /** 日志内容 */ - public final static String FIELD_MESSAGE = "message"; - public final static String FIELD_TRACEID = "tlogTraceId"; - // 定义颜色值 - /** 文本颜色：黑色 */ - public final static String COLOR_BLACK = "\u001B[30m"; - /** 文本颜色：红色 */ - public final static String COLOR_RED = "\u001B[31m"; - /** 文本颜色：绿色 */ - public final static String COLOR_GREEN = "\u001B[32m"; - /** 文本颜色：黄色 */ - public final static String COLOR_YELLOW = "\u001B[33m"; - /** 文本颜色：蓝色 */ - public final static String COLOR_BLUE = "\u001B[34m"; - /** 文本颜色：品红色 */ - public final static String COLOR_MAGENTA = "\u001B[35m"; - /** 文本颜色：青色 */ - public final static String COLOR_CYAN = "\u001B[36m"; - /** 文本颜色：白色 */ - public final static String COLOR_WHITE = "\u001B[37m"; - /** 文本颜色重置 */ - public final static String COLOR_RESET = "\u001B[0m"; - /** 背景颜色：黄色 */ - public final static String BACKGROUND_YELLOW = "\u001B[43m"; - /** 索引路径 */ - public final static String INDEX_DIR = "E:\\lucene\\index"; - -} diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java deleted file mode 100644 index a4c204ad..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.nl.config.lucene; -/** - * @author ldjun - * @version 1.0 - * @date 2023年08月24日 13:00 - * @desc desc - */ - -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.AppenderBase; -import com.alibaba.ttl.TransmittableThreadLocal; -import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.*; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.nl.common.utils.YmlConfigFileUtil; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import java.io.IOException; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -public class LuceneAppender extends AppenderBase { - - public static final TransmittableThreadLocal traceIdTL = new TransmittableThreadLocal(); - public LuceneProperties properties; - public static Directory index; - private List encoders; - public static IndexWriter indexWriter; - - - - @Override - public void start() { - super.start(); - try { - // 读取配置文件 - Properties properties = YmlConfigFileUtil.readConfig("config/application.yml"); - - // 获取配置值 - String luceneDir = properties.getProperty("lucene.index.path"); - System.out.println("---index地址----"+luceneDir); - index = FSDirectory.open(Paths.get(luceneDir)); - // 初始化 Lucene 索引 - Analyzer analyzer = new IKAnalyzer(); - IndexWriterConfig config = new IndexWriterConfig(analyzer); - indexWriter = new IndexWriter(index, config); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - @Override - protected void append(ILoggingEvent event) { - Document doc = new Document(); - for (Property property : this.properties.getProperties()) { - LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context); - String encode = encoder.encode(event); - doc.add(new StringField(property.getName(), encode, Field.Store.YES)); - } - Map map = event.getMDCPropertyMap(); - if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){ - doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES)); - }else { - doc.add(new StringField("traceId"," ", Field.Store.YES)); - } - - doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES)); - doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES)); - doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp())); - try { - indexWriter.addDocument(doc); - indexWriter.commit(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Override - public void stop() { - super.stop(); - try { - indexWriter.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public void setProperties(LuceneProperties properties) { - this.properties = properties; - - } -} diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java deleted file mode 100644 index c0442277..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.nl.config.lucene; - - -import java.util.ArrayList; -import java.util.List; - -public class LuceneProperties { - - private List properties; - - public LuceneProperties() { - this.properties = new ArrayList(); - } - - public List getProperties() { - return properties; - } - - public void addProperty(Property property) { - properties.add(property); - } - -} diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java deleted file mode 100644 index 7ad3ecfa..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.nl.config.lucene; - -import ch.qos.logback.classic.PatternLayout; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.Context; -import ch.qos.logback.core.pattern.PatternLayoutBase; - -/* - * @author ZZQ - * @Date 2023/12/22 18:11 - */ -public class LucenePropertyAndEncoder { - - private Property property; - - private PatternLayoutBase layout = new PatternLayout(); - - public LucenePropertyAndEncoder(Property property, Context context) { - this.property = property; - this.layout.setContext(context); - this.layout.setPattern(String.valueOf(property.getValue())); - this.layout.setPostCompileProcessor(null); - this.layout.start(); - } - - public String encode(ILoggingEvent event) { - return layout.doLayout(event); - } - - public String getName() { - return property.getName(); - } - - public boolean allowEmpty() { - return property.isAllowEmpty(); - } -} - diff --git a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/Property.java b/wcs/nladmin-system/src/main/java/org/nl/config/lucene/Property.java deleted file mode 100644 index 4d33144a..00000000 --- a/wcs/nladmin-system/src/main/java/org/nl/config/lucene/Property.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.nl.config.lucene; - -/* - * @author ZZQ - * @Date 2023/12/26 15:30 - */ -public class Property { - private String name; - private String value; - private boolean allowEmpty; - - public Property() { - } - - public Property(String name, String value, boolean allowEmpty) { - this.name = name; - this.value = value; - this.allowEmpty = allowEmpty; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public boolean isAllowEmpty() { - return allowEmpty; - } - - public void setAllowEmpty(boolean allowEmpty) { - this.allowEmpty = allowEmpty; - } -} diff --git a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java index f1176063..909391b2 100644 --- a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java +++ b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java @@ -70,17 +70,12 @@ public class LuceneAppender extends AppenderBase { @Override protected void append(ILoggingEvent event) { String message = event.getFormattedMessage(); - String[] split = message.split("@"); - LuceneLogDto luceneLogDto = JSONObject.parseObject(split[1], LuceneLogDto.class); + LuceneLogDto luceneLogDto = JSONObject.parseObject(message, LuceneLogDto.class); Document document = new Document(); try { //向document对象中添加域。 Map mdcPropertyMap = event.getMDCPropertyMap(); - String traceId = mdcPropertyMap.get("traceId"); - System.out.println("---追踪号---"+traceId); - if (ObjectUtil.isNotEmpty(traceId)) { - document.add(new StringField("trace_id", traceId, Field.Store.YES)); - } + if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())) { document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); } diff --git a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java index c32761d7..10496fef 100644 --- a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java +++ b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java @@ -6,13 +6,11 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.nl.modules.logging.annotation.Log; import org.nl.modules.lucene.service.LuceneService; +import org.nl.modules.lucene.service.dto.LogQuery; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.Map; @@ -27,7 +25,6 @@ public class LuceneController { private final LuceneService luceneService; - @GetMapping("/getAll") @Log("日志检索") @ApiOperation("日志检索") diff --git a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java index d5ef5e07..2c122012 100644 --- a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java +++ b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java @@ -1,6 +1,7 @@ package org.nl.modules.lucene.service; import com.alibaba.fastjson.JSONArray; +import org.nl.modules.lucene.service.dto.LogQuery; import org.springframework.data.domain.Pageable; import java.util.Map; @@ -11,8 +12,6 @@ public interface LuceneService { /** * 获取数据分页 * - * @param whereJson 条件 - * @param page 分页参数 * @return Map */ Map getAll(Map whereJson, Pageable page); diff --git a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/dto/LogQuery.java b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/dto/LogQuery.java new file mode 100644 index 00000000..3aa1b29e --- /dev/null +++ b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/dto/LogQuery.java @@ -0,0 +1,53 @@ +package org.nl.modules.lucene.service.dto; + +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + + +import java.util.Date; + +/* + * @author ZZQ + * @Date 2023/2/8 5:18 下午 + */ +@Data +public class LogQuery { + /** + * 创建时间范围查询 + */ + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Date begin_time; + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Date end_time; + /** + * 追踪id + */ + private String traceId; + private String requestMethod; + /** + * 日志内容模糊匹配 + */ + private String message; + /** + * 日志级别 + */ + private String logLevel; + /** + * 系统标签 + */ + private String system; + /** + * 是否只查询Http相关请求 + */ + private Boolean isRequest = Boolean.TRUE; + /** + * 是否过滤wql日志 + */ + private Boolean filterSql = Boolean.TRUE; + + private Integer size = 20; + + private Integer page = 1; +} diff --git a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java index afa3d6ae..856b33c6 100644 --- a/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java +++ b/wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java @@ -1,17 +1,30 @@ package org.nl.modules.lucene.service.impl; -import cn.hutool.core.util.CharsetUtil; -import cn.hutool.http.HttpUtil; import com.alibaba.fastjson.JSONArray; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; import org.nl.modules.lucene.service.LuceneService; import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import java.util.Map; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; @Service @@ -22,57 +35,117 @@ public class LuceneServiceImpl implements LuceneService { //日志索引目录 @Value("${lucene.index.path}") - private String luceneUrl; + private String luceneUrl; - /** - * 获取labels和values树 - * - * @return - */ - public JSONArray getLabelsValues() { - JSONArray result = new JSONArray(); - // 获取所有标签 - String labelString = HttpUtil.get(luceneUrl + "/labels", CharsetUtil.CHARSET_UTF_8); - JSONObject parse = (JSONObject) JSONObject.parse(labelString); - JSONArray labels = parse.getJSONArray("data"); - for (int i=0; i getAll(Map whereJson, Pageable page) { JSONObject jo = new JSONObject(); try { - JSONObject jsonObject = null;//(JSONObject) Searcher.search(luceneUrl, "", whereJson); - JSONArray array = jsonObject.getJSONArray("content"); - int totalElements = Integer.parseInt(jsonObject.get("totalElements").toString()); - jo.put("content", array); - jo.put("totalElements", totalElements); - } catch (Exception e) { - log.error("索引查询为空", e); - throw new NullPointerException("索引查询为空"); + //获取要查询的路径，也就是索引所在的位置 + try { + Resource resource = new ClassPathResource("config/application.yml"); + YamlPropertiesFactoryBean yamlPropertiesFactoryBean = new YamlPropertiesFactoryBean(); + yamlPropertiesFactoryBean.setResources(resource); + Properties properties = yamlPropertiesFactoryBean.getObject(); + // 获取配置值 + String luceneDir = properties.getProperty("lucene.index.path"); + FSDirectory directory = FSDirectory.open(Paths.get(luceneDir)); + DirectoryReader open = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(open); + // 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数： + int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数 + int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码 + + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + //时间范围查询 + String startDate = (String) whereJson.get("begin_time"); + String endDate = (String) whereJson.get("end_time"); + + if (startDate == null) { + Calendar calendar = Calendar.getInstance(); + calendar.set(1970, 0, 1); + startDate = DateUtil.format(calendar.getTime(), "yyyy-MM-dd HH:mm:ss.SSS"); + } else { + startDate = getDate(startDate); + } + if (endDate == null) { + endDate = DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS"); + } else { + endDate = getDate(endDate); + } + TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true); + booleanQueryBuilder.add(termRangeQuery, BooleanClause.Occur.MUST); + if (whereJson.get("device_code") != null) { + Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code"))); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (whereJson.get("method") != null) { + Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method"))); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (whereJson.get("status_code") != null) { + Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code"))); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (whereJson.get("requestparam") != null) { + WildcardQuery query = new WildcardQuery(new Term("requestparam", "*" + (String) whereJson.get("requestparam") + "*")); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + if (whereJson.get("responseparam") != null) { + WildcardQuery query = new WildcardQuery(new Term("responseparam", "*" + (String) whereJson.get("responseparam") + "*")); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + if (whereJson.get("blurry") != null) { + WildcardQuery query = new WildcardQuery(new Term("fieldContent", "*" + (String) whereJson.get("blurry") + "*")); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + + TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("logTime", SortField.Type.LONG, true)), 20000, 0); + searcher.search(booleanQueryBuilder.build(), collector); + TopDocs topDocs = collector.topDocs(pageNum * pageSize, pageSize); + int totalSize = collector.getTotalHits(); + ScoreDoc[] scoreDocs = topDocs.scoreDocs; + + List list = new ArrayList<>(); + for (ScoreDoc scoreDoc : scoreDocs) { + Document doc = open.document(scoreDoc.doc); + JSONObject object = new JSONObject(); + object.put("content", doc.get("fieldContent")); + object.put("device_code", doc.get("device_code")); + object.put("logTime", doc.get("logTime")); + object.put("method", doc.get("method")); + object.put("status_code", doc.get("status_code")); + object.put("requestparam", doc.get("requestparam")); + object.put("responseparam", doc.get("responseparam")); + if (doc.get("fieldContent") != null) { + list.add(object); + } + } + open.close(); + directory.close(); + jo.put("content", list); + jo.put("totalElements", totalSize); + return jo; + } catch (Exception e) { + log.error("索引查询为空", e); + throw new NullPointerException("索引查询为空"); + } + } catch (NumberFormatException e) { + e.printStackTrace(); + } catch (NullPointerException e) { + e.printStackTrace(); } return jo; } + + public static String getDate(String timeString) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");//时间格式 + Date date = sdf.parse(timeString); + timeString = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss.SSS");//格式化后的时间 + return timeString; + } + } diff --git a/wcs/nladmin-system/src/main/resources/logback-spring.xml b/wcs/nladmin-system/src/main/resources/logback-spring.xml index 51db030d..195bedac 100644 --- a/wcs/nladmin-system/src/main/resources/logback-spring.xml +++ b/wcs/nladmin-system/src/main/resources/logback-spring.xml @@ -64,39 +64,8 @@ https://juejin.cn/post/6844903775631572999 500 - - - - system - lms - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - - + + 512 @@ -124,28 +93,25 @@ https://juejin.cn/post/6844903775631572999 - + - - - + + + - - -