diff --git a/mes/hd/nladmin-system/pom.xml b/mes/hd/nladmin-system/pom.xml index 1d1affdc..2cbc55d4 100644 --- a/mes/hd/nladmin-system/pom.xml +++ b/mes/hd/nladmin-system/pom.xml @@ -45,19 +45,50 @@ + - com.internetitem - logback-elasticsearch-appender - 1.6 + com.yomahub + tlog-all-spring-boot-starter + 1.5.0 + + + hutool-core + cn.hutool + + - - - org.springframework.boot - spring-boot-starter-data-elasticsearch - 2.7.5 + org.apache.lucene + lucene-core + 8.2.0 + + + org.apache.lucene + lucene-highlighter + 8.2.0 + + + org.apache.lucene + lucene-analyzers-common + 8.2.0 + + + com.github.magese + ik-analyzer + 8.2.0 + + + + org.apache.lucene + lucene-analyzers-smartcn + 8.2.0 + + + + org.apache.lucene + lucene-queryparser + 8.2.0 - e-iceblue diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/aspect/LogAspect.java b/mes/hd/nladmin-system/src/main/java/org/nl/aspect/LogAspect.java index e681aa4e..740d4727 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/aspect/LogAspect.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/aspect/LogAspect.java @@ -25,6 +25,7 @@ import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Pointcut; import org.aspectj.lang.reflect.MethodSignature; +import org.nl.config.lucene.LuceneAppender; import org.nl.modules.system.util.BaseCode; import org.nl.utils.RequestHolder; import org.nl.utils.SecurityUtils; @@ -53,7 +54,10 @@ public class LogAspect { //org.nl.ext.lk.rest org.nl.pda.pdm.rest @Pointcut("execution(* org.nl.*.*..rest..*.*(..)) " + - "&& !execution(* org.nl.pda.pdm.rest.PfScreenController.*(..)) ") + "&& !execution(* org.nl.pda.pdm.rest.PfScreenController.*(..)) " + + "&& !execution(* org.nl.modules.system.rest.EsLogController.*(..)) " + ) + public void logPointCut() { } @@ -65,12 +69,12 @@ public class LogAspect { String params = getParameter(method, joinPoint.getArgs()); HttpServletRequest request = RequestHolder.getHttpServletRequest(); String url = request.getRequestURI(); - + String traceId = BaseCode.intToChars(IdUtil.getSnowflake(1, 1).nextId()); MDC.put("requestMethod",url); MDC.put("requestIp", StringUtils.getIp(request)); - MDC.put("traceId", BaseCode.intToChars(IdUtil.getSnowflake(1, 1).nextId())); + MDC.put("traceId", traceId); MDC.put("requestTime", DateUtil.now()); - + LuceneAppender.traceIdTL.set(traceId); Object result; long comming = System.currentTimeMillis(); try { @@ -94,6 +98,7 @@ public class LogAspect { }finally { log.info("[--response--][请求接口:{} 执行结束][耗时:{}s]",url,(System.currentTimeMillis() - comming)/1000); MDC.clear(); + LuceneAppender.traceIdTL.remove(); } return result; } diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java new file mode 100644 index 00000000..6a0573a5 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java @@ -0,0 +1,33 @@ +package org.nl.config.lucene; +/** + * @author ldjun + * @version 1.0 + * @date 2023年08月24日 13:00 + * @desc desc + */ + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.MDC; + +import java.util.Map; + +public class AsyncLuceneAppender extends AspectLogbackAsyncAppender { + + + @Override + protected void append(ILoggingEvent event) { + String traceId = LuceneAppender.traceIdTL.get(); + if (StringUtils.isNotEmpty(traceId)){ + MDC.put("traceId",traceId); + Map mdcPropertyMap = event.getMDCPropertyMap(); + if (mdcPropertyMap.getClass().getName().contains("SynchronizedMap")){ + mdcPropertyMap.put("traceId",traceId); + } + MDC.clear(); + } + super.append(event); + } + +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java new file mode 100644 index 00000000..14ddcca0 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java @@ -0,0 +1,46 @@ +package org.nl.config.lucene; + +/** + * @Author: lyd + * @Description: 定义lucene相关常量 + * @Date: 2023/8/25 + */ +public class LogMessageConstant { + /** */ + public final static String SORT_NAME = "time"; + /** 级别 */ + public final static String FIELD_LEVEL = "level"; + /** 时间 */ + public final static String FIELD_TIMESTAMP = "timestamp"; + /** 类的限定名 */ + public final static String FIELD_CLASS_NAME = "logger"; + /** 线程名 */ + public final static String FIELD_THREAD = "thread"; + /** 日志内容 */ + public final static String FIELD_MESSAGE = "message"; + public final static String FIELD_TRACEID = "tlogTraceId"; + // 定义颜色值 + /** 文本颜色：黑色 */ + public final static String COLOR_BLACK = "\u001B[30m"; + /** 文本颜色：红色 */ + public final static String COLOR_RED = "\u001B[31m"; + /** 文本颜色：绿色 */ + public final static String COLOR_GREEN = "\u001B[32m"; + /** 文本颜色：黄色 */ + public final static String COLOR_YELLOW = "\u001B[33m"; + /** 文本颜色：蓝色 */ + public final static String COLOR_BLUE = "\u001B[34m"; + /** 文本颜色：品红色 */ + public final static String COLOR_MAGENTA = "\u001B[35m"; + /** 文本颜色：青色 */ + public final static String COLOR_CYAN = "\u001B[36m"; + /** 文本颜色：白色 */ + public final static String COLOR_WHITE = "\u001B[37m"; + /** 文本颜色重置 */ + public final static String COLOR_RESET = "\u001B[0m"; + /** 背景颜色：黄色 */ + public final static String BACKGROUND_YELLOW = "\u001B[43m"; + /** 索引路径 */ + public final static String INDEX_DIR = "E:\\lucene\\index"; + +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java new file mode 100644 index 00000000..adb35de0 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java @@ -0,0 +1,98 @@ +package org.nl.config.lucene; +/** + * @author ldjun + * @version 1.0 + * @date 2023年08月24日 13:00 + * @desc desc + */ + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.AppenderBase; +import com.alibaba.ttl.TransmittableThreadLocal; +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class LuceneAppender extends AppenderBase { + + public static final TransmittableThreadLocal traceIdTL = new TransmittableThreadLocal(); + public LuceneProperties properties; + public static Directory index; + private List encoders; + public static IndexWriter indexWriter; + + + + @Override + public void start() { + super.start(); + try { + Resource resource = new ClassPathResource("config/application.yml"); + YamlPropertiesFactoryBean yamlPropertiesFactoryBean = new YamlPropertiesFactoryBean(); + yamlPropertiesFactoryBean.setResources(resource); + Properties properties = yamlPropertiesFactoryBean.getObject(); + String luceneDir = properties.getProperty("lucene.index.path"); + System.out.println("---index地址----"+luceneDir); + index = FSDirectory.open(Paths.get(luceneDir)); + Analyzer analyzer = new IKAnalyzer(); + IndexWriterConfig config = new IndexWriterConfig(analyzer); + indexWriter = new IndexWriter(index, config); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Override + protected void append(ILoggingEvent event) { + Document doc = new Document(); + for (Property property : this.properties.getProperties()) { + LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context); + String encode = encoder.encode(event); + doc.add(new StringField(property.getName(), encode, Field.Store.YES)); + } + Map map = event.getMDCPropertyMap(); + if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){ + doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES)); + } + + doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES)); + doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES)); + doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp())); + try { + indexWriter.addDocument(doc); + indexWriter.commit(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void stop() { + super.stop(); + try { + indexWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void setProperties(LuceneProperties properties) { + this.properties = properties; + + } +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java new file mode 100644 index 00000000..c0442277 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java @@ -0,0 +1,23 @@ +package org.nl.config.lucene; + + +import java.util.ArrayList; +import java.util.List; + +public class LuceneProperties { + + private List properties; + + public LuceneProperties() { + this.properties = new ArrayList(); + } + + public List getProperties() { + return properties; + } + + public void addProperty(Property property) { + properties.add(property); + } + +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java new file mode 100644 index 00000000..7ad3ecfa --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java @@ -0,0 +1,38 @@ +package org.nl.config.lucene; + +import ch.qos.logback.classic.PatternLayout; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Context; +import ch.qos.logback.core.pattern.PatternLayoutBase; + +/* + * @author ZZQ + * @Date 2023/12/22 18:11 + */ +public class LucenePropertyAndEncoder { + + private Property property; + + private PatternLayoutBase layout = new PatternLayout(); + + public LucenePropertyAndEncoder(Property property, Context context) { + this.property = property; + this.layout.setContext(context); + this.layout.setPattern(String.valueOf(property.getValue())); + this.layout.setPostCompileProcessor(null); + this.layout.start(); + } + + public String encode(ILoggingEvent event) { + return layout.doLayout(event); + } + + public String getName() { + return property.getName(); + } + + public boolean allowEmpty() { + return property.isAllowEmpty(); + } +} + diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/Property.java b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/Property.java new file mode 100644 index 00000000..4d33144a --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/config/lucene/Property.java @@ -0,0 +1,44 @@ +package org.nl.config.lucene; + +/* + * @author ZZQ + * @Date 2023/12/26 15:30 + */ +public class Property { + private String name; + private String value; + private boolean allowEmpty; + + public Property() { + } + + public Property(String name, String value, boolean allowEmpty) { + this.name = name; + this.value = value; + this.allowEmpty = allowEmpty; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public boolean isAllowEmpty() { + return allowEmpty; + } + + public void setAllowEmpty(boolean allowEmpty) { + this.allowEmpty = allowEmpty; + } +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/domain/LogRepositoryDTO.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/domain/LogRepositoryDTO.java index 5cf6b031..47e6cd5f 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/domain/LogRepositoryDTO.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/domain/LogRepositoryDTO.java @@ -2,13 +2,10 @@ package org.nl.modules.system.domain; import lombok.Data; import org.springframework.data.annotation.Id; -import org.springframework.data.elasticsearch.annotations.Document; - /* * @author ZZQ * @Date 2023/2/8 4:06 下午 */ -@Document(indexName = "#{@ESConfig.index}", type = "whxr_log") @Data public class LogRepositoryDTO { diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/repository/EsLogRepository.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/repository/EsLogRepository.java deleted file mode 100644 index 2c67e75e..00000000 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/repository/EsLogRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.nl.modules.system.repository; - -import org.nl.modules.system.domain.LogRepositoryDTO; -import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; -import org.springframework.stereotype.Repository; - -/* - * @author ZZQ - * @Date 2023/2/8 4:11 下午 - */ -@Repository -public interface EsLogRepository extends ElasticsearchRepository { - -} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/rest/AutoRunController.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/rest/AutoRunController.java index b4eb1bcd..78189fac 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/rest/AutoRunController.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/rest/AutoRunController.java @@ -3,7 +3,9 @@ package org.nl.modules.system.rest; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.nl.annotation.Log; +import org.nl.modules.system.service.dto.LogQuery; import org.nl.start.auto.run.AutoRunService; import org.nl.start.auto.run.ThreadDto; import org.nl.utils.PageUtil; @@ -23,6 +25,7 @@ import java.util.Map; @RequestMapping("/api/autorun") @Api(tags = "系统：查询自动线程") @RequiredArgsConstructor +@Slf4j public class AutoRunController { @Autowired @@ -52,4 +55,21 @@ public class AutoRunController { autoRunService.stopThread(code); return new ResponseEntity<>(HttpStatus.NO_CONTENT); } + + @GetMapping("/thread") + //("清空日志") + public ResponseEntity thread(@RequestBody LogQuery query) { + log.info("线程链路测试"+Thread.currentThread().getName()); + Thread thread = new Thread(()->{ + try { + Thread.sleep(5000); + log.info("线程链路测试"+Thread.currentThread().getName()); + log.error("线程链路测试"+Thread.currentThread().getName()); + }catch (Exception ex){ + ex.printStackTrace(); + } + }); + thread.start(); + return new ResponseEntity<>(HttpStatus.OK); + } } diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/dto/LogQuery.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/dto/LogQuery.java index f0d54186..6c77f1d0 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/dto/LogQuery.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/dto/LogQuery.java @@ -19,6 +19,7 @@ public class LogQuery { * 追踪id */ private String traceId; + private String requestMethod; /** * 日志内容模糊匹配 */ diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/EsLogServiceImpl.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/EsLogServiceImpl.java index 2a224afd..7eff33af 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/EsLogServiceImpl.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/EsLogServiceImpl.java @@ -1,37 +1,35 @@ package org.nl.modules.system.service.impl; import cn.hutool.core.date.DateField; +import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; import cn.hutool.db.PageResult; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.script.Script; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.nl.exception.BadRequestException; import org.nl.modules.system.domain.LogRepositoryDTO; -import org.nl.modules.system.enums.LevelEnum; -import org.nl.modules.system.repository.EsLogRepository; import org.nl.modules.system.service.EsLogService; import org.nl.modules.system.service.dto.LogQuery; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; -import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl; -import org.springframework.data.elasticsearch.core.query.*; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; +import org.wltea.analyzer.lucene.IKAnalyzer; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import java.util.List; @@ -42,108 +40,138 @@ import java.util.List; * @desc desc */ @Service -@RequiredArgsConstructor @Slf4j public class EsLogServiceImpl implements EsLogService { - private final EsLogRepository esLogRepository; + static String[] INFO_LEVEL = new String[]{"DEBUG","INFO","WARN","ERROR"}; - private final ElasticsearchRestTemplate elasticsearchRestTemplate; + @Value("${lucene.index.path}") + private String indexUrl; @Override - public JSONObject query(LogQuery logQuery){ + public JSONObject query(LogQuery logQuery) { JSONObject res = new JSONObject(); PageResult page = new PageResult(); - if (logQuery != null){ - BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod - NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); - queryBuilder.withSort(SortBuilders.fieldSort("@timestamp").unmappedType("date").order(SortOrder.DESC)); - extractedParam(logQuery, query); - queryBuilder.withQuery(query); - queryBuilder.withPageable(PageRequest.of(logQuery.getPage()-1,logQuery.getSize())); - NativeSearchQuery build = queryBuilder.build(); - Iterable all = esLogRepository.search(build); - page.addAll(((AggregatedPageImpl) all).getContent()); - page.setTotal((int) ((AggregatedPageImpl) all).getTotalElements()); - page.setPage(logQuery.getPage()); - page.setPageSize(logQuery.getSize()); - res.put("total",page.getTotal()); + page.setPage(logQuery.getPage()); + page.setPageSize(logQuery.getSize()); + try { + FSDirectory directory = FSDirectory.open(Paths.get(indexUrl)); + DirectoryReader open = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(open); + // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + //时间范围查询 + Date startDate = logQuery.getStartTime(); + Date endDate = logQuery.getEndTime(); + + if (startDate == null){ + Calendar calendar=Calendar.getInstance(); + calendar.set(1970, 0, 1); + startDate = calendar.getTime(); } + if (endDate == null){ endDate = new DateTime(); } + TermRangeQuery termRangeQuery = new TermRangeQuery( + "requestTime", + new BytesRef(DateUtil.format(startDate, "yyyy-MM-dd HH:mm:ss.SSS")), + new BytesRef(DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + + // 字段之间的与或非关系，MUST表示and，MUST_NOT表示not，SHOULD表示or，有几个fields就必须有几个clauses + if (ObjectUtil.isNotEmpty(logQuery.getTraceId())){ + TermQuery termQuery = new TermQuery(new Term("traceId", logQuery.getTraceId())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getLogLevel())){ + TermQuery termQuery = new TermQuery(new Term("logLevel", logQuery.getLogLevel())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getSystem())) { + TermQuery termQuery = new TermQuery(new Term("system", logQuery.getSystem())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getRequestMethod())) { + TermQuery termQuery = new TermQuery(new Term("requestMethod", logQuery.getRequestMethod())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (Boolean.TRUE.equals(logQuery.getIsRequest())) { + Term traceid = new Term("traceId"); + TermQuery termQuery = new TermQuery(traceid); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST_NOT); + } + if (ObjectUtil.isNotEmpty(logQuery.getMessage())){ + //查询解析器 + QueryParser queryParser = new QueryParser("message", new IKAnalyzer(true)); + Query query = queryParser.parse(logQuery.getMessage()); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + + TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("time", SortField.Type.LONG,true)), 20000, 0); + searcher.search(booleanQueryBuilder.build(), collector); + TopDocs topDocs = collector.topDocs((logQuery.getPage()-1)*logQuery.getSize(), logQuery.getSize()); + int totalSize = collector.getTotalHits(); + ScoreDoc[] scoreDocs = topDocs.scoreDocs; + + List list = new ArrayList<>(); + for (ScoreDoc scoreDoc : scoreDocs) { + Document doc = open.document(scoreDoc.doc); + LogRepositoryDTO dto = new LogRepositoryDTO(); + dto.setSystem(doc.get("system")); + dto.setTraceId(doc.get("traceId")); + dto.setLogger(doc.get("logger")); + dto.setLogLevel(doc.get("logLevel")); + dto.setMessage(doc.get("message")); + dto.setRequestIp(doc.get("requestIp")); + dto.setRequestMethod(doc.get("requestMethod")); + dto.setRequestTime(doc.get("requestTime")); + dto.setThread(doc.get("thread")); + list.add(dto); + } + open.close(); + directory.close(); + page.addAll(list); + page.setTotal(scoreDocs.length); + res.put("total", totalSize); + }catch (Exception ex){ + ex.printStackTrace(); } res.put("page",page); return res; + } @Override @Async public void clearLogs(LogQuery logQuery) { - String system = logQuery.getSystem(); - if (StringUtils.isEmpty(system)){ - return; - } - log.info("清除日志"+system); - BoolQueryBuilder query = QueryBuilders.boolQuery(); - if (!StringUtils.isEmpty(system)){ - query.must().add(QueryBuilders.matchQuery("system", system)); - } - DeleteQuery deleteQuery = new DeleteQuery(); - deleteQuery.setQuery(query); - elasticsearchRestTemplate.delete(deleteQuery,new LogRepositoryDTO().getClass()); - } - - - private void extractedQueryParam(LogQuery logQuery, BoolQueryBuilder query) { - extractedParam(logQuery,query); - if (logQuery.getIsRequest()){ - query.must().add(QueryBuilders.existsQuery("requestMethod")); - } - if (logQuery.getFilterSql()){ - query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*")); + try { + // 初始化 Lucene 索引 + Analyzer analyzer = new IKAnalyzer(); + IndexWriterConfig config = new IndexWriterConfig(analyzer); + IndexWriter indexWriter = new IndexWriter(FSDirectory.open(Paths.get(indexUrl)), config); + if (indexWriter != null){ + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + DateTime offset = DateUtil.offset(new Date(), DateField.DAY_OF_YEAR, -10); + TermRangeQuery termRangeQuery = new TermRangeQuery( + "requestTime",null, + new BytesRef(DateUtil.format(offset, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + indexWriter.deleteDocuments(termRangeQuery); + indexWriter.commit(); + indexWriter.close(); + } + }catch (Exception ex){ + throw new BadRequestException("删除失败："+ex.getMessage()); } } - private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) { - if (StringUtils.isNotEmpty(logQuery.getLogLevel())){ - query.must().add(QueryBuilders.matchQuery("logLevel.keyword", LevelEnum.checkLevel(logQuery.getLogLevel()))); - } - if (StringUtils.isNotEmpty(logQuery.getSystem())){ - query.must().add(QueryBuilders.matchQuery("system.keyword", logQuery.getSystem())); - } - query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient")); - if (StringUtils.isNotEmpty(logQuery.getTraceId())){ - query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId())); - } - if (StringUtils.isNotEmpty(logQuery.getMessage())){ - query.must().add(QueryBuilders.matchPhraseQuery("message", logQuery.getMessage())); - } - if (logQuery.getEndTime()!=null ){ - String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L"; - query.must().add(QueryBuilders.scriptQuery(new Script(script))); - } - if (logQuery.getStartTime()!=null){ - String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L"; - query.must().add(QueryBuilders.scriptQuery(new Script(script))); - } - } @Override public JSONArray getLabelsValues(String type) { JSONArray result = new JSONArray(); - FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null); - NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); - queryBuilder.withCollapseField(type+".keyword"); - queryBuilder.withSourceFilter(fetchSourceFilter); - queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100)); - Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations); - Terms terms = agg.get(type); - List buckets = terms.getBuckets(); - if (!CollectionUtils.isEmpty(buckets)){ - buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> { - JSONObject item = new JSONObject(); - item.put("label", v); - item.put("value", v); - result.add(item); - }); + for (String v : INFO_LEVEL) { + JSONObject item = new JSONObject(); + item.put("label", v); + item.put("value", v); + result.add(item); } return result; } diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/MonitorServiceImpl.java b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/MonitorServiceImpl.java index 5b8c2fce..0d3ac32e 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/MonitorServiceImpl.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/modules/system/service/impl/MonitorServiceImpl.java @@ -173,11 +173,11 @@ public class MonitorServiceImpl implements MonitorService { long time = ManagementFactory.getRuntimeMXBean().getStartTime(); Date date = new Date(time); // 计算项目运行时间 - String formatBetween = DateUtil.formatBetween(date, new Date(),BetweenFormater.Level.HOUR); + String formatBetween = DateUtil.formatBetween(date, new Date(), BetweenFormater.Level.HOUR); // 系统信息 systemInfo.put("os", os.toString()); systemInfo.put("day", formatBetween); systemInfo.put("ip", StringUtils.getLocalIp()); return systemInfo; } -} \ No newline at end of file +} diff --git a/mes/hd/nladmin-system/src/main/resources/config/application-dev.yml b/mes/hd/nladmin-system/src/main/resources/config/application-dev.yml index c4a5b97f..3cd89dcb 100644 --- a/mes/hd/nladmin-system/src/main/resources/config/application-dev.yml +++ b/mes/hd/nladmin-system/src/main/resources/config/application-dev.yml @@ -15,21 +15,6 @@ erp: #配置数据源 spring: - data: - elasticsearch: - repositories: - enabled: true - client: - reactive: - #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网 - endpoints: 127.0.0.1:9200 #外网 -# endpoints: http://10.1.3.90:9200 #外网 - elasticsearch: - rest: - #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网 - uris: 127.0.0.1:9200 #外网 -# uris: http://10.1.3.90:9200 #外网 - datasource: druid: db-type: com.alibaba.druid.pool.DruidDataSource diff --git a/mes/hd/nladmin-system/src/main/resources/config/application-prod.yml b/mes/hd/nladmin-system/src/main/resources/config/application-prod.yml index 186d64bb..1eb889a7 100644 --- a/mes/hd/nladmin-system/src/main/resources/config/application-prod.yml +++ b/mes/hd/nladmin-system/src/main/resources/config/application-prod.yml @@ -14,22 +14,6 @@ erp: password: 123456 #配置数据源 spring: - data: - elasticsearch: - repositories: - enabled: true - client: - reactive: - #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网 - endpoints: 127.0.0.1:9200 #外网 - # endpoints: http://10.1.3.90:9200 #外网 - elasticsearch: - rest: - #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网 - uris: 127.0.0.1:9200 #外网 - # uris: http://10.1.3.90:9200 #外网 -# username: elastic -# password: 123456 datasource: druid: db-type: com.alibaba.druid.pool.DruidDataSource diff --git a/mes/hd/nladmin-system/src/main/resources/config/application.yml b/mes/hd/nladmin-system/src/main/resources/config/application.yml index f56c0c14..0d975984 100644 --- a/mes/hd/nladmin-system/src/main/resources/config/application.yml +++ b/mes/hd/nladmin-system/src/main/resources/config/application.yml @@ -45,3 +45,6 @@ logging: file: path: C:\log\wms config: classpath:logback-spring.xml +lucene: + index: + path: E:\lms\lucene\index diff --git a/mes/hd/nladmin-system/src/main/resources/log/LKToMes.xml b/mes/hd/nladmin-system/src/main/resources/log/LKToMes.xml index 8d8a3734..fe79f6d1 100644 --- a/mes/hd/nladmin-system/src/main/resources/log/LKToMes.xml +++ b/mes/hd/nladmin-system/src/main/resources/log/LKToMes.xml @@ -2,65 +2,6 @@ - - http://10.16.1.24:9200/_bulk - ${esIndex} - whxr_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - mes - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - @@ -83,7 +24,6 @@ - diff --git a/mes/hd/nladmin-system/src/main/resources/log/MesToErp.xml b/mes/hd/nladmin-system/src/main/resources/log/MesToErp.xml index bcc2752a..c6295e45 100644 --- a/mes/hd/nladmin-system/src/main/resources/log/MesToErp.xml +++ b/mes/hd/nladmin-system/src/main/resources/log/MesToErp.xml @@ -2,65 +2,6 @@ - - http://127.0.0.1:9200/_bulk - ${esIndex} - whxr_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - mes - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - @@ -82,7 +23,6 @@ - diff --git a/mes/hd/nladmin-system/src/main/resources/log/MesToLK.xml b/mes/hd/nladmin-system/src/main/resources/log/MesToLK.xml index 30c19d67..42e11023 100644 --- a/mes/hd/nladmin-system/src/main/resources/log/MesToLK.xml +++ b/mes/hd/nladmin-system/src/main/resources/log/MesToLK.xml @@ -2,65 +2,6 @@ - - http://10.16.1.24:9200/_bulk - ${esIndex} - whxr_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - mes - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - @@ -82,7 +23,6 @@ - diff --git a/mes/hd/nladmin-system/src/main/resources/log/XrToMes.xml b/mes/hd/nladmin-system/src/main/resources/log/XrToMes.xml index 0cefa98f..252cbcd2 100644 --- a/mes/hd/nladmin-system/src/main/resources/log/XrToMes.xml +++ b/mes/hd/nladmin-system/src/main/resources/log/XrToMes.xml @@ -2,65 +2,6 @@ - - http://10.16.1.24:9200/_bulk - ${esIndex} - whxr_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - mes - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - diff --git a/mes/hd/nladmin-system/src/main/resources/logback-spring.xml b/mes/hd/nladmin-system/src/main/resources/logback-spring.xml index 7cb6d0e1..dd4dfced 100644 --- a/mes/hd/nladmin-system/src/main/resources/logback-spring.xml +++ b/mes/hd/nladmin-system/src/main/resources/logback-spring.xml @@ -13,25 +13,8 @@ https://juejin.cn/post/6844903775631572999 - - - - - - - - - - - ${log.pattern} - ${log.charset} - - - - - - +s @@ -47,30 +30,22 @@ https://juejin.cn/post/6844903775631572999 %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - http://127.0.0.1:9200/_bulk - ${esIndex} - whxr_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - + + + + true + + ${log.pattern} + ${log.charset} + + + + system - mes + lms traceId @@ -101,14 +76,12 @@ https://juejin.cn/post/6844903775631572999 %logger - -
- Content-Type - application/json -
- + + + 512 + @@ -119,118 +92,51 @@ https://juejin.cn/post/6844903775631572999 - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mes/qd/src/views/tools/es/view/index.vue b/mes/qd/src/views/tools/es/view/index.vue index 5517ec00..f3b7c6d6 100644 --- a/mes/qd/src/views/tools/es/view/index.vue +++ b/mes/qd/src/views/tools/es/view/index.vue @@ -33,6 +33,14 @@ /> + + + - - - -