From ca77135394a5930927f47d908cfa83884c4aa89b Mon Sep 17 00:00:00 2001 From: zhangzq Date: Thu, 28 Dec 2023 19:34:08 +0800 Subject: [PATCH] =?UTF-8?q?opt:=E6=97=A5=E5=BF=97=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lms/nladmin-system/pom.xml | 44 +++-- .../main/java/org/nl/ElasticSearchTest.java | 104 ---------- .../main/java/org/nl/config/DruidFilter.java | 42 ++-- .../nl/modules/logging/aspect/LogAspect.java | 4 +- .../logging/repository/EsLogRepository.java | 14 -- .../modules/logging/rest/EsLogController.java | 24 +++ .../modules/logging/service/EsLogService.java | 12 +- .../modules/logging/service/dto/LogQuery.java | 2 + .../logging/service/dto/LogRepositoryDTO.java | 2 - .../service/impl/EsLogServiceImpl.java | 134 ------------- .../service/impl/LuceneLogServiceImpl.java | 185 ++++++++++++++++++ .../main/resources/config/application-dev.yml | 19 +- .../resources/config/application-prod.yml | 19 +- .../resources/config/application-test.yml | 3 + .../src/main/resources/config/application.yml | 7 +- .../src/main/resources/log/AcsToLms.xml | 69 +------ .../src/main/resources/log/LmsToAcs.xml | 68 +------ .../src/main/resources/log/LmsToMes.xml | 69 +------ .../src/main/resources/log/LmsToSap.xml | 69 +------ .../src/main/resources/log/MesToLms.xml | 69 +------ .../src/main/resources/log/SapToLms.xml | 70 +------ .../src/main/resources/logback-spring.xml | 162 ++++----------- lms/nladmin-ui/src/views/loki/view/index.vue | 30 +-- 23 files changed, 370 insertions(+), 851 deletions(-) delete mode 100644 lms/nladmin-system/src/main/java/org/nl/ElasticSearchTest.java delete mode 100644 lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java delete mode 100644 lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java create mode 100644 lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/LuceneLogServiceImpl.java diff --git a/lms/nladmin-system/pom.xml b/lms/nladmin-system/pom.xml index fea41a2d3..cb7cb459a 100644 --- a/lms/nladmin-system/pom.xml +++ b/lms/nladmin-system/pom.xml @@ -37,13 +37,41 @@ tlog-all-spring-boot-starter 1.5.0 - - + - org.springframework.boot - spring-boot-starter-data-elasticsearch - 2.7.5 + org.apache.lucene + lucene-core + 8.2.0 + + org.apache.lucene + lucene-highlighter + 8.2.0 + + + org.apache.lucene + lucene-analyzers-common + 8.2.0 + + + com.github.magese + ik-analyzer + 8.2.0 + + + + + org.apache.lucene + lucene-analyzers-smartcn + 8.2.0 + + + + org.apache.lucene + lucene-queryparser + 8.2.0 + + com.github.oshi @@ -51,12 +79,6 @@ 6.1.4 - - - com.internetitem - logback-elasticsearch-appender - 1.6 - org.reflections diff --git a/lms/nladmin-system/src/main/java/org/nl/ElasticSearchTest.java b/lms/nladmin-system/src/main/java/org/nl/ElasticSearchTest.java deleted file mode 100644 index 251f7c365..000000000 --- a/lms/nladmin-system/src/main/java/org/nl/ElasticSearchTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.nl; - - -import org.apache.http.HttpHost; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; - -public class ElasticSearchTest { - - public final String ES_URL = "127.0.0.1"; - - public final int ES_PORT = 9200; - - - public static RestHighLevelClient getClientConnection() { - - RestHighLevelClient client = new RestHighLevelClient( - RestClient.builder( - new HttpHost("localhost", 9200, "http") - ) - ); - - return client; - } - - public static void searchById() throws IOException { - RestHighLevelClient client = getClientConnection(); - GetRequest getRequest = null;//new GetRequest("gateway_log", "DceJqGwBqlIig5BB05Z-"); - GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT); - System.out.println(getResponse.getSourceAsString()); - client.close(); - } - - /** - * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-search.html - * - * @throws IOException - */ - public static void paginationSearch() throws IOException { - - SearchRequest searchRequest = new SearchRequest(); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(QueryBuilders.matchPhraseQuery("eventType", "WAN_ONOFF")); - sourceBuilder.from(0); - sourceBuilder.size(1); - sourceBuilder.timeout(new TimeValue(1000)); - sourceBuilder.trackTotalHits(true); - searchRequest.source(sourceBuilder); - - - RestHighLevelClient client = getClientConnection(); - SearchResponse response = client.search(new SearchRequest("gateway_log") - .source(sourceBuilder), RequestOptions.DEFAULT); - - System.out.println(response.toString()); - client.close(); - } - - public static void paginationSearch2() throws IOException { - RestHighLevelClient client = getClientConnection(); - - - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - - RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("count").gte(8); - boolQuery.filter(rangeQuery); - - MatchQueryBuilder matchQuery = new MatchQueryBuilder("eventType", "WAN_ONOFF"); - boolQuery.must(matchQuery); - - SearchResponse response = client.search(new SearchRequest("gateway_log") - .source(new SearchSourceBuilder() - .query(boolQuery) - .from(0) - .size(2) - .trackTotalHits(true) - ), RequestOptions.DEFAULT); - - System.out.println(response.getHits().getTotalHits()); - System.out.println(response.toString()); - client.close(); - } - - - public static void main(String[] args) throws IOException { - //searchById(); - //paginationSearch(); - paginationSearch2(); - } - -} diff --git a/lms/nladmin-system/src/main/java/org/nl/config/DruidFilter.java b/lms/nladmin-system/src/main/java/org/nl/config/DruidFilter.java index a9a0c514c..e69791f70 100644 --- a/lms/nladmin-system/src/main/java/org/nl/config/DruidFilter.java +++ b/lms/nladmin-system/src/main/java/org/nl/config/DruidFilter.java @@ -39,7 +39,6 @@ public class DruidFilter extends FilterEventAdapter { @Override protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) { - String traceId = MDC.get("traceId"); int size = statement.getParametersSize(); String executeSql = sql; int count = 0; @@ -47,7 +46,7 @@ public class DruidFilter extends FilterEventAdapter { count = statement.getUpdateCount(); } catch (Exception ex) { } - if (StringUtils.isNotEmpty(traceId) && count > 0) { + if ( count > 0) { if (size > 0) { Collection values = statement.getParameters().values(); List params = new ArrayList<>(); @@ -65,28 +64,25 @@ public class DruidFilter extends FilterEventAdapter { public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException { ResultSetProxy rs = super.statement_getResultSet(chain, statement); String executeSql = statement.getLastExecuteSql(); - String traceId = MDC.get("traceId"); - if (StringUtils.isNotEmpty(traceId)) { - int result = 0; - if (rs != null) { - ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class); - result = rss.getRows().size(); - } - try { - int size = statement.getParametersSize(); - if (size > 0) { - Collection values = statement.getParameters().values(); - List params = new ArrayList<>(); - for (JdbcParameter value : values) { - params.add(value.getValue()); - } - executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); - } - } catch (Exception ex) { - log.warn("[-SQL解析异常-][{}]", ex.getMessage()); - } - log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]", result, executeSql); + int result = 0; + if (rs != null) { + ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class); + result = rss.getRows().size(); } + try { + int size = statement.getParametersSize(); + if (size > 0) { + Collection values = statement.getParameters().values(); + List params = new ArrayList<>(); + for (JdbcParameter value : values) { + params.add(value.getValue()); + } + executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); + } + } catch (Exception ex) { + log.warn("[-SQL解析异常-][{}]", ex.getMessage()); + } + log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]", result, executeSql); return rs; } diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java index 9728e66d0..fede5af97 100644 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java @@ -28,6 +28,7 @@ import org.aspectj.lang.reflect.MethodSignature; import org.nl.common.utils.BaseCode; import org.nl.common.utils.IdUtil; import org.nl.common.utils.SecurityUtils; +import org.nl.config.lucene.LuceneAppender; import org.nl.modules.common.utils.RequestHolder; import org.nl.modules.common.utils.StringUtils; import org.nl.modules.common.utils.ThrowableUtil; @@ -81,7 +82,7 @@ public class LogAspect { MDC.put("requestIp", StringUtils.getIp(request)); MDC.put("traceId", BaseCode.intToChars(IdUtil.getLongId())); MDC.put("requestTime", DateUtil.now()); - + LuceneAppender.traceIdTL.set(BaseCode.intToChars(IdUtil.getLongId())); Object result = null; long comming = System.currentTimeMillis(); try { @@ -139,6 +140,7 @@ public class LogAspect { } finally { log.info("[--response--][请求接口:{} 执行结束][耗时:{}s]", url, (System.currentTimeMillis() - comming) / 1000); MDC.clear(); + LuceneAppender.traceIdTL.remove(); } return result; } diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java deleted file mode 100644 index 9d1b55fbd..000000000 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.nl.modules.logging.repository; - -import org.nl.modules.logging.service.dto.LogRepositoryDTO; -import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; -import org.springframework.stereotype.Repository; - -/* - * @author ZZQ - * @Date 2023/2/8 4:11 下午 - */ -@Repository -public interface EsLogRepository extends ElasticsearchRepository { - -} diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java index 74afc8513..7c645fa90 100644 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java @@ -2,6 +2,8 @@ package org.nl.modules.logging.rest; import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.nl.modules.logging.annotation.Log; import org.nl.modules.logging.service.EsLogService; import org.nl.modules.logging.service.dto.LogQuery; import org.springframework.http.HttpStatus; @@ -18,6 +20,7 @@ import org.springframework.web.bind.annotation.*; @RestController @RequiredArgsConstructor @RequestMapping("/api/esLog") +@Slf4j public class EsLogController { private final EsLogService esLogService; @@ -41,4 +44,25 @@ public class EsLogController { return new ResponseEntity<>(HttpStatus.OK); } + + @DeleteMapping("/thread") + @Log("清空日志") + public ResponseEntity thread() { + log.info("Async线程链路测试"+Thread.currentThread().getName()); + log.error("Async线程链路测试"+Thread.currentThread().getName()); + log.info("Async线程链路测试"+Thread.currentThread().getName()); + log.info("线程链路测试"+Thread.currentThread().getName()); + + esLogService.syncdemo(); + new Thread(()->{ + log.info("线程链路测试"+Thread.currentThread().getName()); + log.error("线程链路测试"+Thread.currentThread().getName()); + log.info("线程链路测试"+Thread.currentThread().getName()); + log.info("线程链路测试"+Thread.currentThread().getName()); + + }).start(); + return new ResponseEntity<>(HttpStatus.OK); + } + + } diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java index 61776e52e..5a91293e1 100644 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java @@ -1,7 +1,7 @@ package org.nl.modules.logging.service; import com.alibaba.fastjson.JSONArray; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.alibaba.fastjson.JSONObject; import org.nl.modules.logging.service.dto.LogQuery; /** @@ -24,11 +24,7 @@ public interface EsLogService { */ JSONArray getLabelsValues(String type); - /** - * 日志查询 - * - * @param logQuery - * @return - */ - Page query(LogQuery logQuery); + JSONObject query(LogQuery logQuery); + + void syncdemo(); } diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java index e5e1d8a66..457bb9b07 100644 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java @@ -19,6 +19,8 @@ public class LogQuery { * 追踪id */ private String traceId; + private String requestMethod; + /** * 日志内容模糊匹配 */ diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java index 5f0ba2659..2198b56aa 100644 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java @@ -2,13 +2,11 @@ package org.nl.modules.logging.service.dto; import lombok.Data; import org.springframework.data.annotation.Id; -import org.springframework.data.elasticsearch.annotations.Document; /* * @author ZZQ * @Date 2023/2/8 4:06 下午 */ -@Document(indexName = "#{@ESConfig.index}", type = "lms_log") @Data public class LogRepositoryDTO { diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java deleted file mode 100644 index 38629a6ae..000000000 --- a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.nl.modules.logging.service.impl; - -import cn.hutool.core.date.DateField; -import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.apache.poi.ss.formula.functions.T; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.script.Script; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.nl.common.enums.LevelEnum; -import org.nl.modules.logging.repository.EsLogRepository; -import org.nl.modules.logging.service.EsLogService; -import org.nl.modules.logging.service.dto.LogQuery; -import org.nl.modules.logging.service.dto.LogRepositoryDTO; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; -import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl; -import org.springframework.data.elasticsearch.core.query.DeleteQuery; -import org.springframework.data.elasticsearch.core.query.FetchSourceFilter; -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.util.Date; -import java.util.List; - -/** - * @author ldjun - * @version 1.0 - * @date 2023年02月07日 14:35 - * @desc desc - */ -@Service -@RequiredArgsConstructor -public class EsLogServiceImpl implements EsLogService { - - private final EsLogRepository esLogRepository; - - private final ElasticsearchRestTemplate elasticsearchRestTemplate; - - @Override - public Page query(LogQuery logQuery) { - Page page = new Page<>(); - if (logQuery != null) { - BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod - extractedParam(logQuery, query); - Iterable all = esLogRepository.search(query, PageRequest.of(logQuery.getPage() - 1, logQuery.getSize(), Sort.by("@timestamp").descending())); - page.setRecords(((AggregatedPageImpl) all).getContent()); - page.setTotal(((AggregatedPageImpl) all).getTotalElements()); - page.setPages(logQuery.getPage()); - page.setSize(logQuery.getSize()); - } - return page; - } - - - private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) { - if (StringUtils.isNotEmpty(logQuery.getLogLevel())) { - query.must().add(QueryBuilders.matchQuery("logLevel", LevelEnum.checkLevel(logQuery.getLogLevel()))); - } - if (StringUtils.isNotEmpty(logQuery.getSystem())) { - query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem())); - } - if (logQuery.getIsRequest()) { - query.must().add(QueryBuilders.existsQuery("requestMethod")); - } - if (logQuery.getFilterSql()) { - query.mustNot().add(QueryBuilders.wildcardQuery("logger", "org.nl.modules.wql.core.engine.*")); - } - query.mustNot().add(QueryBuilders.matchPhraseQuery("logger", "org.elasticsearch.client.RestClient")); - if (StringUtils.isNotEmpty(logQuery.getTraceId())) { - query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId())); - } - if (StringUtils.isNotEmpty(logQuery.getMessage())) { - query.must().add(QueryBuilders.matchPhraseQuery("message", logQuery.getMessage())); - } - if (logQuery.getEndTime() != null) { - String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L"; - query.must().add(QueryBuilders.scriptQuery(new Script(script))); - } - if (logQuery.getStartTime() != null) { - String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L"; - query.must().add(QueryBuilders.scriptQuery(new Script(script))); - } - } - - @Override - public void clearLogs(LogQuery logQuery) { - String system = logQuery.getSystem(); - BoolQueryBuilder query = QueryBuilders.boolQuery(); - if (!StringUtils.isEmpty(system)) { - query.must().add(QueryBuilders.matchQuery("system", system)); - } - long time = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -10).getTime(); - String script = "doc['@timestamp'].value.millis < " + time + "L"; - query.must().add(QueryBuilders.scriptQuery(new Script(script))); - DeleteQuery deleteQuery = new DeleteQuery(); - deleteQuery.setQuery(query); - elasticsearchRestTemplate.delete(deleteQuery, new LogRepositoryDTO().getClass()); - } - - - @Override - public JSONArray getLabelsValues(String type) { - JSONArray result = new JSONArray(); - FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null); - NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); - queryBuilder.withCollapseField(type + ".keyword"); - queryBuilder.withSourceFilter(fetchSourceFilter); - queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type + ".keyword").size(100)); - Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations); - Terms terms = agg.get(type); - List buckets = terms.getBuckets(); - if (!CollectionUtils.isEmpty(buckets)) { - buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v -> { - JSONObject item = new JSONObject(); - item.put("label", v); - item.put("value", v); - result.add(item); - }); - } - return result; - } - -} diff --git a/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/LuceneLogServiceImpl.java b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/LuceneLogServiceImpl.java new file mode 100644 index 000000000..90f6e65c5 --- /dev/null +++ b/lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/LuceneLogServiceImpl.java @@ -0,0 +1,185 @@ +package org.nl.modules.logging.service.impl; + +import cn.hutool.core.date.DateField; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.db.PageResult; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.nl.config.lucene.LuceneAppender; +import org.nl.modules.common.exception.BadRequestException; +import org.nl.modules.logging.service.EsLogService; +import org.nl.modules.logging.service.dto.LogQuery; +import org.nl.modules.logging.service.dto.LogRepositoryDTO; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +/** + * @author ldjun + * @version 1.0 + * @date 2023年02月07日 14:35 + * @desc desc + */ +@Service +@Slf4j +public class LuceneLogServiceImpl implements EsLogService { + + static String[] INFO_LEVEL = new String[]{"DEBUG","INFO","WARN","ERROR"}; + + @Value("${lucene.index.path}") + private String indexUrl; + + @Override + public void clearLogs(LogQuery query) { + try { + // 初始化 Lucene 索引 + IndexWriter indexWriter = LuceneAppender.indexWriter; + if (indexWriter != null){ + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + DateTime offset = DateUtil.offset(new Date(), DateField.HOUR_OF_DAY, -10); + TermRangeQuery termRangeQuery = new TermRangeQuery( + "requestTime",null, + new BytesRef(DateUtil.format(offset, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + indexWriter.deleteDocuments(termRangeQuery); + indexWriter.commit(); + } + }catch (Exception ex){ + throw new BadRequestException("删除失败："+ex.getMessage()); + } + } + + @Override + public JSONArray getLabelsValues(String type) { + JSONArray result = new JSONArray(); + for (String v : INFO_LEVEL) { + JSONObject item = new JSONObject(); + item.put("label", v); + item.put("value", v); + result.add(item); + } + return result; + } + + + @Override + public JSONObject query(LogQuery logQuery) { + JSONObject res = new JSONObject(); + PageResult page = new PageResult(); + page.setPage(logQuery.getPage()); + page.setPageSize(logQuery.getSize()); + try { + FSDirectory directory = FSDirectory.open(Paths.get(indexUrl)); + DirectoryReader open = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(open); + // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + //时间范围查询 + Date startDate = logQuery.getStartTime(); + Date endDate = logQuery.getEndTime(); + + if (startDate == null){ + Calendar calendar=Calendar.getInstance(); + calendar.set(1970, 0, 1); + startDate = calendar.getTime(); } + if (endDate == null){ endDate = new DateTime(); } + TermRangeQuery termRangeQuery = new TermRangeQuery( + "requestTime", + new BytesRef(DateUtil.format(startDate, "yyyy-MM-dd HH:mm:ss.SSS")), + new BytesRef(DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss.SSS")), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + + // 字段之间的与或非关系，MUST表示and，MUST_NOT表示not，SHOULD表示or，有几个fields就必须有几个clauses + if (ObjectUtil.isNotEmpty(logQuery.getTraceId())){ + TermQuery termQuery = new TermQuery(new Term("traceId", logQuery.getTraceId())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getLogLevel())){ + TermQuery termQuery = new TermQuery(new Term("logLevel", logQuery.getLogLevel())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getSystem())) { + TermQuery termQuery = new TermQuery(new Term("system", logQuery.getSystem())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (ObjectUtil.isNotEmpty(logQuery.getRequestMethod())) { + TermQuery termQuery = new TermQuery(new Term("requestMethod", logQuery.getRequestMethod())); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); + } + if (Boolean.TRUE.equals(logQuery.getIsRequest())) { + Term traceid = new Term("traceId"," "); + TermQuery termQuery = new TermQuery(traceid); + booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST_NOT); + } + if (ObjectUtil.isNotEmpty(logQuery.getMessage())){ + //查询解析器 + QueryParser queryParser = new QueryParser("message", new IKAnalyzer(true)); + Query query = queryParser.parse(logQuery.getMessage()); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + + TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("time", SortField.Type.LONG,true)), 20000, 0); + searcher.search(booleanQueryBuilder.build(), collector); + TopDocs topDocs = collector.topDocs((logQuery.getPage()-1)*logQuery.getSize(), logQuery.getSize()); + int totalSize = collector.getTotalHits(); + ScoreDoc[] scoreDocs = topDocs.scoreDocs; + + List list = new ArrayList<>(); + for (ScoreDoc scoreDoc : scoreDocs) { + Document doc = open.document(scoreDoc.doc); + LogRepositoryDTO dto = new LogRepositoryDTO(); + dto.setSystem(doc.get("system")); + dto.setTraceId(doc.get("traceId")); + dto.setLogger(doc.get("logger")); + dto.setLogLevel(doc.get("logLevel")); + dto.setMessage(doc.get("message")); + dto.setRequestIp(doc.get("requestIp")); + dto.setRequestMethod(doc.get("requestMethod")); + dto.setRequestTime(doc.get("requestTime")); + dto.setThread(doc.get("thread")); + list.add(dto); + } + open.close(); + directory.close(); + page.addAll(list); + page.setTotal(scoreDocs.length); + res.put("total", totalSize); + }catch (Exception ex){ + ex.printStackTrace(); + } + res.put("page",page); + return res; + + } + + + @Override + @Async + public void syncdemo() { + log.info("Async线程链路测试"+Thread.currentThread().getName()); + log.error("Async线程链路测试"+Thread.currentThread().getName()); + log.info("Async线程链路测试"+Thread.currentThread().getName()); + log.info("线程链路测试"+Thread.currentThread().getName()); + } + +} diff --git a/lms/nladmin-system/src/main/resources/config/application-dev.yml b/lms/nladmin-system/src/main/resources/config/application-dev.yml index 263ccda44..b48bc21b7 100644 --- a/lms/nladmin-system/src/main/resources/config/application-dev.yml +++ b/lms/nladmin-system/src/main/resources/config/application-dev.yml @@ -2,22 +2,6 @@ server: port: 8010 #配置数据源 spring: - data: - elasticsearch: - repositories: - enabled: true - client: - reactive: - #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网 -# endpoints: 47.96.133.178:8200 #外网 - endpoints: http://10.1.3.90:9200 #外网 - elasticsearch: - rest: - #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网 -# uris: 47.96.133.178:8200 #外网 - uris: http://10.1.3.90:9200 #外网 -# username: elastic -# password: 123456 datasource: druid: db-type: com.alibaba.druid.pool.DruidDataSource @@ -186,3 +170,6 @@ loki: systemName: lms es: index: lms_log +lucene: + index: + path: D:\lms\lucene\index diff --git a/lms/nladmin-system/src/main/resources/config/application-prod.yml b/lms/nladmin-system/src/main/resources/config/application-prod.yml index 7eebf35e9..02edcca57 100644 --- a/lms/nladmin-system/src/main/resources/config/application-prod.yml +++ b/lms/nladmin-system/src/main/resources/config/application-prod.yml @@ -2,22 +2,6 @@ server: port: 8013 #配置数据源 spring: - data: - elasticsearch: - repositories: - enabled: true - client: - reactive: - #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网 - # endpoints: 47.96.133.178:8200 #外网 - endpoints: http://10.1.3.90:9200 #外网 - elasticsearch: - rest: - #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网 - # uris: 47.96.133.178:8200 #外网 - uris: http://10.1.3.90:9200 #外网 -# username: elastic -# password: 123456 datasource: druid: db-type: com.alibaba.druid.pool.DruidDataSource @@ -183,3 +167,6 @@ sa-token: cookie: # 配置 Cookie 作用域:根据二级域名实现sso登入如lms.sso.com;acs.sso.com domain: +lucene: + index: + path: D:\lms\lucene\index diff --git a/lms/nladmin-system/src/main/resources/config/application-test.yml b/lms/nladmin-system/src/main/resources/config/application-test.yml index 250e563b1..fcf527009 100644 --- a/lms/nladmin-system/src/main/resources/config/application-test.yml +++ b/lms/nladmin-system/src/main/resources/config/application-test.yml @@ -147,3 +147,6 @@ logging: file: path: C:\log\wms config: classpath:logback-spring.xml +lucene: + index: + path: D:\lms\lucene\index diff --git a/lms/nladmin-system/src/main/resources/config/application.yml b/lms/nladmin-system/src/main/resources/config/application.yml index 7050b3042..0780a00a4 100644 --- a/lms/nladmin-system/src/main/resources/config/application.yml +++ b/lms/nladmin-system/src/main/resources/config/application.yml @@ -119,10 +119,6 @@ mybatis-plus: db-config: id-type: INPUT banner: false -management: - health: - elasticsearch: - enabled: false #取消对elasticsearch的检查 https://www.codeleading.com/article/60643988608/ jetcache: statIntervalMinutes: 15 areaInCacheName: false @@ -143,3 +139,6 @@ jetcache: maxTotal: 50 host: 127.0.0.1 port: 6379 +lucene: + index: + path: D:\lms\lucene\index diff --git a/lms/nladmin-system/src/main/resources/log/AcsToLms.xml b/lms/nladmin-system/src/main/resources/log/AcsToLms.xml index 93946707b..bfed653f5 100644 --- a/lms/nladmin-system/src/main/resources/log/AcsToLms.xml +++ b/lms/nladmin-system/src/main/resources/log/AcsToLms.xml @@ -2,66 +2,6 @@ - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - - @@ -80,10 +20,13 @@ ${log.charset} - + + 2 + 500 + + - - + diff --git a/lms/nladmin-system/src/main/resources/log/LmsToAcs.xml b/lms/nladmin-system/src/main/resources/log/LmsToAcs.xml index 96ccf3733..6429e414f 100644 --- a/lms/nladmin-system/src/main/resources/log/LmsToAcs.xml +++ b/lms/nladmin-system/src/main/resources/log/LmsToAcs.xml @@ -2,66 +2,6 @@ - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - - @@ -81,9 +21,13 @@ + + 2 + 500 + + - - + diff --git a/lms/nladmin-system/src/main/resources/log/LmsToMes.xml b/lms/nladmin-system/src/main/resources/log/LmsToMes.xml index bbf8e9fd3..d7152b0f4 100644 --- a/lms/nladmin-system/src/main/resources/log/LmsToMes.xml +++ b/lms/nladmin-system/src/main/resources/log/LmsToMes.xml @@ -20,71 +20,14 @@ ${log.charset} - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- + + 2 + 500 + - - - - - + + diff --git a/lms/nladmin-system/src/main/resources/log/LmsToSap.xml b/lms/nladmin-system/src/main/resources/log/LmsToSap.xml index a21b51a7d..85529648d 100644 --- a/lms/nladmin-system/src/main/resources/log/LmsToSap.xml +++ b/lms/nladmin-system/src/main/resources/log/LmsToSap.xml @@ -2,66 +2,6 @@ - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - - @@ -80,10 +20,13 @@ ${log.charset} - + + 2 + 500 + + - - + diff --git a/lms/nladmin-system/src/main/resources/log/MesToLms.xml b/lms/nladmin-system/src/main/resources/log/MesToLms.xml index 2ee047f21..cca2fd064 100644 --- a/lms/nladmin-system/src/main/resources/log/MesToLms.xml +++ b/lms/nladmin-system/src/main/resources/log/MesToLms.xml @@ -2,66 +2,6 @@ - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - - @@ -81,9 +21,14 @@ + + 2 + 500 + + + - - + diff --git a/lms/nladmin-system/src/main/resources/log/SapToLms.xml b/lms/nladmin-system/src/main/resources/log/SapToLms.xml index 8ad39d38a..cdd433fd8 100644 --- a/lms/nladmin-system/src/main/resources/log/SapToLms.xml +++ b/lms/nladmin-system/src/main/resources/log/SapToLms.xml @@ -2,66 +2,6 @@ - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - - - - system - lms - - - traceId - %X{traceId} - - - logLevel - %level - - - requestMethod - %X{requestMethod} - - - requestTime - %d{yyyy-MM-dd HH:mm:ss.SSS} - - - requestIp - %X{requestIp} - - - thread - %thread - - - logger - %logger - - - -
- Content-Type - application/json -
- - - @@ -81,9 +21,15 @@ + + + 2 + 500 + + + - - + diff --git a/lms/nladmin-system/src/main/resources/logback-spring.xml b/lms/nladmin-system/src/main/resources/logback-spring.xml index e34bb3bd2..ceb2e3d9e 100644 --- a/lms/nladmin-system/src/main/resources/logback-spring.xml +++ b/lms/nladmin-system/src/main/resources/logback-spring.xml @@ -56,34 +56,19 @@ https://juejin.cn/post/6844903775631572999 + + + 0 + 500 + + - - http://10.1.3.90:9200/_bulk - ${esIndex} - lms_log - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 20000 - + system lms - - traceId - %X{traceId} - logLevel %level @@ -104,92 +89,42 @@ https://juejin.cn/post/6844903775631572999 thread %thread - - logger - %logger - - -
- Content-Type - application/json -
- - - - - error - - - - ${LOG_HOME}/ERROR/%d{yyyy-MM-dd}.%i.log - - 15 - - 200MB - - 20GB - - - ${log.pattern} - ${log.charset} - - - - - - 0 - 500 - + + + 512 - + + - - + + + - - - - - - - - + - - - - - - - - - - - - - + - + - + - - + + - - + + - - + + @@ -197,33 +132,16 @@ https://juejin.cn/post/6844903775631572999 + - - - - - - - - - - - - - - - - - - - + - + @@ -232,13 +150,13 @@ https://juejin.cn/post/6844903775631572999 - + - + - + @@ -247,22 +165,8 @@ https://juejin.cn/post/6844903775631572999 - + + - - - - - - - - - - - - - - - diff --git a/lms/nladmin-ui/src/views/loki/view/index.vue b/lms/nladmin-ui/src/views/loki/view/index.vue index 6380fe875..3b5987867 100644 --- a/lms/nladmin-ui/src/views/loki/view/index.vue +++ b/lms/nladmin-ui/src/views/loki/view/index.vue @@ -33,6 +33,14 @@ /> + + + - - - - - + @@ -163,6 +162,7 @@
{{ log.system }} {{ log.thread }} + {{ log.logger }} {{ log.logLevel }} {{ log.requestIp }} {{ log.requestTime }} @@ -205,6 +205,7 @@ export default { timeRange: [], message: '', traceId: '', + requestMethod: '', size: 20, logData: [], filterSql: '1', @@ -302,6 +303,7 @@ export default { // 清空查询数据 this.clearParam() queryParam.logLevel = this.logLevelValue + queryParam.requestMethod = this.requestMethod const time = new Date() if (this.timeZoneValue !== '') { queryParam.startTime = new Date(((time.getTime() - this.timeZoneValue))) @@ -312,7 +314,7 @@ export default { } queryParam.message = this.message.replace(/^\s*|\s*$/g, '') queryParam.filterSql = this.filterSql === '1' - queryParam.isRequest = this.filterSql === '1' + queryParam.isRequest = this.isRequest === '1' queryParam.traceId = this.traceId queryParam.size = this.size queryParam.page = this.page @@ -323,8 +325,8 @@ export default { this.total = res.total // 清空 this.logs = [] - for (const j in res.records) { // 用push的方式将所有日志数组添加进去 - this.logs.push(res.records[j]) + for (const j in res.page) { // 用push的方式将所有日志数组添加进去 + this.logs.push(res.page[j]) } }) },