root/longdianningxing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opt:日志优化

Browse Source
This commit is contained in:
zhangzq
2023-12-28 19:34:08 +08:00
parent 3bbc5dfef3
commit ca77135394
23 changed files with 370 additions and 851 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java
View File

@@ -28,6 +28,7 @@ import org.aspectj.lang.reflect.MethodSignature;
import org.nl.common.utils.BaseCode;
import org.nl.common.utils.IdUtil;
import org.nl.common.utils.SecurityUtils;
import org.nl.config.lucene.LuceneAppender;
import org.nl.modules.common.utils.RequestHolder;
import org.nl.modules.common.utils.StringUtils;
import org.nl.modules.common.utils.ThrowableUtil;
@@ -81,7 +82,7 @@ public class LogAspect {
MDC.put("requestIp", StringUtils.getIp(request));
MDC.put("traceId", BaseCode.intToChars(IdUtil.getLongId()));
MDC.put("requestTime", DateUtil.now());
LuceneAppender.traceIdTL.set(BaseCode.intToChars(IdUtil.getLongId()));
Object result = null;
long comming = System.currentTimeMillis();
try {
@@ -139,6 +140,7 @@ public class LogAspect {
} finally {
log.info("[--response--][请求接口:{} 执行结束][耗时:{}s]", url, (System.currentTimeMillis() - comming) / 1000);
MDC.clear();
LuceneAppender.traceIdTL.remove();
}
return result;
}

14
lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java
View File

@@ -1,14 +0,0 @@
package org.nl.modules.logging.repository;
import org.nl.modules.logging.service.dto.LogRepositoryDTO;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;
/*
* @author ZZQ
* @Date 2023/2/8 4:11 下午
*/
@Repository
public interface EsLogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> {
}

24
lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java
View File

@@ -2,6 +2,8 @@ package org.nl.modules.logging.rest;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.nl.modules.logging.annotation.Log;
import org.nl.modules.logging.service.EsLogService;
import org.nl.modules.logging.service.dto.LogQuery;
import org.springframework.http.HttpStatus;
@@ -18,6 +20,7 @@ import org.springframework.web.bind.annotation.*;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/esLog")
@Slf4j
public class EsLogController {
private final EsLogService esLogService;
@@ -41,4 +44,25 @@ public class EsLogController {
return new ResponseEntity<>(HttpStatus.OK);
}
@DeleteMapping("/thread")
@Log("清空日志")
public ResponseEntity<Object> thread() {
log.info("Async线程链路测试"+Thread.currentThread().getName());
log.error("Async线程链路测试"+Thread.currentThread().getName());
log.info("Async线程链路测试"+Thread.currentThread().getName());
log.info("线程链路测试"+Thread.currentThread().getName());
esLogService.syncdemo();
new Thread(()->{
log.info("线程链路测试"+Thread.currentThread().getName());
log.error("线程链路测试"+Thread.currentThread().getName());
log.info("线程链路测试"+Thread.currentThread().getName());
log.info("线程链路测试"+Thread.currentThread().getName());
}).start();
return new ResponseEntity<>(HttpStatus.OK);
}
}

12
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java
View File

@@ -1,7 +1,7 @@
package org.nl.modules.logging.service;
import com.alibaba.fastjson.JSONArray;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.alibaba.fastjson.JSONObject;
import org.nl.modules.logging.service.dto.LogQuery;
/**
@@ -24,11 +24,7 @@ public interface EsLogService {
*/
JSONArray getLabelsValues(String type);
/**
* 日志查询
*
* @param logQuery
* @return
*/
Page query(LogQuery logQuery);
JSONObject query(LogQuery logQuery);
void syncdemo();
}

2
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java
View File

@@ -19,6 +19,8 @@ public class LogQuery {
* 追踪id
*/
private String traceId;
private String requestMethod;
/**
* 日志内容模糊匹配
*/

2
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java
View File

@@ -2,13 +2,11 @@ package org.nl.modules.logging.service.dto;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
/*
* @author ZZQ
* @Date 2023/2/8 4:06 下午
*/
@Document(indexName = "#{@ESConfig.index}", type = "lms_log")
@Data
public class LogRepositoryDTO {

134
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java
View File

@@ -1,134 +0,0 @@
package org.nl.modules.logging.service.impl;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.formula.functions.T;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.nl.common.enums.LevelEnum;
import org.nl.modules.logging.repository.EsLogRepository;
import org.nl.modules.logging.service.EsLogService;
import org.nl.modules.logging.service.dto.LogQuery;
import org.nl.modules.logging.service.dto.LogRepositoryDTO;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
import org.springframework.data.elasticsearch.core.query.DeleteQuery;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Date;
import java.util.List;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月07日 14:35
* @desc desc
*/
@Service
@RequiredArgsConstructor
public class EsLogServiceImpl implements EsLogService {
private final EsLogRepository esLogRepository;
private final ElasticsearchRestTemplate elasticsearchRestTemplate;
@Override
public Page query(LogQuery logQuery) {
Page<T> page = new Page<>();
if (logQuery != null) {
BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
extractedParam(logQuery, query);
Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage() - 1, logQuery.getSize(), Sort.by("@timestamp").descending()));
page.setRecords(((AggregatedPageImpl) all).getContent());
page.setTotal(((AggregatedPageImpl) all).getTotalElements());
page.setPages(logQuery.getPage());
page.setSize(logQuery.getSize());
}
return page;
}
private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) {
if (StringUtils.isNotEmpty(logQuery.getLogLevel())) {
query.must().add(QueryBuilders.matchQuery("logLevel", LevelEnum.checkLevel(logQuery.getLogLevel())));
}
if (StringUtils.isNotEmpty(logQuery.getSystem())) {
query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem()));
}
if (logQuery.getIsRequest()) {
query.must().add(QueryBuilders.existsQuery("requestMethod"));
}
if (logQuery.getFilterSql()) {
query.mustNot().add(QueryBuilders.wildcardQuery("logger", "org.nl.modules.wql.core.engine.*"));
}
query.mustNot().add(QueryBuilders.matchPhraseQuery("logger", "org.elasticsearch.client.RestClient"));
if (StringUtils.isNotEmpty(logQuery.getTraceId())) {
query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId()));
}
if (StringUtils.isNotEmpty(logQuery.getMessage())) {
query.must().add(QueryBuilders.matchPhraseQuery("message", logQuery.getMessage()));
}
if (logQuery.getEndTime() != null) {
String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L";
query.must().add(QueryBuilders.scriptQuery(new Script(script)));
}
if (logQuery.getStartTime() != null) {
String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L";
query.must().add(QueryBuilders.scriptQuery(new Script(script)));
}
}
@Override
public void clearLogs(LogQuery logQuery) {
String system = logQuery.getSystem();
BoolQueryBuilder query = QueryBuilders.boolQuery();
if (!StringUtils.isEmpty(system)) {
query.must().add(QueryBuilders.matchQuery("system", system));
}
long time = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -10).getTime();
String script = "doc['@timestamp'].value.millis < " + time + "L";
query.must().add(QueryBuilders.scriptQuery(new Script(script)));
DeleteQuery deleteQuery = new DeleteQuery();
deleteQuery.setQuery(query);
elasticsearchRestTemplate.delete(deleteQuery, new LogRepositoryDTO().getClass());
}
@Override
public JSONArray getLabelsValues(String type) {
JSONArray result = new JSONArray();
FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null);
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
queryBuilder.withCollapseField(type + ".keyword");
queryBuilder.withSourceFilter(fetchSourceFilter);
queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type + ".keyword").size(100));
Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations);
Terms terms = agg.get(type);
List<? extends Terms.Bucket> buckets = terms.getBuckets();
if (!CollectionUtils.isEmpty(buckets)) {
buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v -> {
JSONObject item = new JSONObject();
item.put("label", v);
item.put("value", v);
result.add(item);
});
}
return result;
}
}

185
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/LuceneLogServiceImpl.java Normal file
View File

@@ -0,0 +1,185 @@
package org.nl.modules.logging.service.impl;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.db.PageResult;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.nl.config.lucene.LuceneAppender;
import org.nl.modules.common.exception.BadRequestException;
import org.nl.modules.logging.service.EsLogService;
import org.nl.modules.logging.service.dto.LogQuery;
import org.nl.modules.logging.service.dto.LogRepositoryDTO;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月07日 14:35
* @desc desc
*/
@Service
@Slf4j
public class LuceneLogServiceImpl implements EsLogService {
static String[] INFO_LEVEL = new String[]{"DEBUG","INFO","WARN","ERROR"};
@Value("${lucene.index.path}")
private String indexUrl;
@Override
public void clearLogs(LogQuery query) {
try {
// 初始化 Lucene 索引
IndexWriter indexWriter = LuceneAppender.indexWriter;
if (indexWriter != null){
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
DateTime offset = DateUtil.offset(new Date(), DateField.HOUR_OF_DAY, -10);
TermRangeQuery termRangeQuery = new TermRangeQuery(
"requestTime",null,
new BytesRef(DateUtil.format(offset, "yyyy-MM-dd HH:mm:ss.SSS")), true, true);
booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST);
indexWriter.deleteDocuments(termRangeQuery);
indexWriter.commit();
}
}catch (Exception ex){
throw new BadRequestException("删除失败："+ex.getMessage());
}
}
@Override
public JSONArray getLabelsValues(String type) {
JSONArray result = new JSONArray();
for (String v : INFO_LEVEL) {
JSONObject item = new JSONObject();
item.put("label", v);
item.put("value", v);
result.add(item);
}
return result;
}
@Override
public JSONObject query(LogQuery logQuery) {
JSONObject res = new JSONObject();
PageResult page = new PageResult();
page.setPage(logQuery.getPage());
page.setPageSize(logQuery.getSize());
try {
FSDirectory directory = FSDirectory.open(Paths.get(indexUrl));
DirectoryReader open = DirectoryReader.open(directory);
IndexSearcher searcher = new IndexSearcher(open);
// 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
//时间范围查询
Date startDate = logQuery.getStartTime();
Date endDate = logQuery.getEndTime();
if (startDate == null){
Calendar calendar=Calendar.getInstance();
calendar.set(1970, 0, 1);
startDate = calendar.getTime(); }
if (endDate == null){ endDate = new DateTime(); }
TermRangeQuery termRangeQuery = new TermRangeQuery(
"requestTime",
new BytesRef(DateUtil.format(startDate, "yyyy-MM-dd HH:mm:ss.SSS")),
new BytesRef(DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss.SSS")), true, true);
booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST);
// 字段之间的与或非关系，MUST表示and，MUST_NOT表示not，SHOULD表示or，有几个fields就必须有几个clauses
if (ObjectUtil.isNotEmpty(logQuery.getTraceId())){
TermQuery termQuery = new TermQuery(new Term("traceId", logQuery.getTraceId()));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (ObjectUtil.isNotEmpty(logQuery.getLogLevel())){
TermQuery termQuery = new TermQuery(new Term("logLevel", logQuery.getLogLevel()));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (ObjectUtil.isNotEmpty(logQuery.getSystem())) {
TermQuery termQuery = new TermQuery(new Term("system", logQuery.getSystem()));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (ObjectUtil.isNotEmpty(logQuery.getRequestMethod())) {
TermQuery termQuery = new TermQuery(new Term("requestMethod", logQuery.getRequestMethod()));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (Boolean.TRUE.equals(logQuery.getIsRequest())) {
Term traceid = new Term("traceId"," ");
TermQuery termQuery = new TermQuery(traceid);
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST_NOT);
}
if (ObjectUtil.isNotEmpty(logQuery.getMessage())){
//查询解析器
QueryParser queryParser = new QueryParser("message", new IKAnalyzer(true));
Query query = queryParser.parse(logQuery.getMessage());
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST);
}
TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("time", SortField.Type.LONG,true)), 20000, 0);
searcher.search(booleanQueryBuilder.build(), collector);
TopDocs topDocs = collector.topDocs((logQuery.getPage()-1)*logQuery.getSize(), logQuery.getSize());
int totalSize = collector.getTotalHits();
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
List<LogRepositoryDTO> list = new ArrayList<>();
for (ScoreDoc scoreDoc : scoreDocs) {
Document doc = open.document(scoreDoc.doc);
LogRepositoryDTO dto = new LogRepositoryDTO();
dto.setSystem(doc.get("system"));
dto.setTraceId(doc.get("traceId"));
dto.setLogger(doc.get("logger"));
dto.setLogLevel(doc.get("logLevel"));
dto.setMessage(doc.get("message"));
dto.setRequestIp(doc.get("requestIp"));
dto.setRequestMethod(doc.get("requestMethod"));
dto.setRequestTime(doc.get("requestTime"));
dto.setThread(doc.get("thread"));
list.add(dto);
}
open.close();
directory.close();
page.addAll(list);
page.setTotal(scoreDocs.length);
res.put("total", totalSize);
}catch (Exception ex){
ex.printStackTrace();
}
res.put("page",page);
return res;
}
@Override
@Async
public void syncdemo() {
log.info("Async线程链路测试"+Thread.currentThread().getName());
log.error("Async线程链路测试"+Thread.currentThread().getName());
log.info("Async线程链路测试"+Thread.currentThread().getName());
log.info("线程链路测试"+Thread.currentThread().getName());
}
}