root/rl_mg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add:es日志

Browse Source
This commit is contained in:
zhangzhiqiang
2023-02-10 17:28:42 +08:00
parent 142b3fac27
commit 2f52130762
11 changed files with 411 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
nladmin-system/nlsso-server/pom.xml
View File

@@ -33,20 +33,11 @@
</properties>
<dependencies>
<!-- elasticsearch-->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.6.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.1</version>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
<version>2.7.5</version>
</dependency>
<!-- logback appender日志-->
@@ -271,7 +262,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>runtime</scope>
<version>8.0.20</version>
</dependency>
<!-- druid数据源驱动 -->

95
nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java Normal file
View File

@@ -0,0 +1,95 @@
package org.nl.config;
import com.alibaba.druid.filter.FilterChain;
import com.alibaba.druid.filter.FilterEventAdapter;
import com.alibaba.druid.proxy.jdbc.JdbcParameter;
import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy;
import com.alibaba.druid.proxy.jdbc.ResultSetProxy;
import com.alibaba.druid.proxy.jdbc.StatementProxy;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcUtils;
import com.mysql.cj.jdbc.result.ResultSetImpl;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.MDC;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/*
* @author ZZQ
* @Date 2023/2/10 11:27 上午
*/
@Slf4j
public class DruidFilter extends FilterEventAdapter {
@Override
public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException {
return super.preparedStatement_executeUpdate(chain, statement);
}
@Override
public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException {
return super.statement_executeUpdate(chain, statement, sql);
}
@Override
protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) {
String traceId = MDC.get("traceId");
int size = statement.getParametersSize();
String executeSql = sql;
int count = 0;
try {
count=statement.getUpdateCount();
}catch (Exception ex){ }
if (StringUtils.isNotEmpty(traceId) && count>0) {
if (size > 0) {
Collection<JdbcParameter> values = statement.getParameters().values();
List<Object> params = new ArrayList<>();
for (JdbcParameter value : values) {
params.add(value.getValue());
}
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
}
log.info("[----SQL----][update][ SQL: {} ]", executeSql);
}
super.statementExecuteAfter(statement, sql, result);
}
@Override
public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException {
ResultSetProxy rs = super.statement_getResultSet(chain, statement);
String executeSql = statement.getLastExecuteSql();
String traceId = MDC.get("traceId");
if (StringUtils.isNotEmpty(traceId)){
int result = 0;
if (rs != null) {
ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class);
result = rss.getRows().size();
}
try {
int size = statement.getParametersSize();
if (size>0){
Collection<JdbcParameter> values = statement.getParameters().values();
List<Object> params = new ArrayList<>();
for (JdbcParameter value : values) {
params.add(value.getValue());
}
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
}
}catch (Exception ex){
log.warn("[-SQL解析异常-][{}]",ex.getMessage());
}
log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql);
}
return rs;
}
}

37
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java Normal file
View File

@@ -0,0 +1,37 @@
package org.nl.monitor.controller.log;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.nl.monitor.server.LogService;
import org.nl.monitor.server.log.dto.LogQuery;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
/**
* @author ldjun
* @version 1.0
* @date 2023年01月29日 18:55
* @desc desc
*/
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/esLog")
public class LogController {
private final LogService esLogService;
@GetMapping("/labels/{type}")
@ApiOperation("获取标签")
public ResponseEntity<Object> labelsValues(@PathVariable String type) {
return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK);
}
@PostMapping("/query")
@ApiOperation("日志查询")
public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) {
return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK);
}
}

26
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java Normal file
View File

@@ -0,0 +1,26 @@
package org.nl.monitor.server;
import com.alibaba.fastjson.JSONArray;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.nl.monitor.server.log.dto.LogQuery;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月07日 14:34
* @desc desc
*/
public interface LogService {
/**
* 获取labels和values树
* @return
*/
JSONArray getLabelsValues(String type);
/**
* 日志查询
* @param logQuery
* @return
*/
Page query(LogQuery logQuery);
}

46
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java Normal file
View File

@@ -0,0 +1,46 @@
package org.nl.monitor.server.log.dto;
import lombok.Data;
import java.util.Date;
/*
* @author ZZQ
* @Date 2023/2/8 5:18 下午
*/
@Data
public class LogQuery {
/**
* 创建时间范围查询
*/
private Date startTime;
private Date endTime;
/**
* 追踪id
*/
private String traceId;
/**
* 日志内容模糊匹配
*/
private String message;
/**
* 日志级别
*/
private String logLevel;
/**
* 系统标签
*/
private String system;
/**
* 是否只查询Http相关请求
*/
private Boolean isRequest = Boolean.TRUE;
/**
* 是否过滤wql日志
*/
private Boolean filterSql = Boolean.TRUE;
private Integer size = 20;
private Integer page = 1;
}

28
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java Normal file
View File

@@ -0,0 +1,28 @@
package org.nl.monitor.server.log.dto;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
/*
* @author ZZQ
* @Date 2023/2/8 4:06 下午
*/
@Document(indexName = "lms_log", type = "lms_log")
@Data
public class LogRepositoryDTO {
private String message;
private String host;
private String logLevel;
private String logger;
private String requestTime;
private String requestIp;
@Id
private String id;
private String traceId;
private String requestMethod;
private String thread;
private String system;
}

115
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java Normal file
View File

@@ -0,0 +1,115 @@
package org.nl.monitor.server.log.impl;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.formula.functions.T;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.nl.monitor.server.LogService;
import org.nl.monitor.server.log.dto.LogQuery;
import org.nl.monitor.server.log.dto.LogRepositoryDTO;
import org.nl.monitor.server.log.repository.LogRepository;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月07日 14:35
* @desc desc
*/
@Service
@RequiredArgsConstructor
public class LogServiceImpl implements LogService {
private final LogRepository esLogRepository;
private final ElasticsearchRestTemplate elasticsearchRestTemplate;
@Override
public Page query(LogQuery logQuery){
Page<T> page = new Page<>();
if (logQuery != null){
BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
extractedParam(logQuery, query);
Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending()));
page.setRecords(((AggregatedPageImpl) all).getContent());
page.setTotal(((AggregatedPageImpl) all).getTotalElements());
page.setPages(logQuery.getPage());
page.setSize(logQuery.getSize());
}
return page;
}
private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) {
if (StringUtils.isNotEmpty(logQuery.getLogLevel())){
query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel()));
}
if (StringUtils.isNotEmpty(logQuery.getSystem())){
query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem()));
}
if (logQuery.getIsRequest()){
query.must().add(QueryBuilders.existsQuery("requestMethod"));
}
if (logQuery.getFilterSql()){
query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*"));
}
query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient"));
if (StringUtils.isNotEmpty(logQuery.getTraceId())){
query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId()));
}
if (StringUtils.isNotEmpty(logQuery.getMessage())){
query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%"));
}
if (logQuery.getEndTime()!=null ){
String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L";
query.must().add(QueryBuilders.scriptQuery(new Script(script)));
}
if (logQuery.getStartTime()!=null){
String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L";
query.must().add(QueryBuilders.scriptQuery(new Script(script)));
}
}
@Override
public JSONArray getLabelsValues(String type) {
JSONArray result = new JSONArray();
FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null);
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
queryBuilder.withCollapseField(type+".keyword");
queryBuilder.withSourceFilter(fetchSourceFilter);
queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100));
Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations);
Terms terms = agg.get(type);
List<? extends Terms.Bucket> buckets = terms.getBuckets();
if (!CollectionUtils.isEmpty(buckets)){
buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> {
JSONObject item = new JSONObject();
item.put("label", v);
item.put("value", v);
result.add(item);
});
}
return result;
}
}

14
nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java Normal file
View File

@@ -0,0 +1,14 @@
package org.nl.monitor.server.log.repository;
import org.nl.monitor.server.log.dto.LogRepositoryDTO;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;
/*
* @author ZZQ
* @Date 2023/2/8 4:11 下午
*/
@Repository
public interface LogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> {
}

1
nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties Normal file
View File

@@ -0,0 +1 @@
druid.filters.DruidFilter=org.nl.config.DruidFilter

16
nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
View File

@@ -7,6 +7,22 @@ server:
min-spare-threads: 50
#配置数据源
spring:
data:
elasticsearch:
repositories:
enabled: true
client:
reactive:
#endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网
# endpoints: 47.96.133.178:8200 #外网
endpoints: http://10.1.3.90:9200 #外网
elasticsearch:
rest:
#uris: 172.31.185.110:9200,172.31.154.9:9200 #内网
# uris: 47.96.133.178:8200 #外网
uris: http://10.1.3.90:9200 #外网
# username: elastic
# password: 123456
datasource:
druid:
db-type: com.alibaba.druid.pool.DruidDataSource

46
nladmin-system/nlsso-server/src/main/resources/logback-spring.xml
View File

@@ -55,10 +55,10 @@ https://juejin.cn/post/6844903775631572999
<appender-ref ref="FILE"/>
</appender>
<appender name="plumelog" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
<url>http://47.111.78.178:27017/_bulk</url>
<index>logs-%date{yyyy-MM-dd}</index>
<type>tester</type>
<appender name="esLogAppender" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
<url>http://47.96.133.178:8200/_bulk</url>
<index>lms_log</index>
<type>lms_log</type>
<loggerName>es-logger</loggerName> <!-- optional -->
<errorLoggerName>es-error-logger</errorLoggerName> <!-- optional -->
<connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
@@ -72,33 +72,40 @@ https://juejin.cn/post/6844903775631572999
<rawJsonMessage>false</rawJsonMessage> <!-- optional (default false) -->
<includeMdc>false</includeMdc> <!-- optional (default false) -->
<maxMessageSize>20000</maxMessageSize> <!-- optional (default -1 -->
<!-- <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> &lt;!&ndash; optional &ndash;&gt;-->
<!-- <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> &lt;!&ndash; optional &ndash;&gt;-->
<properties>
<property>
<name>host</name>
<value>${HOSTNAME}</value>
<allowEmpty>false</allowEmpty>
<name>system</name>
<value>lms</value>
</property>
<property>
<name>severity</name>
<name>traceId</name>
<value>%X{traceId}</value>
</property>
<property>
<name>logLevel</name>
<value>%level</value>
</property>
<property>
<name>requestMethod</name>
<value>%X{requestMethod}</value>
</property>
<property>
<name>requestTime</name>
<value>%d{yyyy-MM-dd HH:mm:ss.SSS}</value>
</property>
<property>
<name>requestIp</name>
<value>%X{requestIp}</value>
</property>
<property>
<name>thread</name>
<value>%thread</value>
</property>
<property>
<name>stacktrace</name>
<value>%ex</value>
</property>
<property>
<name>logger</name>
<value>%logger</value>
</property>
<property>
<name>traceId</name>
<value>%X{traceId}</value>
</property>
</properties>
<headers>
<header>
@@ -113,8 +120,11 @@ https://juejin.cn/post/6844903775631572999
<springProfile name="dev">
<root level="debug">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="plumelog" />
<appender-ref ref="esLogAppender" />
</root>
<logger name="es-logger" level="warn" additivity="false">
<appender-ref ref="esLogAppender" />
</logger>
<!--<logger name="org.springframework" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>