root/longdianningxing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add:es

Browse Source
This commit is contained in:
zhangzhiqiang
2023-02-09 13:07:57 +08:00
parent fdb4a846c3
commit de549c2a1a
14 changed files with 277 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
lms/nladmin-system/pom.xml
View File

@@ -32,7 +32,12 @@
</properties>
<dependencies>
<!-- elasticsearch-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
<version>2.7.5</version>
</dependency>
<!-- 获取系统信息 -->
<dependency>
<groupId>com.github.oshi</groupId>
@@ -40,21 +45,22 @@
<version>6.1.4</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.6.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.1</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
<!-- <version>7.6.1</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-client</artifactId>-->
<!-- <version>7.6.1</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch</groupId>-->
<!-- <artifactId>elasticsearch</artifactId>-->
<!-- <version>7.6.1</version>-->
<!-- </dependency>-->
<!-- logback appender日志-->
<dependency>
<groupId>com.internetitem</groupId>

2
lms/nladmin-system/src/main/java/org/nl/AppRun.java
View File

@@ -12,6 +12,7 @@ import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactor
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@@ -36,6 +37,7 @@ import org.springframework.web.bind.annotation.RestController;
@EnableMethodCache(basePackages = "org.nl")
@EnableCreateCacheAnnotation
@MapperScan("org.nl.**.mapper")
//@EnableElasticsearchRepositories(basePackages = {"org.nl.modules.logging.repository.*"})
public class AppRun {
public static void main(String[] args) {

2
lms/nladmin-system/src/main/java/org/nl/ElasticSearchTest.java
View File

@@ -38,7 +38,7 @@ public class ElasticSearchTest {
public static void searchById() throws IOException {
RestHighLevelClient client = getClientConnection();
GetRequest getRequest = new GetRequest("gateway_log", "DceJqGwBqlIig5BB05Z-");
GetRequest getRequest = null;//new GetRequest("gateway_log", "DceJqGwBqlIig5BB05Z-");
GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
System.out.println(getResponse.getSourceAsString());
client.close();

20
lms/nladmin-system/src/main/java/org/nl/common/enums/LevelEnum.java Normal file
View File

@@ -0,0 +1,20 @@
package org.nl.common.enums;
import org.apache.commons.lang3.StringUtils;
public enum LevelEnum{
DEBUG,
INFO,
WARN,
ERROR;
public static LevelEnum checkLevel(String level){
if (!StringUtils.isEmpty(level)){
for (LevelEnum value : LevelEnum.values()) {
if (value.name().equals(level)){
return value;
};
}
}
return LevelEnum.DEBUG;
}
}

72
lms/nladmin-system/src/main/java/org/nl/modules/logging/aspect/LogAspect.java
View File

@@ -35,6 +35,8 @@ import org.nl.modules.common.utils.ThrowableUtil;
import org.nl.modules.logging.domain.Log;
import org.nl.modules.logging.service.LogService;
import org.nl.modules.wql.core.bean.WQLObject;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
@@ -53,52 +55,39 @@ import java.util.*;
@Slf4j
public class LogAspect {
private final LogService logService;
@Autowired
private LogService logService;
ThreadLocal<Long> currentTime = new ThreadLocal<>();
public LogAspect(LogService logService) {
this.logService = logService;
}
/**
* 配置切入点
*/
@Pointcut("@annotation(org.nl.modules.logging.annotation.Log)")
public void logPointcut() {
// 该方法无方法体,主要为了让同类中其他方法使用此切入点
}
/**
* 配置环绕通知,使用在方法logPointcut()上注册的切入点
*
* @param joinPoint join point for advice
*/
@Around("logPointcut()")
public Object logAround(ProceedingJoinPoint joinPoint) throws Throwable {
@Around("@annotation(logInfo)")
public Object logAround(ProceedingJoinPoint joinPoint,org.nl.modules.logging.annotation.Log logInfo) throws Throwable {
String trackId = UUID.randomUUID().toString();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();
// 方法路径
String methodName = joinPoint.getTarget().getClass().getName() + "." + signature.getName() + "()";
String params = getParameter(method, joinPoint.getArgs());
org.nl.modules.logging.annotation.Log logInfo = method.getAnnotation(org.nl.modules.logging.annotation.Log.class);
//是否输出到日志文件
if (logInfo.isPrintToLogFile()) {
log.info("track_id:{},请求方法:{},请求方法参数:{}",trackId,methodName,params);
}
HttpServletRequest request = RequestHolder.getHttpServletRequest();
String requestIp = StringUtils.getIp(request);
Object result;
currentTime.set(System.currentTimeMillis());
MDC.put("requestMethod",methodName);
MDC.put("requestIp", StringUtils.getIp(request));
MDC.put("trackId", UUID.randomUUID().toString());
MDC.put("requestTime", DateUtil.now());
Object result = null;
long comming = System.currentTimeMillis();
try {
log.info("[--request--][请求方法:{}][请求参数:{}]",methodName,params);
result = joinPoint.proceed();
//是否把日志存到日志表
if (logInfo.isAddLogTable()) {
Log log = new Log("INFO", System.currentTimeMillis() - currentTime.get());
currentTime.remove();
Log log = new Log("INFO", System.currentTimeMillis() - comming);
logService.save(getUsername(), StringUtils.getBrowser(request), requestIp, joinPoint, log);
}
if (logInfo.isInterfaceLog()) {
@@ -112,25 +101,29 @@ public class LogAspect {
json.put("method", methodName);
json.put("params", getParameter(method, joinPoint.getArgs()));
json.put("request_ip", StringUtils.getIp(request));
json.put("time", System.currentTimeMillis() - currentTime.get());
json.put("time", System.currentTimeMillis() - comming);
json.put("username", getUsername());
json.put("address", StringUtils.getCityInfo(requestIp));
json.put("browser", StringUtils.getBrowser(request));
json.put("exception_detail", IdUtil.getStringId());
json.put("create_time", DateUtil.now());
json.put("return_result", JSONObject.parse(result.toString()));
Object parse = JSONObject.parse(result.toString());
json.put("return_result", parse);
log.info("请求结果:{}",parse);
interfaceLog.insert(json);
} catch (Exception e) {
}
}
}catch (Exception ex){
log.error("track_id:{},error:{}",trackId,ex.getMessage());
Log log = new Log("ERROR", System.currentTimeMillis() - currentTime.get());
currentTime.remove();
log.info("[--requestError--][请求方法:{}]【异常信息 :{}】", methodName, ex);
Log log = new Log("ERROR", System.currentTimeMillis() - comming);
log.setExceptionDetail(ThrowableUtil.getStackTrace(ex).getBytes());
logService.save(getUsername(), StringUtils.getBrowser(request), StringUtils.getIp(request), (ProceedingJoinPoint) joinPoint, log);
throw ex;
}finally {
log.info("[--response--][请求方法:{}][请求参数:{}]【返回结果 :{}】",methodName,params,result);
MDC.clear();
}
return result;
}
@@ -165,21 +158,6 @@ public class LogAspect {
return argList.size() == 1 ? JSONUtil.toJsonStr(argList.get(0)) : JSONUtil.toJsonStr(argList);
}
/**
* 配置异常通知
*
* @param joinPoint join point for advice
* @param e exception
*/
// @AfterThrowing(pointcut = "logPointcut()", throwing = "e")
public void logAfterThrowing(JoinPoint joinPoint, Throwable e) {
Log log = new Log("ERROR", System.currentTimeMillis() - currentTime.get());
currentTime.remove();
log.setExceptionDetail(ThrowableUtil.getStackTrace(e).getBytes());
HttpServletRequest request = RequestHolder.getHttpServletRequest();
logService.save(getUsername(), StringUtils.getBrowser(request), StringUtils.getIp(request), (ProceedingJoinPoint) joinPoint, log);
}
public String getUsername() {
try {
return SecurityUtils.getCurrentUsername();

14
lms/nladmin-system/src/main/java/org/nl/modules/logging/repository/EsLogRepository.java Normal file
View File

@@ -0,0 +1,14 @@
package org.nl.modules.logging.repository;
import org.nl.modules.logging.service.dto.LogRepositoryDTO;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;
/*
* @author ZZQ
* @Date 2023/2/8 4:11 下午
*/
@Repository
public interface EsLogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> {
}

13
lms/nladmin-system/src/main/java/org/nl/modules/logging/rest/EsLogController.java
View File

@@ -2,12 +2,12 @@ package org.nl.modules.logging.rest;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.nl.common.domain.query.PageQuery;
import org.nl.modules.logging.service.EsLogService;
import org.nl.modules.logging.service.dto.LogQuery;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
/**
* @author ldjun
@@ -22,10 +22,17 @@ import org.springframework.web.bind.annotation.RestController;
public class EsLogController {
private final EsLogService esLogService;
@GetMapping("/labels")
@ApiOperation("获取标签")
public ResponseEntity<Object> labelsValues() {
return new ResponseEntity<>(esLogService.getLabelsValues(), HttpStatus.OK);
}
@PostMapping("/query")
@ApiOperation("日志查询")
public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) {
return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK);
}
}

10
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/EsLogService.java
View File

@@ -1,6 +1,9 @@
package org.nl.modules.logging.service;
import com.alibaba.fastjson.JSONArray;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.nl.common.domain.query.PageQuery;
import org.nl.modules.logging.service.dto.LogQuery;
/**
* @author ldjun
@@ -14,4 +17,11 @@ public interface EsLogService {
* @return
*/
JSONArray getLabelsValues();
/**
* 日志查询
* @param logQuery
* @return
*/
Page query(LogQuery logQuery);
}

42
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogQuery.java Normal file
View File

@@ -0,0 +1,42 @@
package org.nl.modules.logging.service.dto;
import lombok.Data;
import java.util.Date;
/*
* @author ZZQ
* @Date 2023/2/8 5:18 下午
*/
@Data
public class LogQuery {
/**
* 创建时间范围查询
*/
private Date startTime;
private Date endTime;
/**
* 追踪id
*/
private String trackId;
/**
* 日志内容模糊匹配
*/
private String message;
/**
* 日志级别
*/
private String logLevel;
/**
* 是否只查询Http相关请求
*/
private Boolean isRequest = Boolean.TRUE;
/**
* 是否过滤wql日志
*/
private Boolean filterSql = Boolean.TRUE;
private Integer size = 20;
private Integer page = 0;
}

27
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/dto/LogRepositoryDTO.java Normal file
View File

@@ -0,0 +1,27 @@
package org.nl.modules.logging.service.dto;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
/*
* @author ZZQ
* @Date 2023/2/8 4:06 下午
*/
@Document(indexName = "lms_log", type = "lms_log")
@Data
public class LogRepositoryDTO {
private String message;
private String host;
private String logLevel;
private String logger;
private String requestTime;
private String requestIp;
@Id
private String id;
private String trackId;
private String requestMethod;
private String thread;
}

115
lms/nladmin-system/src/main/java/org/nl/modules/logging/service/impl/EsLogServiceImpl.java
View File

@@ -4,9 +4,23 @@ import cn.hutool.http.HttpRequest;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.formula.functions.T;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.nl.common.domain.query.PageQuery;
import org.nl.common.enums.LevelEnum;
import org.nl.modules.logging.repository.EsLogRepository;
import org.nl.modules.logging.service.EsLogService;
import org.nl.modules.logging.service.dto.LogQuery;
import org.nl.modules.logging.service.dto.LogRepositoryDTO;
import org.nl.wms.ext.acs.service.impl.AcsToWmsServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
import org.springframework.stereotype.Service;
import java.util.Map;
@@ -20,57 +34,62 @@ import java.util.Map;
@Service
@RequiredArgsConstructor
public class EsLogServiceImpl implements EsLogService {
private final EsLogRepository esLogRepository;
@Override
public Page query(LogQuery logQuery){
Page<T> page = new Page<>();
if (logQuery != null){
BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
extractedParam(logQuery, query);
Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage(),logQuery.getSize(), Sort.by("@timestamp").descending()));
page.setRecords(((AggregatedPageImpl) all).getContent());
page.setTotal(((AggregatedPageImpl) all).getTotalElements());
page.setPages(logQuery.getPage());
page.setSize(logQuery.getSize());
}
return page;
}
private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) {
if (StringUtils.isNotEmpty(logQuery.getLogLevel())){
query.must().add(QueryBuilders.matchQuery("logLevel", LevelEnum.checkLevel(logQuery.getLogLevel())));
}
if (logQuery.getIsRequest()){
query.must().add(QueryBuilders.existsQuery("requestMethod"));
}
if (logQuery.getFilterSql()){
query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*"));
}
query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*"));
if (StringUtils.isNotEmpty(logQuery.getTrackId())){
query.must().add(QueryBuilders.matchQuery("trackId", logQuery.getTrackId()));
}
if (StringUtils.isNotEmpty(logQuery.getMessage())){
query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("50%"));
}
if (logQuery.getStartTime()!=null){
query.must().add(QueryBuilders.rangeQuery("requestTime").gte(logQuery.getStartTime()));
}
if (logQuery.getEndTime()!=null){
query.must().add(QueryBuilders.rangeQuery("requestTime").lte(logQuery.getStartTime()));
}
}
@Override
public JSONArray getLabelsValues() {
/**
* [{
* label:
* value:
* children:[{
* label
* value
* }]
* }]
*/
//获取所有索引
// String url = "http://47.111.78.178:27017/_cat/indices";
String url = "http://47.111.78.178:27017/_aliases";
String resultMsg = HttpRequest.get(url)
.execute().body();
JSONObject jsonObject = JSON.parseObject(resultMsg);
JSONArray arr = new JSONArray();
for (Map.Entry entry : jsonObject.entrySet()) {
System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue());
JSONObject json = new JSONObject();
json.put("label", entry.getKey());
json.put("Value", entry.getValue());
arr.add(json);
JSONArray result = new JSONArray();
for (LevelEnum value : LevelEnum.values()) {
JSONObject level = new JSONObject();
level.put("label", value.name());
level.put("Value", value.name());
result.add(level);
}
return arr;
return result;
}
public static void main(String[] args) {
// String url = "http://47.111.78.178:27017/_cat/indices";
String url = "http://47.111.78.178:27017/_aliases";
String resultMsg = HttpRequest.get(url)
.execute().body();
JSONObject jsonObject = JSON.parseObject(resultMsg);
JSONArray arr = new JSONArray();
for (Map.Entry entry : jsonObject.entrySet()) {
System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue());
JSONObject json = new JSONObject();
json.put("label", entry.getKey());
json.put("Value", entry.getValue());
arr.add(json);
}
}
}

20
lms/nladmin-system/src/main/resources/config/application-dev.yml
View File

@@ -2,16 +2,30 @@ server:
port: 8010
#配置数据源
spring:
data:
elasticsearch:
repositories:
enabled: true
client:
reactive:
#endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网
endpoints: 47.96.133.178:8200 #外网
elasticsearch:
rest:
#uris: 172.31.185.110:9200,172.31.154.9:9200 #内网
uris: 47.96.133.178:8200 #外网
# username: elastic
# password: 123456
datasource:
druid:
db-type: com.alibaba.druid.pool.DruidDataSource
driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
# url: jdbc:log4jdbc:mysql://${DB_HOST:10.1.3.91}:${DB_PORT:3306}/${DB_NAME:lms_test2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true
url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:hl_tb_lms}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true
url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:lms}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true
# username: ${DB_USER:root}
username: ${DB_USER:root}
# password: ${DB_PWD:123456}
password: ${DB_PWD:root}
password: ${DB_PWD:942464Yy}
# 初始连接数
initial-size: 5
# 最小连接数
@@ -93,7 +107,7 @@ login:
# 字体大小
font-size: 25
#jwt
#jwt/api/esLog
jwt:
header: Authorization
# 令牌前缀

1
lms/nladmin-system/src/main/resources/config/application.yml
View File

@@ -83,6 +83,7 @@ security:
# 参数
- /api/param/getValueByCode
- /plumelog/**
- /api/esLog/**
# Sa-Token配置
sa-token:
# token 名称 (同时也是cookie名称)

36
lms/nladmin-system/src/main/resources/logback-spring.xml
View File

@@ -12,7 +12,7 @@ https://juejin.cn/post/6844903775631572999
<contextName>nlAdmin</contextName>
<property name="log.charset" value="utf-8"/>
<property name="log.pattern"
value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss.SSS}) %green([%thread]) %highlight(%-5level) %boldMagenta(%logger{36}) - %gray(%msg%n)"/>
value="%black(%contextName-) %X{traceId} %red(%d{yyyy-MM-dd HH:mm:ss.SSS}) %green([%thread]) %highlight(%-5level) %boldMagenta(%logger{36}) - %gray(%msg%n)"/>
<springProperty scope="context" name="logPath" source="logging.file.path" defaultValue="logs"/>
<springProperty scope="context" name="lokiUrl" source="loki.url"/>
<springProperty scope="context" name="systemName" source="loki.systemName"/>
@@ -49,9 +49,9 @@ https://juejin.cn/post/6844903775631572999
<appender name="esLogAppender" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
<url>http://10.1.3.90:9200/_bulk</url>
<index>lms_root</index>
<type>lms_root</type>
<url>http://47.96.133.178:8200/_bulk</url>
<index>lms_log</index>
<type>lms_log</type>
<loggerName>es-logger</loggerName> <!-- optional -->
<errorLoggerName>es-error-logger</errorLoggerName> <!-- optional -->
<connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
@@ -68,30 +68,33 @@ https://juejin.cn/post/6844903775631572999
<!-- <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> &lt;!&ndash; optional &ndash;&gt;-->
<properties>
<property>
<name>host</name>
<value>${HOSTNAME}</value>
<allowEmpty>false</allowEmpty>
<name>trackId</name>
<value>%X{trackId}</value>
</property>
<property>
<name>severity</name>
<name>logLevel</name>
<value>%level</value>
</property>
<property>
<name>requestMethod</name>
<value>%X{requestMethod}</value>
</property>
<property>
<name>requestTime</name>
<value>%X{requestTime}</value>
</property>
<property>
<name>requestIp</name>
<value>%X{requestIp}</value>
</property>
<property>
<name>thread</name>
<value>%thread</value>
</property>
<property>
<name>stacktrace</name>
<value>%ex</value>
</property>
<property>
<name>logger</name>
<value>%logger</value>
</property>
<property>
<name>traceId</name>
<value>%X{traceId}</value>
</property>
</properties>
<headers>
<header>
@@ -133,7 +136,6 @@ https://juejin.cn/post/6844903775631572999
<root level="debug">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="asyncFileAppender"/>
<appender-ref ref="ERROR" />
<appender-ref ref="esLogAppender" />
</root>
<logger name="org.springframework" level="ERROR" additivity="false">