add:es日志

nladmin-system/nlsso-server/pom.xml

@@ -33,20 +33,11 @@
     </properties>
 
     <dependencies>
+        <!-- elasticsearch-->
         <dependency>
-            <groupId>org.elasticsearch.client</groupId>
+            <groupId>org.springframework.boot</groupId>
-            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
-            <version>7.6.1</version>
+            <version>2.7.5</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>elasticsearch-rest-client</artifactId>
-            <version>7.6.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <version>7.6.1</version>
         </dependency>
 
         <!--        logback appender日志-->
@@ -271,7 +262,7 @@
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
-            <scope>runtime</scope>
+            <version>8.0.20</version>
         </dependency>
 
         <!-- druid数据源驱动 -->

nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java

@@ -0,0 +1,95 @@
+package org.nl.config;
+
+import com.alibaba.druid.filter.FilterChain;
+import com.alibaba.druid.filter.FilterEventAdapter;
+import com.alibaba.druid.proxy.jdbc.JdbcParameter;
+import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy;
+import com.alibaba.druid.proxy.jdbc.ResultSetProxy;
+import com.alibaba.druid.proxy.jdbc.StatementProxy;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.util.JdbcUtils;
+import com.mysql.cj.jdbc.result.ResultSetImpl;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.MDC;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/10 11:27 上午
+ */
+@Slf4j
+public class DruidFilter extends FilterEventAdapter {
+
+    @Override
+    public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException {
+
+        return super.preparedStatement_executeUpdate(chain, statement);
+    }
+
+    @Override
+    public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException {
+
+        return super.statement_executeUpdate(chain, statement, sql);
+    }
+
+    @Override
+    protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) {
+        String traceId = MDC.get("traceId");
+        int size = statement.getParametersSize();
+        String executeSql = sql;
+        int count = 0;
+        try {
+            count=statement.getUpdateCount();
+        }catch (Exception ex){ }
+        if (StringUtils.isNotEmpty(traceId) && count>0) {
+            if (size > 0) {
+                Collection<JdbcParameter> values = statement.getParameters().values();
+                List<Object> params = new ArrayList<>();
+                for (JdbcParameter value : values) {
+                    params.add(value.getValue());
+                }
+                executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
+            }
+            log.info("[----SQL----][update][ SQL: {} ]", executeSql);
+        }
+        super.statementExecuteAfter(statement, sql, result);
+    }
+    @Override
+    public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException {
+        ResultSetProxy rs = super.statement_getResultSet(chain, statement);
+        String executeSql = statement.getLastExecuteSql();
+        String traceId = MDC.get("traceId");
+        if (StringUtils.isNotEmpty(traceId)){
+            int result = 0;
+            if (rs != null) {
+                ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class);
+                result = rss.getRows().size();
+            }
+            try {
+                int size = statement.getParametersSize();
+                if (size>0){
+                    Collection<JdbcParameter> values = statement.getParameters().values();
+                    List<Object> params = new ArrayList<>();
+                    for (JdbcParameter value : values) {
+                        params.add(value.getValue());
+                    }
+                    executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
+                }
+            }catch (Exception ex){
+                log.warn("[-SQL解析异常-][{}]",ex.getMessage());
+            }
+            log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql);
+        }
+        return rs;
+    }
+
+
+
+
+
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java

@@ -0,0 +1,37 @@
+package org.nl.monitor.controller.log;
+
+import io.swagger.annotations.ApiOperation;
+import lombok.RequiredArgsConstructor;
+import org.nl.monitor.server.LogService;
+import org.nl.monitor.server.log.dto.LogQuery;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年01月29日 18:55
+ * @desc desc
+ */
+
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/api/esLog")
+public class LogController {
+    private final LogService esLogService;
+
+
+    @GetMapping("/labels/{type}")
+    @ApiOperation("获取标签")
+    public ResponseEntity<Object> labelsValues(@PathVariable String type) {
+        return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK);
+    }
+
+    @PostMapping("/query")
+    @ApiOperation("日志查询")
+   public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) {
+        return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK);
+    }
+
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java

@@ -0,0 +1,26 @@
+package org.nl.monitor.server;
+
+import com.alibaba.fastjson.JSONArray;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.nl.monitor.server.log.dto.LogQuery;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年02月07日 14:34
+ * @desc desc
+ */
+public interface LogService {
+    /**
+     * 获取labels和values树
+     * @return
+     */
+    JSONArray getLabelsValues(String type);
+
+    /**
+     * 日志查询
+     * @param logQuery
+     * @return
+     */
+    Page query(LogQuery logQuery);
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java

@@ -0,0 +1,46 @@
+package org.nl.monitor.server.log.dto;
+
+import lombok.Data;
+
+import java.util.Date;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 5:18 下午
+ */
+@Data
+public class LogQuery {
+    /**
+     * 创建时间范围查询
+     */
+    private Date startTime;
+    private Date endTime;
+    /**
+     * 追踪id
+     */
+    private String traceId;
+    /**
+     * 日志内容模糊匹配
+     */
+    private String message;
+    /**
+     * 日志级别
+     */
+    private String logLevel;
+    /**
+     * 系统标签
+     */
+    private String system;
+    /**
+     * 是否只查询Http相关请求
+     */
+    private Boolean isRequest = Boolean.TRUE;
+    /**
+     * 是否过滤wql日志
+     */
+    private Boolean filterSql = Boolean.TRUE;
+
+    private Integer size = 20;
+
+    private Integer page = 1;
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java

@@ -0,0 +1,28 @@
+package org.nl.monitor.server.log.dto;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.elasticsearch.annotations.Document;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 4:06 下午
+ */
+@Document(indexName = "lms_log", type = "lms_log")
+@Data
+public class LogRepositoryDTO {
+
+    private String message;
+    private String host;
+    private String logLevel;
+    private String logger;
+    private String requestTime;
+    private String requestIp;
+    @Id
+    private String id;
+    private String traceId;
+    private String requestMethod;
+    private String thread;
+    private String system;
+
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java

@@ -0,0 +1,115 @@
+package org.nl.monitor.server.log.impl;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.ss.formula.functions.T;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.nl.monitor.server.LogService;
+import org.nl.monitor.server.log.dto.LogQuery;
+import org.nl.monitor.server.log.dto.LogRepositoryDTO;
+import org.nl.monitor.server.log.repository.LogRepository;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
+import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
+import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
+import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年02月07日 14:35
+ * @desc desc
+ */
+@Service
+@RequiredArgsConstructor
+public class LogServiceImpl implements LogService {
+
+    private final LogRepository esLogRepository;
+
+    private final ElasticsearchRestTemplate elasticsearchRestTemplate;
+
+    @Override
+    public Page query(LogQuery logQuery){
+        Page<T> page = new Page<>();
+        if (logQuery != null){
+            BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
+            extractedParam(logQuery, query);
+            Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending()));
+            page.setRecords(((AggregatedPageImpl) all).getContent());
+            page.setTotal(((AggregatedPageImpl) all).getTotalElements());
+            page.setPages(logQuery.getPage());
+            page.setSize(logQuery.getSize());
+        }
+        return page;
+    }
+
+
+    private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) {
+        if (StringUtils.isNotEmpty(logQuery.getLogLevel())){
+            query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel()));
+        }
+        if (StringUtils.isNotEmpty(logQuery.getSystem())){
+            query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem()));
+        }
+        if (logQuery.getIsRequest()){
+            query.must().add(QueryBuilders.existsQuery("requestMethod"));
+        }
+        if (logQuery.getFilterSql()){
+            query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*"));
+        }
+        query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient"));
+        if (StringUtils.isNotEmpty(logQuery.getTraceId())){
+            query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId()));
+        }
+        if (StringUtils.isNotEmpty(logQuery.getMessage())){
+            query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%"));
+        }
+        if (logQuery.getEndTime()!=null ){
+            String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L";
+            query.must().add(QueryBuilders.scriptQuery(new Script(script)));
+        }
+        if (logQuery.getStartTime()!=null){
+            String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L";
+            query.must().add(QueryBuilders.scriptQuery(new Script(script)));
+        }
+    }
+
+
+
+    @Override
+    public JSONArray getLabelsValues(String type) {
+        JSONArray result = new JSONArray();
+        FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null);
+        NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
+        queryBuilder.withCollapseField(type+".keyword");
+        queryBuilder.withSourceFilter(fetchSourceFilter);
+        queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100));
+        Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations);
+        Terms terms = agg.get(type);
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
+        if (!CollectionUtils.isEmpty(buckets)){
+            buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> {
+                JSONObject item = new JSONObject();
+                item.put("label", v);
+                item.put("value", v);
+                result.add(item);
+            });
+        }
+        return result;
+    }
+
+}

nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java

@@ -0,0 +1,14 @@
+package org.nl.monitor.server.log.repository;
+
+import org.nl.monitor.server.log.dto.LogRepositoryDTO;
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
+import org.springframework.stereotype.Repository;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 4:11 下午
+ */
+@Repository
+public interface LogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> {
+
+}

nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties

@@ -0,0 +1 @@
+druid.filters.DruidFilter=org.nl.config.DruidFilter

nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml

@@ -7,6 +7,22 @@ server:
     min-spare-threads: 50
 #配置数据源
 spring:
+  data:
+    elasticsearch:
+      repositories:
+        enabled: true
+      client:
+        reactive:
+          #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网
+          #          endpoints: 47.96.133.178:8200 #外网
+          endpoints: http://10.1.3.90:9200 #外网
+  elasticsearch:
+    rest:
+      #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网
+      #      uris: 47.96.133.178:8200 #外网
+      uris: http://10.1.3.90:9200 #外网
+#      username: elastic
+#      password: 123456
   datasource:
     druid:
       db-type: com.alibaba.druid.pool.DruidDataSource

nladmin-system/nlsso-server/src/main/resources/logback-spring.xml

@@ -55,10 +55,10 @@ https://juejin.cn/post/6844903775631572999
         <appender-ref ref="FILE"/>
     </appender>
 
-    <appender name="plumelog" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
+    <appender name="esLogAppender" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
-        <url>http://47.111.78.178:27017/_bulk</url>
+        <url>http://47.96.133.178:8200/_bulk</url>
-        <index>logs-%date{yyyy-MM-dd}</index>
+        <index>lms_log</index>
-        <type>tester</type>
+        <type>lms_log</type>
         <loggerName>es-logger</loggerName> <!-- optional -->
         <errorLoggerName>es-error-logger</errorLoggerName> <!-- optional -->
         <connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
@@ -72,33 +72,40 @@ https://juejin.cn/post/6844903775631572999
         <rawJsonMessage>false</rawJsonMessage> <!-- optional (default false) -->
         <includeMdc>false</includeMdc> <!-- optional (default false) -->
         <maxMessageSize>20000</maxMessageSize> <!-- optional (default -1 -->
-<!--        <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> &lt;!&ndash; optional &ndash;&gt;-->
+        <!--        <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> &lt;!&ndash; optional &ndash;&gt;-->
         <properties>
             <property>
-                <name>host</name>
+                <name>system</name>
-                <value>${HOSTNAME}</value>
+                <value>lms</value>
-                <allowEmpty>false</allowEmpty>
             </property>
             <property>
-                <name>severity</name>
+                <name>traceId</name>
+                <value>%X{traceId}</value>
+            </property>
+            <property>
+                <name>logLevel</name>
                 <value>%level</value>
             </property>
+            <property>
+                <name>requestMethod</name>
+                <value>%X{requestMethod}</value>
+            </property>
+            <property>
+                <name>requestTime</name>
+                <value>%d{yyyy-MM-dd HH:mm:ss.SSS}</value>
+            </property>
+            <property>
+                <name>requestIp</name>
+                <value>%X{requestIp}</value>
+            </property>
             <property>
                 <name>thread</name>
                 <value>%thread</value>
             </property>
-            <property>
-                <name>stacktrace</name>
-                <value>%ex</value>
-            </property>
             <property>
                 <name>logger</name>
                 <value>%logger</value>
             </property>
-            <property>
-                <name>traceId</name>
-                <value>%X{traceId}</value>
-            </property>
         </properties>
         <headers>
             <header>
@@ -113,8 +120,11 @@ https://juejin.cn/post/6844903775631572999
     <springProfile name="dev">
         <root level="debug">
             <appender-ref ref="CONSOLE"/>
-            <appender-ref ref="plumelog" />
+            <appender-ref ref="esLogAppender" />
         </root>
+        <logger name="es-logger" level="warn" additivity="false">
+            <appender-ref ref="esLogAppender" />
+        </logger>
         <!--<logger name="org.springframework" level="ERROR" additivity="false">
             <appender-ref ref="CONSOLE"/>
         </logger>