rev:集群日志修改

mes/hd/nladmin-system/pom.xml

@@ -300,20 +300,6 @@
             <version>1.6.2</version>
         </dependency>
 
-        <!-- 解析客户端操作系统、浏览器信息 -->
-<!--        <dependency>-->
-<!--            <groupId>nl.basjes.parse.useragent</groupId>-->
-<!--            <artifactId>yauaa</artifactId>-->
-<!--            <version>5.23</version>-->
-<!--        </dependency>-->
-
-<!--        <dependency>-->
-<!--            <groupId>eu.bitwalker</groupId>-->
-<!--            <artifactId>UserAgentUtils</artifactId>-->
-<!--            <version>1.21</version>-->
-<!--        </dependency>-->
-
-        <!--WQL-->
         <dependency>
             <groupId>com.hynnet</groupId>
             <artifactId>jxl</artifactId>

mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java

@@ -6,6 +6,7 @@ package org.nl.common.lucene;
  * @desc desc
  */
 
+import ch.qos.logback.classic.AsyncAppender;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender;
 import org.apache.commons.lang3.StringUtils;
@@ -14,7 +15,8 @@ import org.slf4j.MDC;
 
 import java.util.Map;
 
-public class AsyncLuceneAppender extends AspectLogbackAsyncAppender {
+//public class AsyncLuceneAppender extends AspectLogbackAsyncAppender {
+public class AsyncLuceneAppender extends AsyncAppender {
 
 
     public AsyncLuceneAppender() {

mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LogEntity.java

@@ -0,0 +1,19 @@
+package org.nl.common.lucene;
+
+import ch.qos.logback.classic.spi.LoggingEvent;
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/*
+ * @author ZZQ
+ * @Date 2024/1/24 11:26
+ */
+@Data
+public class LogEntity extends LoggingEvent {
+   private Map<String,String> propertiys = new HashMap<>();
+    private long timeStamp;
+    private String msg;
+    private String traceId;
+}

mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java

@@ -8,6 +8,7 @@ package org.nl.common.lucene;
 
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.AppenderBase;
+import ch.qos.logback.core.Context;
 import com.alibaba.ttl.TransmittableThreadLocal;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.lucene.analysis.Analyzer;
@@ -23,6 +24,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
 
 import java.io.IOException;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -32,7 +34,7 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
     public static final TransmittableThreadLocal<String> traceIdTL = new TransmittableThreadLocal();
     public LuceneProperties properties;
     public static Directory index;
-    private List<LucenePropertyAndEncoder>  encoders;
+    public static List<LucenePropertyAndEncoder>  encoders =new ArrayList<>();
     public static IndexWriter indexWriter;
 
 
@@ -43,7 +45,10 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
         try {
             // 读取配置文件
             Properties properties = YmlConfigFileUtil.readConfig("config/application.yml");
-
+             for (Property property : this.properties.getProperties()) {
+                LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context);
+                 encoders.add(encoder);
+            }
             // 获取配置值
             String luceneDir = properties.getProperty("lucene.index.path");
             System.out.println("---index地址----"+luceneDir);
@@ -61,26 +66,22 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
     @Override
     protected void append(ILoggingEvent event) {
         Document doc = new Document();
-        for (Property property : this.properties.getProperties()) {
+        if (event instanceof LogEntity){
-            LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context);
+            LogEntity logEntity = (LogEntity) event;
-            String encode = encoder.encode(event);
+            Map<String, String> propertiys = logEntity.getPropertiys();
-            doc.add(new StringField(property.getName(), encode, Field.Store.YES));
+            propertiys.forEach((a,b)->{
-        }
+                doc.add(new StringField(a, b, Field.Store.YES));
-        Map<String, String> map = event.getMDCPropertyMap();
+            });
-        if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){
+            doc.add(new StringField("traceId",logEntity.getTraceId(), Field.Store.YES));
-            doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES));
+            doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, logEntity.getMsg(), Field.Store.YES));
-        }else {
+            doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(logEntity.getTimeStamp()),Field.Store.YES));
-            doc.add(new StringField("traceId"," ", Field.Store.YES));
+            doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME,logEntity.getTimeStamp()));
-        }
+            try {
-
+                indexWriter.addDocument(doc);
-        doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES));
+                indexWriter.commit();
-        doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES));
+            } catch (Exception e) {
-        doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp()));
+                e.printStackTrace();
-        try {
+            }
-            indexWriter.addDocument(doc);
-            indexWriter.commit();
-        } catch (Exception e) {
-            e.printStackTrace();
         }
     }
 

mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/LuceneJsonWriter.java

@@ -0,0 +1,64 @@
+package org.nl.common.lucene.netty;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.jetbrains.annotations.NotNull;
+import org.nl.common.lucene.LogEntity;
+import org.nl.common.lucene.LuceneAppender;
+import org.nl.common.lucene.LucenePropertyAndEncoder;
+import org.nl.common.utils.MapOf;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Map;
+
+/*
+ * @author ZZQ
+ * @Date 2024/1/24 11:07
+ */
+public class LuceneJsonWriter extends Writer {
+
+    private StringBuilder sendBuffer;
+    private JsonGenerator jsonGenerator;
+
+    public LuceneJsonWriter() throws IOException {
+        JsonFactory jsonFactory = new JsonFactory();
+        jsonFactory.setRootValueSeparator(null);
+        this.jsonGenerator = jsonFactory.createGenerator(this);
+    }
+
+    public static LogEntity convertLog(ILoggingEvent event){
+        LogEntity logEntity = new LogEntity();
+        Map<String, String> map = logEntity.getPropertiys();
+        for (LucenePropertyAndEncoder encoder : LuceneAppender.encoders) {
+            map.put(encoder.getName(),encoder.encode(event));
+        }
+        String msg = String.format(event.getFormattedMessage(), event.getArgumentArray());
+        logEntity.setMsg(msg);
+        logEntity.setTimeStamp(event.getTimeStamp());
+        Map<String, String> mdc = event.getMDCPropertyMap();
+        if (!mdc.isEmpty() && StringUtils.isNotEmpty(mdc.get("traceId"))){
+            logEntity.setTraceId(mdc.get("traceId"));
+        }else {
+            logEntity.setTraceId(" ");
+        }
+        return logEntity;
+    }
+    @Override
+    public void write(@NotNull char[] cbuf, int off, int len) throws IOException {
+        sendBuffer.append(cbuf, off, len);
+    }
+
+    @Override
+    public void flush() throws IOException {
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+}

mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java

@@ -3,12 +3,14 @@ package org.nl.common.lucene.netty;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.serializer.SerializerFeature;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.util.CharsetUtil;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
 import org.nl.common.lucene.AsyncLuceneAppender;
+import org.nl.common.lucene.LogEntity;
 import org.nl.common.lucene.netty.AbstraceServer;
 import org.nl.common.lucene.netty.impl.ClientServer;
 import org.nl.common.lucene.netty.impl.RemoteServer;
@@ -46,10 +48,12 @@ public class RemoteLogServer implements SmartLifecycle {
     public static AsyncLuceneAppender asyncLuceneAppender;
 
     public static void writeLog(ILoggingEvent event){
+        LogEntity logEntity = LuceneJsonWriter.convertLog(event);
         if (LOCAL_LOG){
-            asyncLuceneAppender.appendSync(event);
+            asyncLuceneAppender.appendSync(logEntity);
         }else {
-            ByteBuf log = Unpooled.copiedBuffer(JSON.toJSONString(event), CharsetUtil.UTF_8);
+            String json = JSON.toJSONString(logEntity, SerializerFeature.IgnoreErrorGetter);
+            ByteBuf log = Unpooled.copiedBuffer(json, CharsetUtil.UTF_8);
             server.channel.writeAndFlush(log);
         }
     };
@@ -58,15 +62,15 @@ public class RemoteLogServer implements SmartLifecycle {
     @Override
     public void start() {
         try {
-            String provider = redisTemplate.opsForValue().get("providers");
+            String provider = redisTemplate.opsForValue().get("provider");
             if (StringUtils.isEmpty(provider)){
                 String ip = Inet4Address.getLocalHost().getHostAddress();
-                Map<String,Object> config = MapOf.of("ip", ip, "port", port);
+                Map<String,Object> config = MapOf.of("ip", ip, "port", String.valueOf(port));
                 redisTemplate.opsForValue().set("provider", JSON.toJSONString(config));
                 Runtime.getRuntime().addShutdownHook(new Thread(() ->{
                     System.out.println("------服务关闭-升级从变主-------");
                     server.doClose();
-                    redisTemplate.delete("providers");
+                    redisTemplate.delete("provider");
                     try {
                         Thread.sleep(5000);
                     }catch (Exception ex){}

mes/hd/nladmin-system/src/main/resources/logback-spring.xml

@@ -34,11 +34,12 @@
         </encoder>
     </appender>
     <!--异步到文件-->
-    <appender name="asyncFileAppender" class="com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender">
+    <appender name="asyncFileAppender" class="ch.qos.logback.classic.AsyncAppender">
         <discardingThreshold>2</discardingThreshold>
         <queueSize>500</queueSize>
         <appender-ref ref="FILE"/>
     </appender>
+
     <appender name="luceneAppender" class="org.nl.common.lucene.LuceneAppender" >
         <properties>
             <property>
@@ -76,36 +77,6 @@
         <queueSize>512</queueSize>
     </appender>
 
-    <!--开发环境:打印控制台-->
-    <springProfile name="dev">
-        <root level="debug">
-            <appender-ref ref="CONSOLE"/>
-        </root>
-        <logger name="jdbc" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.springframework" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.hibernate" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.quartz" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="com.google" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.redisson" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.nl.modules.wql" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-        <logger name="org.springframework.data" level="ERROR" additivity="true">
-            <appender-ref ref="asyncFileAppender"/>
-        </logger>
-    </springProfile>
 
     <springProfile name="dev3">
         <root level="info">