diff --git a/mes/hd/nladmin-system/pom.xml b/mes/hd/nladmin-system/pom.xml index 1f74e8e8..f420ddfa 100644 --- a/mes/hd/nladmin-system/pom.xml +++ b/mes/hd/nladmin-system/pom.xml @@ -300,20 +300,6 @@ 1.6.2 - - - - - - - - - - - - - - com.hynnet jxl diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java index 37454a61..b627f4da 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java @@ -6,6 +6,7 @@ package org.nl.common.lucene; * @desc desc */ +import ch.qos.logback.classic.AsyncAppender; import ch.qos.logback.classic.spi.ILoggingEvent; import com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender; import org.apache.commons.lang3.StringUtils; @@ -14,7 +15,8 @@ import org.slf4j.MDC; import java.util.Map; -public class AsyncLuceneAppender extends AspectLogbackAsyncAppender { +//public class AsyncLuceneAppender extends AspectLogbackAsyncAppender { +public class AsyncLuceneAppender extends AsyncAppender { public AsyncLuceneAppender() { diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LogEntity.java b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LogEntity.java new file mode 100644 index 00000000..5fd4dc48 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LogEntity.java @@ -0,0 +1,19 @@ +package org.nl.common.lucene; + +import ch.qos.logback.classic.spi.LoggingEvent; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +/* + * @author ZZQ + * @Date 2024/1/24 11:26 + */ +@Data +public class LogEntity extends LoggingEvent { + private Map propertiys = new HashMap<>(); + private long timeStamp; + private String msg; + private String traceId; +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java index 9e1ee338..117743e9 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java @@ -8,6 +8,7 @@ package org.nl.common.lucene; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.AppenderBase; +import ch.qos.logback.core.Context; import com.alibaba.ttl.TransmittableThreadLocal; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.analysis.Analyzer; @@ -23,6 +24,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer; import java.io.IOException; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -32,7 +34,7 @@ public class LuceneAppender extends AppenderBase { public static final TransmittableThreadLocal traceIdTL = new TransmittableThreadLocal(); public LuceneProperties properties; public static Directory index; - private List encoders; + public static List encoders =new ArrayList<>(); public static IndexWriter indexWriter; @@ -43,7 +45,10 @@ public class LuceneAppender extends AppenderBase { try { // 读取配置文件 Properties properties = YmlConfigFileUtil.readConfig("config/application.yml"); - + for (Property property : this.properties.getProperties()) { + LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context); + encoders.add(encoder); + } // 获取配置值 String luceneDir = properties.getProperty("lucene.index.path"); System.out.println("---index地址----"+luceneDir); @@ -61,26 +66,22 @@ public class LuceneAppender extends AppenderBase { @Override protected void append(ILoggingEvent event) { Document doc = new Document(); - for (Property property : this.properties.getProperties()) { - LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context); - String encode = encoder.encode(event); - doc.add(new StringField(property.getName(), encode, Field.Store.YES)); - } - Map map = event.getMDCPropertyMap(); - if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){ - doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES)); - }else { - doc.add(new StringField("traceId"," ", Field.Store.YES)); - } - - doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES)); - doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES)); - doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp())); - try { - indexWriter.addDocument(doc); - indexWriter.commit(); - } catch (Exception e) { - e.printStackTrace(); + if (event instanceof LogEntity){ + LogEntity logEntity = (LogEntity) event; + Map propertiys = logEntity.getPropertiys(); + propertiys.forEach((a,b)->{ + doc.add(new StringField(a, b, Field.Store.YES)); + }); + doc.add(new StringField("traceId",logEntity.getTraceId(), Field.Store.YES)); + doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, logEntity.getMsg(), Field.Store.YES)); + doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(logEntity.getTimeStamp()),Field.Store.YES)); + doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME,logEntity.getTimeStamp())); + try { + indexWriter.addDocument(doc); + indexWriter.commit(); + } catch (Exception e) { + e.printStackTrace(); + } } } diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/LuceneJsonWriter.java b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/LuceneJsonWriter.java new file mode 100644 index 00000000..862d1009 --- /dev/null +++ b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/LuceneJsonWriter.java @@ -0,0 +1,64 @@ +package org.nl.common.lucene.netty; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.jetbrains.annotations.NotNull; +import org.nl.common.lucene.LogEntity; +import org.nl.common.lucene.LuceneAppender; +import org.nl.common.lucene.LucenePropertyAndEncoder; +import org.nl.common.utils.MapOf; + +import java.io.IOException; +import java.io.Writer; +import java.util.Map; + +/* + * @author ZZQ + * @Date 2024/1/24 11:07 + */ +public class LuceneJsonWriter extends Writer { + + private StringBuilder sendBuffer; + private JsonGenerator jsonGenerator; + + public LuceneJsonWriter() throws IOException { + JsonFactory jsonFactory = new JsonFactory(); + jsonFactory.setRootValueSeparator(null); + this.jsonGenerator = jsonFactory.createGenerator(this); + } + + public static LogEntity convertLog(ILoggingEvent event){ + LogEntity logEntity = new LogEntity(); + Map map = logEntity.getPropertiys(); + for (LucenePropertyAndEncoder encoder : LuceneAppender.encoders) { + map.put(encoder.getName(),encoder.encode(event)); + } + String msg = String.format(event.getFormattedMessage(), event.getArgumentArray()); + logEntity.setMsg(msg); + logEntity.setTimeStamp(event.getTimeStamp()); + Map mdc = event.getMDCPropertyMap(); + if (!mdc.isEmpty() && StringUtils.isNotEmpty(mdc.get("traceId"))){ + logEntity.setTraceId(mdc.get("traceId")); + }else { + logEntity.setTraceId(" "); + } + return logEntity; + } + @Override + public void write(@NotNull char[] cbuf, int off, int len) throws IOException { + sendBuffer.append(cbuf, off, len); + } + + @Override + public void flush() throws IOException { + } + + @Override + public void close() throws IOException { + + } +} diff --git a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java index ecefe3a0..0efd7eb2 100644 --- a/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java +++ b/mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java @@ -3,12 +3,14 @@ package org.nl.common.lucene.netty; import ch.qos.logback.classic.spi.ILoggingEvent; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.util.CharsetUtil; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; import org.nl.common.lucene.AsyncLuceneAppender; +import org.nl.common.lucene.LogEntity; import org.nl.common.lucene.netty.AbstraceServer; import org.nl.common.lucene.netty.impl.ClientServer; import org.nl.common.lucene.netty.impl.RemoteServer; @@ -46,10 +48,12 @@ public class RemoteLogServer implements SmartLifecycle { public static AsyncLuceneAppender asyncLuceneAppender; public static void writeLog(ILoggingEvent event){ + LogEntity logEntity = LuceneJsonWriter.convertLog(event); if (LOCAL_LOG){ - asyncLuceneAppender.appendSync(event); + asyncLuceneAppender.appendSync(logEntity); }else { - ByteBuf log = Unpooled.copiedBuffer(JSON.toJSONString(event), CharsetUtil.UTF_8); + String json = JSON.toJSONString(logEntity, SerializerFeature.IgnoreErrorGetter); + ByteBuf log = Unpooled.copiedBuffer(json, CharsetUtil.UTF_8); server.channel.writeAndFlush(log); } }; @@ -58,15 +62,15 @@ public class RemoteLogServer implements SmartLifecycle { @Override public void start() { try { - String provider = redisTemplate.opsForValue().get("providers"); + String provider = redisTemplate.opsForValue().get("provider"); if (StringUtils.isEmpty(provider)){ String ip = Inet4Address.getLocalHost().getHostAddress(); - Map config = MapOf.of("ip", ip, "port", port); + Map config = MapOf.of("ip", ip, "port", String.valueOf(port)); redisTemplate.opsForValue().set("provider", JSON.toJSONString(config)); Runtime.getRuntime().addShutdownHook(new Thread(() ->{ System.out.println("------服务关闭-升级从变主-------"); server.doClose(); - redisTemplate.delete("providers"); + redisTemplate.delete("provider"); try { Thread.sleep(5000); }catch (Exception ex){} diff --git a/mes/hd/nladmin-system/src/main/resources/logback-spring.xml b/mes/hd/nladmin-system/src/main/resources/logback-spring.xml index 9af22fa9..2af1330c 100644 --- a/mes/hd/nladmin-system/src/main/resources/logback-spring.xml +++ b/mes/hd/nladmin-system/src/main/resources/logback-spring.xml @@ -34,11 +34,12 @@ - + 2 500 + @@ -76,36 +77,6 @@ 512 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -