root/hl_one
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rev:集群日志修改

Browse Source
This commit is contained in:
zhangzq
2024-01-24 15:09:40 +08:00
parent 48a45ba56d
commit c9ea2f112b
7 changed files with 120 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
mes/hd/nladmin-system/pom.xml
View File

@@ -300,20 +300,6 @@
<version>1.6.2</version>
</dependency>
<!-- 解析客户端操作系统、浏览器信息 -->
<!-- <dependency>-->
<!-- <groupId>nl.basjes.parse.useragent</groupId>-->
<!-- <artifactId>yauaa</artifactId>-->
<!-- <version>5.23</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>eu.bitwalker</groupId>-->
<!-- <artifactId>UserAgentUtils</artifactId>-->
<!-- <version>1.21</version>-->
<!-- </dependency>-->
<!--WQL-->
<dependency>
<groupId>com.hynnet</groupId>
<artifactId>jxl</artifactId>

4
mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/AsyncLuceneAppender.java
View File

@@ -6,6 +6,7 @@ package org.nl.common.lucene;
* @desc desc
*/
import ch.qos.logback.classic.AsyncAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender;
import org.apache.commons.lang3.StringUtils;
@@ -14,7 +15,8 @@ import org.slf4j.MDC;
import java.util.Map;
public class AsyncLuceneAppender extends AspectLogbackAsyncAppender {
//public class AsyncLuceneAppender extends AspectLogbackAsyncAppender {
public class AsyncLuceneAppender extends AsyncAppender {
public AsyncLuceneAppender() {

19
mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LogEntity.java Normal file
View File

@@ -0,0 +1,19 @@
package org.nl.common.lucene;
import ch.qos.logback.classic.spi.LoggingEvent;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/*
* @author ZZQ
* @Date 2024/1/24 11:26
*/
@Data
public class LogEntity extends LoggingEvent {
private Map<String,String> propertiys = new HashMap<>();
private long timeStamp;
private String msg;
private String traceId;
}

45
mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/LuceneAppender.java
View File

@@ -8,6 +8,7 @@ package org.nl.common.lucene;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.AppenderBase;
import ch.qos.logback.core.Context;
import com.alibaba.ttl.TransmittableThreadLocal;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.analysis.Analyzer;
@@ -23,6 +24,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -32,7 +34,7 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
public static final TransmittableThreadLocal<String> traceIdTL = new TransmittableThreadLocal();
public LuceneProperties properties;
public static Directory index;
private List<LucenePropertyAndEncoder> encoders;
public static List<LucenePropertyAndEncoder> encoders =new ArrayList<>();
public static IndexWriter indexWriter;
@@ -43,7 +45,10 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
try {
// 读取配置文件
Properties properties = YmlConfigFileUtil.readConfig("config/application.yml");
for (Property property : this.properties.getProperties()) {
LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context);
encoders.add(encoder);
}
// 获取配置值
String luceneDir = properties.getProperty("lucene.index.path");
System.out.println("---index地址----"+luceneDir);
@@ -61,26 +66,22 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
@Override
protected void append(ILoggingEvent event) {
Document doc = new Document();
for (Property property : this.properties.getProperties()) {
LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context);
String encode = encoder.encode(event);
doc.add(new StringField(property.getName(), encode, Field.Store.YES));
}
Map<String, String> map = event.getMDCPropertyMap();
if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){
doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES));
}else {
doc.add(new StringField("traceId"," ", Field.Store.YES));
}
doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES));
doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES));
doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp()));
try {
indexWriter.addDocument(doc);
indexWriter.commit();
} catch (Exception e) {
e.printStackTrace();
if (event instanceof LogEntity){
LogEntity logEntity = (LogEntity) event;
Map<String, String> propertiys = logEntity.getPropertiys();
propertiys.forEach((a,b)->{
doc.add(new StringField(a, b, Field.Store.YES));
});
doc.add(new StringField("traceId",logEntity.getTraceId(), Field.Store.YES));
doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, logEntity.getMsg(), Field.Store.YES));
doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(logEntity.getTimeStamp()),Field.Store.YES));
doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME,logEntity.getTimeStamp()));
try {
indexWriter.addDocument(doc);
indexWriter.commit();
} catch (Exception e) {
e.printStackTrace();
}
}
}

64
mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/LuceneJsonWriter.java Normal file
View File

@@ -0,0 +1,64 @@
package org.nl.common.lucene.netty;
import ch.qos.logback.classic.spi.ILoggingEvent;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.jetbrains.annotations.NotNull;
import org.nl.common.lucene.LogEntity;
import org.nl.common.lucene.LuceneAppender;
import org.nl.common.lucene.LucenePropertyAndEncoder;
import org.nl.common.utils.MapOf;
import java.io.IOException;
import java.io.Writer;
import java.util.Map;
/*
* @author ZZQ
* @Date 2024/1/24 11:07
*/
public class LuceneJsonWriter extends Writer {
private StringBuilder sendBuffer;
private JsonGenerator jsonGenerator;
public LuceneJsonWriter() throws IOException {
JsonFactory jsonFactory = new JsonFactory();
jsonFactory.setRootValueSeparator(null);
this.jsonGenerator = jsonFactory.createGenerator(this);
}
public static LogEntity convertLog(ILoggingEvent event){
LogEntity logEntity = new LogEntity();
Map<String, String> map = logEntity.getPropertiys();
for (LucenePropertyAndEncoder encoder : LuceneAppender.encoders) {
map.put(encoder.getName(),encoder.encode(event));
}
String msg = String.format(event.getFormattedMessage(), event.getArgumentArray());
logEntity.setMsg(msg);
logEntity.setTimeStamp(event.getTimeStamp());
Map<String, String> mdc = event.getMDCPropertyMap();
if (!mdc.isEmpty() && StringUtils.isNotEmpty(mdc.get("traceId"))){
logEntity.setTraceId(mdc.get("traceId"));
}else {
logEntity.setTraceId(" ");
}
return logEntity;
}
@Override
public void write(@NotNull char[] cbuf, int off, int len) throws IOException {
sendBuffer.append(cbuf, off, len);
}
@Override
public void flush() throws IOException {
}
@Override
public void close() throws IOException {
}
}

14
mes/hd/nladmin-system/src/main/java/org/nl/common/lucene/netty/RemoteLogServer.java
View File

@@ -3,12 +3,14 @@ package org.nl.common.lucene.netty;
import ch.qos.logback.classic.spi.ILoggingEvent;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.util.CharsetUtil;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.nl.common.lucene.AsyncLuceneAppender;
import org.nl.common.lucene.LogEntity;
import org.nl.common.lucene.netty.AbstraceServer;
import org.nl.common.lucene.netty.impl.ClientServer;
import org.nl.common.lucene.netty.impl.RemoteServer;
@@ -46,10 +48,12 @@ public class RemoteLogServer implements SmartLifecycle {
public static AsyncLuceneAppender asyncLuceneAppender;
public static void writeLog(ILoggingEvent event){
LogEntity logEntity = LuceneJsonWriter.convertLog(event);
if (LOCAL_LOG){
asyncLuceneAppender.appendSync(event);
asyncLuceneAppender.appendSync(logEntity);
}else {
ByteBuf log = Unpooled.copiedBuffer(JSON.toJSONString(event), CharsetUtil.UTF_8);
String json = JSON.toJSONString(logEntity, SerializerFeature.IgnoreErrorGetter);
ByteBuf log = Unpooled.copiedBuffer(json, CharsetUtil.UTF_8);
server.channel.writeAndFlush(log);
}
};
@@ -58,15 +62,15 @@ public class RemoteLogServer implements SmartLifecycle {
@Override
public void start() {
try {
String provider = redisTemplate.opsForValue().get("providers");
String provider = redisTemplate.opsForValue().get("provider");
if (StringUtils.isEmpty(provider)){
String ip = Inet4Address.getLocalHost().getHostAddress();
Map<String,Object> config = MapOf.of("ip", ip, "port", port);
Map<String,Object> config = MapOf.of("ip", ip, "port", String.valueOf(port));
redisTemplate.opsForValue().set("provider", JSON.toJSONString(config));
Runtime.getRuntime().addShutdownHook(new Thread(() ->{
System.out.println("------服务关闭-升级从变主-------");
server.doClose();
redisTemplate.delete("providers");
redisTemplate.delete("provider");
try {
Thread.sleep(5000);
}catch (Exception ex){}

33
mes/hd/nladmin-system/src/main/resources/logback-spring.xml
View File

@@ -34,11 +34,12 @@
</encoder>
</appender>
<!--异步到文件-->
<appender name="asyncFileAppender" class="com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender">
<appender name="asyncFileAppender" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>2</discardingThreshold>
<queueSize>500</queueSize>
<appender-ref ref="FILE"/>
</appender>
<appender name="luceneAppender" class="org.nl.common.lucene.LuceneAppender" >
<properties>
<property>
@@ -76,36 +77,6 @@
<queueSize>512</queueSize>
</appender>
<!--开发环境:打印控制台-->
<springProfile name="dev">
<root level="debug">
<appender-ref ref="CONSOLE"/>
</root>
<logger name="jdbc" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.springframework" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.hibernate" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.quartz" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="com.google" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.redisson" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.nl.modules.wql" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
<logger name="org.springframework.data" level="ERROR" additivity="true">
<appender-ref ref="asyncFileAppender"/>
</logger>
</springProfile>
<springProfile name="dev3">
<root level="info">