root/hl_one
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rev:lucene

Browse Source
This commit is contained in:
gengby
2024-03-15 08:53:31 +08:00
parent ddc10ecc10
commit 1bfc70d6c7
12 changed files with 181 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
wcs/nladmin-system/src/main/java/org/nl/config/lucene/AsyncLuceneAppender.java
View File

@@ -1,32 +0,0 @@
package org.nl.config.lucene;
/**
* @author ldjun
* @version 1.0
* @date 2023年08月24日 13:00
* @desc desc
*/
import ch.qos.logback.classic.AsyncAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.MDC;
import java.util.Map;
public class AsyncLuceneAppender extends AsyncAppender {
@Override
protected void append(ILoggingEvent event) {
String traceId = LuceneAppender.traceIdTL.get();
if (StringUtils.isNotEmpty(traceId)){
MDC.put("traceId",traceId);
Map<String, String> mdcPropertyMap = event.getMDCPropertyMap();
if (mdcPropertyMap.getClass().getName().contains("SynchronizedMap")){
mdcPropertyMap.put("traceId",traceId);
}
MDC.clear();
}
super.append(event);
}
}

46
wcs/nladmin-system/src/main/java/org/nl/config/lucene/LogMessageConstant.java
View File

@@ -1,46 +0,0 @@
package org.nl.config.lucene;
/**
* @Author: lyd
* @Description: 定义lucene相关常量
* @Date: 2023/8/25
*/
public class LogMessageConstant {
/** */
public final static String SORT_NAME = "time";
/** 级别 */
public final static String FIELD_LEVEL = "level";
/** 时间 */
public final static String FIELD_TIMESTAMP = "timestamp";
/** 类的限定名 */
public final static String FIELD_CLASS_NAME = "logger";
/** 线程名 */
public final static String FIELD_THREAD = "thread";
/** 日志内容 */
public final static String FIELD_MESSAGE = "message";
public final static String FIELD_TRACEID = "tlogTraceId";
// 定义颜色值
/** 文本颜色：黑色 */
public final static String COLOR_BLACK = "\u001B[30m";
/** 文本颜色：红色 */
public final static String COLOR_RED = "\u001B[31m";
/** 文本颜色：绿色 */
public final static String COLOR_GREEN = "\u001B[32m";
/** 文本颜色：黄色 */
public final static String COLOR_YELLOW = "\u001B[33m";
/** 文本颜色：蓝色 */
public final static String COLOR_BLUE = "\u001B[34m";
/** 文本颜色：品红色 */
public final static String COLOR_MAGENTA = "\u001B[35m";
/** 文本颜色：青色 */
public final static String COLOR_CYAN = "\u001B[36m";
/** 文本颜色：白色 */
public final static String COLOR_WHITE = "\u001B[37m";
/** 文本颜色重置 */
public final static String COLOR_RESET = "\u001B[0m";
/** 背景颜色：黄色 */
public final static String BACKGROUND_YELLOW = "\u001B[43m";
/** 索引路径 */
public final static String INDEX_DIR = "E:\\lucene\\index";
}

99
wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneAppender.java
View File

@@ -1,99 +0,0 @@
package org.nl.config.lucene;
/**
* @author ldjun
* @version 1.0
* @date 2023年08月24日 13:00
* @desc desc
*/
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.AppenderBase;
import com.alibaba.ttl.TransmittableThreadLocal;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.nl.common.utils.YmlConfigFileUtil;
import org.wltea.analyzer.lucene.IKAnalyzer;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class LuceneAppender extends AppenderBase<ILoggingEvent> {
public static final TransmittableThreadLocal<String> traceIdTL = new TransmittableThreadLocal();
public LuceneProperties properties;
public static Directory index;
private List<LucenePropertyAndEncoder> encoders;
public static IndexWriter indexWriter;
@Override
public void start() {
super.start();
try {
// 读取配置文件
Properties properties = YmlConfigFileUtil.readConfig("config/application.yml");
// 获取配置值
String luceneDir = properties.getProperty("lucene.index.path");
System.out.println("---index地址----"+luceneDir);
index = FSDirectory.open(Paths.get(luceneDir));
// 初始化 Lucene 索引
Analyzer analyzer = new IKAnalyzer();
IndexWriterConfig config = new IndexWriterConfig(analyzer);
indexWriter = new IndexWriter(index, config);
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
protected void append(ILoggingEvent event) {
Document doc = new Document();
for (Property property : this.properties.getProperties()) {
LucenePropertyAndEncoder encoder = new LucenePropertyAndEncoder(property, this.context);
String encode = encoder.encode(event);
doc.add(new StringField(property.getName(), encode, Field.Store.YES));
}
Map<String, String> map = event.getMDCPropertyMap();
if (!map.isEmpty() && StringUtils.isNotEmpty(map.get("traceId"))){
doc.add(new StringField("traceId",map.get("traceId"), Field.Store.YES));
}else {
doc.add(new StringField("traceId"," ", Field.Store.YES));
}
doc.add(new TextField(LogMessageConstant.FIELD_MESSAGE, event.getFormattedMessage(), Field.Store.YES));
doc.add(new StringField(LogMessageConstant.FIELD_TIMESTAMP, String.valueOf(event.getTimeStamp()),Field.Store.YES));
doc.add(new NumericDocValuesField(LogMessageConstant.SORT_NAME, event.getTimeStamp()));
try {
indexWriter.addDocument(doc);
indexWriter.commit();
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void stop() {
super.stop();
try {
indexWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public void setProperties(LuceneProperties properties) {
this.properties = properties;
}
}

23
wcs/nladmin-system/src/main/java/org/nl/config/lucene/LuceneProperties.java
View File

@@ -1,23 +0,0 @@
package org.nl.config.lucene;
import java.util.ArrayList;
import java.util.List;
public class LuceneProperties {
private List<Property> properties;
public LuceneProperties() {
this.properties = new ArrayList<Property>();
}
public List<Property> getProperties() {
return properties;
}
public void addProperty(Property property) {
properties.add(property);
}
}

38
wcs/nladmin-system/src/main/java/org/nl/config/lucene/LucenePropertyAndEncoder.java
View File

@@ -1,38 +0,0 @@
package org.nl.config.lucene;
import ch.qos.logback.classic.PatternLayout;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Context;
import ch.qos.logback.core.pattern.PatternLayoutBase;
/*
* @author ZZQ
* @Date 2023/12/22 18:11
*/
public class LucenePropertyAndEncoder {
private Property property;
private PatternLayoutBase layout = new PatternLayout();
public LucenePropertyAndEncoder(Property property, Context context) {
this.property = property;
this.layout.setContext(context);
this.layout.setPattern(String.valueOf(property.getValue()));
this.layout.setPostCompileProcessor(null);
this.layout.start();
}
public String encode(ILoggingEvent event) {
return layout.doLayout(event);
}
public String getName() {
return property.getName();
}
public boolean allowEmpty() {
return property.isAllowEmpty();
}
}

44
wcs/nladmin-system/src/main/java/org/nl/config/lucene/Property.java
View File

@@ -1,44 +0,0 @@
package org.nl.config.lucene;
/*
* @author ZZQ
* @Date 2023/12/26 15:30
*/
public class Property {
private String name;
private String value;
private boolean allowEmpty;
public Property() {
}
public Property(String name, String value, boolean allowEmpty) {
this.name = name;
this.value = value;
this.allowEmpty = allowEmpty;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public boolean isAllowEmpty() {
return allowEmpty;
}
public void setAllowEmpty(boolean allowEmpty) {
this.allowEmpty = allowEmpty;
}
}

9
wcs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java
View File

@@ -70,17 +70,12 @@ public class LuceneAppender extends AppenderBase<ILoggingEvent> {
@Override
protected void append(ILoggingEvent event) {
String message = event.getFormattedMessage();
String[] split = message.split("@");
LuceneLogDto luceneLogDto = JSONObject.parseObject(split[1], LuceneLogDto.class);
LuceneLogDto luceneLogDto = JSONObject.parseObject(message, LuceneLogDto.class);
Document document = new Document();
try {
//向document对象中添加域。
Map<String, String> mdcPropertyMap = event.getMDCPropertyMap();
String traceId = mdcPropertyMap.get("traceId");
System.out.println("---追踪号---"+traceId);
if (ObjectUtil.isNotEmpty(traceId)) {
document.add(new StringField("trace_id", traceId, Field.Store.YES));
}
if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())) {
document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES));
}

7
wcs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java
View File

@@ -6,13 +6,11 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.nl.modules.logging.annotation.Log;
import org.nl.modules.lucene.service.LuceneService;
import org.nl.modules.lucene.service.dto.LogQuery;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
@@ -27,7 +25,6 @@ public class LuceneController {
private final LuceneService luceneService;
@GetMapping("/getAll")
@Log("日志检索")
@ApiOperation("日志检索")

3
wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java
View File

@@ -1,6 +1,7 @@
package org.nl.modules.lucene.service;
import com.alibaba.fastjson.JSONArray;
import org.nl.modules.lucene.service.dto.LogQuery;
import org.springframework.data.domain.Pageable;
import java.util.Map;
@@ -11,8 +12,6 @@ public interface LuceneService {
/**
* 获取数据分页
*
* @param whereJson 条件
* @param page 分页参数
* @return Map<String, Object>
*/
Map<String, Object> getAll(Map whereJson, Pageable page);

53
wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/dto/LogQuery.java Normal file
View File

@@ -0,0 +1,53 @@
package org.nl.modules.lucene.service.dto;
import lombok.Data;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
import java.util.Date;
/*
* @author ZZQ
* @Date 2023/2/8 5:18 下午
*/
@Data
public class LogQuery {
/**
* 创建时间范围查询
*/
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
private Date begin_time;
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
private Date end_time;
/**
* 追踪id
*/
private String traceId;
private String requestMethod;
/**
* 日志内容模糊匹配
*/
private String message;
/**
* 日志级别
*/
private String logLevel;
/**
* 系统标签
*/
private String system;
/**
* 是否只查询Http相关请求
*/
private Boolean isRequest = Boolean.TRUE;
/**
* 是否过滤wql日志
*/
private Boolean filterSql = Boolean.TRUE;
private Integer size = 20;
private Integer page = 1;
}

161
wcs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java
View File

@@ -1,17 +1,30 @@
package org.nl.modules.lucene.service.impl;
import cn.hutool.core.util.CharsetUtil;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import com.alibaba.fastjson.JSONObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.nl.modules.lucene.service.LuceneService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import java.util.Map;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
@Service
@@ -22,57 +35,117 @@ public class LuceneServiceImpl implements LuceneService {
//日志索引目录
@Value("${lucene.index.path}")
private String luceneUrl;
private String luceneUrl;
/**
* 获取labels和values树
*
* @return
*/
public JSONArray getLabelsValues() {
JSONArray result = new JSONArray();
// 获取所有标签
String labelString = HttpUtil.get(luceneUrl + "/labels", CharsetUtil.CHARSET_UTF_8);
JSONObject parse = (JSONObject) JSONObject.parse(labelString);
JSONArray labels = parse.getJSONArray("data");
for (int i=0; i<labels.size(); i++) {
// 获取标签下的所有值
String valueString = HttpUtil.get(luceneUrl + "/label/" + labels.getString(i) + "/values", CharsetUtil.CHARSET_UTF_8);
JSONObject parse2 = (JSONObject) JSONObject.parse(valueString);
JSONArray values = parse2.getJSONArray("data");
JSONArray children = new JSONArray();
// 组成树形状态 两级
for (int j=0; j<values.size(); j++) {
JSONObject leaf = new JSONObject();
leaf.put("label", values.getString(j));
leaf.put("value", values.getString(j));
children.add(leaf);
}
JSONObject node = new JSONObject();
node.put("label", labels.getString(i));
node.put("value", labels.getString(i));
node.put("children", children);
result.add(node);
}
return result;
}
@Override
public Map<String, Object> getAll(Map whereJson, Pageable page) {
JSONObject jo = new JSONObject();
try {
JSONObject jsonObject = null;//(JSONObject) Searcher.search(luceneUrl, "", whereJson);
JSONArray array = jsonObject.getJSONArray("content");
int totalElements = Integer.parseInt(jsonObject.get("totalElements").toString());
jo.put("content", array);
jo.put("totalElements", totalElements);
} catch (Exception e) {
log.error("索引查询为空", e);
throw new NullPointerException("索引查询为空");
//获取要查询的路径，也就是索引所在的位置
try {
Resource resource = new ClassPathResource("config/application.yml");
YamlPropertiesFactoryBean yamlPropertiesFactoryBean = new YamlPropertiesFactoryBean();
yamlPropertiesFactoryBean.setResources(resource);
Properties properties = yamlPropertiesFactoryBean.getObject();
// 获取配置值
String luceneDir = properties.getProperty("lucene.index.path");
FSDirectory directory = FSDirectory.open(Paths.get(luceneDir));
DirectoryReader open = DirectoryReader.open(directory);
IndexSearcher searcher = new IndexSearcher(open);
// 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数：
int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数
int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
//时间范围查询
String startDate = (String) whereJson.get("begin_time");
String endDate = (String) whereJson.get("end_time");
if (startDate == null) {
Calendar calendar = Calendar.getInstance();
calendar.set(1970, 0, 1);
startDate = DateUtil.format(calendar.getTime(), "yyyy-MM-dd HH:mm:ss.SSS");
} else {
startDate = getDate(startDate);
}
if (endDate == null) {
endDate = DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS");
} else {
endDate = getDate(endDate);
}
TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true);
booleanQueryBuilder.add(termRangeQuery, BooleanClause.Occur.MUST);
if (whereJson.get("device_code") != null) {
Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code")));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (whereJson.get("method") != null) {
Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method")));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (whereJson.get("status_code") != null) {
Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code")));
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST);
}
if (whereJson.get("requestparam") != null) {
WildcardQuery query = new WildcardQuery(new Term("requestparam", "*" + (String) whereJson.get("requestparam") + "*"));
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST);
}
if (whereJson.get("responseparam") != null) {
WildcardQuery query = new WildcardQuery(new Term("responseparam", "*" + (String) whereJson.get("responseparam") + "*"));
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST);
}
if (whereJson.get("blurry") != null) {
WildcardQuery query = new WildcardQuery(new Term("fieldContent", "*" + (String) whereJson.get("blurry") + "*"));
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST);
}
TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("logTime", SortField.Type.LONG, true)), 20000, 0);
searcher.search(booleanQueryBuilder.build(), collector);
TopDocs topDocs = collector.topDocs(pageNum * pageSize, pageSize);
int totalSize = collector.getTotalHits();
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
List<JSONObject> list = new ArrayList<>();
for (ScoreDoc scoreDoc : scoreDocs) {
Document doc = open.document(scoreDoc.doc);
JSONObject object = new JSONObject();
object.put("content", doc.get("fieldContent"));
object.put("device_code", doc.get("device_code"));
object.put("logTime", doc.get("logTime"));
object.put("method", doc.get("method"));
object.put("status_code", doc.get("status_code"));
object.put("requestparam", doc.get("requestparam"));
object.put("responseparam", doc.get("responseparam"));
if (doc.get("fieldContent") != null) {
list.add(object);
}
}
open.close();
directory.close();
jo.put("content", list);
jo.put("totalElements", totalSize);
return jo;
} catch (Exception e) {
log.error("索引查询为空", e);
throw new NullPointerException("索引查询为空");
}
} catch (NumberFormatException e) {
e.printStackTrace();
} catch (NullPointerException e) {
e.printStackTrace();
}
return jo;
}
public static String getDate(String timeString) throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");//时间格式
Date date = sdf.parse(timeString);
timeString = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss.SSS");//格式化后的时间
return timeString;
}
}

46
wcs/nladmin-system/src/main/resources/logback-spring.xml
View File

@@ -64,39 +64,8 @@ https://juejin.cn/post/6844903775631572999
<queueSize>500</queueSize>
<appender-ref ref="FILE"/>
</appender>
<appender name="luceneAppender" class="org.nl.config.lucene.LuceneAppender" >
<properties>
<property>
<name>system</name>
<value>lms</value>
</property>
<property>
<name>logLevel</name>
<value>%level</value>
</property>
<property>
<name>requestMethod</name>
<value>%X{requestMethod}</value>
</property>
<property>
<name>requestTime</name>
<value>%d{yyyy-MM-dd HH:mm:ss.SSS}</value>
</property>
<property>
<name>requestIp</name>
<value>%X{requestIp}</value>
</property>
<property>
<name>thread</name>
<value>%thread</value>
</property>
<property>
<name>logger</name>
<value>%logger</value>
</property>
</properties>
</appender>
<appender name="asyncLuceneAppender" class="org.nl.config.lucene.AsyncLuceneAppender">
<appender name="luceneAppender" class="org.nl.modules.lucene.config.LuceneAppender" ></appender>
<appender name="asyncLuceneAppender" class="org.nl.modules.lucene.config.AsyncLuceneAppender">
<appender-ref ref="luceneAppender" />
<queueSize>512</queueSize>
</appender>
@@ -124,28 +93,25 @@ https://juejin.cn/post/6844903775631572999
<!--开发环境:打印控制台-->
<springProfile name="dev3">
<springProfile name="dev2">
<root level="info">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="asyncFileAppender"/>
</root>
<!--logmanage -->
<logger name="org.nl.acs.log.service.impl.DeviceExecuteLogServiceImpl" level="info" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>
<logger name="org.nl.modules.lucene.service.impl" level="info" additivity="false">
<appender-ref ref="luceneAppender"/>
</logger>
<logger name="jdbc.resultsettable" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>
<logger name="org.openscada.opc.lib.da.Server" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>
<!--logmanage -->
<logger name="jdbc.audit" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>