root/rl_mg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add:es日志

Browse Source
This commit is contained in:
zhangzhiqiang
2023-02-13 09:31:26 +08:00
parent 2f52130762
commit 45a0645559
9 changed files with 166 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nladmin-system/nlsso-server/src/main/java/org/nl/AppRun.java
View File

@@ -11,6 +11,7 @@ import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactor
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.web.bind.annotation.RestController;

68
nladmin-system/nlsso-server/src/main/java/org/nl/ElasticSearchClientConfig.java
View File

@@ -1,68 +0,0 @@
package org.nl;
import org.apache.http.HttpHost;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.io.IOException;
import java.util.Map;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月06日 18:49
* @desc desc
*/
@Configuration
public class ElasticSearchClientConfig {
//配置RestHighLevelClient依赖到spring容器中待用
@Bean
public RestHighLevelClient restHighLevelClient() {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
//绑定本机，端口，协议，如果是ES集群，就配置多个
new HttpHost("127.0.0.1", 9200, "http")));
return client;
}
public static void main(String[] args) throws IOException {
// 指定ip 端口
HttpHost[] httpHosts = {new HttpHost("47.111.78.178", 27017, "http")};
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(httpHosts));
SearchRequest searchRequest = new SearchRequest("logs-2023-02-06");
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.query(QueryBuilders.matchQuery("_id", "HzAeJoYBlkwLvExN1Vg4"));
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
RestStatus restStatus = searchResponse.status();
System.out.println(restStatus);
if (restStatus == RestStatus.OK) {
SearchHits searchHits = searchResponse.getHits();
for (SearchHit searchHit: searchHits) {
System.out.println("id:" + searchHit.getId());
System.out.println("index:" + searchHit.getIndex());
System.out.println("score:" + searchHit.getScore());
Map<String, Object> map = searchHit.getSourceAsMap();
System.out.println("name:" + (String) map.get("name"));
System.out.println("city:" + (String) map.get("city"));
System.out.println("price:" + (Double) map.get("price"));
}
}
restHighLevelClient.close();
}
}

45
nladmin-system/nlsso-server/src/main/java/org/nl/common/LogMdcFilter.java
View File

@@ -1,45 +0,0 @@
package org.nl.common;
/**
* @author ldjun
* @version 1.0
* @date 2023年02月07日 11:05
* @desc desc
*/
import org.nl.common.utils.IdUtil;
import org.slf4j.MDC;
import javax.servlet.*;
import javax.servlet.annotation.WebFilter;
import java.io.IOException;
@WebFilter(urlPatterns = "/*", filterName = "logMdcFilter")
public class LogMdcFilter implements Filter {
private static final String UNIQUE_ID = "traceId";
@Override
public void init(FilterConfig filterConfig) {
}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
boolean bInsertMDC = insertMDC();
try {
chain.doFilter(request, response);
} finally {
if(bInsertMDC) {
MDC.remove(UNIQUE_ID);
}
}
}
@Override
public void destroy() {
}
private boolean insertMDC() {
String uniqueId = IdUtil.getStringId();
MDC.put(UNIQUE_ID, uniqueId);
return true;
}
}

16
nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
View File

@@ -21,8 +21,8 @@ spring:
#uris: 172.31.185.110:9200,172.31.154.9:9200 #内网
# uris: 47.96.133.178:8200 #外网
uris: http://10.1.3.90:9200 #外网
# username: elastic
# password: 123456
# username: elastic
# password: 123456
datasource:
druid:
db-type: com.alibaba.druid.pool.DruidDataSource
@@ -64,16 +64,8 @@ spring:
enabled: true
url-pattern: /druid/*
reset-enable: false
filter:
stat:
enabled: true
# 记录慢SQL
log-slow-sql: true
slow-sql-millis: 1000
merge-sql: true
wall:
config:
multi-statement-allow: true
filters:
DruidFilter,stat
redis:
#数据库索引
database: ${REDIS_DB:2}

12
nladmin-system/nlsso-server/src/main/resources/config/application-prod.yml
View File

@@ -41,16 +41,8 @@ spring:
reset-enable: false
login-username: admin
login-password: 123456
filter:
stat:
enabled: true
# 记录慢SQL
log-slow-sql: true
slow-sql-millis: 1000
merge-sql: true
wall:
config:
multi-statement-allow: true
filters:
DruidFilter,stat
redis:
#数据库索引
database: ${REDIS_DB:15}

1
nladmin-system/nlsso-server/src/main/resources/config/application.yml
View File

@@ -60,6 +60,7 @@ security:
- /api/localStorage/pictures
# 参数
- /api/param/getValueByCode
- /api/esLog/**
mybatis-plus:
configuration:
map-underscore-to-camel-case: true

6
nladmin-system/nlsso-server/src/main/resources/logback-spring.xml
View File

@@ -56,7 +56,7 @@ https://juejin.cn/post/6844903775631572999
</appender>
<appender name="esLogAppender" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
<url>http://47.96.133.178:8200/_bulk</url>
<url>http://10.1.3.90:9200/_bulk</url>
<index>lms_log</index>
<type>lms_log</type>
<loggerName>es-logger</loggerName> <!-- optional -->
@@ -76,7 +76,7 @@ https://juejin.cn/post/6844903775631572999
<properties>
<property>
<name>system</name>
<value>lms</value>
<value>sso</value>
</property>
<property>
<name>traceId</name>
@@ -118,7 +118,7 @@ https://juejin.cn/post/6844903775631572999
<!--开发环境:打印控制台-->
<springProfile name="dev">
<root level="debug">
<root level="info">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="esLogAppender" />
</root>

6
nladmin-ui/src/views/loki/api/loki.js
View File

@@ -2,15 +2,15 @@ import request from '@/utils/request'
export function getLogData(param) {
return request({
url: 'api/loki/logs',
url: '/api/esLog/query',
method: 'post',
data: param
})
}
export function labelsValues() {
export function labelsValues(type) {
return request({
url: 'api/loki/labels/values',
url: '/api/esLog/labels/' + type,
method: 'get'
})
}

424
nladmin-ui/src/views/loki/view/index.vue
View File

@@ -2,25 +2,73 @@
<div class="app-container">
<div class="head-container">
<!--工具栏-->
<el-form :inline="true" class="demo-form-inline" label-suffix=":" label-width="80px">
<el-form-item label="日志标签">
<el-cascader
v-model="labelAndValue"
:options="labelsOptions"
placeholder="请选择标签"
@change="queryData"
/>
<el-form :inline="true" class="demo-form-inline" label-suffix=":" label-width="90px">
<el-form-item label="标签">
<el-select
v-model="system"
clearable
style="width: 100px; height: 35px;top: -5px;"
placeholder="所属标签"
>
<el-option
v-for="item in systemOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
<el-form-item label="级别">
<el-select
v-model="logLevelValue"
clearable
style="width: 100px; height: 35px;top: -5px;"
placeholder="日志级别"
>
<el-option
v-for="item in labelsOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
<el-form-item label="关键字">
<label slot="label">关&nbsp;&nbsp;键&nbsp;字:</label>
<el-input
v-model="text"
v-model="message"
size="mini"
placeholder="请输入内容"
clearable
/>
</el-form-item>
<el-form-item label="链路追踪">
<el-input
v-model="traceId"
size="mini"
placeholder="请输入链路id"
clearable
/>
</el-form-item>
<el-form-item label="SQL日志" prop="filterSql">
<el-switch
v-model="filterSql"
active-color="#F56C6C"
inactive-color="#409EFF"
active-value="1"
inactive-valu="0"
/>
</el-form-item>
<el-form-item label="HTTP日志" prop="isRequest">
<el-switch
v-model="isRequest"
active-color="#409EFF"
inactive-color="#F56C6C"
active-value="1"
inactive-valu="0"
/>
</el-form-item>
<el-form-item v-show="!showOptions" label="时间范围">
<el-date-picker
v-model="timeRange"
@@ -32,6 +80,7 @@
end-placeholder="结束日期"
align="right"
@change="queryData"
@blur="queryData"
/>
</el-form-item>
<el-form-item v-show="showOptions" label="时间段">
@@ -49,36 +98,29 @@
<span class="el-icon-sort" @click="changeShow" />
</el-tooltip>
</el-form-item>
<el-form-item label="方向">
<label slot="label">方&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;向:</label>
<el-radio-group v-model="direction" size="mini" @change="queryData">
<el-radio label="backward">backward</el-radio>
<el-radio label="forward">forward</el-radio>
</el-radio-group>
</el-form-item>
<el-form-item label="显示条数">
<el-input-number
v-model="limits"
v-model="size"
size="mini"
controls-position="right"
:min="100"
:min="20"
:max="5000"
:step="100"
:step="10"
/>
</el-form-item>
<el-form-item label="滚动步数">
<el-form-item label="当前页">
<el-input-number
v-model="scrollStep"
v-model="page"
size="mini"
controls-position="right"
:min="10"
:min="1"
:max="2000"
:step="10"
:step="1"
/>
</el-form-item>
<el-form-item>
<el-dropdown split-button type="primary" size="mini" @click="queryData">
查询{{ runStatu }}
查询
<el-dropdown-menu slot="dropdown">
<el-dropdown-item v-for="(item, index) in runStatuOptions" :key="index" @click.native="startInterval(item)">{{ item.label }}</el-dropdown-item>
</el-dropdown-menu>
@@ -90,10 +132,19 @@
<!--数据判空-->
<el-empty v-if="showEmpty" :description="emptyText" />
<!--数据加载-->
<el-card v-else shadow="hover" style="width: 100%" class="log-warpper">
<el-card v-else shadow="hover" style="width: 100%;overflow-x: scroll" class="log-warpper">
<div style="width: 100%">
<div v-for="(log, index) in logs" :key="index">
<div style="margin-bottom: 5px; font-size: 12px;" v-html="log[1]" />
<div>
<span style="color: #6c0a99;font-weight: 700">{{ log.system }}</span>
<span style="color: #13ce66">{{ log.thread }}</span>
<span :style="fontType(log.logLevel)">{{ log.logLevel }}</span>
<span>{{ log.requestIp }}</span>
<span style="color: #7c8db0">{{ log.requestTime }}</span>
<span style="color: chocolate">{{ log.traceId }}</span>
<span style="color: #7a6df0">{{ log.requestMethod }}</span>
<span style="margin: 5px;font-size: 15px" v-html="log.message">{{ log.message }}</span>
</div>
</div>
</div>
</el-card>
@@ -104,39 +155,38 @@
<script>
import logOperation from '@/views/loki/api/loki'
import { default as AnsiUp } from 'ansi_up'
let queryParam = {
logLabel: null,
logLabelValue: null,
start: null,
end: null,
text: null,
limits: 100,
direction: 'backward'
logLevel: null,
system: null,
startTime: null,
endTime: null,
traceId: null,
message: null,
filterSql: true,
isRequest: true,
page: null,
size: 20
}
export default {
name: 'Loki',
name: 'ES',
data() {
return {
labelAndValue: [], // 搜索的值
labelsOptions: [], // 所有标签和对应所有值数据
systemOptions: [], // 所有标签和对应所有值数据
logLevelValue: '',
system: '',
timeRange: [],
text: '',
limits: 100,
direction: 'backward',
message: '',
traceId: '',
size: 20,
logData: [],
filterSql: '1',
isRequest: '1',
logs: [], // 所有日志
showEmpty: true,
emptyText: '请选择标签',
displayDirection: [{
value: 'backward',
label: '新的在前'
}, {
value: 'forward',
label: '旧的在前'
}],
scrollStep: 10,
page: 1,
runStatu: 'off',
runStatuOptions: [{
label: 'off',
@@ -195,247 +245,77 @@ export default {
showOptions: true
}
},
mounted() {
window.addEventListener('scroll', this.handleScroll)
},
created() {
this.initLabelsValues()
},
// 关闭定时器 - 加了缓存就必须使用deactivated
deactivated() {
// js提供的clearInterval方法用来清除定时器
console.log('定时任务销毁')
clearInterval(this.timer)
window.removeEventListener('scroll', this.handleScroll)
},
beforeDestroy() {
// js提供的clearInterval方法用来清除定时器
console.log('定时任务销毁')
clearInterval(this.timer)
window.removeEventListener('scroll', this.handleScroll)
this.initLabelsValues('logLevel')
this.initLabelsValues('system')
},
methods: {
initLabelsValues() {
logOperation.labelsValues().then(res => {
this.labelsOptions = res
fontType(level) {
if (level === 'INFO') {
return { 'color': '#1b6cc4' }
}
if (level === 'ERROR') {
return { 'color': '#e30a0a' }
}
if (level === 'DEBUG') {
return { 'color': '#1e2022' }
}
return { 'color': '#a74dc6' }
},
initLabelsValues(type) {
logOperation.labelsValues(type).then(res => {
if (type === 'logLevel') {
this.labelsOptions = res
}
if (type === 'system') {
this.systemOptions = res
}
})
},
queryData() {
console.log(this.labelAndValue)
// 清空查询数据
this.clearParam()
if (this.labelAndValue.length > 0) {
queryParam.logLabel = this.labelAndValue[0]
queryParam.logLabelValue = this.labelAndValue[1]
queryParam.logLevel = this.logLevelValue
const time = new Date()
if (this.timeZoneValue !== '') {
queryParam.startTime = new Date(((time.getTime() - this.timeZoneValue)))
}
if (queryParam.logLabelValue === null) { // 判空
this.$message({
showClose: true,
message: '请选择标签',
type: 'warning'
})
this.showEmpty = true
this.emptyText = '请选择标签'
return
if (this.timeRange !== '' && this.timeRange.length > 0) {
queryParam.startTime = this.timeRange[0]
queryParam.endTime = this.timeRange[1]
}
if (this.timeRange.length !== 0) { // 如果是输入时间范围
queryParam.start = (new Date(this.timeRange[0]).getTime() * 1000000).toString()
queryParam.end = (new Date(this.timeRange[1]).getTime() * 1000000).toString()
}
if (this.timeZoneValue) {
const time = new Date()
queryParam.start = ((time.getTime() - this.timeZoneValue) * 1000000).toString()
queryParam.end = (time.getTime() * 1000000).toString()
}
if (this.text) {
queryParam.text = this.text.replace(/^\s*|\s*$/g, '')
}
if (this.limits) {
queryParam.limits = this.limits
}
queryParam.direction = this.direction
var ansi_up = new AnsiUp()
queryParam.message = this.message.replace(/^\s*|\s*$/g, '')
queryParam.filterSql = this.filterSql === '1'
queryParam.isRequest = this.filterSql === '1'
queryParam.traceId = this.traceId
queryParam.size = this.size
queryParam.page = this.page
queryParam.system = this.system
logOperation.getLogData(queryParam).then(res => {
this.showEmpty = false
if (res.data.result.length === 1) {
this.logs = res.data.result[0].values
for (const i in res.data.result[0].values) {
this.logs[i][1] = ansi_up.ansi_to_html(res.data.result[0].values[i][1])
}
} else if (res.data.result.length > 1) {
// 清空
this.logs = []
for (const j in res.data.result) { // 用push的方式将所有日志数组添加进去
for (const values_index in res.data.result[j].values) {
this.logs.push(res.data.result[j].values[values_index])
}
}
for (const k in this.logs) {
this.logs[k][1] = ansi_up.ansi_to_html(this.logs[k][1])
}
if (this.direction === 'backward') { // 由于使用公共标签会导致时间顺序错乱，因此对二维数组进行排序
this.logs.sort((a, b) => b[0] - a[0])
} else {
this.logs.sort((a, b) => a[0] - b[0])
}
} else {
this.showEmpty = true
this.emptyText = '暂无日志信息，请选择时间段试试'
// 清空
this.logs = []
for (const j in res.records) { // 用push的方式将所有日志数组添加进去
this.logs.push(res.records[j])
}
})
},
changetype() {
},
clearParam() {
queryParam = {
logLabel: null,
logLabelValue: null,
start: null,
end: null,
text: null,
limits: 100,
direction: 'backward'
logLevel: null,
startTime: null,
endTime: null,
message: null,
traceId: null,
isRequest: true,
filterSql: true,
size: 20
}
},
handleScroll() { // 滚动事件
const scrollTop = document.documentElement.scrollTop// 滚动高度
const clientHeight = document.documentElement.clientHeight// 可视高度
const scrollHeight = document.documentElement.scrollHeight// 内容高度
const bottomest = Math.ceil(scrollTop + clientHeight)
if (bottomest >= scrollHeight) {
// 加载新数据
queryParam.limits = this.scrollStep
queryParam.direction = this.direction
// 获取时间差
let zone = queryParam.end - queryParam.start
if (this.timeRange.length) { // 如果是输入时间范围
zone = ((new Date(this.timeRange[1]).getTime() - new Date(this.timeRange[0]).getTime()) * 1000000).toString()
}
if (this.timeZoneValue) {
zone = this.timeZoneValue * 1000000
}
if (zone === 0) {
zone = 3600 * 1000 * 6
}
if (this.direction === 'backward') { // 设置时间区间
queryParam.start = (this.logs[this.logs.length - 1][0] - zone).toString()
queryParam.end = this.logs[this.logs.length - 1][0]
} else {
queryParam.start = this.logs[this.logs.length - 1][0]
queryParam.end = (parseFloat(this.logs[this.logs.length - 1][0]) + parseFloat(zone.toString())).toString()
}
var ansi_up = new AnsiUp()
logOperation.getLogData(queryParam).then(res => {
console.log(res)
this.showEmpty = false
if (res.data.result.length === 1) {
// 如果返回的日志是一样的就不显示
if (res.data.result[0].values.length === 1 && ansi_up.ansi_to_html(res.data.result[0].values[0][1]) === this.logs[this.logs.length - 1][1]) {
this.$notify({
title: '警告',
duration: 1000,
message: '当前时间段日志已最新！',
type: 'warning'
})
return
}
const log = res.data.result[0].values
for (const i in res.data.result[0].values) {
log[i][1] = ansi_up.ansi_to_html(res.data.result[0].values[i][1])
this.logs.push(log[i])
}
} else if (res.data.result.length > 1) {
const tempArray = [] // 数据需要处理，由于是追加数组，所以需要用额外变量来存放
// 刷新就是添加，不清空原数组
for (const j in res.data.result) { // 用push的方式将所有日志数组添加进去
for (const values_index in res.data.result[j].values) {
tempArray.push(res.data.result[j].values[values_index])
}
}
if (this.direction === 'backward') { // 由于使用公共标签会导致时间顺序错乱，因此对二维数组进行排序
tempArray.sort((a, b) => b[0] - a[0])
} else {
tempArray.sort((a, b) => a[0] - b[0])
}
for (const k in tempArray) {
tempArray[k][1] = ansi_up.ansi_to_html(tempArray[k][1]) // 数据转换
this.logs.push(tempArray[k]) // 追加数据
}
} else {
this.$notify({
title: '警告',
duration: 1000,
message: '暂无以往日志数据！',
type: 'warning'
})
}
})
}
},
startInterval(item) {
this.runStatu = item.label
console.log(item.value)
if (item.value !== 0) {
this.timer = setInterval(() => { // 定时刷新
this.intervalLogs()
}, item.value)
} else {
console.log('销毁了')
clearInterval(this.timer)
}
},
intervalLogs() { // 定时器的方法
// 组织参数
// 设置开始时间和结束时间
// 开始为现在时间
const start = new Date()
const end = new Date()
// 时差判断
let zone = queryParam.end - queryParam.start
if (this.timeRange.length) { // 如果是输入时间范围
zone = ((new Date(this.timeRange[1]).getTime() - new Date(this.timeRange[0]).getTime()) * 1000000).toString()
}
if (this.timeZoneValue) {
zone = this.timeZoneValue * 1000000
}
if (zone === 0) { // 防止空指针
start.setTime(start.getTime() - 3600 * 1000 * 6)
queryParam.start = (start.getTime() * 1000000).toString()
} else {
queryParam.start = (start.getTime() * 1000000 - zone).toString()
}
queryParam.end = (end.getTime() * 1000000).toString()
queryParam.limits = this.limits
console.log('定时器最后参数：', queryParam)
var ansi_up = new AnsiUp() // 后端日志格式转化
logOperation.getLogData(queryParam).then(res => {
console.log('res', res)
this.showEmpty = false
if (res.data.result.length === 1) {
this.logs = res.data.result[0].values
for (const i in res.data.result[0].values) { // 格式转换
this.logs[i][1] = ansi_up.ansi_to_html(res.data.result[0].values[i][1])
}
} else if (res.data.result.length > 1) {
// 清空
this.logs = []
for (const j in res.data.result) { // 用push的方式将所有日志数组添加进去
for (const values_index in res.data.result[j].values) {
this.logs.push(res.data.result[j].values[values_index])
}
}
for (const k in this.logs) {
this.logs[k][1] = ansi_up.ansi_to_html(this.logs[k][1])
}
if (this.direction === 'backward') { // 由于使用公共标签会导致时间顺序错乱，因此对二维数组进行排序
this.logs.sort((a, b) => b[0] - a[0])
} else {
this.logs.sort((a, b) => a[0] - b[0])
}
} else {
this.showEmpty = true
this.emptyText = '暂无日志信息，请选择时间段试试'
}
})
},
changeShow() {
// 清空数据
this.timeZoneValue = ''
@@ -448,7 +328,7 @@ export default {
<style scoped>
.log-warpper {
white-space: nowrap;
word-break: break-word;
word-wrap: break-word
}
</style>