wyl/hertzbeat
1
0
Fork 0
mirror of https://github.com/dromara/hertzbeat.git synced 2026-09-17 09:40:58 +00:00
Code Issues Projects Releases Wiki Activity

[monitor] enable powerful jsonpath parser, add es metrics(#295)

Browse Source 
jsonpath支持不同json层级的获取，es添加cpu、mem、heap监听修复2.x版本es采集nodes出现空指针
[monitor] support calculates evaluate expression or-> |
Add @Ceilzcx as a contributor
Add @VampireAchao as a contributor
This commit is contained in:
zcx
2022-09-18 09:43:31 +08:00
committed by tomsun28
parent b1d8ef5674
commit 90ab1f76d9
8 changed files with 148 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.all-contributorsrc
+18
View File
@@ -247,6 +247,24 @@
"contributions": [
"doc"
]
},
{
"login": "VampireAchao",
"name": "VampireAchao",
"avatar_url": "https://avatars.githubusercontent.com/u/52746628?v=4",
"profile": "https://vampireachao.github.io/",
"contributions": [
"code"
]
},
{
"login": "Ceilzcx",
"name": "zcx",
"avatar_url": "https://avatars.githubusercontent.com/u/48920254?v=4",
"profile": "https://github.com/Ceilzcx",
"contributions": [
"code"
]
}
],
"contributorsPerLine": 7
.github/ISSUE_TEMPLATE/task.yml
+4 -4
View File
@@ -13,10 +13,10 @@ body:
attributes:
label: Task List
placeholder: |
1. Task1...
2. Task2...
3. Task3...
4. Task4...
-[ ] Task1...
-[ ] Task2...
-[ ] Task3...
-[ ] Task4...
validations:
required: false
- type: markdown
README.md
+2
View File
@@ -130,6 +130,8 @@ Thanks these wonderful people, welcome to join us:
<td align="center"><a href="https://github.com/oyiyou"><img src="https://avatars.githubusercontent.com/u/39228891?v=4?s=100" width="100px;" alt=""/><br /><sub><b>lambert</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=oyiyou" title="Code">💻</a></td>
<td align="center"><a href="http://mroldx.xyz/"><img src="https://avatars.githubusercontent.com/u/34847828?v=4?s=100" width="100px;" alt=""/><br /><sub><b>mroldx</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=mroldx" title="Documentation">📖</a></td>
<td align="center"><a href="https://github.com/woshiniusange"><img src="https://avatars.githubusercontent.com/u/91513022?v=4?s=100" width="100px;" alt=""/><br /><sub><b>woshiniusange</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=woshiniusange" title="Documentation">📖</a></td>
<td align="center"><a href="https://vampireachao.github.io/"><img src="https://avatars.githubusercontent.com/u/52746628?v=4?s=100" width="100px;" alt=""/><br /><sub><b>VampireAchao</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=VampireAchao" title="Code">💻</a></td>
<td align="center"><a href="https://github.com/Ceilzcx"><img src="https://avatars.githubusercontent.com/u/48920254?v=4?s=100" width="100px;" alt=""/><br /><sub><b>zcx</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=Ceilzcx" title="Code">💻</a></td>
</tr>
</table>
README_CN.md
+2
View File
@@ -132,6 +132,8 @@ Thanks these wonderful people, welcome to join us:
<td align="center"><a href="https://github.com/oyiyou"><img src="https://avatars.githubusercontent.com/u/39228891?v=4?s=100" width="100px;" alt=""/><br /><sub><b>lambert</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=oyiyou" title="Code">💻</a></td>
<td align="center"><a href="http://mroldx.xyz/"><img src="https://avatars.githubusercontent.com/u/34847828?v=4?s=100" width="100px;" alt=""/><br /><sub><b>mroldx</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=mroldx" title="Documentation">📖</a></td>
<td align="center"><a href="https://github.com/woshiniusange"><img src="https://avatars.githubusercontent.com/u/91513022?v=4?s=100" width="100px;" alt=""/><br /><sub><b>woshiniusange</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=woshiniusange" title="Documentation">📖</a></td>
<td align="center"><a href="https://vampireachao.github.io/"><img src="https://avatars.githubusercontent.com/u/52746628?v=4?s=100" width="100px;" alt=""/><br /><sub><b>VampireAchao</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=VampireAchao" title="Code">💻</a></td>
<td align="center"><a href="https://github.com/Ceilzcx"><img src="https://avatars.githubusercontent.com/u/48920254?v=4?s=100" width="100px;" alt=""/><br /><sub><b>zcx</b></sub></a><br /><a href="https://github.com/tomsun28/hertzbeat/commits?author=Ceilzcx" title="Code">💻</a></td>
</tr>
</table>
alerter/src/main/java/com/usthe/alert/AlerterConfiguration.java
+27 -2
View File
@@ -18,9 +18,10 @@
package com.usthe.alert;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.lexer.token.OperatorType;
import com.googlecode.aviator.runtime.function.AbstractFunction;
import com.googlecode.aviator.runtime.type.AviatorBoolean;
import com.googlecode.aviator.runtime.type.AviatorObject;
import com.googlecode.aviator.runtime.type.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -31,6 +32,7 @@ import java.util.Map;
*
*/
@Configuration
@Slf4j
public class AlerterConfiguration {
private static final int AVIATOR_LRU_CACHE_SIZE = 1024;
@@ -41,6 +43,29 @@ public class AlerterConfiguration {
AviatorEvaluator.getInstance()
.useLRUExpressionCache(AVIATOR_LRU_CACHE_SIZE)
.addFunction(new StrEqualFunction());
AviatorEvaluator.getInstance().addOpFunction(OperatorType.BIT_OR, new AbstractFunction() {
@Override
public AviatorObject call(final Map<String, Object> env, final AviatorObject arg1,
final AviatorObject arg2) {
try {
Object value1 = arg1.getValue(env);
Object value2 = arg2.getValue(env);
Object currentValue = value1 == null ? value2 : value1;
if (arg1.getAviatorType() == AviatorType.String) {
return new AviatorString(String.valueOf(currentValue));
} else {
return AviatorDouble.valueOf(currentValue);
}
} catch (Exception e) {
log.error(e.getMessage());
}
return arg1.bitOr(arg2, env);
}
@Override
public String getName() {
return OperatorType.BIT_OR.getToken();
}
});
}
/**
collector/src/main/java/com/usthe/collector/collect/http/HttpCollectImpl.java
+15 -2
View File
@@ -308,14 +308,27 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.MetricsData.Builder builder, Long responseTime) {
List<Map<String, Object>> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
for (Map<String, Object> stringMap : results) {
for (int i = 0; i < results.size(); i++) {
Map<String, Object> stringMap = results.get(i);
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
// 监控目标版本问题可能出现属性不存在，stringMap为空时过滤。参考app-elasticsearch.yml的name: nodes
if (stringMap == null) {
continue;
}
for (String alias : aliasFields) {
Object value = stringMap.get(alias);
if (value != null) {
valueRowBuilder.addColumns(String.valueOf(value));
} else {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
if (alias.startsWith("$.")) {
List<Map<String, Object>> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
if (subResults != null && subResults.size() > i) {
Object resultValue = subResults.get(i);
valueRowBuilder.addColumns(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
} else {
valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
}
} else if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumns(Integer.toString(keywordNum));
collector/src/main/java/com/usthe/collector/dispatch/MetricsCollect.java
+6 -8
View File
@@ -259,6 +259,8 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
String aliasFieldValue = aliasRow.getColumns(aliasIndex);
if (!CommonConstants.NULL_VALUE.equals(aliasFieldValue)) {
aliasFieldValueMap.put(aliasFields.get(aliasIndex), aliasFieldValue);
} else {
aliasFieldValueMap.put(aliasFields.get(aliasIndex), null);
}
}
StringBuilder instanceBuilder = new StringBuilder();
@@ -270,18 +272,14 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
// If there is a calculation expression, calculate the value
// 存在计算表达式 则计算值
if (CommonConstants.TYPE_NUMBER == field.getType()) {
for (String variable : expression.getVariableNames()) {
for (String variable : expression.getVariableFullNames()) {
Double doubleValue = CommonUtil.parseStrDouble(aliasFieldValueMap.get(variable));
if (doubleValue != null) {
fieldValueMap.put(variable, doubleValue);
}
fieldValueMap.put(variable, doubleValue);
}
} else {
for (String variable : expression.getVariableNames()) {
for (String variable : expression.getVariableFullNames()) {
String strValue = aliasFieldValueMap.get(variable);
if (strValue != null && !"".equals(strValue)) {
fieldValueMap.put(variable, strValue);
}
fieldValueMap.put(variable, strValue);
}
}
try {
manager/src/main/resources/define/app/app-elasticsearch.yml
+74 -4
View File
@@ -87,8 +87,8 @@ metrics:
- initializing_shards
- unassigned_shards
- active_shards_percent_as_number
# (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
# (非必须)指标计算表达式,与上面的别名一起作用,计算出最终需要的指标值
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- cluster_name=cluster_name
- status=status
@@ -100,7 +100,7 @@ metrics:
- initializing_shards=initializing_shards
- unassigned_shards=unassigned_shards
protocol: http
# 当protocol为http协议时具体的采集配置
# 当protocol为http协议时具体的采集配置
http:
# 主机host: ipv4 ipv6 域名
host: ^_^host^_^
@@ -155,4 +155,74 @@ metrics:
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: jsonPath
parseScript: '$._nodes'
parseScript: '$._nodes'
- name: nodes_detail
priority: 2
fields:
- field: node_name
type: 1
instance: true
- field: ip
type: 1
- field: cpu_load_average
type: 0
- field: cpu_percent
type: 0
- field: heap_used
type: 0
- field: heap_used_percent
type: 0
- field: heap_total
type: 0
- field: disk_free
type: 0
- field: disk_total
type: 0
- field: disk_used_percent
type: 0
aliasFields:
- $.name
- $.ip
# $.os.cpu.load_average.1m 支持 5.x 及以上版本；$.os.load_average 支持 2.x 及以上版本
- $.os.cpu.load_average.1m
- $.os.load_average
# $.os.cpu.percent 支持 5.x 及以上版本；$.os.cpu_percent 支持 2.x 及以上版本
- $.os.cpu_percent
- $.os.cpu.percent
- $.jvm.mem.heap_used_in_bytes
- $.jvm.mem.heap_used_percent
- $.jvm.mem.heap_max_in_bytes
- $.fs.total.free_in_bytes
- $.fs.total.total_in_bytes
calculates:
- node_name=$.name
- ip=$.ip
- cpu_load_average=#`$.os.cpu.load_average.1m` | $.os.load_average
- cpu_percent=$.os.cpu.percent | $.os.cpu_percent
- heap_used=$.jvm.mem.heap_used_in_bytes
- heap_used_percent=$.jvm.mem.heap_used_percent
- heap_total=$.jvm.mem.heap_max_in_bytes
- disk_free=$.fs.total.free_in_bytes
- disk_total=$.fs.total.total_in_bytes
- disk_used_percent=(1-$.fs.total.free_in_bytes/$.fs.total.total_in_bytes)*100
protocol: http
# 当protocol为http协议时具体的采集配置
http:
# 主机host: ipv4 ipv6 域名
host: ^_^host^_^
# 端口
port: ^_^port^_^
# url请求接口路径
url: /_nodes/stats
# 请求方式 GET POST PUT DELETE PATCH
method: GET
# 认证
authorization:
# 认证方式: Basic Auth, Digest Auth, Bearer Token
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: jsonPath
parseScript: '$.nodes.*'