Compare commits

..
Author SHA1 Message Date
tomsun28 491aa5da94 [security] update hertzbeat security model 2025-06-10 22:26:24 +08:00
749 changed files with 8749 additions and 30877 deletions
-19
View File
@@ -6,7 +6,6 @@
"files": [
"README.md",
"README_CN.md",
"README_JP.md"
"home/src/pages/team/index.jsx"
],
"imageSize": 100,
@@ -2505,24 +2504,6 @@
"code",
"test"
]
},
{
"login": "RainBondsongyg",
"name": "songyg",
"avatar_url": "https://avatars.githubusercontent.com/u/94501396?v=4",
"profile": "https://github.com/RainBondsongyg",
"contributions": [
"doc"
]
},
{
"login": "lx1229",
"name": "Liuxin",
"avatar_url": "https://avatars.githubusercontent.com/u/44620005?v=4",
"profile": "https://github.com/lx1229",
"contributions": [
"code"
]
}
],
"contributorsPerLine": 7,
+1 -4
View File
@@ -16,7 +16,7 @@
#
github:
description: Real-time observability system with agentless, performance cluster, prometheus-compatible, custom monitoring and status page building capabilities.
description: Apache HertzBeat(incubating) is a real-time monitoring system with agentless, performance cluster, prometheus-compatible, custom monitoring and status page building capabilities.
homepage: https://hertzbeat.apache.org/
labels:
- monitoring
@@ -52,9 +52,6 @@ github:
required_pull_request_reviews:
dismiss_stale_reviews: true
required_approving_review_count: 1
# enable GitHub Dependabot to create PRs for security alerts but not for every dependency update
dependabot_alerts: true
dependabot_updates: false
notifications:
commits: notifications@hertzbeat.apache.org
issues: notifications@hertzbeat.apache.org
+1 -1
View File
@@ -46,7 +46,7 @@ jobs:
- uses: ./script/ci/github-actions/setup-deps
- name: Build with Maven
run: mvnd clean -B package -Prelease -Dmaven.test.skip=false --file pom.xml
run: mvn clean -B package -Prelease -Dmaven.test.skip=false --file pom.xml
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4.0.1
+5 -3
View File
@@ -49,9 +49,11 @@ jobs:
- name: Dead Link Check
run: |
sudo npm install -g markdown-link-check@3.8.7
find ./home -name "*.md" > all_md_files.txt
grep -vFf ./script/ci/exclude_files.txt all_md_files.txt > to_check.txt
xargs -P 8 -a to_check.txt -I{} markdown-link-check -c ./script/ci/link_check.json -q "{}"
for file in $(find ./home -name "*.md"); do
if ! grep -Fxq "$file" ./script/ci/exclude_files.txt; then
markdown-link-check -c ./script/ci/link_check.json -q "$file"
fi
done
- name: NPM INSTALL
working-directory: home
-1
View File
@@ -67,7 +67,6 @@ header:
- 'karma.conf.js'
- 'proxy.conf.js'
- '.helmignore'
- 'script/helm/**'
- 'web-app/src/app/core/**'
- 'web-app/src/app/layout/**'
- 'web-app/src/app/routes/exception/**'
+10
View File
@@ -0,0 +1,10 @@
Apache HertzBeat (incubating) is an effort undergoing incubation at the Apache
Software Foundation (ASF), sponsored by the Apache Incubator PMC.
Incubation is required of all newly accepted projects until a further review
indicates that the infrastructure, communications, and decision making process
have stabilized in a manner consistent with other successful ASF projects.
While incubation status is not necessarily a reflection of the completeness
or stability of the code, it does indicate that the project has yet to be
fully endorsed by the ASF.
+1 -1
View File
@@ -1,4 +1,4 @@
Apache HertzBeat
Apache HertzBeat (incubating)
Copyright 2024-2025 The Apache Software Foundation
This product includes software developed at
+4 -6
View File
@@ -9,7 +9,7 @@
<b><a href="README.md">English</a></b> | <a href="README_CN.md">中文</a> | <a href="README_JP.md">日本語</a>
</p>
> A real-time observability system with agentless, performance cluster, prometheus-compatible, custom monitoring and status page building capabilities.
> A real-time monitoring system with agentless, performance cluster, prometheus-compatible, custom monitoring and status page building capabilities.
[![Discord](https://img.shields.io/badge/Chat-Discord-7289DA?logo=discord)](https://discord.gg/Fb6M73htGr)
[![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
@@ -28,7 +28,7 @@
## 🎡 <font color="green">Introduction</font>
[Apache HertzBeat](https://github.com/apache/hertzbeat) is an easy-to-use, open source, real-time observability system with agentless, high performance cluster, prometheus-compatible, offers powerful custom monitoring and status page building capabilities.
[Apache HertzBeat](https://github.com/apache/hertzbeat) (incubating) is an easy-to-use, open source, real-time monitoring system with agentless, high performance cluster, prometheus-compatible, offers powerful custom monitoring and status page building capabilities.
### Features
@@ -527,8 +527,6 @@ Thanks to these wonderful people, welcome to join us:
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/xiaomizhou2"><img src="https://avatars.githubusercontent.com/u/47807926?v=4?s=100" width="100px;" alt="zhangyaxi"/><br /><sub><b>zhangyaxi</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Tests">⚠️</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/RainBondsongyg"><img src="https://avatars.githubusercontent.com/u/94501396?v=4?s=100" width="100px;" alt="songyg"/><br /><sub><b>songyg</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=RainBondsongyg" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
</tr>
</tbody>
</table>
@@ -572,8 +570,8 @@ HertzBeat is built on so many great open source projects, thanks to them!
<p align="left">
<img src="./home/static/img/home/cncf-landscape-left-logo.svg" width="300">&nbsp;&nbsp;<img src="./home/static/img/home/cncf-right-logo.svg" width="345" />
<br /><br />
HertzBeat has been included in the <a href="https://landscape.cncf.io/?item=observability-and-analysis--observability--hertzbeat">
CNCF Observability And Analysis - Observability Landscape.</a>
HertzBeat has been included in the <a href="https://landscape.cncf.io/guide?item=observability-and-analysis--monitoring--hertzbeat#observability-and-analysis--monitoring">
CNCF Observability And Analysis - Monitoring Landscape.</a>
</p>
## 🛡️ License
+8 -10
View File
@@ -9,7 +9,7 @@
<a href="README.md">English</a> | <b><a href="README_CN.md">中文</a></b> | <a href="README_JP.md">日本語</a>
</p>
> 实时观测系统,无需 Agent,性能集群,兼容 Prometheus,自定义监控和状态页构建能力。
> 实时监控系统,无需 Agent,性能集群,兼容 Prometheus,自定义监控和状态页构建能力。
[![Discord](https://img.shields.io/badge/Chat-Discord-7289DA?logo=discord)](https://discord.gg/Fb6M73htGr)
[![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
@@ -28,16 +28,16 @@
## 🎡 <font color="green">介绍</font>
[Apache HertzBeat](https://github.com/apache/hertzbeat) 是一个易用友好的开源实时监控告警系统,无需 Agent,高性能集群,兼容 Prometheus,提供强大的自定义监控和状态页构建能力。
[Apache HertzBeat](https://github.com/apache/hertzbeat) incubating是一个易用友好的开源实时监控告警系统,无需 Agent,高性能集群,兼容 Prometheus,提供强大的自定义监控和状态页构建能力。
### 特点
-**监控+告警+通知** 为一体,支持对应用服务,应用程序,数据库,缓存,操作系统,大数据,中间件,Web 服务器,云原生,网络,自定义等监控阈值告警通知一步到位。
- 易用友好,无需 `Agent`,全 `WEB` 页面操作,鼠标点一点就能监控告警,无需学习成本。
-`Http, Jmx, Ssh, Snmp, Jdbc, Prometheus` 等协议规范可配置化,只需在浏览器配置监控模板 `YML` 就能使用这些协议去自定义采集想要的指标。您相信只需简单配置即可快速适配一款 `K8s``Docker` 等新的监控类型吗?
- 易用友好,无需 `Agent`,全 `WEB` 页面操作,鼠标点一点就能监控告警,零上手学习成本。
-`Http, Jmx, Ssh, Snmp, Jdbc, Prometheus` 等协议规范可配置化,只需在浏览器配置监控模板 `YML` 就能使用这些协议去自定义采集想要的指标。您相信只需配置下就能立刻适配一款 `K8s``Docker` 等新的监控类型吗?
- 兼容 `Prometheus` 的系统生态并且更多,只需页面操作就可以监控 `Prometheus` 所能监控的。
- 高性能,支持多采集器集群横向扩展,支持多隔离网络监控,云边协同。
- 灵活的告警阈值规则,`邮件` `Discord` `Slack` `Telegram` `钉钉` `微信` `飞书` `短信` `Webhook` `Server酱` 等方式消息及时送达。
- 自由的告警阈值规则,`邮件` `Discord` `Slack` `Telegram` `钉钉` `微信` `飞书` `短信` `Webhook` `Server酱` 等方式消息及时送达。
- 提供强大的状态页构建能力,轻松向用户传达您产品服务的实时状态。
@@ -126,7 +126,7 @@
- `-e IDENTITY=custom-collector-name` : 配置此采集器的唯一性标识符名称,多个采集器名称不能相同,建议自定义英文名称。
- `-e MODE=public` : 配置运行模式(public or private), 公共集群模式或私有云边模式。
- `-e MANAGER_HOST=127.0.0.1` : 配置连接主 HertzBeat 服务的对外 IP。
- `-e MANAGER_HOST=127.0.0.1` : 配置连接主 HertaBeat 服务的对外 IP。
- `-e MANAGER_PORT=1158` : 配置连接主 HertzBeat 服务的对外端口,默认1158。
@@ -525,8 +525,6 @@ Thanks these wonderful people, welcome to join us:
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/xiaomizhou2"><img src="https://avatars.githubusercontent.com/u/47807926?v=4?s=100" width="100px;" alt="zhangyaxi"/><br /><sub><b>zhangyaxi</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Tests">⚠️</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/RainBondsongyg"><img src="https://avatars.githubusercontent.com/u/94501396?v=4?s=100" width="100px;" alt="songyg"/><br /><sub><b>songyg</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=RainBondsongyg" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
</tr>
</tbody>
</table>
@@ -572,8 +570,8 @@ HertzBeat is built on so many great open source projects, thanks to them!
<p align="left">
<img src="./home/static/img/home/cncf-landscape-left-logo.svg" width="300">&nbsp;&nbsp;<img src="./home/static/img/home/cncf-right-logo.svg" width="345" />
<br /><br />
HertzBeat has been included in the <a href="https://landscape.cncf.io/?item=observability-and-analysis--observability--hertzbeat">
CNCF Observability And Analysis - Observability Landscape.</a>
HertzBeat has been included in the <a href="https://landscape.cncf.io/guide?item=observability-and-analysis--monitoring--hertzbeat#observability-and-analysis--monitoring">
CNCF Observability And Analysis - Monitoring Landscape.</a>
</p>
## 🛡️ License
+2 -8
View File
@@ -525,12 +525,6 @@ Thanks these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/boyucjz"><img src="https://avatars.githubusercontent.com/u/18730041?v=4?s=100" width="100px;" alt="boyucjz"/><br /><sub><b>boyucjz</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=boyucjz" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Cyanty"><img src="https://avatars.githubusercontent.com/u/153884653?v=4?s=100" width="100px;" alt="Cyanty"/><br /><sub><b>Cyanty</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Cyanty" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=Cyanty" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/KevinLLF"><img src="https://avatars.githubusercontent.com/u/85452733?v=4?s=100" width="100px;" alt="Jay丿167"/><br /><sub><b>Jay丿167</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=KevinLLF" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Duansg"><img src="https://avatars.githubusercontent.com/u/112607719?v=4?s=100" width="100px;" alt="Duansg"/><br /><sub><b>Duansg</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Duansg" title="Documentation">📖</a></td>
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/xiaomizhou2"><img src="https://avatars.githubusercontent.com/u/47807926?v=4?s=100" width="100px;" alt="zhangyaxi"/><br /><sub><b>zhangyaxi</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=xiaomizhou2" title="Tests">⚠️</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/RainBondsongyg"><img src="https://avatars.githubusercontent.com/u/94501396?v=4?s=100" width="100px;" alt="songyg"/><br /><sub><b>songyg</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=RainBondsongyg" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
</tr>
</tbody>
</table>
@@ -576,8 +570,8 @@ HertzBeat is built on so many great open source projects, thanks to them!
<p align="left">
<img src="./home/static/img/home/cncf-landscape-left-logo.svg" width="300">&nbsp;&nbsp;<img src="./home/static/img/home/cncf-right-logo.svg" width="345" />
<br /><br />
HertzBeat has been included in the <a href="https://landscape.cncf.io/?item=observability-and-analysis--observability--hertzbeat">
CNCF Observability And Analysis - Observability Landscape.</a>
HertzBeat has been included in the <a href="https://landscape.cncf.io/guide?item=observability-and-analysis--monitoring--hertzbeat#observability-and-analysis--monitoring">
CNCF Observability And Analysis - Monitoring Landscape.</a>
</p>
## 🛡️ License
+3 -3
View File
@@ -1,6 +1,6 @@
# Security Policy
Apache HertzBeat follows the [ASF vulnerability handling process](https://apache.org/security/#vulnerability-handling)
Apache HertzBeat follows the [ASF vulnerability handling process](https://apache.org/security/#vulnerability-handling)
## Supported Versions
@@ -17,6 +17,6 @@ To report a new vulnerability you have discovered please follow the [ASF vulnera
## Extra Information
- [Apache HertzBeat Security Model](https://hertzbeat.apache.org/docs/help/security_model)
- [Apache HertzBeat Security Model](https://hertzbeat.apache.org/docs/help/security_model)
- [ASF Security](https://www.apache.org/security/)
- [Apache HertzBeat Document](https://hertzbeat.apache.org/docs/)
- [Apache HertzBeat Document](https://hertzbeat.apache.org/docs/)
-85
View File
@@ -1,85 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat</artifactId>
<version>2.0-SNAPSHOT</version>
</parent>
<artifactId>hertzbeat-ai-agent</artifactId>
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.1</spring-ai.version>
<java.version>17</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-mcp-server-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common</artifactId>
</dependency>
<dependency>
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-bom</artifactId>
<version>${spring-ai.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.adapters;
import org.springframework.data.domain.Page;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import java.util.List;
/**
* Interface that provides access to monitor information by retrieving monitor data
* through the underlying monitor service.
*/
public interface MonitorServiceAdapter {
Page<Monitor> getMonitors(
List<Long> ids,
String app,
String search,
Byte status,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
String labels
);
}
@@ -1,100 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.adapters.impl;
import com.usthe.sureness.subject.SubjectSum;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.agent.adapters.MonitorServiceAdapter;
import org.apache.hertzbeat.ai.agent.config.McpContextHolder;
import org.springframework.data.domain.Page;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.springframework.stereotype.Component;
import java.lang.reflect.Method;
import java.util.List;
/**
* Implementation of the MonitorServiceAdapter interface that provides access to monitor information
* through reflection by invoking the underlying monitor service implementation.
*/
@Slf4j
@Component
public class MonitorServiceAdapterImpl implements MonitorServiceAdapter {
@Override
public Page<Monitor> getMonitors(
List<Long> ids,
String app,
String search,
Byte status,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
String labels) {
try {
// Provide default values for all nullable parameters
if (sort == null || sort.trim().isEmpty()) {
sort = "gmtCreate";
}
if (order == null || order.trim().isEmpty()) {
order = "desc";
}
if (pageIndex == null) {
pageIndex = 0;
}
if (pageSize == null) {
pageSize = 8;
}
Object monitorService = null;
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current security subject: {}", subjectSum);
try {
monitorService = SpringContextHolder.getBean("monitorServiceImpl");
} catch (Exception e) {
log.debug("Could not find bean by name 'monitorServiceImpl', trying by class name");
}
assert monitorService != null;
log.debug("MonitorService bean found: {}", monitorService.getClass().getSimpleName());
Method method = monitorService.getClass().getMethod(
"getMonitors",
List.class, String.class, String.class, Byte.class,
String.class, String.class, int.class, int.class, String.class);
@SuppressWarnings("unchecked")
Page<Monitor> result = (Page<Monitor>) method.invoke(
monitorService,
ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorServiceAdapter.getMonitors result: {}", result.getContent());
return result;
} catch (NoSuchMethodException e) {
throw new RuntimeException("Method not found: getMonitors", e);
} catch (Exception e) {
log.debug("Failed to invoke getMonitors via adapter", e);
throw new RuntimeException("Failed to invoke getMonitors via adapter", e);
}
}
}
@@ -1,246 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.usthe.sureness.mgt.SurenessSecurityManager;
import com.usthe.sureness.subject.SubjectSum;
import io.modelcontextprotocol.spec.McpError;
import io.modelcontextprotocol.spec.McpSchema;
import io.modelcontextprotocol.spec.McpServerSession;
import io.modelcontextprotocol.spec.McpServerTransport;
import io.modelcontextprotocol.spec.McpServerTransportProvider;
import io.modelcontextprotocol.util.Assert;
import java.io.IOException;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import jakarta.servlet.http.HttpServletRequest;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.RouterFunctions;
import org.springframework.web.servlet.function.ServerRequest;
import org.springframework.web.servlet.function.ServerResponse;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* Custom Server-Sent Events transport provider for Model Context Protocol.
*/
@Slf4j
public class CustomSseServerTransport implements McpServerTransportProvider {
private final ObjectMapper objectMapper;
private final String messageEndpoint;
private final String sseEndpoint;
private final String baseUrl;
@Getter
private final RouterFunction<ServerResponse> routerFunction;
@Setter
private McpServerSession.Factory sessionFactory;
private final Map<String, Object> sessionRequest = new HashMap<>();
private final ConcurrentHashMap<String, McpServerSession> sessions;
private volatile boolean isClosing;
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint) {
this(objectMapper, messageEndpoint, "/sse");
}
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint, String sseEndpoint) {
this(objectMapper, "", messageEndpoint, sseEndpoint);
}
public CustomSseServerTransport(ObjectMapper objectMapper, String baseUrl, String messageEndpoint, String sseEndpoint) {
this.sessions = new ConcurrentHashMap();
this.isClosing = false;
Assert.notNull(objectMapper, "ObjectMapper must not be null");
Assert.notNull(baseUrl, "Message base URL must not be null");
Assert.notNull(messageEndpoint, "Message endpoint must not be null");
Assert.notNull(sseEndpoint, "SSE endpoint must not be null");
this.objectMapper = objectMapper;
this.baseUrl = baseUrl;
this.messageEndpoint = messageEndpoint;
this.sseEndpoint = sseEndpoint;
this.routerFunction = RouterFunctions.route().GET(this.sseEndpoint, this::handleSseConnection).POST(this.messageEndpoint, this::handleMessage).build();
}
public Mono<Void> notifyClients(String method, Object params) {
if (this.sessions.isEmpty()) {
log.debug("No active sessions to broadcast message to");
return Mono.empty();
} else {
log.debug("Attempting to broadcast message to {} active sessions", this.sessions.size());
return Flux.fromIterable(this.sessions.values())
.flatMap((session) -> session.sendNotification(method, params)
.doOnError((e) -> log.error("Failed to send message to session {}: {}", session.getId(), e.getMessage()))
.onErrorComplete())
.then();
}
}
public Mono<Void> closeGracefully() {
return Flux.fromIterable(this.sessions.values()).doFirst(() -> {
this.isClosing = true;
log.debug("Initiating graceful shutdown with {} active sessions", this.sessions.size());
}).flatMap(McpServerSession::closeGracefully).then().doOnSuccess((v) -> log.debug("Graceful shutdown completed"));
}
private ServerResponse handleSseConnection(ServerRequest request) {
log.debug("Handling SSE connection for request: {}", request);
HttpServletRequest servletRequest = request.servletRequest();
try {
log.debug("Processing SSE connection for servlet request: {}", servletRequest);
log.debug("Authorization header: {}", servletRequest.getHeader("Authorization"));
} catch (Exception e) {
log.error("Authentication failed for SSE connection: {}", e.getMessage());
return ServerResponse.status(HttpStatus.UNAUTHORIZED).body("Unauthorized: " + e.getMessage());
}
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else {
String sessionId = UUID.randomUUID().toString();
log.debug("Generated session ID for SSE connection: {}", sessionId);
log.debug("Creating new SSE connection for session: {}", sessionId);
return ServerResponse.sse((sseBuilder) -> {
sseBuilder.onComplete(() -> {
log.debug("SSE connection completed for session: {}", sessionId);
this.sessions.remove(sessionId);
});
sseBuilder.onTimeout(() -> {
log.debug("SSE connection timed out for session: {}", sessionId);
this.sessions.remove(sessionId);
});
CustomSseServerTransport.WebMvcMcpSessionTransport sessionTransport = new CustomSseServerTransport.WebMvcMcpSessionTransport(sessionId, sseBuilder);
McpServerSession session = this.sessionFactory.create(sessionTransport);
this.sessionRequest.put(sessionId, request.servletRequest());
this.sessions.put(sessionId, session);
try {
sseBuilder.id(sessionId).event("endpoint").data(this.baseUrl + this.messageEndpoint + "?sessionId=" + sessionId);
} catch (Exception e) {
log.error("Failed to send initial endpoint event: {}", e.getMessage());
sseBuilder.error(e);
}
}, Duration.ZERO);
}
}
private ServerResponse handleMessage(ServerRequest request) {
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else if (request.param("sessionId").isEmpty()) {
return ServerResponse.badRequest().body(new McpError("Session ID missing in message endpoint"));
} else {
String sessionId = (String) request.param("sessionId").get();
McpServerSession session = (McpServerSession) this.sessions.get(sessionId);
log.debug("Authorization header for message request: {}", request.servletRequest().getHeader("Authorization"));
SubjectSum subject = SurenessSecurityManager.getInstance().checkIn(sessionRequest.get(sessionId));
McpContextHolder.setSubject(subject);
if (session == null) {
return ServerResponse.status(HttpStatus.NOT_FOUND).body(new McpError("Session not found: " + sessionId));
} else {
try {
String body = request.body(String.class);
McpSchema.JSONRPCMessage message = McpSchema.deserializeJsonRpcMessage(this.objectMapper, body);
session.handle(message).block();
return ServerResponse.ok().build();
} catch (IOException | IllegalArgumentException e) {
log.error("Failed to deserialize message: {}", ((Exception) e).getMessage());
return ServerResponse.badRequest().body(new McpError("Invalid message format"));
} catch (Exception e) {
log.error("Error handling message: {}", e.getMessage());
return ServerResponse.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new McpError(e.getMessage()));
}
}
}
}
private class WebMvcMcpSessionTransport implements McpServerTransport {
private final String sessionId;
private final ServerResponse.SseBuilder sseBuilder;
WebMvcMcpSessionTransport(String sessionId, ServerResponse.SseBuilder sseBuilder) {
this.sessionId = sessionId;
this.sseBuilder = sseBuilder;
log.debug("Session transport {} initialized with SSE builder", sessionId);
}
public Mono<Void> sendMessage(McpSchema.JSONRPCMessage message) {
return Mono.fromRunnable(() -> {
try {
String jsonText = CustomSseServerTransport.this.objectMapper.writeValueAsString(message);
this.sseBuilder.id(this.sessionId).event("message").data(jsonText);
log.debug("Message sent to session {}", this.sessionId);
} catch (Exception e) {
log.error("Failed to send message to session {}: {}", this.sessionId, e.getMessage());
this.sseBuilder.error(e);
}
});
}
public <T> T unmarshalFrom(Object data, TypeReference<T> typeRef) {
return (T) CustomSseServerTransport.this.objectMapper.convertValue(data, typeRef);
}
public Mono<Void> closeGracefully() {
return Mono.fromRunnable(() -> {
log.debug("Closing session transport: {}", this.sessionId);
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
});
}
public void close() {
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
}
}
}
@@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Configuration class for Large Language Model (LLM) settings.
*/
@Configuration
public class LlmConfig {
@Bean
public ChatClient openAiChatClient(OpenAiChatModel chatModel) {
return ChatClient.create(chatModel);
}
}
@@ -1,53 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import com.usthe.sureness.subject.SubjectSum;
import org.springframework.core.NamedInheritableThreadLocal;
/**
* Context holder for AI agent security context.
*/
public final class McpContextHolder {
private static final ThreadLocal<SubjectSum> subjectHolder =
new NamedInheritableThreadLocal<>("MCP Security and User Identification Context");
private McpContextHolder() {}
/**
* Attaches the user's context to the current thread.
*/
public static void setSubject(SubjectSum subject) {
subjectHolder.set(subject);
}
/**
* Retrieves the context from the current thread.
*/
public static SubjectSum getSubject() {
return subjectHolder.get();
}
/**
* Clears the context from the thread to prevent memory leaks.
*/
public static void clear() {
subjectHolder.remove();
}
}
@@ -1,53 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import org.springframework.stereotype.Component;
/**
* Provider for system prompts used in the AI agent
*/
@Component
public class PromptProvider {
/**
* Static version of the HertzBeat monitoring prompt
*/
public static final String HERTZBEAT_MONITORING_PROMPT = """
You are an AI assistant specialized in monitoring infrastructure and applications with HertzBeat.
Your role is to help users manage and analyze their monitoring data using the available tools.
You have access to the following HertzBeat monitoring tools:
- list_monitors: Query monitor information with flexible filtering and pagination
- add_monitor: Add a new monitor to the system
When users ask questions about their monitoring setup or data, identify which tool would be most helpful
and use it to provide relevant information. Always provide clear explanations of the monitoring data and
suggest next steps or insights based on the results.
For monitoring-related queries:
1. If users want to see their monitors, use list_monitors with appropriate filters
2. If users want to add a new monitor, use add_monitor with the necessary details
3. If the monitoring information shows potential issues, highlight them and suggest troubleshooting steps
For parameters that accept specific values:
- Monitor status values: 0 (no monitor), 1 (usable), 2 (disabled), 9 (all)
- Sort fields typically include: name, host, app, gmtCreate
- Sort order should be 'asc' or 'desc'
Keep responses focused on monitoring topics and HertzBeat capabilities.
If you're unsure about specific monitoring details, ask clarifying questions before using the tools.
""";
}
@@ -1,70 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.controller;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.http.MediaType;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
import org.apache.hertzbeat.ai.agent.service.ChatClientProviderService;
/**
* Controller class for handling chat-related HTTP requests.
*/
@RestController
@RequestMapping("/api/chat")
public class ChatController {
private final ChatClientProviderService chatClientProviderService;
@Autowired
public ChatController(@Qualifier("openAiChatClient") ChatClient openAiChatClient,
ChatClientProviderService chatClientProviderService) {
this.chatClientProviderService = chatClientProviderService;
}
/**
* Send a message and get a streaming response
*
* @param context The chat request context containing message and optional
* conversationId
* @return SSE emitter for streaming response
*/
@PostMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter streamChat(@RequestBody ChatRequestContext context) {
SseEmitter emitter = new SseEmitter();
new Thread(() -> {
try {
String aiResponse = chatClientProviderService.streamChat(context);
emitter.send(aiResponse);
emitter.complete();
} catch (Exception e) {
emitter.completeWithError(e);
}
}).start();
return emitter;
}
}
@@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.controller;
/**
* Controller for managing conversations.
*/
public class ConversationController {
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for Conversation entities.
*/
public interface ConversationDao {
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for Message entities.
*/
public interface MessageDao {
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for UserPreference entities.
*/
public interface UserPreferenceDao {
}
@@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.pojo.dto;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Chat request context for AI chat endpoint.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ChatRequestContext {
/**
* The user's message (required)
*/
private String message;
/**
* Optional conversation ID for context
*/
private String conversationId;
}
@@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
/**
* Service interface for agent operations.
*/
public interface AgentService {
}
@@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
/**
* Service for interacting with LLM providers (like OpenAI, Anthropic, etc.)
*/
public interface ChatClientProviderService {
String complete(String message);
String streamChat(ChatRequestContext context);
}
@@ -1,71 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.List;
import java.util.Map;
/**
* Service for managing chat conversations and interactions with LLM providers.
*/
public interface ConversationService {
/**
* Send a message and receive a streaming response
*
* @param message The user's message
* @param conversationId Optional conversation ID for continuing a chat
* @return SseEmitter for streaming the response
*/
SseEmitter streamChat(String message, String conversationId);
/**
* Send a message and get a complete response
*
* @param message The user's message
* @param conversationId Optional conversation ID for continuing a chat
* @return Response object containing the AI's response and conversation metadata
*/
Map<String, Object> chat(String message, String conversationId);
/**
* Get conversation history for a specific conversation
*
* @param conversationId Conversation ID
* @return Conversation data including messages
*/
Map<String, Object> getConversation(String conversationId);
/**
* Get all conversations for the current user
*
* @return List of conversations
*/
List<Map<String, Object>> getAllConversations();
/**
* Delete a conversation
*
* @param conversationId Conversation ID to delete
*/
void deleteConversation(String conversationId);
}
@@ -1,28 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.springframework.ai.tool.ToolCallbackProvider;
/**
* Service interface for MCP server operations.
*/
public interface McpServerService {
ToolCallbackProvider hertzbeatTools();
}
@@ -1,30 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.service.AgentService;
import org.springframework.stereotype.Service;
/**
* Implementation of the AgentService interface.
* This service provides functionality for handling AI agent operations.
*/
@Service
public class AgentServiceImpl implements AgentService {
}
@@ -1,70 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.config.PromptProvider;
import org.apache.hertzbeat.ai.agent.service.ChatClientProviderService;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
/**
* Implementation of the {@link ChatClientProviderService}.
* Provides functionality to interact with the ChatClient for handling chat
* messages.
*/
@Service
public class ChatClientProviderServiceImpl implements ChatClientProviderService {
private final ChatClient chatClient;
@Qualifier("hertzbeatTools")
@Autowired
private ToolCallbackProvider toolCallbackProvider;
@Autowired
public ChatClientProviderServiceImpl(@Qualifier("openAiChatClient") ChatClient openAiChatClient) {
this.chatClient = openAiChatClient;
}
@Override
public String complete(String message) {
return this.chatClient.prompt()
.user(message)
.call()
.content();
}
@Override
public String streamChat(ChatRequestContext context) {
try {
return this.chatClient.prompt(PromptProvider.HERTZBEAT_MONITORING_PROMPT)
.user(context.getMessage())
.toolCallbacks(toolCallbackProvider)
.call()
.content();
} catch (Exception e) {
return "Error: " + e.getMessage();
}
}
}
@@ -1,29 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.springframework.stereotype.Service;
/**
* Implementation of the ConversationService interface for managing chat conversations.
*/
@Service
public class ConversationServiceImpl {
}
@@ -1,82 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.config.CustomSseServerTransport;
import org.apache.hertzbeat.ai.agent.service.McpServerService;
import org.springframework.ai.mcp.server.autoconfigure.McpServerProperties;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.tools.impl.MonitorToolsImpl;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.ai.tool.method.MethodToolCallbackProvider;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.ServerResponse;
/**
* Implementation of the McpServerService interface.
* This service provides functionality for handling MCP server operations.
*/
@Service
@Configuration
public class McpServerServiceImpl implements McpServerService {
@Autowired
private MonitorToolsImpl monitorTools;
@Bean
public ToolCallbackProvider hertzbeatTools() {
return MethodToolCallbackProvider.builder().toolObjects(monitorTools).build();
}
/**
* Provides a custom SSE server transport for the MCP server.
*
* @param objectMapper the ObjectMapper instance for JSON serialization
* @param serverProperties the properties for the MCP server configuration
* @return a CustomSseServerTransport instance configured with the provided properties
*/
@Bean
public CustomSseServerTransport webMvcSseServerTransportProvider(
ObjectMapper objectMapper,
McpServerProperties serverProperties
) {
return new CustomSseServerTransport(
objectMapper,
serverProperties.getBaseUrl(),
serverProperties.getSseMessageEndpoint(),
serverProperties.getSseEndpoint()
);
}
/**
* Provides the MCP server transport bean.
*
* @param transport the custom SSE server transport
* @return the MCP server transport instance
*/
@Primary
@Bean
public RouterFunction<ServerResponse> mvcMcpRouterFunction(CustomSseServerTransport transport) {
return transport.getRouterFunction();
}
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
/**
* Tools for alert operations
*/
public interface AlertTools {
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
/**
* Tools for metrics operations
*/
public interface MetricsTools {
}
@@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
import org.springframework.ai.chat.model.ToolContext;
import java.util.List;
/**
* Interface for Monitoring Tools
*/
public interface MonitorTools {
String addMonitor(String name, ToolContext context);
/**
* Query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
* pagination.
* Returns results as plain JSON.
*/
String listMonitors(
List<Long> ids,
String app,
Byte status,
String search,
String labels,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
ToolContext context);
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
/**
* Implementation of Alert Tools functionality
*/
public class AlertToolsImpl {
}
@@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
/**
* Implementation of Metrics Tools functionality
*/
public class MetricsToolsImpl {
}
@@ -1,87 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.agent.adapters.MonitorServiceAdapter;
import org.apache.hertzbeat.ai.agent.config.McpContextHolder;
import org.springframework.ai.chat.model.ToolContext;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.tools.MonitorTools;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import java.util.List;
/**
* Implementation of Monitoring Tools functionality
*/
@Slf4j
@Service
public class MonitorToolsImpl implements MonitorTools {
@Autowired
private MonitorServiceAdapter monitorServiceAdapter;
/**
* Tool to query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
* pagination.
* Returns monitor names as string.
*/
@Override
@Tool(name = "list_monitors", returnDirect = true, description = """
Query monitor information with flexible filtering and pagination.
Supports filtering by monitor IDs, type, status, host, labels, sorting, and pagination.
Returns results as String. When no parameters are available, pass the default value as mentioned below. If the user doesn't provide any specific parameter, the default value will be used.
""")
public String listMonitors(
@ToolParam(description = "List of monitor IDs to filter (default: empty list)", required = false) List<Long> ids,
@ToolParam(description = "Monitor type, e.g., 'linux' (default: null)", required = false) String app,
@ToolParam(description = "Monitor status (0: no monitor, 1: usable, 2: disabled, 9: all) (default: null)", required = false) Byte status,
@ToolParam(description = "Fuzzy search for host or name (default: null)", required = false) String search,
@ToolParam(description = "Monitor labels, e.g., 'env:prod,instance:22' (default: null)", required = false) String labels,
@ToolParam(description = "Sort field, e.g., 'name' (default: gmtCreate)", required = false) String sort,
@ToolParam(description = "Sort order, 'asc' or 'desc' (default: desc)", required = false) String order,
@ToolParam(description = "Page index (default: 0)", required = false) Integer pageIndex,
@ToolParam(description = "Page size (default: 8)", required = false) Integer pageSize,
ToolContext context) {
try {
Page<Monitor> result = monitorServiceAdapter.getMonitors(ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorServiceAdapter.getMonitors result: {}", result);
return result.getContent().stream().map(Monitor::getName).toList().toString();
} catch (Exception e) {
return "error is" + e.getMessage();
}
}
@Override
@Tool(name = "add_monitor", description = "Add a new monitor")
public String addMonitor(@ToolParam(description = "Name of the monitor") String name, ToolContext context) {
log.debug("Adding monitor with name: {}", name);
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current subject in tool: {}", subjectSum);
return "Monitor added: " + name;
}
}
@@ -17,9 +17,6 @@
package org.apache.hertzbeat.alert.calculate;
import com.google.common.collect.Table;
import com.google.common.collect.Tables;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
import org.apache.hertzbeat.alert.util.AlertUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
@@ -27,6 +24,7 @@ import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -35,83 +33,49 @@ import java.util.concurrent.ConcurrentHashMap;
@Component
public class AlarmCacheManager {
private static final String CUSTOM_FIRING_ROW_KEY = "CUSTOM_FIRING_";
/**
* The alarm in the process is triggered
* rowKey - define id
* columnKey - labels fingerprint
* key - labels fingerprint
*/
private final Table<String, String, SingleAlert> pendingAlertMap;
private final Map<String, SingleAlert> pendingAlertMap;
/**
* The not recover alert
* rowKey - define id
* columnKey - labels fingerprint
* key - labels fingerprint
*/
private final Table<String, String, SingleAlert> firingAlertMap;
private final Map<String, SingleAlert> firingAlertMap;
public AlarmCacheManager(SingleAlertDao singleAlertDao) {
this.pendingAlertMap = Tables.newCustomTable(new ConcurrentHashMap<>(8), ConcurrentHashMap::new);
this.firingAlertMap = Tables.newCustomTable(new ConcurrentHashMap<>(8), ConcurrentHashMap::new);
this.pendingAlertMap = new ConcurrentHashMap<>(8);
this.firingAlertMap = new ConcurrentHashMap<>(8);
List<SingleAlert> singleAlerts = singleAlertDao.querySingleAlertsByStatus(CommonConstants.ALERT_STATUS_FIRING);
for (SingleAlert singleAlert : singleAlerts) {
String fingerprint = AlertUtil.calculateFingerprint(singleAlert.getLabels());
String defineId = singleAlert.getLabels().get(CommonConstants.LABEL_DEFINE_ID);
if (StringUtils.isBlank(defineId)) {
defineId = getCustomKey(fingerprint);
}
singleAlert.setId(null);
this.firingAlertMap.put(defineId, fingerprint, singleAlert);
this.firingAlertMap.put(fingerprint, singleAlert);
}
}
public void putPending(Long defineId, String fingerPrint, SingleAlert alert) {
this.pendingAlertMap.put(String.valueOf(defineId), fingerPrint, alert);
public void putPending(String fingerPrint, SingleAlert alert) {
this.pendingAlertMap.put(fingerPrint, alert);
}
public SingleAlert getPending(Long defineId, String fingerPrint) {
return this.pendingAlertMap.get(String.valueOf(defineId), fingerPrint);
public SingleAlert getPending(String fingerPrint) {
return this.pendingAlertMap.get(fingerPrint);
}
public void removePending(Long defineId, String fingerPrint) {
this.pendingAlertMap.remove(String.valueOf(defineId), fingerPrint);
}
public void putFiring(Long defineId, String fingerPrint, SingleAlert alert) {
this.firingAlertMap.put(String.valueOf(defineId), fingerPrint, alert);
public SingleAlert removePending(String fingerPrint) {
return this.pendingAlertMap.remove(fingerPrint);
}
public void putFiring(String fingerPrint, SingleAlert alert) {
this.firingAlertMap.put(getCustomKey(fingerPrint), fingerPrint, alert);
}
public SingleAlert getFiring(Long defineId, String fingerPrint) {
SingleAlert singleAlert = this.firingAlertMap.get(String.valueOf(defineId), fingerPrint);
if (null != singleAlert) {
return singleAlert;
}
return getFiring(fingerPrint);
}
public SingleAlert removeFiring(Long defineId, String fingerPrint) {
SingleAlert singleAlert = this.firingAlertMap.remove(String.valueOf(defineId), fingerPrint);
if (null == singleAlert) {
return this.firingAlertMap.remove(getCustomKey(fingerPrint), fingerPrint);
}
return singleAlert;
this.firingAlertMap.put(fingerPrint, alert);
}
public SingleAlert getFiring(String fingerPrint) {
return this.firingAlertMap.get(getCustomKey(fingerPrint), fingerPrint);
return this.firingAlertMap.get(fingerPrint);
}
private String getCustomKey(String fingerPrint) {
return CUSTOM_FIRING_ROW_KEY + fingerPrint;
}
public SingleAlert removeFiring(String fingerPrint) {
return this.firingAlertMap.remove(getCustomKey(fingerPrint), fingerPrint);
return this.firingAlertMap.remove(fingerPrint);
}
}
@@ -76,12 +76,11 @@ public class CollectorAlertHandler {
fingerPrints.put(KEY_COLLECTOR_VERSION, collector.getVersion());
fingerPrints.put(KEY_COLLECTOR_HOST, collector.getIp());
String fingerprint = AlertUtil.calculateFingerprint(fingerPrints);
SingleAlert firingAlert = alarmCacheManager.removeFiring(fingerprint);
SingleAlert firingAlert = alarmCacheManager.getFiring(fingerprint);
if (firingAlert != null) {
firingAlert.setTriggerTimes(1);
firingAlert.setEndAt(System.currentTimeMillis());
firingAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
firingAlert.setContent(this.bundle.getString("alerter.availability.collector.recover"));
alarmCommonReduce.reduceAndSendAlarm(firingAlert.clone());
}
}
@@ -17,9 +17,8 @@
package org.apache.hertzbeat.alert.calculate;
import java.util.HashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.alert.util.AlertTemplateUtil;
@@ -27,11 +26,11 @@ import org.apache.hertzbeat.alert.util.AlertUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
/**
* Periodic Alert Calculator
@@ -55,9 +54,9 @@ public class PeriodicAlertCalculator {
this.alarmCacheManager = alarmCacheManager;
}
public void calculate(AlertDefine define) {
if (!define.isEnable() || StringUtils.isEmpty(define.getExpr())) {
log.error("Periodic define {} is disabled or expression is empty", define.getName());
public void calculate(AlertDefine rule) {
if (!rule.isEnable() || StringUtils.isEmpty(rule.getExpr())) {
log.error("Periodic rule {} is disabled or expression is empty", rule.getName());
return;
}
long currentTimeMilli = System.currentTimeMillis();
@@ -67,8 +66,8 @@ public class PeriodicAlertCalculator {
// the return result should be matched with threshold
try {
List<Map<String, Object>> results = dataSourceService.calculate(
define.getDatasource(),
define.getExpr()
rule.getDatasource(),
rule.getExpr()
);
// if no match the expr threshold, the results item map {'value': null} should be null and others field keep
// if results has multi list, should trigger multi alert
@@ -78,9 +77,8 @@ public class PeriodicAlertCalculator {
for (Map<String, Object> result : results) {
Map<String, String> fingerPrints = new HashMap<>(8);
// here use the alert name as finger, not care the alert name may be changed
fingerPrints.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(define.getId()));
fingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
fingerPrints.putAll(define.getLabels());
fingerPrints.put(CommonConstants.LABEL_ALERT_NAME, rule.getName());
fingerPrints.putAll(rule.getLabels());
for (Map.Entry<String, Object> entry : result.entrySet()) {
if (entry.getValue() != null && !VALUE.equals(entry.getKey())
&& !TIMESTAMP.equals(entry.getKey())) {
@@ -89,33 +87,32 @@ public class PeriodicAlertCalculator {
}
if (result.get(VALUE) == null) {
// recovery the alert
handleRecoveredAlert(define.getId(), fingerPrints);
handleRecoveredAlert(fingerPrints);
continue;
}
Map<String, Object> fieldValueMap = new HashMap<>(8);
fieldValueMap.putAll(define.getLabels());
fieldValueMap.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
fieldValueMap.putAll(rule.getLabels());
fieldValueMap.put(CommonConstants.LABEL_ALERT_NAME, rule.getName());
for (Map.Entry<String, Object> entry : result.entrySet()) {
if (entry.getValue() != null) {
fieldValueMap.put(entry.getKey(), entry.getValue());
}
}
afterThresholdRuleMatch(currentTimeMilli, fingerPrints, fieldValueMap, define);
afterThresholdRuleMatch(currentTimeMilli, fingerPrints, fieldValueMap, rule);
}
} catch (Exception ignored) {
// ignore the query exception eg: no result, timeout, etc
return;
}
} catch (Exception e) {
log.error("Calculate periodic define {} failed: {}", define.getName(), e.getMessage());
log.error("Calculate periodic rule {} failed: {}", rule.getName(), e.getMessage());
}
}
private void afterThresholdRuleMatch(long currentTimeMilli, Map<String, String> fingerPrints,
Map<String, Object> fieldValueMap, AlertDefine define) {
Long defineId = define.getId();
String fingerprint = AlertUtil.calculateFingerprint(fingerPrints);
SingleAlert existingAlert = alarmCacheManager.getPending(defineId, fingerprint);
SingleAlert existingAlert = alarmCacheManager.getPending(fingerprint);
Map<String, String> labels = new HashMap<>(8);
fieldValueMap.putAll(define.getLabels());
labels.putAll(fingerPrints);
@@ -136,11 +133,11 @@ public class PeriodicAlertCalculator {
// If required trigger times is 1, set to firing status directly
if (requiredTimes <= 1) {
newAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
alarmCacheManager.putFiring(defineId, fingerprint, newAlert);
alarmCacheManager.putFiring(fingerprint, newAlert);
alarmCommonReduce.reduceAndSendAlarm(newAlert.clone());
} else {
// Otherwise put into pending queue first
alarmCacheManager.putPending(defineId, fingerprint, newAlert);
alarmCacheManager.putPending(fingerprint, newAlert);
}
} else {
// Update existing alert
@@ -150,17 +147,17 @@ public class PeriodicAlertCalculator {
// Check if required trigger times reached
if (existingAlert.getStatus().equals(CommonConstants.ALERT_STATUS_PENDING) && existingAlert.getTriggerTimes() >= requiredTimes) {
// Reached trigger times threshold, change to firing status
alarmCacheManager.removePending(defineId, fingerprint);
alarmCacheManager.removePending(fingerprint);
existingAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
alarmCacheManager.putFiring(defineId, fingerprint, existingAlert);
alarmCacheManager.putFiring(fingerprint, existingAlert);
alarmCommonReduce.reduceAndSendAlarm(existingAlert.clone());
}
}
}
private void handleRecoveredAlert(Long defineId, Map<String, String> fingerprints) {
private void handleRecoveredAlert(Map<String, String> fingerprints) {
String fingerprint = AlertUtil.calculateFingerprint(fingerprints);
SingleAlert firingAlert = alarmCacheManager.removeFiring(defineId, fingerprint);
SingleAlert firingAlert = alarmCacheManager.removeFiring(fingerprint);
if (firingAlert != null) {
// todo consider multi times to tig for resolved alert
firingAlert.setTriggerTimes(1);
@@ -168,7 +165,7 @@ public class PeriodicAlertCalculator {
firingAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
alarmCommonReduce.reduceAndSendAlarm(firingAlert.clone());
}
alarmCacheManager.removePending(defineId, fingerprint);
alarmCacheManager.removePending(fingerprint);
}
}
@@ -183,11 +183,9 @@ public class RealTimeAlertCalculator {
if (StringUtils.isBlank(expr)) {
continue;
}
Long defineId = define.getId();
Map<String, String> commonFingerPrints = new HashMap<>(8);
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE, instance);
// here use the alert name as finger, not care the alert name may be changed
commonFingerPrints.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(define.getId()));
commonFingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_NAME, instanceName);
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_HOST, instanceHost);
@@ -202,9 +200,9 @@ public class RealTimeAlertCalculator {
try {
if (match) {
// If the threshold rule matches, the number of times the threshold has been triggered is determined and an alarm is triggered
afterThresholdRuleMatch(defineId, currentTimeMilli, commonFingerPrints, fieldValueMap, define, annotations);
afterThresholdRuleMatch(currentTimeMilli, commonFingerPrints, fieldValueMap, define, annotations);
} else {
handleRecoveredAlert(defineId, commonFingerPrints);
handleRecoveredAlert(commonFingerPrints);
}
// if this threshold pre compile success, ignore blew
continue;
@@ -256,9 +254,9 @@ public class RealTimeAlertCalculator {
boolean match = execAlertExpression(fieldValueMap, expr, false);
try {
if (match) {
afterThresholdRuleMatch(defineId, currentTimeMilli, fingerPrints, fieldValueMap, define, annotations);
afterThresholdRuleMatch(currentTimeMilli, fingerPrints, fieldValueMap, define, annotations);
} else {
handleRecoveredAlert(defineId, fingerPrints);
handleRecoveredAlert(fingerPrints);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
@@ -336,9 +334,9 @@ public class RealTimeAlertCalculator {
.collect(Collectors.toList());
}
private void handleRecoveredAlert(Long defineId, Map<String, String> fingerprints) {
private void handleRecoveredAlert(Map<String, String> fingerprints) {
String fingerprint = AlertUtil.calculateFingerprint(fingerprints);
SingleAlert firingAlert = alarmCacheManager.removeFiring(defineId, fingerprint);
SingleAlert firingAlert = alarmCacheManager.removeFiring(fingerprint);
if (firingAlert != null) {
// todo consider multi times to tig for resolved alert
firingAlert.setTriggerTimes(1);
@@ -346,14 +344,13 @@ public class RealTimeAlertCalculator {
firingAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
alarmCommonReduce.reduceAndSendAlarm(firingAlert.clone());
}
alarmCacheManager.removePending(defineId, fingerprint);
alarmCacheManager.removePending(fingerprint);
}
private void afterThresholdRuleMatch(long defineId, long currentTimeMilli, Map<String, String> fingerPrints,
Map<String, Object> fieldValueMap, AlertDefine define,
Map<String, String> annotations) {
private void afterThresholdRuleMatch(long currentTimeMilli, Map<String, String> fingerPrints,
Map<String, Object> fieldValueMap, AlertDefine define, Map<String, String> annotations) {
String fingerprint = AlertUtil.calculateFingerprint(fingerPrints);
SingleAlert existingAlert = alarmCacheManager.getPending(defineId, fingerprint);
SingleAlert existingAlert = alarmCacheManager.getPending(fingerprint);
fieldValueMap.putAll(define.getLabels());
int requiredTimes = define.getTimes() == null ? 1 : define.getTimes();
if (existingAlert == null) {
@@ -385,11 +382,11 @@ public class RealTimeAlertCalculator {
// If required trigger times is 1, set to firing status directly
if (requiredTimes <= 1) {
newAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
alarmCacheManager.putFiring(defineId, fingerprint, newAlert);
alarmCacheManager.putFiring(fingerprint, newAlert);
alarmCommonReduce.reduceAndSendAlarm(newAlert.clone());
} else {
// Otherwise put into pending queue first
alarmCacheManager.putPending(define.getId(), fingerprint, newAlert);
alarmCacheManager.putPending(fingerprint, newAlert);
}
} else {
// Update existing alert
@@ -399,9 +396,9 @@ public class RealTimeAlertCalculator {
// Check if required trigger times reached
if (existingAlert.getStatus().equals(CommonConstants.ALERT_STATUS_PENDING) && existingAlert.getTriggerTimes() >= requiredTimes) {
// Reached trigger times threshold, change to firing status
alarmCacheManager.removePending(defineId, fingerprint);
alarmCacheManager.removePending(fingerprint);
existingAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
alarmCacheManager.putFiring(defineId, fingerprint, existingAlert);
alarmCacheManager.putFiring(fingerprint, existingAlert);
alarmCommonReduce.reduceAndSendAlarm(existingAlert.clone());
}
}
@@ -17,16 +17,17 @@
package org.apache.hertzbeat.alert.controller;
import static org.apache.hertzbeat.common.constants.CommonConstants.MONITOR_NOT_EXIST_CODE;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import java.util.Objects;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -35,17 +36,8 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE;
import static org.apache.hertzbeat.common.constants.CommonConstants.MONITOR_NOT_EXIST_CODE;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
/**
* Alarm definition management API
*/
@@ -98,18 +90,4 @@ public class AlertDefineController {
return ResponseEntity.ok(Message.success("Delete success"));
}
@GetMapping(path = "/preview/{datasource}")
@Operation(summary = "Alarm definition expression preview",
description = "If the expression is formal, then the result of the query will be returned, otherwise it will respond with an error")
public ResponseEntity<Message<List<Map<String, Object>>>> getDefinePreview(
@Parameter(description = "Data Source Type", example = "promql") @PathVariable("datasource") String datasource,
@Parameter(description = "alert threshold type:realtime,periodic") @RequestParam String type,
@Parameter(description = "alert threshold expression") @RequestParam String expr) {
try {
return ResponseEntity.ok(Message.successWithData(alertDefineService.getDefinePreview(datasource, type, expr)));
} catch (AlertExpressionException ae) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Message.fail(FAIL_CODE, ae.getMessage()));
}
}
}
@@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* volcengine event alert entity class
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class VolcEngineExternEventAlert {
@JsonProperty("Type")
private String type;
@JsonProperty("AccountId")
private String accountId;
@JsonProperty("Source")
private String source;
@JsonProperty("ProjectName")
private String projectName;
@JsonProperty("Id")
private String id;
@JsonProperty("EventType")
private String eventType;
@JsonProperty("DescriptionCN")
private String descriptionCn;
@JsonProperty("HappenedAt")
private Long happenedAt;
@JsonProperty("Region")
private String region;
@JsonProperty("Details")
private EventDetails details;
@JsonProperty("Rules")
private List<EventRule> rules;
/**
* event details
*/
@Data
public static class EventDetails {
@JsonProperty("specversion")
private String specVersion;
@JsonProperty("id")
private String id;
@JsonProperty("source")
private String source;
@JsonProperty("type")
private String type;
@JsonProperty("volcpublishtime")
private String volcPublishTime;
@JsonProperty("volcregion")
private String volcRegion;
@JsonProperty("volcaccountid")
private String volcAccountId;
@JsonProperty("volceventbusname")
private String volcEventBusName;
@JsonProperty("volcresourcename")
private String volcResourceName;
@JsonProperty("subject")
private String subject;
@JsonProperty("time")
private String time;
}
/**
* event rule
*/
@Data
public static class EventRule {
@JsonProperty("RuleId")
private String ruleId;
@JsonProperty("RuleName")
private String ruleName;
@JsonProperty("Level")
private String level;
}
}
@@ -1,203 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* Volcengine alarm entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class VolcEngineExternMetricAlert {
@JsonProperty("Type")
private String type;
@JsonProperty("AccountId")
private String accountId;
@JsonProperty("RuleName")
private String ruleName;
@JsonProperty("RuleId")
private String ruleId;
@JsonProperty("Namespace")
private String namespace;
@JsonProperty("SubNamespace")
private String subNamespace;
@JsonProperty("Level")
private String level;
@JsonProperty("HappenedAt")
private String happenedAt;
@JsonProperty("RuleCondition")
private String ruleCondition;
@JsonProperty("Resources")
private List<Resource> resources;
@JsonProperty("RecoveredResources")
private List<Resource> recoveredResources;
@JsonProperty("NoDataResources")
private List<NoDataResource> noDataResources;
@JsonProperty("NoDataRecoveredResources")
private List<NoDataRecoveredResource> noDataRecoveredResources;
/**
* alert resource entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class Resource {
@JsonProperty("Id")
private String id;
@JsonProperty("Name")
private String name;
@JsonProperty("Region")
private String region;
@JsonProperty("FirstAlertTime")
private Long firstAlertTime;
@JsonProperty("LastAlertTime")
private Long lastAlertTime;
@JsonProperty("Metrics")
private List<Metric> metrics;
@JsonProperty("AlertGroupId")
private String alertGroupId;
@JsonProperty("Dimensions")
private List<Dimension> dimensions;
}
/**
* alert no data resource entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public static class NoDataResource extends Resource {
@JsonProperty("NoDataMetrics")
private List<Metric> noDataMetrics;
}
/**
* alert no data recovered resource entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public static class NoDataRecoveredResource extends Resource {
@JsonProperty("DroppedMetrics")
private List<DroppedMetric> droppedMetrics;
}
/**
* alert metric entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class Metric {
@JsonProperty("Name")
private String name;
@JsonProperty("Unit")
private String unit;
@JsonProperty("Threshold")
private Double threshold;
@JsonProperty("CurrentValue")
private Object currentValue;
@JsonProperty("Description")
private String description;
@JsonProperty("Warning")
private Boolean warning;
}
/**
* alert dropped metric entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public static class DroppedMetric extends Metric {
@JsonProperty("Reason")
private String reason;
}
/**
* resource dimension entity class
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public static class Dimension {
@JsonProperty("Name")
private String name;
@JsonProperty("NameCN")
private String nameCn;
@JsonProperty("Value")
private String value;
@JsonProperty("Description")
private String description;
}
public static final String ALERT_TYPE_METRIC = "Metric";
public static final String ALERT_TYPE_EVENT = "Event";
public static final String ALERT_TYPE_METRIC_RECOVERED = "MetricRecovered";
public static final String ALERT_TYPE_METRICS_NODATA = "MetricsNoData";
public static final String ALERT_TYPE_NO_DATA_RECOVERED = "NoDataRecovered";
}
@@ -27,7 +27,7 @@ import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
* of the available methods.
*
* @param <T> The return type of the visit operation. Use {@link Void} for
* operations with no return type.
* operations with no return type.
*/
@SuppressWarnings("CheckReturnValue")
public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements AlertExpressionVisitor<T> {
@@ -37,10 +37,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitExpression(AlertExpressionParser.ExpressionContext ctx) {
return visitChildren(ctx);
}
@Override public T visitExpression(AlertExpressionParser.ExpressionContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -48,10 +45,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitAndExpr(AlertExpressionParser.AndExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitAndExpr(AlertExpressionParser.AndExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -59,10 +53,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitComparisonExpr(AlertExpressionParser.ComparisonExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitComparisonExpr(AlertExpressionParser.ComparisonExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -70,10 +61,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitUnlessExpr(AlertExpressionParser.UnlessExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitUnlessExpr(AlertExpressionParser.UnlessExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -81,10 +69,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitSqlExpr(AlertExpressionParser.SqlExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitSqlExpr(AlertExpressionParser.SqlExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -92,10 +77,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitSqlCallExpr(AlertExpressionParser.SqlCallExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitSqlCallExpr(AlertExpressionParser.SqlCallExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -103,10 +85,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLiteralExpr(AlertExpressionParser.LiteralExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLiteralExpr(AlertExpressionParser.LiteralExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -114,10 +93,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitParenExpr(AlertExpressionParser.ParenExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitParenExpr(AlertExpressionParser.ParenExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -125,10 +101,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitPromqlCallExpr(AlertExpressionParser.PromqlCallExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitPromqlCallExpr(AlertExpressionParser.PromqlCallExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -136,10 +109,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitPromqlExpr(AlertExpressionParser.PromqlExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitPromqlExpr(AlertExpressionParser.PromqlExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -147,10 +117,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitOrExpr(AlertExpressionParser.OrExprContext ctx) {
return visitChildren(ctx);
}
@Override public T visitOrExpr(AlertExpressionParser.OrExprContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -158,10 +125,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitFunctionCall(AlertExpressionParser.FunctionCallContext ctx) {
return visitChildren(ctx);
}
@Override public T visitFunctionCall(AlertExpressionParser.FunctionCallContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -169,10 +133,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitParameterList(AlertExpressionParser.ParameterListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitParameterList(AlertExpressionParser.ParameterListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -180,10 +141,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitParameter(AlertExpressionParser.ParameterContext ctx) {
return visitChildren(ctx);
}
@Override public T visitParameter(AlertExpressionParser.ParameterContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -191,10 +149,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitNumber(AlertExpressionParser.NumberContext ctx) {
return visitChildren(ctx);
}
@Override public T visitNumber(AlertExpressionParser.NumberContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -202,10 +157,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitString(AlertExpressionParser.StringContext ctx) {
return visitChildren(ctx);
}
@Override public T visitString(AlertExpressionParser.StringContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -213,10 +165,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitDuration(AlertExpressionParser.DurationContext ctx) {
return visitChildren(ctx);
}
@Override public T visitDuration(AlertExpressionParser.DurationContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -224,10 +173,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitFunctionName(AlertExpressionParser.FunctionNameContext ctx) {
return visitChildren(ctx);
}
@Override public T visitFunctionName(AlertExpressionParser.FunctionNameContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -235,10 +181,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitSelectSql(AlertExpressionParser.SelectSqlContext ctx) {
return visitChildren(ctx);
}
@Override public T visitSelectSql(AlertExpressionParser.SelectSqlContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -246,10 +189,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitSelectFieldList(AlertExpressionParser.SelectFieldListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitSelectFieldList(AlertExpressionParser.SelectFieldListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -257,10 +197,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitSelectField(AlertExpressionParser.SelectFieldContext ctx) {
return visitChildren(ctx);
}
@Override public T visitSelectField(AlertExpressionParser.SelectFieldContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -268,10 +205,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitGroupByList(AlertExpressionParser.GroupByListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitGroupByList(AlertExpressionParser.GroupByListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -279,10 +213,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitOrderByList(AlertExpressionParser.OrderByListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitOrderByList(AlertExpressionParser.OrderByListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -290,10 +221,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitOrderByField(AlertExpressionParser.OrderByFieldContext ctx) {
return visitChildren(ctx);
}
@Override public T visitOrderByField(AlertExpressionParser.OrderByFieldContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -301,10 +229,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLimitClause(AlertExpressionParser.LimitClauseContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLimitClause(AlertExpressionParser.LimitClauseContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -312,10 +237,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitRelList(AlertExpressionParser.RelListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitRelList(AlertExpressionParser.RelListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -323,10 +245,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitRelation(AlertExpressionParser.RelationContext ctx) {
return visitChildren(ctx);
}
@Override public T visitRelation(AlertExpressionParser.RelationContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -334,10 +253,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitConditionList(AlertExpressionParser.ConditionListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitConditionList(AlertExpressionParser.ConditionListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -345,10 +261,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitCompOp(AlertExpressionParser.CompOpContext ctx) {
return visitChildren(ctx);
}
@Override public T visitCompOp(AlertExpressionParser.CompOpContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -356,10 +269,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitCondition(AlertExpressionParser.ConditionContext ctx) {
return visitChildren(ctx);
}
@Override public T visitCondition(AlertExpressionParser.ConditionContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -367,10 +277,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitConditionUnit(AlertExpressionParser.ConditionUnitContext ctx) {
return visitChildren(ctx);
}
@Override public T visitConditionUnit(AlertExpressionParser.ConditionUnitContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -378,10 +285,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitPromql(AlertExpressionParser.PromqlContext ctx) {
return visitChildren(ctx);
}
@Override public T visitPromql(AlertExpressionParser.PromqlContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -389,10 +293,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitMetricSelector(AlertExpressionParser.MetricSelectorContext ctx) {
return visitChildren(ctx);
}
@Override public T visitMetricSelector(AlertExpressionParser.MetricSelectorContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -400,10 +301,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLabelMatcherList(AlertExpressionParser.LabelMatcherListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLabelMatcherList(AlertExpressionParser.LabelMatcherListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -411,10 +309,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLabelMatcherItem(AlertExpressionParser.LabelMatcherItemContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLabelMatcherItem(AlertExpressionParser.LabelMatcherItemContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -422,10 +317,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLabelMatcherOp(AlertExpressionParser.LabelMatcherOpContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLabelMatcherOp(AlertExpressionParser.LabelMatcherOpContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -433,10 +325,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitLabelList(AlertExpressionParser.LabelListContext ctx) {
return visitChildren(ctx);
}
@Override public T visitLabelList(AlertExpressionParser.LabelListContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -444,10 +333,7 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitInstantVectorOp(AlertExpressionParser.InstantVectorOpContext ctx) {
return visitChildren(ctx);
}
@Override public T visitInstantVectorOp(AlertExpressionParser.InstantVectorOpContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
@@ -455,8 +341,13 @@ public class AlertExpressionBaseVisitor<T> extends AbstractParseTreeVisitor<T> i
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override
public T visitAggregationOperator(AlertExpressionParser.AggregationOperatorContext ctx) {
return visitChildren(ctx);
}
@Override public T visitAggregationOperator(AlertExpressionParser.AggregationOperatorContext ctx) { return visitChildren(ctx); }
/**
* {@inheritDoc}
*
* <p>The default implementation returns the result of calling
* {@link #visitChildren} on {@code ctx}.</p>
*/
@Override public T visitBinaryOperator(AlertExpressionParser.BinaryOperatorContext ctx) { return visitChildren(ctx); }
}
@@ -18,26 +18,21 @@
package org.apache.hertzbeat.alert.expr;
import org.antlr.v4.runtime.CommonTokenStream;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.hertzbeat.common.support.exception.ExpressionVisitorException;
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Alert expression visitor implement
*/
public class AlertExpressionEvalVisitor extends AlertExpressionBaseVisitor<List<Map<String, Object>>> {
private static final String SCALAR = "__scalar__";
private static final String NAME = "__name__";
private static final String THRESHOLD = "__threshold__";
private static final String VALUE = "__value__";
private static final String TIMESTAMP = "__timestamp__";
private final QueryExecutor executor;
private final CommonTokenStream tokens;
@@ -61,126 +56,70 @@ public class AlertExpressionEvalVisitor extends AlertExpressionBaseVisitor<List<
public List<Map<String, Object>> visitComparisonExpr(AlertExpressionParser.ComparisonExprContext ctx) {
List<Map<String, Object>> leftResult = visit(ctx.left);
List<Map<String, Object>> rightResult = visit(ctx.right);
int type = ctx.op.getType();
boolean boolModifier = ctx.BOOL() != null;
boolean leftIsScalar = isScalar(leftResult);
boolean rightIsScalar = isScalar(rightResult);
List<Map<String, Object>> results = new ArrayList<>();
if (rightResult.size() == 1 && rightResult.get(0).containsKey(THRESHOLD)) {
double threshold = (double) rightResult.get(0).get(THRESHOLD);
String operator = ctx.op.getText();
// scalar and scalar
if (leftIsScalar && rightIsScalar) {
if (!boolModifier) {
// Between two scalars,
// the bool modifier must be provided and these operators result in another scalar that is either 0 (false) or 1 (true), depending on the comparison result.
return results;
}
Object leftVal = leftResult.get(0).get(SCALAR);
Object rightVal = rightResult.get(0).get(SCALAR);
Boolean match = compareOp(leftVal, type, rightVal);
// returns a result only if the comparison condition is met.
Map<String, Object> result = new HashMap<>();
result.put(VALUE, match ? 1 : 0);
results.add(result);
return results;
}
// scalar and vector
if (leftIsScalar) {
Object leftVal = leftResult.get(0).get(SCALAR);
for (Map<String, Object> rightItem : rightResult) {
Object rightVal = rightItem.getOrDefault(VALUE, null);
if (isValidValue(rightVal)) {
List<Map<String, Object>> result = new ArrayList<>();
for (Map<String, Object> item : leftResult) {
Object queryValues = item.get(VALUE);
if (queryValues == null) {
// ignore the query result data is empty
continue;
}
Boolean match = compareOp(leftVal, type, rightVal);
Map<String, Object> result = new HashMap<>(rightItem);
if (boolModifier) {
result.put(VALUE, match ? 1 : 0);
results.add(result);
} else {
result.put(VALUE, match ? rightVal : null);
results.add(result);
}
// queryValues may be a list of values, or a single value
Object matchValue = evaluateCondition(queryValues, operator, threshold);
Map<String, Object> resultMap = new HashMap(item);
resultMap.put(VALUE, matchValue);
// if matchValue is null, mean not match the threshold
// if not null, mean match the threshold
result.add(resultMap);
}
return results;
return result;
}
// vector and scalar
if (rightIsScalar) {
Object rightVal = rightResult.get(0).get(SCALAR);
for (Map<String, Object> leftItem : leftResult) {
Object leftVal = leftItem.getOrDefault(VALUE, null);
if (isValidValue(leftVal)) {
continue;
}
Boolean match = compareOp(leftVal, type, rightVal);
Map<String, Object> result = new HashMap<>(leftItem);
if (boolModifier) {
result.put(VALUE, match ? 1 : 0);
results.add(result);
} else {
result.put(VALUE, match ? leftVal : null);
results.add(result);
}
}
return results;
}
// vector and vector
Map<String, Map<String, Object>> rightMap = rightResult.stream()
.filter(item -> item.get(VALUE) != null)
.collect(Collectors.toMap(this::labelKey, item -> item, (existing, replacement) -> existing));
for (Map<String, Object> leftItem : leftResult) {
Object leftVal = leftItem.getOrDefault(VALUE, null);
if (isValidValue(leftVal)) {
continue;
}
Map<String, Object> rightItem = rightMap.get(labelKey(leftItem));
if (rightItem == null) {
continue;
}
Object rightVal = rightItem.get(VALUE);
if (isValidValue(rightVal)) {
continue;
}
Boolean match = compareOp(leftVal, type, rightVal);
Map<String, Object> result = new HashMap<>(leftItem);
if (boolModifier) {
result.put(VALUE, match ? 1 : 0);
results.add(result);
} else {
result.put(VALUE, match ? leftVal : null);
results.add(result);
}
}
return results;
return new LinkedList<>();
}
@Override
public List<Map<String, Object>> visitAndExpr(AlertExpressionParser.AndExprContext ctx) {
List<Map<String, Object>> leftOperand = visit(ctx.left);
List<Map<String, Object>> rightOperand = visit(ctx.right);
List<Map<String, Object>> results = new ArrayList<>();
// build a hash set of the right-side tag collection
Set<String> rightLabelsSet = rightOperand.stream()
.filter(item -> item.get(VALUE) != null)
.map(this::labelKey)
.collect(Collectors.toSet());
// iterate over the left side, O(1) match
for (Map<String, Object> leftItem : leftOperand) {
Object leftVal = leftItem.get(VALUE);
if (leftVal == null) {
continue;
Map<String, Object> leftMap = null;
boolean leftMatch = false;
Map<String, Object> rightMap = null;
boolean rightMatch = false;
for (Map<String, Object> item : leftOperand) {
if (leftMap == null) {
leftMap = item;
}
String labelKey = labelKey(leftItem);
if (rightLabelsSet.contains(labelKey)) {
results.add(new HashMap<>(leftItem));
if (item.get(VALUE) != null) {
leftMap = item;
leftMatch = true;
break;
}
}
return results;
for (Map<String, Object> item : rightOperand) {
if (rightMap == null) {
rightMap = item;
}
if (item.get(VALUE) != null) {
rightMap = item;
rightMatch = true;
break;
}
}
if (leftMatch && rightMatch) {
rightMap.putAll(leftMap);
return new LinkedList<>(List.of(rightMap));
} else if (leftMap != null) {
leftMap.put(VALUE, null);
return new LinkedList<>(List.of(leftMap));
} else if (rightMap != null) {
rightMap.put(VALUE, null);
return new LinkedList<>(List.of(rightMap));
}
return new LinkedList<>();
}
@Override
@@ -188,51 +127,92 @@ public class AlertExpressionEvalVisitor extends AlertExpressionBaseVisitor<List<
List<Map<String, Object>> leftOperand = visit(ctx.left);
List<Map<String, Object>> rightOperand = visit(ctx.right);
// build a hashMap of the left-hand label collection
Map<String, Map<String, Object>> leftLabelMap = leftOperand.stream()
.filter(item -> item.get(VALUE) != null)
.collect(Collectors.toMap(this::labelKey, HashMap::new, (k1, k2) -> k1));
// first add all the non-empty items on the left side
List<Map<String, Object>> results = new ArrayList<>(leftLabelMap.values());
// add the term that has a value on the right side and not on the left side
for (Map<String, Object> rightItem : rightOperand) {
Object rightVal = rightItem.get(VALUE);
if (rightVal == null) {
continue;
Map<String, Object> leftMap = null;
boolean leftMatch = false;
Map<String, Object> rightMap = null;
boolean rightMatch = false;
for (Map<String, Object> item : leftOperand) {
if (leftMap == null) {
leftMap = item;
}
String key = labelKey(rightItem);
if (!leftLabelMap.containsKey(key)) {
results.add(new HashMap<>(rightItem));
if (item.get(VALUE) != null) {
leftMap = item;
leftMatch = true;
break;
}
}
return results;
for (Map<String, Object> item : rightOperand) {
if (rightMap == null) {
rightMap = item;
}
if (item.get(VALUE) != null) {
rightMap = item;
rightMatch = true;
break;
}
}
if (leftMatch && rightMatch) {
rightMap.putAll(leftMap);
return new LinkedList<>(List.of(rightMap));
} else if (leftMatch) {
return new LinkedList<>(List.of(leftMap));
} else if (rightMatch) {
return new LinkedList<>(List.of(rightMap));
} else {
if (leftMap != null && rightMap != null) {
rightMap.putAll(leftMap);
return new LinkedList<>(List.of(rightMap));
} else if (leftMap != null) {
return new LinkedList<>(List.of(leftMap));
} else if (rightMap != null) {
return new LinkedList<>(List.of(rightMap));
}
}
return new LinkedList<>();
}
@Override
public List<Map<String, Object>> visitUnlessExpr(AlertExpressionParser.UnlessExprContext ctx) {
List<Map<String, Object>> leftOperand = visit(ctx.left);
List<Map<String, Object>> rightOperand = visit(ctx.right);
List<Map<String, Object>> results = new ArrayList<>();
// build a hash set of the right-side tag collection
Set<String> rightLabelSet = rightOperand.stream()
.filter(item -> item.get(VALUE) != null)
.map(this::labelKey)
.collect(Collectors.toSet());
// iterate over the left side, O(1) match
for (Map<String, Object> leftItem : leftOperand) {
Object leftVal = leftItem.get(VALUE);
if (leftVal == null) {
continue;
Map<String, Object> leftMap = null;
boolean leftMatch = false;
Map<String, Object> rightMap = null;
boolean rightMatch = false;
for (Map<String, Object> item : leftOperand) {
if (leftMap == null) {
leftMap = item;
}
if (!rightLabelSet.contains(labelKey(leftItem))) {
results.add(new HashMap<>(leftItem));
if (item.get(VALUE) != null) {
leftMap = item;
leftMatch = true;
break;
}
}
return results;
for (Map<String, Object> item : rightOperand) {
if (rightMap == null) {
rightMap = item;
}
if (item.get(VALUE) != null) {
rightMap = item;
rightMatch = true;
break;
}
}
if (leftMatch && !rightMatch) {
return new LinkedList<>(List.of(leftMap));
} else {
if (leftMap != null) {
leftMap.put(VALUE, null);
return new LinkedList<>(List.of(leftMap));
} else {
if (rightMap != null) {
rightMap.put(VALUE, null);
return new LinkedList<>(List.of(rightMap));
}
}
}
return new LinkedList<>();
}
@Override
@@ -240,7 +220,7 @@ public class AlertExpressionEvalVisitor extends AlertExpressionBaseVisitor<List<
double value = Double.parseDouble(ctx.number().getText());
List<Map<String, Object>> numAsList = new ArrayList<>();
Map<String, Object> valueMap = new HashMap<>();
valueMap.put(SCALAR, value);
valueMap.put(THRESHOLD, value);
numAsList.add(valueMap);
return numAsList;
}
@@ -267,56 +247,84 @@ public class AlertExpressionEvalVisitor extends AlertExpressionBaseVisitor<List<
return callSqlOrPromql(tokens.getText(ctx.string()));
}
private List<Map<String, Object>> callSqlOrPromql(String text) {
private Object evaluateCondition(Object value, String operator, Double threshold) {
// value may be a list of values, or a single value
switch (operator) {
case ">":
// if value is list, return the max value
if (value instanceof List<?> values) {
Double doubleValue = values.stream().map(v -> Double.valueOf(v.toString())).max(Double::compareTo).orElse(null);
if (doubleValue != null) {
return doubleValue > threshold ? doubleValue : null;
} else {
return null;
}
} else {
return Double.parseDouble(value.toString()) > threshold ? value : null;
}
case ">=":
if (value instanceof List<?> values) {
Double doubleValue = values.stream().map(v -> Double.valueOf(v.toString())).max(Double::compareTo).orElse(null);
if (doubleValue != null) {
return doubleValue >= threshold ? doubleValue : null;
} else {
return null;
}
} else {
return Double.parseDouble(value.toString()) >= threshold ? value : null;
}
case "<":
if (value instanceof List<?> values) {
Double doubleValue = values.stream().map(v -> Double.valueOf(v.toString())).min(Double::compareTo).orElse(null);
if (doubleValue != null) {
return doubleValue < threshold ? doubleValue : null;
} else {
return null;
}
} else {
return Double.parseDouble(value.toString()) < threshold ? value : null;
}
case "<=":
if (value instanceof List<?> values) {
Double doubleValue = values.stream().map(v -> Double.valueOf(v.toString())).min(Double::compareTo).orElse(null);
if (doubleValue != null) {
return doubleValue <= threshold ? doubleValue : null;
} else {
return null;
}
} else {
return Double.parseDouble(value.toString()) <= threshold ? value : null;
}
case "==":
if (value instanceof List<?> values) {
for (Object v : values) {
if (v.equals(threshold)) {
return v;
}
}
return null;
} else {
return value.equals(threshold) ? value : null;
}
case "!=":
if (value instanceof List<?> values) {
for (Object v : values) {
if (v.equals(threshold)) {
return null;
}
}
return value;
} else {
return value.equals(threshold) ? null : value;
}
default:
// unsupported operator todo add more operator
return null;
}
}
private List<Map<String, Object>> callSqlOrPromql(String text){
String script = text.substring(1, text.length() - 1);
return executor.execute(script);
}
/**
* Generate tag key (excluding `__name__` and `__value__` and `__timestamp__`)
*/
private String labelKey(Map<String, Object> labelsMap) {
if (null == labelsMap || labelsMap.isEmpty()) {
return "-";
}
String key = labelsMap.entrySet().stream()
.filter(e -> !e.getKey().equals(VALUE) && !e.getKey().equals(NAME) && !e.getKey().equals(TIMESTAMP))
.sorted(Map.Entry.comparingByKey())
.map(e -> e.getKey() + "=" + (e.getValue() == null ? "" : e.getValue()))
.collect(Collectors.joining(","));
return key.isEmpty() ? "-" : key;
}
private boolean isScalar(List<Map<String, Object>> context) {
return CollectionUtils.isNotEmpty(context)
&& context.size() == 1
&& context.get(0).containsKey(SCALAR)
&& null != context.get(0).get(SCALAR);
}
private double parseStrToDouble(String text) {
try {
return Double.parseDouble(text);
} catch (NumberFormatException e) {
throw new ExpressionVisitorException("number format exception", e);
}
}
private boolean isValidValue(Object val) {
return val == null || val instanceof List<?>;
}
private Boolean compareOp(Object leftVal, int opType, Object rightVal) {
double left = parseStrToDouble(leftVal.toString());
double right = parseStrToDouble(rightVal.toString());
return switch (opType) {
case AlertExpressionParser.GT -> left > right;
case AlertExpressionParser.GE -> left >= right;
case AlertExpressionParser.LT -> left < right;
case AlertExpressionParser.LE -> left <= right;
case AlertExpressionParser.EQ -> left == right;
case AlertExpressionParser.NE -> left != right;
default -> false;
};
}
}
}
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -26,13 +26,12 @@ import org.antlr.v4.runtime.tree.ParseTreeVisitor;
* by {@link AlertExpressionParser}.
*
* @param <T> The return type of the visit operation. Use {@link Void} for
* operations with no return type.
* operations with no return type.
*/
public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#expression}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -41,7 +40,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code AndExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -50,7 +48,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code ComparisonExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -59,7 +56,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code UnlessExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -68,7 +64,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code SqlExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -77,7 +72,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code SqlCallExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -86,7 +80,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code LiteralExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -95,7 +88,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code ParenExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -104,7 +96,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code PromqlCallExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -113,7 +104,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code PromqlExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -122,7 +112,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by the {@code OrExpr}
* labeled alternative in {@link AlertExpressionParser#expr}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -130,7 +119,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#functionCall}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -138,7 +126,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#parameterList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -146,7 +133,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#parameter}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -154,7 +140,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#number}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -162,7 +147,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#string}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -170,7 +154,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#duration}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -178,7 +161,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#functionName}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -186,7 +168,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#selectSql}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -194,7 +175,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#selectFieldList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -202,7 +182,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#selectField}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -210,7 +189,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#groupByList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -218,7 +196,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#orderByList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -226,7 +203,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#orderByField}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -234,7 +210,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#limitClause}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -242,7 +217,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#relList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -250,7 +224,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#relation}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -258,7 +231,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#conditionList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -266,7 +238,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#compOp}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -274,7 +245,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#condition}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -282,7 +252,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#conditionUnit}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -290,7 +259,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#promql}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -298,7 +266,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#metricSelector}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -306,7 +273,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#labelMatcherList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -314,7 +280,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#labelMatcherItem}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -322,7 +287,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#labelMatcherOp}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -330,7 +294,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#labelList}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -338,7 +301,6 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#instantVectorOp}.
*
* @param ctx the parse tree
* @return the visitor result
*/
@@ -346,9 +308,15 @@ public interface AlertExpressionVisitor<T> extends ParseTreeVisitor<T> {
/**
* Visit a parse tree produced by {@link AlertExpressionParser#aggregationOperator}.
*
* @param ctx the parse tree
* @return the visitor result
*/
T visitAggregationOperator(AlertExpressionParser.AggregationOperatorContext ctx);
/**
* Visit a parse tree produced by {@link AlertExpressionParser#binaryOperator}.
* @param ctx the parse tree
* @return the visitor result
*/
T visitBinaryOperator(AlertExpressionParser.BinaryOperatorContext ctx);
}
@@ -17,6 +17,8 @@
package org.apache.hertzbeat.alert.notice.impl;
import java.util.Arrays;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -27,14 +29,11 @@ import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.stream.Collectors;
/**
* Send alert information through FeiShu
*/
@@ -48,18 +47,19 @@ final class FlyBookAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
*/
private static final String[] TITLE_COLOR = {"red", "yellow", "orange"};
@Override
public void send(NoticeReceiver receiver, NoticeTemplate noticeTemplate, GroupAlert alert) {
try {
String notificationContent = JsonUtil.toJson(renderContent(noticeTemplate, alert));
// todo priority custom the color
// todo priority custom the color
String cardMessage = createLarkMessage(receiver.getUserId(), notificationContent, (byte) 1);
String webHookUrl = alerterProperties.getFlyBookWebhookUrl() + receiver.getAccessToken();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<String> flyEntity = new HttpEntity<>(cardMessage, headers);
ResponseEntity<CommonRobotNotifyResp> entity = restTemplate.postForEntity(webHookUrl,
flyEntity, CommonRobotNotifyResp.class);
flyEntity, CommonRobotNotifyResp.class);
if (entity.getStatusCode() == HttpStatus.OK) {
assert entity.getBody() != null;
if (entity.getBody().getCode() == null || entity.getBody().getCode() == 0) {
@@ -87,120 +87,115 @@ final class FlyBookAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
*/
private String createLarkMessage(String userId, String notificationContent, byte priority) {
String larkCardMessage = """
{
"msg_type": "interactive",
"card": {
"schema": "2.0",
"config": {
"update_multi": true,
"locales": [
"en_us",
"zh_cn"
],
"style": {
"text_size": {
"normal_v2": {
"default": "normal",
"pc": "normal",
"mobile": "heading"
}
}
}
},
"body": {
"direction": "vertical",
"padding": "12px 12px 12px 12px",
"elements": [
{
"tag": "markdown",
"content": "%s",
"i18n_content": {
"en_us": ""
},
"text_align": "left",
"text_size": "normal_v2",
"margin": "0px 0px 0px 0px"
},
{
"tag": "hr",
"margin": "0px 0px 0px 0px"
},
{
"tag": "column_set",
"horizontal_align": "left",
"columns": [
{
"tag": "column",
"width": "weighted",
"elements": [
{
"tag": "button",
"text": {
"tag": "plain_text",
"content": "登入控制台",
"i18n_content": {
"en_us": "Login In"
}
},
"type": "default",
"width": "default",
"size": "medium",
"behaviors": [
{
"type": "open_url",
"default_url": "%s",
"pc_url": "",
"ios_url": "",
"android_url": ""
}
]
}
],
"direction": "horizontal",
"vertical_spacing": "8px",
"horizontal_align": "left",
"vertical_align": "top",
"weight": 1
}
],
"margin": "0px 0px 0px 0px"
}
]
},
"header": {
"title": {
"tag": "plain_text",
"content": "HertzBeat 告警",
"i18n_content": {
"en_us": "HertzBeat Alarm"
}
},
"subtitle": {
"tag": "plain_text",
"content": ""
},
"template": "%s",
"padding": "12px 12px 12px 12px"
}
}
{
"msg_type": "interactive",
"card": {
"config": {
"update_multi": true
},
"i18n_elements": {
"zh_cn": [
{
"tag": "column_set",
"flex_mode": "none",
"horizontal_spacing": "default",
"background_style": "default",
"columns": [
{
"tag": "column",
"elements": [
{
"tag": "div",
"text": {
"tag": "plain_text",
"content": "",
"text_size": "normal",
"text_align": "left",
"text_color": "default"
}
}
],
"width": "weighted",
"weight": 1
}
]
},
{
"tag": "column_set",
"flex_mode": "none",
"horizontal_spacing": "default",
"background_style": "default",
"columns": [
{
"tag": "column",
"elements": [
{
"tag": "div",
"text": {
"tag": "plain_text",
"content": %s,
"text_size": "normal",
"text_align": "left",
"text_color": "default"
}
}
],
"width": "weighted",
"weight": 1
}
]
},
%s
{
"tag": "action",
"actions": [
{
"tag": "button",
"text": {
"tag": "plain_text",
"content": "登入控制台"
},
"type": "default",
"complex_interaction": true,
"width": "default",
"size": "medium",
"multi_url": {
"url": "%s"
}
}
]
}
]
},
"i18n_header": {
"zh_cn": {
"title": {
"tag": "plain_text",
"content": "HertzBeat 告警"
},
"template": "%s"
}
}
}
""";
}
""";
String atUserElement = "";
if (StringUtils.isNotBlank(userId)) {
atUserElement = "\\n" + Arrays.stream(userId.split(","))
.map(id -> "<at id=" + id + "></at>")
.collect(Collectors.joining(" "));
String atUserId = Arrays.stream(userId.split(","))
.map(id -> "<at id=" + id + "></at>")
.collect(Collectors.joining(" "));
atUserElement = String.format("""
{
"tag": "div",
"text": {
"content": "%s",
"tag": "lark_md"
}
},
""", atUserId);
}
if (notificationContent.startsWith("\"") && notificationContent.endsWith("\"")) {
notificationContent = StringUtils.removeStart(notificationContent, "\"");
notificationContent = StringUtils.removeEnd(notificationContent, "\"");
}
return String.format(larkCardMessage,
notificationContent.replace("\"", "\\\"") + atUserElement,
alerterProperties.getConsoleUrl(), TITLE_COLOR[priority]);
return String.format(larkCardMessage, notificationContent, atUserElement, alerterProperties.getConsoleUrl(), TITLE_COLOR[priority]);
}
@Override
@@ -60,7 +60,7 @@ public class ServerChanAlertNotifyHandlerImpl extends AbstractAlertNotifyHandler
String webHookUrl = String.format(alerterProperties.getServerChanWebhookUrl(), sanitizedToken);
// Validate the constructed URL against a whitelist
List<String> allowedBaseUrls = List.of("https://api.serverchan.com", "https://serverchan.example.com", "https://sctapi.ftqq.com");
List<String> allowedBaseUrls = List.of("https://api.serverchan.com", "https://serverchan.example.com");
boolean isValidUrl = allowedBaseUrls.stream().anyMatch(webHookUrl::startsWith);
if (!isValidUrl) {
throw new AlertNoticeException("Invalid webhook URL: " + webHookUrl);
@@ -18,15 +18,12 @@
package org.apache.hertzbeat.alert.service;
import jakarta.servlet.http.HttpServletResponse;
import java.util.List;
import java.util.Set;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.springframework.data.domain.Page;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Alarm define manager service
*/
@@ -110,11 +107,5 @@ public interface AlertDefineService {
* @return Real-time alarm definition list
*/
List<AlertDefine> getRealTimeAlertDefines();
/**
* Get define preview
* @return Data queried based on expressions
* @throws AlertExpressionException expression error
*/
List<Map<String, Object>> getDefinePreview(String datasource, String type, String expr);
}
@@ -28,9 +28,6 @@ import org.apache.hertzbeat.alert.dto.ExportAlertDefineDTO;
import org.apache.hertzbeat.alert.service.AlertDefineImExportService;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.util.LogUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.util.CollectionUtils;
@@ -44,15 +41,12 @@ public abstract class AlertDefineAbstractImExportServiceImpl implements AlertDef
@Lazy
private AlertDefineService alertDefineService;
private static final Logger logger = LoggerFactory.getLogger(AlertDefineAbstractImExportServiceImpl.class);
@Override
public void importConfig(InputStream is) {
var formList = parseImport(is)
.stream()
.map(this::convert)
.toList();
LogUtil.info(logger, "Importing alert defines from {0}", formList);
if (!CollectionUtils.isEmpty(formList)) {
formList.forEach(alertDefine -> {
alertDefineService.validate(alertDefine, false);
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.alert.service.impl;
import static org.apache.hertzbeat.common.constants.CommonConstants.ALERT_THRESHOLD_TYPE_REALTIME;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -28,9 +29,7 @@ import org.apache.hertzbeat.alert.calculate.PeriodicAlertRuleScheduler;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.alert.service.AlertDefineImExportService;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.common.cache.CacheFactory;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.ExportFileConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
@@ -73,21 +72,18 @@ public class AlertDefineServiceImpl implements AlertDefineService {
@Autowired
private PeriodicAlertRuleScheduler periodicAlertRuleScheduler;
private final DataSourceService dataSourceService;
private final Map<String, AlertDefineImExportService> alertDefineImExportServiceMap = new HashMap<>();
private static final String CONTENT_TYPE = MediaType.APPLICATION_OCTET_STREAM_VALUE + SignConstants.SINGLE_MARK + "charset=" + StandardCharsets.UTF_8;
public AlertDefineServiceImpl(List<AlertDefineImExportService> alertDefineImExportServiceList, DataSourceService dataSourceService) {
public AlertDefineServiceImpl(List<AlertDefineImExportService> alertDefineImExportServiceList) {
alertDefineImExportServiceList.forEach(it -> alertDefineImExportServiceMap.put(it.type(), it));
this.dataSourceService = dataSourceService;
}
@Override
public void validate(AlertDefine alertDefine, boolean isModify) throws IllegalArgumentException {
if (StringUtils.hasText(alertDefine.getExpr())) {
if (CommonConstants.ALERT_THRESHOLD_TYPE_REALTIME.equals(alertDefine.getType())) {
if (ALERT_THRESHOLD_TYPE_REALTIME.equals(alertDefine.getType())) {
try {
JexlExpressionRunner.compile(alertDefine.getExpr());
} catch (Exception e) {
@@ -217,23 +213,9 @@ public class AlertDefineServiceImpl implements AlertDefineService {
public List<AlertDefine> getRealTimeAlertDefines() {
List<AlertDefine> alertDefines = CacheFactory.getAlertDefineCache();
if (alertDefines == null) {
alertDefines = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(CommonConstants.ALERT_THRESHOLD_TYPE_REALTIME);
alertDefines = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(ALERT_THRESHOLD_TYPE_REALTIME);
CacheFactory.setAlertDefineCache(alertDefines);
}
return alertDefines;
}
@Override
public List<Map<String, Object>> getDefinePreview(String datasource, String type, String expr) {
if (!StringUtils.hasText(expr) || !StringUtils.hasText(datasource) || !StringUtils.hasText(type)) {
return Collections.emptyList();
}
switch (type) {
case CommonConstants.ALERT_THRESHOLD_TYPE_PERIODIC:
return dataSourceService.calculate(datasource, expr);
default:
log.error("Get define preview unsupported type: {}", type);
return Collections.emptyList();
}
}
}
@@ -27,14 +27,11 @@ import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.support.exception.SendMessageException;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.LogUtil;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
@@ -65,7 +62,6 @@ public class AlibabaSmsClientImpl implements SmsClient {
private final String accessKeySecret;
private final String signName;
private final String templateCode;
private static final Logger logger = LoggerFactory.getLogger(AlibabaSmsClientImpl.class);
public AlibabaSmsClientImpl(AlibabaSmsProperties config) {
if (config != null) {
@@ -177,7 +173,7 @@ public class AlibabaSmsClientImpl implements SmsClient {
log.info("Successfully sent SMS to phone: {}", phoneNumber);
}
} catch (Exception e) {
LogUtil.warn(logger, "Failed to send SMS: {0}", e.getMessage());
log.warn("Failed to send SMS: {}", e.getMessage());
throw new SendMessageException(e.getMessage());
}
}
@@ -196,7 +192,6 @@ public class AlibabaSmsClientImpl implements SmsClient {
// Step 4: Build authorization header
return ALGORITHM + " Credential=" + accessKeyId + ",SignedHeaders=host;x-acs-action;x-acs-content-sha256;x-acs-date;" + "x-acs-signature-nonce;x-acs-version,Signature=" + signature;
} catch (Exception e) {
LogUtil.warn(logger, "Failed to calculate authorization {0}", e.getMessage());
throw new RuntimeException("Failed to calculate authorization", e);
}
}
@@ -24,14 +24,11 @@ import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.hertzbeat.alert.expr.AlertExpressionEvalVisitor;
import org.apache.hertzbeat.alert.expr.AlertExpressionLexer;
import org.apache.hertzbeat.alert.expr.AlertExpressionParser;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.apache.hertzbeat.common.util.ResourceBundleUtil;
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -39,7 +36,6 @@ import org.springframework.util.StringUtils;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import java.util.concurrent.TimeUnit;
/**
@@ -49,8 +45,6 @@ import java.util.concurrent.TimeUnit;
@Slf4j
public class DataSourceServiceImpl implements DataSourceService {
protected ResourceBundle bundle = ResourceBundleUtil.getBundle("alerter");
@Setter
@Autowired(required = false)
private List<QueryExecutor> executors;
@@ -75,7 +69,7 @@ public class DataSourceServiceImpl implements DataSourceService {
throw new IllegalArgumentException("Empty expression");
}
if (executors == null || executors.isEmpty()) {
throw new IllegalArgumentException(bundle.getString("alerter.datasource.executor.not.found"));
throw new IllegalArgumentException("No query executor found");
}
QueryExecutor executor = executors.stream().filter(e -> e.support(datasource)).findFirst().orElse(null);
@@ -86,9 +80,6 @@ public class DataSourceServiceImpl implements DataSourceService {
expr = expr.replaceAll("\\s+", " ");
try {
return evaluate(expr, executor);
} catch (AlertExpressionException ae) {
log.error("Calculate query parse error {}: {}", datasource, ae.getMessage());
throw ae;
} catch (Exception e) {
log.error("Error executing query on datasource {}: {}", datasource, e.getMessage());
throw new RuntimeException("Query execution failed", e);
@@ -99,11 +90,9 @@ public class DataSourceServiceImpl implements DataSourceService {
CommonTokenStream tokens = tokenStreamCache.get(expr, this::createTokenStream);
AlertExpressionParser parser = new AlertExpressionParser(tokens);
ParseTree tree = expressionCache.get(expr, e -> parser.expr());
if (null != tokens && tokens.LA(1) != Token.EOF) {
throw new AlertExpressionException(bundle.getString("alerter.calculate.parse.error"));
}
AlertExpressionEvalVisitor visitor = new AlertExpressionEvalVisitor(executor, tokens);
return visitor.visit(tree);
}
private CommonTokenStream createTokenStream(String expr) {
@@ -1,244 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.hertzbeat.alert.dto.VolcEngineExternEventAlert;
import org.apache.hertzbeat.alert.dto.VolcEngineExternMetricAlert;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.service.ExternAlertService;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Volcengine alarm entity class
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class VolcEngineExternAlertService implements ExternAlertService {
private final AlarmCommonReduce alarmCommonReduce;
private static final Map<String, Integer> severityOrder = Map.of(
CommonConstants.ALERT_SEVERITY_CRITICAL, 1,
CommonConstants.ALERT_SEVERITY_WARNING, 2,
CommonConstants.ALERT_SEVERITY_INFO, 3
);
@Override
@SneakyThrows
public void addExternAlert(String content) {
ObjectMapper mapper = new ObjectMapper();
JsonNode root = mapper.readTree(content);
String type = root.get("Type").asText();
if (VolcEngineExternMetricAlert.ALERT_TYPE_EVENT.equals(type)) {
VolcEngineExternEventAlert eventAlert = JsonUtil.fromJson(content, VolcEngineExternEventAlert.class);
if (eventAlert == null) {
log.warn("parse extern event alert content failed! content: {}", content);
return;
}
SingleAlert singleAlert = new VolcEngineAlertConverter().convertEventToSingleAlert(eventAlert);
alarmCommonReduce.reduceAndSendAlarm(singleAlert);
} else {
// deal with metric alert
VolcEngineExternMetricAlert report = JsonUtil.fromJson(content, VolcEngineExternMetricAlert.class);
if (report == null) {
log.warn("parse extern metrics alert content failed! content: {}", content);
return;
}
for (SingleAlert singleAlert : new VolcEngineAlertConverter().convertMetricAlertToSingeAlert(report)) {
alarmCommonReduce.reduceAndSendAlarm(singleAlert);
}
}
}
@Override
public String supportSource() {
return "volcengine";
}
/**
* Converter: VolcEngine alert to SingleAlert
*/
public static class VolcEngineAlertConverter {
/**
* Convert VolcEngine metric alert to SingleAlert List
*/
public List<SingleAlert> convertMetricAlertToSingeAlert(VolcEngineExternMetricAlert alert) {
String status = convertStatus(alert);
if (Objects.equals(alert.getType(), VolcEngineExternMetricAlert.ALERT_TYPE_METRIC)) {
return convertMetricAlert(alert, status, alert.getResources());
}
if (Objects.equals(alert.getType(), VolcEngineExternMetricAlert.ALERT_TYPE_METRICS_NODATA)) {
return convertMetricAlert(alert, status, alert.getNoDataResources());
}
if (Objects.equals(alert.getType(), VolcEngineExternMetricAlert.ALERT_TYPE_NO_DATA_RECOVERED)) {
return convertMetricAlert(alert, status, alert.getNoDataRecoveredResources());
}
if (Objects.equals(alert.getType(), VolcEngineExternMetricAlert.ALERT_TYPE_METRIC_RECOVERED)) {
return convertMetricAlert(alert, status, alert.getRecoveredResources());
}
return List.of();
}
public SingleAlert convertEventToSingleAlert(VolcEngineExternEventAlert alert) {
SingleAlert.SingleAlertBuilder builder = SingleAlert.builder()
.status(CommonConstants.ALERT_STATUS_FIRING)
.startAt(alert.getHappenedAt() * 1000)
.activeAt(alert.getHappenedAt() * 1000)
.labels(buildEventLabels(alert))
.content(alert.getDescriptionCn())
.annotations(new HashMap<>());
return builder.build();
}
/**
* build labels for volcengine event alert
*/
private Map<String, String> buildEventLabels(VolcEngineExternEventAlert event) {
Map<String, String> labels = new HashMap<>();
labels.put("severity", convertEventSeverity(event));
labels.put("__source__", "volcengine");
labels.put("resource_name", event.getDetails().getVolcResourceName());
labels.put("account_id", event.getAccountId());
labels.put("region", event.getRegion());
labels.put("event_type", event.getEventType());
labels.put("source", event.getSource());
return labels;
}
/**
* convert volcengine event alert status to heartbeat alert status
* use the most severe level in the rules
*
* @param alert volcengine event alert
* @return status
*/
private String convertEventSeverity(VolcEngineExternEventAlert alert) {
return alert.getRules().stream().map(VolcEngineExternEventAlert.EventRule::getLevel)
.min((o1, o2) -> severityOrder.get(convertCommonSeverity(o1)).compareTo(severityOrder.get(convertCommonSeverity(o2))))
.orElse(CommonConstants.ALERT_SEVERITY_INFO);
}
private String convertCommonSeverity(String level) {
return switch (level) {
case "critical" -> CommonConstants.ALERT_SEVERITY_CRITICAL;
case "warning" -> CommonConstants.ALERT_SEVERITY_WARNING;
default -> CommonConstants.ALERT_SEVERITY_INFO;
};
}
/**
* create SingleAlert for each resources in volcengine alert
*
* @param alert volcengine alert
* @param status status
* @param resources resources
* @return List of SingleAlert
*/
private List<SingleAlert> convertMetricAlert(VolcEngineExternMetricAlert alert, String status,
List<? extends VolcEngineExternMetricAlert.Resource> resources) {
if (CollectionUtils.isEmpty(resources)) {
return List.of();
}
List<SingleAlert> result = new ArrayList<>();
for (VolcEngineExternMetricAlert.Resource resource : resources) {
SingleAlert.SingleAlertBuilder builder = SingleAlert.builder()
.status(status)
.startAt(resource.getFirstAlertTime() * 1000)
.endAt(resource.getLastAlertTime() * 1000)
.labels(buildLabels(alert, resource))
.activeAt(convertHappenAt(alert.getHappenedAt()))
.content(resource.getName() + alert.getRuleCondition())
.annotations(buildAnnotations(resource));
result.add(builder.build());
}
return result;
}
private Long convertHappenAt(String happenAt) {
String cleanedStr = happenAt.replace("UTC", "").replace("(", "").replace(")", "");
DateTimeFormatter altFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssXXX");
OffsetDateTime odt = OffsetDateTime.parse(cleanedStr, altFormatter);
return odt.toEpochSecond() * 1000;
}
private Map<String, String> buildLabels(VolcEngineExternMetricAlert alert, VolcEngineExternMetricAlert.Resource resource) {
Map<String, String> labels = new HashMap<>();
labels.put("__source__", "volcengine");
labels.put("resource_name", resource.getName());
labels.put("resource_id", resource.getId());
labels.put("rule_id", alert.getRuleId());
return labels;
}
private Map<String, String> buildAnnotations(VolcEngineExternMetricAlert.Resource resource) {
Map<String, String> annotations = new HashMap<>();
if (resource instanceof VolcEngineExternMetricAlert.NoDataResource noDataResource) {
for (VolcEngineExternMetricAlert.Metric noDataMetric : noDataResource.getNoDataMetrics()) {
annotations.put(noDataMetric.getName(), "N/A");
}
}
for (VolcEngineExternMetricAlert.Metric metric : resource.getMetrics()) {
annotations.put(metric.getName(), metric.getCurrentValue() + metric.getUnit());
}
annotations.put("region", resource.getRegion());
List<VolcEngineExternMetricAlert.Dimension> dimensions = resource.getDimensions();
if (CollectionUtils.isNotEmpty(dimensions)) {
for (VolcEngineExternMetricAlert.Dimension dimension : dimensions) {
annotations.put(dimension.getNameCn(), dimension.getValue());
}
}
return annotations;
}
/**
* convert volcengine alert status to heartbeat alert status
*
* @param alert volcengine alert
* @return status
*/
private String convertStatus(VolcEngineExternMetricAlert alert) {
String type = alert.getType();
if (Objects.equals(type, VolcEngineExternMetricAlert.ALERT_TYPE_METRIC)
|| Objects.equals(type, VolcEngineExternMetricAlert.ALERT_TYPE_EVENT)
|| Objects.equals(type, VolcEngineExternMetricAlert.ALERT_TYPE_METRICS_NODATA)) {
return CommonConstants.ALERT_STATUS_FIRING;
}
return CommonConstants.ALERT_STATUS_RESOLVED;
}
}
}
@@ -32,5 +32,3 @@ alerter.notify.console = Console Login
alerter.priority.0 = Emergency Alert
alerter.priority.1 = Critical Alert
alerter.priority.2 = Warning Alert
alerter.calculate.parse.error = Expression is not fully parsed, may have syntax errors or incomplete inputs
alerter.datasource.executor.not.found = No query executor found
@@ -32,5 +32,3 @@ alerter.notify.console = 登入控制台
alerter.priority.0 = 紧急告警
alerter.priority.1 = 严重告警
alerter.priority.2 = 警告告警
alerter.calculate.parse.error = 表达式未完全解析,可能存在语法错误或输入不完整
alerter.datasource.executor.not.found = 未找到查询执行器
@@ -32,5 +32,3 @@ alerter.notify.console = 控制台登錄
alerter.priority.0 = 緊急警報
alerter.priority.1 = 嚴重警報
alerter.priority.2 = 警告警報
alerter.calculate.parse.error = 表達式未完全解析,可能存在語法錯誤或輸入不完整
alerter.datasource.executor.not.found = 未找到查詢執行器
@@ -23,7 +23,7 @@ expression
expr
: LPAREN expr RPAREN # ParenExpr
| left=expr op=(GE|LE|GT|LT|EQ|NE) BOOL? right=expr # ComparisonExpr
| left=expr op=(GE|LE|GT|LT|EQ|NE) right=expr # ComparisonExpr
| left=expr AND right=expr # AndExpr
| left=expr UNLESS right=expr # UnlessExpr
| left=expr OR right=expr # OrExpr
@@ -156,6 +156,7 @@ conditionUnit
promql
: metricSelector instantVectorOp?
| aggregationOperator LPAREN promql (BY labelList)? RPAREN
| promql binaryOperator promql
| functionCall
| promql LBRACKET duration RBRACKET
| promql LBRACKET duration COLON duration RBRACKET
@@ -191,6 +192,11 @@ aggregationOperator
: SUM | AVG | COUNT | MIN | MAX | STDDEV | STDVAR | TOPK | BOTTOMK | QUANTILE
;
binaryOperator
: EQ | NE | GT | LT | GE | LE
| AND | OR | UNLESS
;
// Lexer rules
// Boolean operators
@@ -254,7 +260,6 @@ LT : '<' ;
LE : '<=' ;
EQ : '==' | '=' ;
NE : '!=' ;
BOOL : 'bool';
// Delimiters
LPAREN : '(' ;
@@ -1,135 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.calculate;
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
import org.apache.hertzbeat.alert.util.AlertUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.Mockito.when;
/**
* alert cache manager test
*/
@ExtendWith(MockitoExtension.class)
public class AlarmCacheManagerTest {
@Mock
private SingleAlertDao singleAlertDao;
private AlarmCacheManager alarmCacheManager;
@BeforeEach
public void setUp() {
Map<String, String> labels = new HashMap<>();
labels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
labels.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(1L));
SingleAlert alert = new SingleAlert();
alert.setContent("Alert cache manager test");
alert.setLabels(labels);
when(singleAlertDao.querySingleAlertsByStatus(CommonConstants.ALERT_STATUS_FIRING)).thenReturn(Collections.singletonList(alert));
alarmCacheManager = new AlarmCacheManager(singleAlertDao);
}
@Test
void testInit() {
Map<String, String> labels = new HashMap<>();
labels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
labels.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(1L));
String fingerprint = AlertUtil.calculateFingerprint(labels);
SingleAlert firingSingleAlert = alarmCacheManager.getFiring(1L, fingerprint);
assertNotNull(firingSingleAlert);
assertEquals("Alert cache manager test", firingSingleAlert.getContent());
alarmCacheManager.removeFiring(1L, fingerprint);
firingSingleAlert = alarmCacheManager.getFiring(1L, fingerprint);
assertNull(firingSingleAlert);
}
@Test
void testPending() {
Map<String, String> labels = new HashMap<>();
labels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
labels.put(CommonConstants.ALERT_SEVERITY_INFO, CommonConstants.ALERT_STATUS_PENDING);
labels.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(2L));
SingleAlert alert = new SingleAlert();
alert.setContent("Alert cache manager test");
alert.setLabels(labels);
String fingerprint = AlertUtil.calculateFingerprint(alert.getLabels());
alarmCacheManager.putPending(2L, fingerprint, alert);
SingleAlert pendingSingleAlert = alarmCacheManager.getPending(2L, fingerprint);
assertNotNull(pendingSingleAlert);
alarmCacheManager.removePending(2L, fingerprint);
pendingSingleAlert = alarmCacheManager.getPending(2L, fingerprint);
assertNull(pendingSingleAlert);
}
@Test
void testFiring() {
Map<String, String> labels = new HashMap<>();
labels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
labels.put(CommonConstants.ALERT_SEVERITY_INFO, CommonConstants.ALERT_STATUS_PENDING);
labels.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(3L));
SingleAlert alert = new SingleAlert();
alert.setContent("Alert cache manager test");
alert.setLabels(labels);
String fingerprint = AlertUtil.calculateFingerprint(alert.getLabels());
alarmCacheManager.putFiring(3L, fingerprint, alert);
SingleAlert firingSingleAlert = alarmCacheManager.getFiring(3L, fingerprint);
assertNotNull(firingSingleAlert);
alarmCacheManager.removeFiring(3L, fingerprint);
firingSingleAlert = alarmCacheManager.getFiring(3L, fingerprint);
assertNull(firingSingleAlert);
}
@Test
void testHistorical() {
SingleAlert alert = new SingleAlert();
alert.setContent("Alert cache manager test");
alert.setLabels(Collections.singletonMap(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL));
when(singleAlertDao.querySingleAlertsByStatus(CommonConstants.ALERT_STATUS_FIRING)).thenReturn(Collections.singletonList(alert));
alarmCacheManager = new AlarmCacheManager(singleAlertDao);
String fingerprint = AlertUtil.calculateFingerprint(alert.getLabels());
SingleAlert historicalSingleAlert = alarmCacheManager.getFiring(4L, fingerprint);
assertNotNull(historicalSingleAlert);
SingleAlert singleAlert = alarmCacheManager.removeFiring(4L, fingerprint);
assertNotNull(singleAlert);
historicalSingleAlert = alarmCacheManager.getFiring(4L, fingerprint);
assertNull(historicalSingleAlert);
}
}
@@ -1,147 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.calculate;
import org.apache.hertzbeat.alert.dao.AlertCollectorDao;
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.util.AlertUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.entity.manager.Collector;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Collector alert handler test
*/
@ExtendWith(MockitoExtension.class)
public class CollectorAlertHandlerTest {
@Mock
private AlertCollectorDao alertCollectorDao;
@Mock
private SingleAlertDao singleAlertDao;
@Mock
private AlarmCommonReduce alarmCommonReduce;
private AlarmCacheManager alarmCacheManager;
private CollectorAlertHandler collectorAlertHandler;
@BeforeEach
void setUp() {
alarmCacheManager = Mockito.spy(new AlarmCacheManager(singleAlertDao));
collectorAlertHandler = new CollectorAlertHandler(alarmCommonReduce, alertCollectorDao, alarmCacheManager);
}
@Test
public void testOnline() {
String identity = "localhost-collector";
Collector collector = Collector.builder()
.name(identity)
.ip("127.0.0.1")
.version("1.0")
.build();
SingleAlert singleAlert = SingleAlert.builder().build();
when(alertCollectorDao.findCollectorByName(any(String.class))).thenReturn(collector);
when(alarmCacheManager.removeFiring(any(String.class))).thenReturn(singleAlert);
collectorAlertHandler.online(identity);
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, singleAlert.getStatus());
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any(SingleAlert.class));
}
@Test
public void testOffline() {
String identity = "localhost-collector";
Collector collector = Collector.builder()
.name(identity)
.ip("127.0.0.1")
.version("1.0")
.build();
when(alertCollectorDao.findCollectorByName(any(String.class))).thenReturn(collector);
when(alarmCacheManager.getFiring(any(String.class))).thenReturn(null);
collectorAlertHandler.offline(identity);
verify(alarmCacheManager, times(1)).putFiring(any(String.class), any(SingleAlert.class));
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any(SingleAlert.class));
}
@Test
void testOfflineAndOnlineLifecycle() {
String identity = "localhost-collector";
Collector collector = Collector.builder()
.name(identity)
.ip("127.0.0.1")
.version("1.0")
.build();
Map<String, String> fingerPrints = new HashMap<>(8);
fingerPrints.put("collectorName", collector.getName());
fingerPrints.put("collectorVersion", collector.getVersion());
fingerPrints.put("collectorHost", collector.getIp());
String fingerprint = AlertUtil.calculateFingerprint(fingerPrints);
when(alertCollectorDao.findCollectorByName(identity)).thenReturn(collector);
// step1. first time offline
collectorAlertHandler.offline(identity);
assertNotNull(alarmCacheManager.getFiring(fingerprint));
// step2. online
collectorAlertHandler.online(identity);
assertNull(alarmCacheManager.getFiring(fingerprint));
// step3. second time offline
collectorAlertHandler.offline(identity);
assertNotNull(alarmCacheManager.getFiring(fingerprint));
// Verify that the push status is correct.
ArgumentCaptor<SingleAlert> captor = ArgumentCaptor.forClass(SingleAlert.class);
verify(alarmCommonReduce, atLeast(3)).reduceAndSendAlarm(captor.capture());
List<SingleAlert> alerts = captor.getAllValues();
assertEquals(CommonConstants.ALERT_STATUS_FIRING, alerts.get(0).getStatus());
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alerts.get(1).getStatus());
assertEquals(CommonConstants.ALERT_STATUS_FIRING, alerts.get(2).getStatus());
}
}
@@ -42,7 +42,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -93,12 +92,12 @@ class PeriodicAlertCalculatorTest {
result.put("__value__", 95.0); // Non-null, matched with threshold
result.put("__timestamp__", System.currentTimeMillis());
when(dataSourceService.calculate(anyString(), anyString())).thenReturn(List.of(result));
when(alarmCacheManager.getPending(eq(rule.getId()), anyString())).thenReturn(null);
when(alarmCacheManager.getPending(anyString())).thenReturn(null);
periodicAlertCalculator.calculate(rule);
// Verify that putFiring is called
ArgumentCaptor<String> idCaptor = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<SingleAlert> alertCaptor = ArgumentCaptor.forClass(SingleAlert.class);
verify(alarmCacheManager).putFiring(eq(rule.getId()), idCaptor.capture(), alertCaptor.capture());
verify(alarmCacheManager).putFiring(idCaptor.capture(), alertCaptor.capture());
// Assertion alarm status and content
SingleAlert alert = alertCaptor.getValue();
assertAll(() -> assertEquals(CommonConstants.ALERT_STATUS_FIRING, alert.getStatus()),
@@ -113,7 +112,7 @@ class PeriodicAlertCalculatorTest {
result.put("__timestamp__", System.currentTimeMillis());
when(dataSourceService.calculate(anyString(), anyString())).thenReturn(List.of(result));
periodicAlertCalculator.calculate(rule);
verify(alarmCacheManager, times(0)).putFiring(any(), any(), any());
verify(alarmCacheManager, times(0)).putFiring(any(), any());
}
@Test
@@ -127,7 +126,7 @@ class PeriodicAlertCalculatorTest {
.triggerTimes(2).startAt(System.currentTimeMillis() - 60000)
.activeAt(System.currentTimeMillis() - 30000)
.build();
when(alarmCacheManager.removeFiring(eq(rule.getId()), anyString())).thenReturn(pendingAlert);
when(alarmCacheManager.removeFiring(anyString())).thenReturn(pendingAlert);
when(dataSourceService.calculate(anyString(), anyString())).thenReturn(List.of(result));
periodicAlertCalculator.calculate(rule);
ArgumentCaptor<SingleAlert> resolvedCaptor = ArgumentCaptor.forClass(SingleAlert.class);
@@ -132,7 +132,6 @@ public class RealTimeAlertCalculatorMatchTest {
AlertDefine matchDefine = new AlertDefine();
matchDefine.setId(1L);
matchDefine.setName("test");
matchDefine.setExpr(
"equals(__app__,\"prometheus\") && "
@@ -152,8 +151,8 @@ public class RealTimeAlertCalculatorMatchTest {
Thread.sleep(3000);
verify(alarmCacheManager, times(1)).getPending(any(), any());
verify(alarmCacheManager, times(1)).putFiring(any(), any(), any());
verify(alarmCacheManager, times(1)).getPending(any());
verify(alarmCacheManager, times(1)).putFiring(any(), any());
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
}
@@ -181,7 +180,6 @@ public class RealTimeAlertCalculatorMatchTest {
CollectRep.MetricsData metricsData = builder.build();
AlertDefine matchDefine = new AlertDefine();
matchDefine.setId(1L);
matchDefine.setName("test");
matchDefine.setExpr("equals(__app__,\"prometheus\") && equals(__metrics__,\"canal_instance\") && metric_value > 0");
matchDefine.setTemplate("Canal instance val: ${value}%");
@@ -196,8 +194,8 @@ public class RealTimeAlertCalculatorMatchTest {
Thread.sleep(3000);
verify(alarmCacheManager, times(1)).getPending(any(), any());
verify(alarmCacheManager, times(1)).putFiring(any(), any(), any());
verify(alarmCacheManager, times(1)).getPending(any());
verify(alarmCacheManager, times(1)).putFiring(any(), any());
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
}
@@ -231,7 +229,6 @@ public class RealTimeAlertCalculatorMatchTest {
CollectRep.MetricsData metricsData = builder.build();
AlertDefine matchDefine = new AlertDefine();
matchDefine.setId(1L);
matchDefine.setName("test");
matchDefine.setExpr("equals(__app__,\"springboot3\") && equals(__metrics__,\"available\") && equals(__instance__, \"518679137103104\") && responseTime > 0");
matchDefine.setTemplate("Canal instance val: ${value}%");
@@ -246,8 +243,8 @@ public class RealTimeAlertCalculatorMatchTest {
Thread.sleep(3000);
verify(alarmCacheManager, times(1)).getPending(any(), any());
verify(alarmCacheManager, times(1)).putFiring(any(), any(), any());
verify(alarmCacheManager, times(1)).getPending(any());
verify(alarmCacheManager, times(1)).putFiring(any(), any());
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
}
@@ -17,22 +17,15 @@
package org.apache.hertzbeat.alert.controller;
import static org.mockito.ArgumentMatchers.anyString;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hertzbeat.alert.service.impl.AlertDefineServiceImpl;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.alerter.AlertDefineMonitorBind;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -107,37 +100,6 @@ class AlertDefineControllerTest {
.andReturn();
}
@Test
void testGetDefinePreview() throws Exception {
List<Map<String, Object>> previewData = new ArrayList<>();
Map<String, Object> row = new HashMap<>();
row.put("__value__", 123);
row.put("job", "spring-boot");
previewData.add(row);
Mockito.when(alertDefineService.getDefinePreview(anyString(), anyString(), anyString()))
.thenReturn(previewData);
mockMvc.perform(MockMvcRequestBuilders.get("/api/alert/define/preview/{datasource}", "promql")
.param("type", "periodic")
.param("expr", "up == 1"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value(0))
.andExpect(jsonPath("$.data[0].__value__").value(123))
.andExpect(jsonPath("$.data[0].job").value("spring-boot"));
Mockito.when(alertDefineService.getDefinePreview(anyString(), anyString(), anyString()))
.thenThrow(new AlertExpressionException("Expression error"));
mockMvc.perform(MockMvcRequestBuilders.get("/api/alert/define/preview/{datasource}", "promql")
.param("type", "periodic")
.param("expr", "http_server_requests_seconds_count{!@~!!#$%^&}"))
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").exists())
.andExpect(jsonPath("$.msg").value("Expression error"));
}
@Test
void modifyAlertDefine() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.put("/api/alert/define")
@@ -24,14 +24,12 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
/**
@@ -181,10 +179,12 @@ class AlertExpressionEvalVisitorTest {
List.of(new HashMap<>(Map.of("__value__", List.of(10.0, 20.0, 30.0)))));
// promql
List<Map<String, Object>> result = evaluate("multi_val > 25");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertEquals(30.0, result.get(0).get("__value__"));
// sql
result = evaluate("(select values from multi_metrics where group_id = 'test_group') > 25");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertEquals(30.0, result.get(0).get("__value__"));
}
@Test
@@ -194,10 +194,12 @@ class AlertExpressionEvalVisitorTest {
List.of(new HashMap<>(Map.of("__value__", List.of(10.0, 20.0, 30.0)))));
// promql
List<Map<String, Object>> result = evaluate("multi_val < 15");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertEquals(10.0, result.get(0).get("__value__"));
// sql
result = evaluate("(select response_times from performance_data where service = 'api') < 15");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertEquals(10.0, result.get(0).get("__value__"));
}
@Test
@@ -207,10 +209,12 @@ class AlertExpressionEvalVisitorTest {
List.of(new HashMap<>(Map.of("__value__", List.of()))));
// promql
List<Map<String, Object>> result = evaluate("empty_list > 50");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
// sql
result = evaluate("(select error_codes from error_log where date = '2024-01-01') > 50");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -236,187 +240,11 @@ class AlertExpressionEvalVisitorTest {
assertEquals(10.0, result.get(0).get("__value__"));
}
@Test
void testUnlessOpPromql() {
String promql = "http_server_requests_seconds_count > 10 unless http_server_requests_seconds_max > 0";
Map<String, Object> countValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> maxValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 10.007799125);
put("method", "GET");
put("__name__", "http_server_requests_seconds_max");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/health");
put("outcome", "SUCCESS");
put("status", "200");
}
};
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(List.of(countValue1));
when(mockExecutor.execute("http_server_requests_seconds_max")).thenReturn(List.of(maxValue1));
List<Map<String, Object>> result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1307, result.get(0).get("__value__"));
maxValue1.put("uri", "/actuator/prometheus");
result = evaluate(promql);
assertEquals(0, result.size());
Map<String, Object> sumValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 20.018);
put("method", "GET");
put("__name__", "http_server_requests_seconds_sum");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/**");
put("outcome", "SUCCESS");
put("status", "200");
}
};
maxValue1.put("uri", "/actuator/health");
when(mockExecutor.execute("http_server_requests_seconds_sum")).thenReturn(List.of(sumValue1));
promql = "(http_server_requests_seconds_count > 10 unless http_server_requests_seconds_max > 0) unless http_server_requests_seconds_sum > 10";
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1307, result.get(0).get("__value__"));
}
@Test
void testOrOpPromql() {
String promql = "http_server_requests_seconds_count > 10 or jvm_threads_states_threads > 0";
Map<String, Object> countValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> countValue2 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 16);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/**");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> countValue3 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 7);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/health");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> threadsValue1 = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 10.007799125);
put("__name__", "jvm_threads_states_threads");
put("__timestamp__", "1.750320922467E9");
put("job", "spring-boot-app");
put("state=", "runnable");
}
};
Map<String, Object> threadsValue2 = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 1);
put("__name__", "jvm_threads_states_threads");
put("__timestamp__", "1.750320922467E9");
put("job", "spring-boot-app");
put("state=", "timed-waiting");
}
};
Map<String, Object> threadsValue3 = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 19.02);
put("__name__", "jvm_threads_states_threads");
put("__timestamp__", "1.750320922467E9");
put("job", "spring-boot-app");
put("state=", "waiting");
}
};
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(List.of(countValue1, countValue2, countValue3));
when(mockExecutor.execute("jvm_threads_states_threads")).thenReturn(List.of(threadsValue1, threadsValue2, threadsValue3));
List<Map<String, Object>> result = evaluate(promql);
assertEquals(5, result.size());
assertTrue(result.stream().allMatch(t -> null != t.get("__value__")));
when(mockExecutor.execute("jvm_threads_states_threads")).thenReturn(new ArrayList<>());
result = evaluate(promql);
assertEquals(2, result.size());
assertTrue(result.stream().allMatch(t -> null != t.get("__value__")));
assertEquals(1307, result.get(0).get("__value__"));
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(new ArrayList<>());
when(mockExecutor.execute("jvm_threads_states_threads")).thenReturn(List.of(threadsValue1, threadsValue2, threadsValue3));
result = evaluate(promql);
assertEquals(3, result.size());
assertTrue(result.stream().allMatch(t -> null != t.get("__value__")));
assertEquals(10.007799125, result.get(0).get("__value__"));
}
@Test
void testMultipleUnlessConditions() {
when(mockExecutor.execute("metric1")).thenReturn(List.of(new HashMap<>(Map.of("job", "api", "__value__", 40.0))));
when(mockExecutor.execute("metric2")).thenReturn(List.of(new HashMap<>(Map.of("job", "web", "__value__", 50.0))));
when(mockExecutor.execute("metric3")).thenReturn(List.of(new HashMap<>(Map.of("job", "api", "__value__", 60.0))));
when(mockExecutor.execute("metric1")).thenReturn(List.of(new HashMap<>(Map.of("__value__", 40.0))));
when(mockExecutor.execute("metric2")).thenReturn(List.of(new HashMap<>(Map.of("__value__", 50.0))));
when(mockExecutor.execute("metric3")).thenReturn(List.of(new HashMap<>(Map.of("__value__", 60.0))));
when(mockExecutor.execute("select cpu_usage from metrics where service = 'web'")).thenReturn(
List.of(new HashMap<>(Map.of("__value__", 40.0))));
when(mockExecutor.execute("select memory_usage from metrics where service = 'db'")).thenReturn(
@@ -424,13 +252,15 @@ class AlertExpressionEvalVisitorTest {
when(mockExecutor.execute("select disk_usage from metrics where service = 'cache'")).thenReturn(
List.of(new HashMap<>(Map.of("__value__", 60.0))));
// promql
List<Map<String, Object>> result = evaluate("(metric1 > 30 unless metric2 > 45) unless metric3 > 50");
assertEquals(0, result.size());
List<Map<String, Object>> result = evaluate("metric1 > 30 unless metric2 > 45 unless metric3 < 70");
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
// sql
result = evaluate("((select cpu_usage from metrics where service = 'web') > 30"
+ " unless (select memory_usage from metrics where service = 'db') > 55)"
+ " unless (select disk_usage from metrics where service = 'cache') > 50");
assertEquals(0, result.size());
result = evaluate("(select cpu_usage from metrics where service = 'web') > 30"
+ " unless (select memory_usage from metrics where service = 'db') > 45"
+ " unless (select disk_usage from metrics where service = 'cache') < 70");
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -490,11 +320,11 @@ class AlertExpressionEvalVisitorTest {
List.of(new HashMap<>(Map.of("__value__", 250.0))));
when(mockExecutor.execute("select min(response_time) from api_metrics where endpoint = '/api/users'")).thenReturn(
List.of(new HashMap<>(Map.of("__value__", 50.0))));
List<Map<String, Object>> result = evaluate("(select max(response_time) from api_metrics where endpoint = '/api/users') > 200");
assertEquals(1, result.size());
assertEquals(250.0, result.get(0).get("__value__"));
result = evaluate("(select min(response_time) from api_metrics where endpoint = '/api/users') < 100");
assertEquals(1, result.size());
assertEquals(50.0, result.get(0).get("__value__"));
@@ -643,227 +473,6 @@ class AlertExpressionEvalVisitorTest {
assertEquals(80, result.get(0).get("__value__"));
}
@Test
void testAndOpPromql() {
String promql = "http_server_requests_seconds_count > 10 and http_server_requests_seconds_max > 5";
Map<String, Object> countValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> countValue2 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 16);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/**");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> countValue3 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 7);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/health");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> maxValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 10.007799125);
put("method", "GET");
put("__name__", "http_server_requests_seconds_max");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> maxValue2 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 10);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/**");
put("outcome", "SUCCESS");
put("status", "200");
}
};
Map<String, Object> maxValue3 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 0);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/health");
put("outcome", "SUCCESS");
put("status", "200");
}
};
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(List.of(countValue1, countValue2, countValue3));
when(mockExecutor.execute("http_server_requests_seconds_max")).thenReturn(List.of(maxValue1, maxValue2, maxValue3));
List<Map<String, Object>> result = evaluate(promql);
assertEquals(2, result.size());
assertEquals(1307, result.get(0).get("__value__"));
assertEquals(16, result.get(1).get("__value__"));
}
@Test
void testComparisonExpr() {
String promql = "3 > bool 2";
// scalar and scalar
List<Map<String, Object>> result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1, result.get(0).get("__value__"));
// scalar and vector
promql = "0 < http_server_requests_seconds_count";
Map<String, Object> vector = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("uri", "/actuator/prometheus");
put("status", "200");
}
};
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(List.of(vector));
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1307, result.get(0).get("__value__"));
// scalar and vector bool
promql = "0 < bool http_server_requests_seconds_count";
// true
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1, result.get(0).get("__value__"));
// false
vector.put("__value__", -1);
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(0, result.get(0).get("__value__"));
// vector and scalar
promql = "http_server_requests_seconds_sum > 0";
vector = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 13);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("uri", "/actuator/prometheus");
put("status", "200");
}
};
when(mockExecutor.execute("http_server_requests_seconds_sum")).thenReturn(List.of(vector));
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(13, result.get(0).get("__value__"));
// vector and scalar bool
promql = "http_server_requests_seconds_sum > bool 0";
// true
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1, result.get(0).get("__value__"));
// false
vector.put("__value__", -1);
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(0, result.get(0).get("__value__"));
// vector and vector
promql = "http_server_requests_seconds_count > http_server_requests_seconds_sum";
Map<String, Object> vectorCount = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 13);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("uri", "/actuator/prometheus");
put("status", "200");
}
};
Map<String, Object> vectorSum = new HashMap<>() {
{
put("instance", "host.docker.internal:8989");
put("__value__", 10);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("uri", "/actuator/prometheus");
put("status", "200");
}
};
when(mockExecutor.execute("http_server_requests_seconds_count")).thenReturn(List.of(vectorCount));
when(mockExecutor.execute("http_server_requests_seconds_sum")).thenReturn(List.of(vectorSum));
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(13, result.get(0).get("__value__"));
// vector and vector bool
promql = "http_server_requests_seconds_count > bool http_server_requests_seconds_sum";
// true
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(1, result.get(0).get("__value__"));
// false
vectorCount.put("__value__", 1);
result = evaluate(promql);
assertEquals(1, result.size());
assertEquals(0, result.get(0).get("__value__"));
}
private List<Map<String, Object>> evaluate(String expression) {
AlertExpressionLexer lexer = new AlertExpressionLexer(CharStreams.fromString(expression));
CommonTokenStream tokens = new CommonTokenStream(lexer);
@@ -17,7 +17,19 @@
package org.apache.hertzbeat.alert.service;
import com.google.common.collect.Lists;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anySet;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import org.apache.hertzbeat.alert.calculate.PeriodicAlertRuleScheduler;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.alert.service.impl.AlertDefineServiceImpl;
@@ -33,27 +45,6 @@ import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.apache.hertzbeat.common.constants.CommonConstants.ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.apache.hertzbeat.common.constants.CommonConstants.ALERT_THRESHOLD_TYPE_REALTIME;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anySet;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test case for {@link AlertDefineService}
*/
@@ -71,9 +62,6 @@ class AlertDefineServiceTest {
@Mock
private List<AlertDefineImExportService> alertDefineImExportServiceList;
@Mock
private DataSourceService dataSourceService;
@InjectMocks
private AlertDefineServiceImpl alertDefineService;
@@ -143,36 +131,4 @@ class AlertDefineServiceTest {
assertNotNull(alertDefineService.getAlertDefines(null, null, "id", "desc", 1, 10));
verify(alertDefineDao, times(1)).findAll(any(Specification.class), any(PageRequest.class));
}
@Test
void getDefinePreview() {
String expr = "http_server_requests_seconds_count > 10";
Map<String, Object> countValue1 = new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
};
when(dataSourceService.calculate(eq("promql"), eq(expr))).thenReturn(Lists.newArrayList(countValue1));
List<Map<String, Object>> result = alertDefineService.getDefinePreview("promql", ALERT_THRESHOLD_TYPE_PERIODIC, expr);
assertNotNull(result);
assertEquals(1307, result.get(0).get("__value__"));
result = alertDefineService.getDefinePreview("promql", ALERT_THRESHOLD_TYPE_PERIODIC, null);
assertEquals(0, result.size());
result = alertDefineService.getDefinePreview("promql", ALERT_THRESHOLD_TYPE_REALTIME, null);
assertEquals(0, result.size());
}
}
@@ -23,6 +23,7 @@ import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.service.impl.AlibabaCloudSlsExternAlertService;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
@@ -42,6 +43,7 @@ import static org.mockito.Mockito.verify;
/**
* unit test for {@link AlibabaCloudSlsExternAlertServiceTest }
*/
@Disabled
@ExtendWith(MockitoExtension.class)
public class AlibabaCloudSlsExternAlertServiceTest {
@@ -17,35 +17,29 @@
package org.apache.hertzbeat.alert.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import java.util.HashMap;
import com.github.benmanes.caffeine.cache.Cache;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.hertzbeat.alert.service.impl.DataSourceServiceImpl;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import org.mockito.Mockito;
/**
* test case for {@link DataSourceService}
*/
class DataSourceServiceTest {
private DataSourceServiceImpl dataSourceService;
@BeforeEach
void setUp() {
dataSourceService = new DataSourceServiceImpl();
@@ -57,12 +51,12 @@ class DataSourceServiceTest {
new HashMap<>(Map.of("__value__", 100.0, "timestamp", 1343554, "instance", "node1")),
new HashMap<>(Map.of("__value__", 200.0, "timestamp", 1343555, "instance", "node2"))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
Mockito.when(mockExecutor.support("promql")).thenReturn(true);
Mockito.when(mockExecutor.execute(Mockito.anyString())).thenReturn(prometheusData);
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total > 150");
assertEquals(2, result.size());
assertNull(result.get(0).get("__value__"));
@@ -302,36 +296,45 @@ class DataSourceServiceTest {
@Test
void calculate15() {
List<Map<String, Object>> prometheusData1 = List.of(
new HashMap<>(Map.of("__value__", 1))
new HashMap<>(Map.of("__value__", 100.0, "timestamp", 1343554, "instance", "node1")),
new HashMap<>(Map.of("__value__", 200.0, "timestamp", 1343555, "instance", "node2"))
);
List<Map<String, Object>> prometheusData2 = List.of(
new HashMap<>(Map.of("__value__", 100.0, "timestamp", 1343554, "instance", "node1")),
new HashMap<>(Map.of("__value__", 200.0, "timestamp", 1343555, "instance", "node2"))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
Mockito.when(mockExecutor.support("promql")).thenReturn(true);
Mockito.when(mockExecutor.execute("count(node_cpu_seconds_total{mode=\"user\"} > 250)")).thenReturn(prometheusData1);
Mockito.when(mockExecutor.execute("count(node_cpu_seconds_total{mode=\"idle\"} < 220 )")).thenReturn(new ArrayList<>());
Mockito.when(mockExecutor.execute("node_cpu_seconds_total{mode=\"user\"}")).thenReturn(prometheusData1);
Mockito.when(mockExecutor.execute("node_cpu_seconds_total{mode=\"idle\"}")).thenReturn(prometheusData2);
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "count(node_cpu_seconds_total{mode=\"user\"} > 250) > 0 and count(node_cpu_seconds_total{mode=\"idle\"} < 220 ) > 0");
assertEquals(0, result.size());
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 250 and node_cpu_seconds_total{mode=\"idle\"} < 220");
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
void calculate16() {
List<Map<String, Object>> prometheusData1 = List.of(
new HashMap<>(Map.of("__value__", 1))
new HashMap<>(Map.of("__value__", 100.0, "timestamp", 1343554, "instance", "node1")),
new HashMap<>(Map.of("__value__", 200.0, "timestamp", 1343555, "instance", "node2"))
);
List<Map<String, Object>> prometheusData2 = List.of(
new HashMap<>(Map.of("__value__", 1))
new HashMap<>(Map.of("__value__", 100.0, "timestamp", 1343554, "instance", "node1")),
new HashMap<>(Map.of("__value__", 200.0, "timestamp", 1343555, "instance", "node2"))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
Mockito.when(mockExecutor.support("promql")).thenReturn(true);
Mockito.when(mockExecutor.execute("count(node_cpu_seconds_total{mode=\"user\"} > 250)")).thenReturn(prometheusData1);
Mockito.when(mockExecutor.execute("count(node_cpu_seconds_total{mode=\"idle\"} < 220 )")).thenReturn(prometheusData2);
Mockito.when(mockExecutor.execute("node_cpu_seconds_total{mode=\"user\"}")).thenReturn(prometheusData1);
Mockito.when(mockExecutor.execute("node_cpu_seconds_total{mode=\"idle\"}")).thenReturn(prometheusData2);
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "count(node_cpu_seconds_total{mode=\"user\"} > 250) > 0 and count(node_cpu_seconds_total{mode=\"idle\"} < 220 ) > 0");
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 50 and node_cpu_seconds_total{mode=\"idle\"} < 20");
assertEquals(1, result.size());
assertNotNull(result.get(0).get("__value__"));
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -396,7 +399,8 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 250 or node_cpu_seconds_total{mode=\"idle\"} < 20");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -417,7 +421,8 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 50 or node_cpu_seconds_total{mode=\"idle\"} < 320");
assertEquals(4, result.size());
assertEquals(1, result.size());
assertNotNull(result.get(0).get("__value__"));
}
@Test
@@ -438,8 +443,8 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 50 unless node_cpu_seconds_total{mode=\"idle\"} < 320");
assertEquals(2, result.size());
assertEquals(100.0, result.get(0).get("__value__"));
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -460,8 +465,8 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 50 unless node_cpu_seconds_total{mode=\"idle\"} < 20");
assertEquals(2, result.size());
assertEquals(100.0, result.get(0).get("__value__"));
assertEquals(1, result.size());
assertNotNull(result.get(0).get("__value__"));
}
@Test
@@ -482,7 +487,8 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "node_cpu_seconds_total{mode=\"user\"} > 250 unless node_cpu_seconds_total{mode=\"idle\"} < 20");
assertEquals(0, result.size());
assertEquals(1, result.size());
assertNull(result.get(0).get("__value__"));
}
@Test
@@ -559,6 +565,7 @@ class DataSourceServiceTest {
tokenStreamCache.invalidateAll();
long beforeHits = tokenStreamCache.stats().hitCount();
dataSourceService.calculate("promql", expr);
expressionCache.invalidateAll();
dataSourceService.calculate("promql", expr);
long actualHits = tokenStreamCache.stats().hitCount() - beforeHits;
assertEquals(1, actualHits, "expression cache should hit but miss");
@@ -609,34 +616,4 @@ class DataSourceServiceTest {
long actualHits = tokenStreamCache.stats().hitCount() - beforeHits;
assertEquals(0, actualHits, "expression cache should miss but hit");
}
@Test
void testAlertExpressionException() {
List<Map<String, Object>> prometheusData = List.of(
new HashMap<>() {
{
put("exception", "none");
put("instance", "host.docker.internal:8989");
put("__value__", 1307);
put("method", "GET");
put("__name__", "http_server_requests_seconds_count");
put("__timestamp__", "1.750320922467E9");
put("error", "none");
put("job", "spring-boot-app");
put("uri", "/actuator/prometheus");
put("outcome", "SUCCESS");
put("status", "200");
}
});
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support(eq("promql"))).thenReturn(true);
when(mockExecutor.execute(eq("http_server_requests_seconds_count"))).thenReturn(prometheusData);
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.calculate("promql", "http_server_requests_seconds_count > 10");
assertNotNull(result);
assertEquals(1307, result.get(0).get("__value__"));
assertThrows(AlertExpressionException.class, () -> dataSourceService.calculate("promql", "http_server_requests_seconds_count{!@~!!#$%^&}"));
}
}
@@ -23,6 +23,7 @@ import org.apache.hertzbeat.alert.service.impl.HuaweiCloudExternAlertService;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
@@ -38,6 +39,7 @@ import static org.mockito.Mockito.verify;
/**
* unit test for {@link AlibabaCloudSlsExternAlertServiceTest }
*/
@Disabled
@ExtendWith(MockitoExtension.class)
public class HuaweiCloudExternAlertServiceTest {
@@ -1,92 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hertzbeat.alert.config.SmsConfig;
import org.apache.hertzbeat.alert.config.SmslocalSmsProperties;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
import org.apache.hertzbeat.common.constants.GeneralConfigTypeEnum;
import org.apache.hertzbeat.common.entity.manager.GeneralConfig;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.Mockito.when;
/**
* unit test for {@link SmsClientFactory }
*/
@ExtendWith(MockitoExtension.class)
public class SmsClientFactoryTest {
@Mock
private GeneralConfigDao generalConfigDao;
@Mock
private ObjectMapper objectMapper;
@Mock
private SmsConfig yamlSmsConfig;
@InjectMocks
private SmsClientFactory smsClientFactory;
@Test
void testloadDbConfig() throws JsonProcessingException {
GeneralConfig generalConfig = new GeneralConfig();
SmsConfig smsConfig = new SmsConfig();
smsConfig.setType("smslocal");
smsConfig.setEnable(true);
smsConfig.setSmslocal(new SmslocalSmsProperties("11"));
generalConfig.setContent(JsonUtil.toJson(smsConfig));
when(objectMapper.readValue(generalConfig.getContent(), SmsConfig.class)).thenReturn(smsConfig);
when(generalConfigDao.findByType(GeneralConfigTypeEnum.sms.name())).thenReturn(generalConfig);
assertNotNull(smsClientFactory.getSmsClient());
}
@Test
void testloadYamlConfig() {
when(generalConfigDao.findByType(GeneralConfigTypeEnum.sms.name())).thenReturn(null);
when(yamlSmsConfig.getType()).thenReturn("smslocal");
when(yamlSmsConfig.isEnable()).thenReturn(true);
when(yamlSmsConfig.getSmslocal()).thenReturn(new SmslocalSmsProperties("11"));
assertNotNull(smsClientFactory.getSmsClient());
}
@Test
void testNull() {
when(generalConfigDao.findByType(GeneralConfigTypeEnum.sms.name())).thenReturn(null);
when(yamlSmsConfig.getType()).thenReturn("");
assertNull(smsClientFactory.getSmsClient());
}
}
@@ -1,73 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service.impl;
import com.google.common.collect.Lists;
import org.apache.hertzbeat.alert.config.SmslocalSmsProperties;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.support.exception.SendMessageException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
/**
* Test case for {@link SmsLocalSmsClientImpl}
*/
@ExtendWith(MockitoExtension.class)
public class SmsLocalSmsClientImplTest {
@Mock
private SmslocalSmsProperties smslocalSmsProperties;
private SmsLocalSmsClientImpl smsLocalSmsClient;
@BeforeEach
void setUp() {
smsLocalSmsClient = new SmsLocalSmsClientImpl(smslocalSmsProperties);
when(smslocalSmsProperties.getApiKey()).thenReturn("2");
}
@Test
void testSendMessage() {
assertEquals("smslocal", smsLocalSmsClient.getType());
assertTrue(smsLocalSmsClient.checkConfig());
//
NoticeReceiver noticeReceiver = new NoticeReceiver();
noticeReceiver.setPhone("13888888888");
SingleAlert singleAlert = new SingleAlert();
singleAlert.setContent("test");
GroupAlert groupAlert = new GroupAlert();
groupAlert.setAlerts(Lists.newArrayList(singleAlert));
assertThrows(SendMessageException.class,
() -> smsLocalSmsClient.sendMessage(noticeReceiver, null, groupAlert));
}
}
@@ -1,85 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service.impl;
import org.apache.hertzbeat.alert.config.UniSmsProperties;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.support.exception.SendMessageException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.HashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
/**
* Test case for {@link UniSmsClientImpl}
*/
@ExtendWith(MockitoExtension.class)
public class UniSmsClientImplTest {
@Mock
private UniSmsProperties uniSmsProperties;
private UniSmsClientImpl uniSmsClient;
@BeforeEach
void setUp() {
uniSmsClient = new UniSmsClientImpl(uniSmsProperties);
when(uniSmsProperties.getSignature()).thenReturn("2");
when(uniSmsProperties.getTemplateId()).thenReturn("any(String.class)");
when(uniSmsProperties.getAuthMode()).thenReturn("hmac");
when(uniSmsProperties.getAccessKeyId()).thenReturn("hmac");
when(uniSmsProperties.getAccessKeySecret()).thenReturn("hmac");
}
@Test
void testSendMessage() {
assertEquals("unisms", uniSmsClient.getType());
assertTrue(uniSmsClient.checkConfig());
//
NoticeReceiver noticeReceiver = new NoticeReceiver();
noticeReceiver.setPhone("13888888888");
Map<String, String> commonLabels = new HashMap<>();
commonLabels.put("instance", "");
commonLabels.put("priority", "unknown");
Map<String, String> commonAnnotations = new HashMap<>();
commonAnnotations.put("test", "test");
GroupAlert groupAlert = new GroupAlert();
groupAlert.setCommonLabels(commonLabels);
groupAlert.setCommonAnnotations(commonAnnotations);
assertThrows(SendMessageException.class,
() -> uniSmsClient.sendMessage(noticeReceiver, null, groupAlert));
}
}
@@ -1,48 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.util;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Test case for {@link CryptoUtils}
*/
public class CryptoUtilsTest {
@Test
void testSha256Hex() {
String sign = CryptoUtils.sha256Hex("Hello world.");
assertEquals("aa3ec16e6acc809d8b2818662276256abfd2f1b441cb51574933f3d4bd115d11", sign);
}
@Test
void testHmacSha256Base64Debug() {
String signature = CryptoUtils.hmacSha256Base64("your-real-key", "your-real-data");
assertEquals("8JrfX0v5Tt3s8PfI85o6jcf5XM3C+vLlMwvFp45LupU=", signature);
}
@Test
void testHmacSha256Hex() {
String signature = CryptoUtils.hmacSha256Hex("your-real-key", "your-real-data");;
assertEquals("41878ccd7ecd795a2dd7ec39be7f33fed4be3ec75f5307689e39dd6f41fdbaac", signature);
}
}
@@ -19,7 +19,6 @@ package org.apache.hertzbeat.alert.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Optional;
import org.junit.jupiter.api.Test;
@@ -59,13 +58,4 @@ class DateUtilTest {
actualTimestamp = DateUtil.getTimeStampFromFormat(date, format);
assertFalse(actualTimestamp.isPresent());
}
@Test
void getZonedTimeStampFromFormat() {
String dataStr = "2025/06/02 22:56:15 GMT+08:00";
Long time = DateUtil.getZonedTimeStampFromFormat(dataStr, "yyyy/MM/dd HH:mm:ss 'GMT'XXX");
assertNotNull(time);
assertEquals(1748876175000L, time);
}
}
@@ -33,7 +33,7 @@
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<mqtt.version>1.2.5</mqtt.version>
<mqtt.version>1.3.3</mqtt.version>
</properties>
<dependencies>
@@ -140,19 +140,10 @@
</dependency>
<!-- mqtt -->
<dependency>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<groupId>com.hivemq</groupId>
<artifactId>hivemq-mqtt-client</artifactId>
<version>${mqtt.version}</version>
</dependency>
<!--Bouncy Castle-->
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.68</version>
</dependency>
<!--plc-->
<dependency>
<groupId>org.apache.plc4x</groupId>
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.collector.collect.database;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -75,7 +74,6 @@ public class JdbcCommonCollect extends AbstractCollect {
// code execution related - may result in remote code execution
"init=", "javaobjectserializer=", "runscript", "serverstatusdiffinterceptor",
"queryinterceptors=", "statementinterceptors=", "exceptioninterceptors=",
"xp_cmdshell", "create function", "dbms_java", "sp_sysexecute", "load_file",
// multiple statement execution - may lead to SQL injection
"allowmultiqueries",
@@ -84,96 +82,6 @@ public class JdbcCommonCollect extends AbstractCollect {
"autodeserialize", "detectcustomcollations",
};
// universal bypass detection mode - applicable to all databases for dangerous command bypass detection
private static final String[] UNIVERSAL_BYPASS_PATTERNS = {
".*create\\s*([/\\\\]|\\\\n|/n|\\n)\\s*trigger.*",
".*create\\s*([/\\\\]|\\\\n|/n|\\n)\\s*function.*",
".*drop\\s*([/\\\\]|\\\\n|/n|\\n)\\s*table.*",
".*drop\\s*([/\\\\]|\\\\n|/n|\\n)\\s*database.*",
".*run\\s*([/\\\\]|\\\\n|/n|\\n)\\s*script.*",
".*alter\\s*([/\\\\]|\\\\n|/n|\\n)\\s*system.*",
".*grant\\s*([/\\\\]|\\\\n|/n|\\n)\\s*all.*",
".*revoke\\s*([/\\\\]|\\\\n|/n|\\n)\\s*all.*",
".*xp\\s*([/\\\\]|\\\\n|/n|\\n)\\s*cmdshell.*",
".*load\\s*([/\\\\]|\\\\n|/n|\\n)\\s*file.*"
};
// database platform specific bypass detection mode
private static final HashMap<String, String[]> PLATFORM_BYPASS_PATTERNS = new HashMap<>();
static {
// H2 database special character bypass mode
PLATFORM_BYPASS_PATTERNS.put("h2", new String[]{
".*(\\\\\\\\|/|\\\\|\\\\n|/n|\\n)\\s*init\\s*=.*",
".*in\\s*([/\\\\]|\\\\n|/n|\\n)\\s*it\\s*=.*",
".*(\\\\\\\\|/|\\\\|\\\\n|/n|\\n)\\s*runscript\\s+from.*",
".*ru\\s*([/\\\\]|\\\\n|/n|\\n)\\s*script\\s+from.*"
});
// MySQL/MariaDB bypass mode
String[] mysqlPatterns = {
".*allow\\s*([/\\\\]|\\\\n|/n|\\n)\\s*load\\s*([/\\\\]|\\\\n|/n|\\n)\\s*local\\s*([/\\\\]|\\\\n|/n|\\n)\\s*infile.*",
".*allow\\s*([/\\\\]|\\\\n|/n|\\n)\\s*multi\\s*([/\\\\]|\\\\n|/n|\\n)\\s*queries.*",
".*query\\s*([/\\\\]|\\\\n|/n|\\n)\\s*interceptors.*",
".*statement\\s*([/\\\\]|\\\\n|/n|\\n)\\s*interceptors.*",
".*exception\\s*([/\\\\]|\\\\n|/n|\\n)\\s*interceptors.*",
".*auto\\s*([/\\\\]|\\\\n|/n|\\n)\\s*deserialize.*"
};
PLATFORM_BYPASS_PATTERNS.put("mysql", mysqlPatterns);
PLATFORM_BYPASS_PATTERNS.put("mariadb", mysqlPatterns);
// PostgreSQL bypass mode
PLATFORM_BYPASS_PATTERNS.put("postgresql", new String[]{
".*socket\\s*([/\\\\]|\\\\n|/n|\\n)\\s*factory.*",
".*logger\\s*([/\\\\]|\\\\n|/n|\\n)\\s*file.*",
".*ssl\\s*([/\\\\]|\\\\n|/n|\\n)\\s*mode.*",
".*logger\\s*([/\\\\]|\\\\n|/n|\\n)\\s*level.*"
});
// SQL Server bypass mode
PLATFORM_BYPASS_PATTERNS.put("sqlserver", new String[]{
".*integrated\\s*([/\\\\]|\\\\n|/n|\\n)\\s*security.*",
".*authentication\\s*([/\\\\]|\\\\n|/n|\\n)\\s*scheme.*",
".*select\\s*([/\\\\]|\\\\n|/n|\\n)\\s*method.*",
".*send\\s*([/\\\\]|\\\\n|/n|\\n)\\s*string\\s*([/\\\\]|\\\\n|/n|\\n)\\s*parameters\\s*([/\\\\]|\\\\n|/n|\\n)\\s*as\\s*([/\\\\]|\\\\n|/n|\\n)\\s*unicode.*",
".*x\\s*([/\\\\]|\\\\n|/n|\\n)\\s*open\\s*([/\\\\]|\\\\n|/n|\\n)\\s*state.*",
".*application\\s*([/\\\\]|\\\\n|/n|\\n)\\s*intent.*"
});
// ClickHouse bypass mode
PLATFORM_BYPASS_PATTERNS.put("clickhouse", new String[]{
".*custom\\s*([/\\\\]|\\\\n|/n|\\n)\\s*http\\s*([/\\\\]|\\\\n|/n|\\n)\\s*params.*",
".*http\\s*([/\\\\]|\\\\n|/n|\\n)\\s*connection\\s*([/\\\\]|\\\\n|/n|\\n)\\s*provider.*",
".*check\\s*([/\\\\]|\\\\n|/n|\\n)\\s*all\\s*([/\\\\]|\\\\n|/n|\\n)\\s*nodes.*",
".*fail\\s*([/\\\\]|\\\\n|/n|\\n)\\s*over.*",
".*use\\s*([/\\\\]|\\\\n|/n|\\n)\\s*objects\\s*([/\\\\]|\\\\n|/n|\\n)\\s*in\\s*([/\\\\]|\\\\n|/n|\\n)\\s*arrays.*"
});
// Oracle bypass mode
PLATFORM_BYPASS_PATTERNS.put("oracle", new String[]{
".*oracle\\s*([/\\\\]|\\\\n|/n|\\n)\\s*jdbc.*",
".*oracle\\s*([/\\\\]|\\\\n|/n|\\n)\\s*net.*",
".*oracle\\.jdbc\\.timezoneinfotable\\s*=.*",
".*oracle\\.net\\.wallet_location\\s*=.*",
".*oracle\\.net\\.ssl_server_dn_match\\s*=\\s*false.*",
".*oracle\\.jdbc\\.enablesqlinjectionattack\\s*=\\s*true.*",
".*oracle\\.jdbc\\.implicitstatementcachesize\\s*=\\s*0.*",
".*oracle\\.jdbc\\.timezoneinfotable\\s*=.*",
".*oracle\\.net\\.wallet_location\\s*=.*",
".*oracle\\.net\\.ssl_server_dn_match\\s*=\\s*false.*",
".*oracle\\.jdbc\\.enablesqlinjectionattack\\s*=\\s*true.*",
".*oracle\\.jdbc\\.implicitstatementcachesize\\s*=\\s*0.*"
});
// DM bypass mode
PLATFORM_BYPASS_PATTERNS.put("dm", new String[]{
".*login\\s*([/\\\\]|\\\\n|/n|\\n)\\s*mode.*",
".*compatible\\s*([/\\\\]|\\\\n|/n|\\n)\\s*mode.*",
".*en\\s*([/\\\\]|\\\\n|/n|\\n)\\s*crypt.*",
".*ci\\s*([/\\\\]|\\\\n|/n|\\n)\\s*pher.*"
});
}
private final GlobalConnectionCache connectionCommonCache = GlobalConnectionCache.getInstance();
@@ -183,9 +91,8 @@ public class JdbcCommonCollect extends AbstractCollect {
throw new IllegalArgumentException("Database collect must has jdbc params");
}
if (StringUtils.hasText(metrics.getJdbc().getUrl())) {
String url = metrics.getJdbc().getUrl().toLowerCase();
for (String keyword : VULNERABLE_KEYWORDS) {
if (url.contains(keyword.toLowerCase())) {
if (metrics.getJdbc().getUrl().contains(keyword)) {
throw new IllegalArgumentException("Jdbc url prohibit contains vulnerable param " + keyword);
}
}
@@ -433,27 +340,6 @@ public class JdbcCommonCollect extends AbstractCollect {
}
}
/**
* Recursively decode the URL to prevent multiple encoding bypasses.
*
* @param url jdbc url
* @return decoded decoded jdbc url
*/
private String recursiveDecode(String url) {
String prev;
String decoded = url;
int max = 5; // Decode it at most 5 times to prevent infinite loops.
do {
prev = decoded;
try {
decoded = java.net.URLDecoder.decode(prev, StandardCharsets.UTF_8);
} catch (Exception e) {
break;
}
} while (!prev.equals(decoded) && --max > 0);
return decoded;
}
/**
* construct jdbc url due the jdbc protocol
*
@@ -469,54 +355,20 @@ public class JdbcCommonCollect extends AbstractCollect {
throw new IllegalArgumentException("JDBC URL length exceeds maximum limit of 2048 characters");
}
// remove special characters
String cleanedUrl = jdbcProtocol.getUrl().replaceAll("[\\x00-\\x1F\\x7F\\xA0]", "");
String url = recursiveDecode(cleanedUrl);
String urlLowerCase = url.toLowerCase();
// url format check
if (!urlLowerCase.matches("^jdbc:[a-zA-Z0-9]+:([^\\s;]+)(;[^\\s;]+)*$")) {
throw new IllegalArgumentException("Invalid JDBC URL format");
}
String cleanedUrl = jdbcProtocol.getUrl().replaceAll("[\\x00-\\x1F\\x7F]", "");
String url = cleanedUrl.toLowerCase();
// backlist check
for (String keyword : BLACK_LIST) {
if (urlLowerCase.contains(keyword.toLowerCase())) {
if (url.contains(keyword)) {
throw new IllegalArgumentException("Invalid JDBC URL: contains potentially malicious parameter: " + keyword);
}
}
// universal detection
String normalizedUrl = urlLowerCase.replaceAll("[\\x00-\\x1F\\x7F\\xA0]", " ");
// universal detection of JDBC injection and deserialization attacks
if (normalizedUrl.matches(".*jndi\\s*[:=].*")
|| normalizedUrl.matches(".*ldap\\s*[:=].*")
|| normalizedUrl.matches(".*rmi\\s*[:=].*")
|| normalizedUrl.matches(".*java\\s*[:=].*")
|| normalizedUrl.matches(".*serialization\\s*[:=].*")
|| normalizedUrl.matches(".*deserializ.*\\s*[:=].*")
|| normalizedUrl.matches(".*objectinputstream\\s*[:=].*")
|| normalizedUrl.matches(".*readobject\\s*[:=].*")) {
throw new IllegalArgumentException("Invalid JDBC URL: contains potentially malicious JNDI or deserialization parameter");
// url format check
if (!url.matches("^jdbc:[a-zA-Z0-9]+://[^\\s]+$")) {
throw new IllegalArgumentException("Invalid JDBC URL format");
}
// universal detection of bypass
for (String pattern : UNIVERSAL_BYPASS_PATTERNS) {
if (normalizedUrl.matches(pattern)) {
throw new IllegalArgumentException("Invalid JDBC URL: contains potentially malicious bypass pattern");
}
}
// database platform specific bypass detection
if (jdbcProtocol.getPlatform() != null) {
String platform = jdbcProtocol.getPlatform().toLowerCase();
// check for specific bypass modes on the platform
String[] platformPatterns = PLATFORM_BYPASS_PATTERNS.get(platform);
if (platformPatterns != null) {
for (String pattern : platformPatterns) {
if (normalizedUrl.matches(pattern)) {
throw new IllegalArgumentException("Invalid " + platform.toUpperCase() + " JDBC URL: contains potentially malicious bypass pattern");
}
}
}
}
return url;
return cleanedUrl;
}
assert jdbcProtocol.getPlatform() != null;
return switch (jdbcProtocol.getPlatform()) {
case "mysql", "mariadb" -> "jdbc:mysql://" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase())
@@ -68,7 +68,6 @@ import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
@@ -145,46 +144,37 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return;
}
long responseTime = System.currentTimeMillis() - startTime;
/*
this could create large objects, potentially impacting JVM memory space significantly.
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
Long responseTime = System.currentTimeMillis() - startTime;
String parseType = metrics.getHttp().getParseType();
HttpEntity entity = response.getEntity();
try {
if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
if (entity != null) {
parseResponseByPrometheusExporter(entity.getContent(), metrics.getAliasFields(), builder);
}
} else if (DispatchConstants.PARSE_HEADER.equals(parseType)) {
parseResponseByHeader(builder, metrics.getAliasFields(), response);
// Consume entity to release connection
EntityUtils.consumeQuietly(entity);
} else {
/*
this could create large objects, potentially impacting JVM memory space significantly.
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8);
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
switch (parseType) {
case DispatchConstants.PARSE_JSON_PATH ->
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
case DispatchConstants.PARSE_PROM_QL ->
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_XML_PATH ->
parseResponseByXmlPath(resp, metrics, builder, responseTime);
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, statusCode);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
case DispatchConstants.PARSE_CONFIG ->
parseResponseByConfig(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
default ->
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
}
switch (parseType) {
case DispatchConstants.PARSE_JSON_PATH ->
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
case DispatchConstants.PARSE_PROM_QL ->
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_PROMETHEUS ->
parseResponseByPrometheusExporter(response.getEntity().getContent(), metrics.getAliasFields(), builder);
case DispatchConstants.PARSE_XML_PATH ->
parseResponseByXmlPath(resp, metrics, builder, responseTime);
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
case DispatchConstants.PARSE_HEADER ->
parseResponseByHeader(builder, metrics.getAliasFields(), response);
case DispatchConstants.PARSE_CONFIG ->
parseResponseByConfig(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
default ->
parseResponseByDefault(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
}
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
@@ -247,15 +237,11 @@ public class HttpCollectImpl extends AbstractCollect {
}
private void parseResponseByWebsite(String resp, Metrics metrics, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime, int statusCode) {
CollectRep.MetricsData.Builder builder, Long responseTime) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
for (String alias : metrics.getAliasFields()) {
if (CollectorConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(statusCode));
} else {
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
}
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
}
builder.addValueRow(valueRowBuilder.build());
}
@@ -604,18 +590,6 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
builder.addValueRow(valueRowBuilder.build());
} else if (objectValue instanceof Number numberValue) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(numberValue.toString());
}
}
builder.addValueRow(valueRowBuilder.build());
}
}
}
@@ -640,13 +614,11 @@ public class HttpCollectImpl extends AbstractCollect {
.collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String aliasField : aliasFields) {
String columnValue = labelMap.get(aliasField);
if (columnValue != null) {
valueRowBuilder.addColumn(columnValue);
} else if (CommonConstants.PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
if ("value".equals(aliasField)) {
valueRowBuilder.addColumn(String.valueOf(metric.getValue()));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
String columnValue = labelMap.get(aliasField);
valueRowBuilder.addColumn(columnValue == null ? CommonConstants.NULL_VALUE : columnValue);
}
}
builder.addValueRow(valueRowBuilder.build());
@@ -880,4 +852,4 @@ public class HttpCollectImpl extends AbstractCollect {
}
return successCodeSet.contains(statusCode);
}
}
}
@@ -60,7 +60,7 @@ public class PrometheusMatrixParser extends AbstractPrometheusParse {
boolean setValueFlag = false;
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String aliasField : aliasFields) {
if (!CollectUtil.assertPromRequireField(aliasField) || CollectUtil.isValueAsLabel(aliasField, r.getMetric())) {
if (!CollectUtil.assertPromRequireField(aliasField)) {
JsonElement jsonElement = r.getMetric().get(aliasField);
if (jsonElement != null) {
valueRowBuilder.addColumn(jsonElement.getAsString());
@@ -93,6 +93,7 @@ public class PrometheusMatrixParser extends AbstractPrometheusParse {
}
builder.addValueRow(valueRowBuilder.build());
}
}
}
}
@@ -59,7 +59,7 @@ public class PrometheusVectorParser extends AbstractPrometheusParse {
for (PromVectorOrMatrix.Result r : result) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String aliasField : aliasFields) {
if (!CollectUtil.assertPromRequireField(aliasField) || CollectUtil.isValueAsLabel(aliasField, r.getMetric())) {
if (!CollectUtil.assertPromRequireField(aliasField)) {
JsonElement jsonElement = r.getMetric().get(aliasField);
if (jsonElement != null) {
valueRowBuilder.addColumn(jsonElement.getAsString());
@@ -42,6 +42,7 @@ import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
import javax.naming.Context;
import javax.rmi.ssl.SslRMIClientSocketFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.cache.AbstractConnection;
import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier;
@@ -53,15 +54,13 @@ import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JmxProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.LogUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
/**
* jmx protocol acquisition implementation
*/
@Slf4j
public class JmxCollectImpl extends AbstractCollect {
private static final String JMX_URL_PREFIX = "service:jmx:rmi:///jndi/rmi://";
@@ -76,8 +75,6 @@ public class JmxCollectImpl extends AbstractCollect {
private final ClassLoader jmxClassLoader;
private static final Logger logger = LoggerFactory.getLogger(JmxCollectImpl.class);
public JmxCollectImpl() {
jmxClassLoader = new JmxClassLoader(ClassLoader.getSystemClassLoader());
}
@@ -198,12 +195,12 @@ public class JmxCollectImpl extends AbstractCollect {
}
} catch (IOException exception) {
String errorMsg = CommonUtil.getMessageFromThrowable(exception);
LogUtil.error(logger, "JMX IOException: {0}", errorMsg);
log.error("JMX IOException :{}", errorMsg);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
LogUtil.error(logger, "JMX Error: {0}", errorMsg);
log.error("JMX Error :{}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
@@ -224,7 +221,7 @@ public class JmxCollectImpl extends AbstractCollect {
for (Attribute attribute : attributeList.asList()) {
Object value = attribute.getValue();
if (value == null) {
LogUtil.info(logger, "attribute {0} value is null.", attribute.getName());
log.info("attribute {} value is null.", attribute.getName());
continue;
}
if (value instanceof Number || value instanceof String || value instanceof ObjectName
@@ -248,7 +245,7 @@ public class JmxCollectImpl extends AbstractCollect {
}
attributeValueMap.put(attribute.getName(), builder.toString());
} else {
LogUtil.warn(logger, "attribute value type {0} not support.", value.getClass().getName());
log.warn("attribute value type {} not support.", value.getClass().getName());
}
}
return attributeValueMap;
@@ -322,7 +319,7 @@ public class JmxCollectImpl extends AbstractCollect {
connectionCommonCache.addCache(identifier, new JmxConnect(conn));
return conn;
} catch (Exception e) {
LogUtil.error(logger, "Failed to connect to JMX connection: {0}", e.getMessage());
log.error("Failed to connect to JMX server: {}", e.getMessage());
throw new IOException("Failed to connect to JMX server: " + e.getMessage(), e);
}
}
@@ -1,195 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.mqtt;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Formats the private key and certificate, supporting concatenation of multiple certificates in PEM format.
*/
public class CertificateFormatter {
public static String formatCertificateChain(String input) {
if (input == null || input.trim().isEmpty()) {
return input;
}
String normalized = normalizeInput(input);
List<String> certificates = extractCertificates(normalized);
if (certificates.isEmpty()) {
return formatAsSingleCertificate(normalized);
}
StringBuilder formattedChain = new StringBuilder();
for (String cert : certificates) {
if (cert.trim().isEmpty()) continue;
String formatted = formatPemBlock(cert);
formattedChain.append(formatted).append("\n");
}
return formattedChain.toString().trim();
}
private static String normalizeInput(String input) {
return input
.replace("\r\n", "\n")
.replace("\r", "\n")
.replaceAll("\\s*\\\\n\\s*", "\n")
.replaceAll("(?m)^\\s+|\\s+$", "")
.trim();
}
private static List<String> extractCertificates(String input) {
List<String> certificates = new ArrayList<>();
String regex = "(-----BEGIN\\s+[\\w\\s]+?-----)[\\s\\S]*?(-----END\\s+[\\w\\s]+?-----)";
Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(input);
int lastEnd = 0;
while (matcher.find()) {
if (matcher.start() > lastEnd) {
String gap = input.substring(lastEnd, matcher.start());
if (!gap.trim().isEmpty()) {
certificates.add(gap);
}
}
certificates.add(matcher.group());
lastEnd = matcher.end();
}
if (lastEnd < input.length()) {
certificates.add(input.substring(lastEnd));
}
return certificates;
}
private static String formatPemBlock(String block) {
try {
Pattern pattern = Pattern.compile(
"(-----BEGIN\\s+[\\w\\s]+?-----)(.*?)(-----END\\s+[\\w\\s]+?-----)",
Pattern.DOTALL | Pattern.CASE_INSENSITIVE
);
Matcher matcher = pattern.matcher(block);
if (matcher.find()) {
String header = matcher.group(1).trim();
String body = matcher.group(2);
String footer = matcher.group(3).trim();
if (body == null) body = "";
String cleanBody = body
.replaceAll("\\s", "")
.replaceAll("\"", "")
.trim();
if (cleanBody.isEmpty() && body != null && !body.trim().isEmpty()) {
cleanBody = body.replaceAll("[^a-zA-Z0-9+/=]", "").trim();
}
String formattedBody = formatBase64Body(cleanBody);
return header + "\n" + formattedBody + "\n" + footer;
} else {
return formatAsCertificate(block);
}
} catch (Exception e) {
return block;
}
}
private static String formatAsCertificate(String content) {
String cleanContent = content.replaceAll("[^a-zA-Z0-9+/=]", "").trim();
if (cleanContent.isEmpty()) {
return content;
}
String formattedBody = formatBase64Body(cleanContent);
if (cleanContent.toLowerCase().contains("private")) {
if (cleanContent.startsWith("MII") || cleanContent.length() > 1000) {
return "-----BEGIN PRIVATE KEY-----\n" + formattedBody + "\n-----END PRIVATE KEY-----";
} else {
return "-----BEGIN RSA PRIVATE KEY-----\n" + formattedBody + "\n-----END RSA PRIVATE KEY-----";
}
} else {
return "-----BEGIN CERTIFICATE-----\n" + formattedBody + "\n-----END CERTIFICATE-----";
}
}
private static String formatAsSingleCertificate(String input) {
String cleanContent = input.replaceAll("[^a-zA-Z0-9+/=]", "").trim();
return formatAsCertificate(cleanContent);
}
private static String formatBase64Body(String body) {
StringBuilder formatted = new StringBuilder();
int index = 0;
while (index < body.length()) {
int end = Math.min(index + 64, body.length());
formatted.append(body.substring(index, end));
if (end < body.length()) {
formatted.append("\n");
}
index = end;
}
return formatted.toString().trim();
}
public static String formatPrivateKey(String input) {
if (input == null || input.trim().isEmpty()) {
return input;
}
String normalized = normalizeInput(input);
if (isPemEncapsulated(normalized)) {
return formatPemBlock(normalized);
}
return formatAsCertificate(normalized);
}
private static boolean isPemEncapsulated(String block) {
return block.contains("-----BEGIN") && block.contains("-----END");
}
}
@@ -17,7 +17,26 @@
package org.apache.hertzbeat.collector.collect.mqtt;
import com.hivemq.client.mqtt.MqttVersion;
import com.hivemq.client.mqtt.datatypes.MqttQos;
import com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClient;
import com.hivemq.client.mqtt.mqtt3.Mqtt3Client;
import com.hivemq.client.mqtt.mqtt3.Mqtt3ClientBuilder;
import com.hivemq.client.mqtt.mqtt3.message.connect.connack.Mqtt3ConnAck;
import com.hivemq.client.mqtt.mqtt5.Mqtt5AsyncClient;
import com.hivemq.client.mqtt.mqtt5.Mqtt5Client;
import com.hivemq.client.mqtt.mqtt5.Mqtt5ClientBuilder;
import com.hivemq.client.mqtt.mqtt5.message.connect.connack.Mqtt5ConnAck;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -27,27 +46,13 @@ import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.entity.message.CollectRep.MetricsData.Builder;
import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
import org.eclipse.paho.client.mqttv3.MqttClientPersistence;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttCallback;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.IMqttToken;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.StopWatch;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* collect mqtt metrics using Eclipse Paho
* collect mqtt metrics
*/
public class MqttCollectImpl extends AbstractCollect {
@@ -56,224 +61,138 @@ public class MqttCollectImpl extends AbstractCollect {
private static final Logger logger = LoggerFactory.getLogger(MqttCollectImpl.class);
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_MQTT;
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
MqttProtocol mqttProtocol = metrics.getMqtt();
Assert.hasText(mqttProtocol.getHost(), "MQTT protocol host is required");
Assert.hasText(mqttProtocol.getPort(), "MQTT protocol port is required");
if ("mqtts".equalsIgnoreCase(mqttProtocol.getProtocol())) {
if (Boolean.parseBoolean(mqttProtocol.getEnableMutualAuth())) {
Assert.hasText(mqttProtocol.getCaCert(), "CA certificate is required for mutual auth");
Assert.hasText(mqttProtocol.getClientCert(), "Client certificate is required for mutual auth");
Assert.hasText(mqttProtocol.getClientKey(), "Client private key is required for mutual auth");
}
}
Assert.hasText(mqttProtocol.getProtocolVersion(), "MQTT protocol version is required");
}
@Override
public void collect(Builder builder, Metrics metrics) {
MqttProtocol mqtt = metrics.getMqtt();
String protocolVersion = mqtt.getProtocolVersion();
MqttVersion mqttVersion = MqttVersion.valueOf(protocolVersion);
if (mqttVersion == MqttVersion.MQTT_3_1_1) {
collectWithVersion3(metrics, builder);
} else if (mqttVersion == MqttVersion.MQTT_5_0) {
collectWithVersion5(metrics, builder);
}
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_MQTT;
}
/**
* collecting data of MQTT 5
*/
private void collectWithVersion5(Metrics metrics, Builder builder) {
MqttProtocol mqttProtocol = metrics.getMqtt();
Map<Object, String> data = new HashMap<>();
try {
MqttAsyncClient client = buildMqttClient(mqttProtocol);
long responseTime = connectClient(client, mqttProtocol);
testSubscribeAndPublish(client, mqttProtocol, data);
convertToMetricsData(builder, metrics, responseTime, data);
client.disconnect();
} catch (Exception e) {
logger.error("MQTT collection error: {}", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Collection failed: " + e.getMessage());
}
}
private MqttAsyncClient buildMqttClient(MqttProtocol protocol) throws Exception {
String clientId = protocol.getClientId();
String serverUri = String.format("%s://%s:%s",
StringUtils.equals(protocol.getProtocol(), "MQTT") ? "tcp" : "ssl",
protocol.getHost(),
protocol.getPort());
MqttClientPersistence persistence = new MemoryPersistence();
return new MqttAsyncClient(serverUri, clientId, persistence);
}
private long connectClient(MqttAsyncClient client, MqttProtocol protocol) throws Exception {
MqttConnectOptions connOpts = new MqttConnectOptions();
if (protocol.hasAuth()) {
connOpts.setUserName(protocol.getUsername());
connOpts.setPassword(protocol.getPassword().toCharArray());
}
connOpts.setKeepAliveInterval(Integer.parseInt(protocol.getKeepalive()));
connOpts.setConnectionTimeout(Integer.parseInt(protocol.getTimeout()) / 1000);
connOpts.setCleanSession(true);
connOpts.setAutomaticReconnect(false);
if ("mqtts".equalsIgnoreCase(protocol.getProtocol())) {
boolean insecureSkipVerify = Boolean.parseBoolean(protocol.getInsecureSkipVerify());
if (insecureSkipVerify) {
connOpts.setHttpsHostnameVerificationEnabled(false);
}
if (Boolean.parseBoolean(protocol.getEnableMutualAuth())) {
connOpts.setSocketFactory(MqttSslFactory.getMslSocketFactory(protocol, insecureSkipVerify));
} else {
connOpts.setSocketFactory(MqttSslFactory.getSslSocketFactory(protocol, insecureSkipVerify));
}
}
StopWatch connectWatch = new StopWatch();
connectWatch.start();
client.connect(connOpts).waitForCompletion(Long.parseLong(protocol.getTimeout()));
connectWatch.stop();
return connectWatch.getTotalTimeMillis();
}
/**
* Test MQTT subscribe and publish capabilities
*/
private void testSubscribeAndPublish(MqttAsyncClient client, MqttProtocol protocol, Map<Object, String> data) {
// 1 test subscribe
if (StringUtils.isNotBlank(protocol.getTopic())) {
String subscribe = testSubscribe(client, protocol.getTopic());
if (StringUtils.isBlank(subscribe)) {
data.put("canSubscribe", "Subscription successful");
} else {
data.put("canSubscribe", String.format("Subscription failed: %s", subscribe));
}
} else {
data.put("canSubscribe", "No topic, subscription test skipped");
}
// 2 test publish
if (StringUtils.isNotBlank(protocol.getTestMessage())) {
String publish = testPublish(client, protocol.getTopic(), protocol.getTestMessage());
if (StringUtils.isBlank(publish)) {
data.put("canPublish", "Message published successfully");
// 3 test receive message
String receivedData = getReceivedData(client, protocol.getTopic());
data.put("canReceive", receivedData);
} else {
data.put("canPublish", String.format("Message publishing failed: %s", publish));
data.put("canReceive", "Message reception skipped due to failed publish");
}
} else {
data.put("canPublish", "No test message, publish test skipped");
data.put("canReceive", "No test message, receive test skipped");
}
// 4 test unsubscribe
if (StringUtils.isNotBlank(protocol.getTopic())) {
String subscribe = testUnSubscribe(client, protocol.getTopic());
if (StringUtils.isBlank(subscribe)) {
data.put("canUnSubscribe", "Unsubscription successful");
} else {
data.put("canUnSubscribe", String.format("Unsubscription failed: %s", subscribe));
}
} else {
data.put("canUnSubscribe", "No topic, unsubscription test skipped");
}
}
private String getReceivedData(MqttAsyncClient client, String topic) {
final CountDownLatch latch = new CountDownLatch(1);
final StringBuilder messageHolder = new StringBuilder();
client.setCallback(new MqttCallback() {
@Override
public void connectionLost(Throwable cause) {
latch.countDown();
}
@Override
public void messageArrived(String arrivedTopic, MqttMessage message) {
if (topic.equals(arrivedTopic)) {
messageHolder.append(new String(message.getPayload()));
latch.countDown();
}
}
@Override
public void deliveryComplete(IMqttDeliveryToken token) {
Mqtt5AsyncClient client = buildMqtt5Client(mqttProtocol);
long responseTime = connectClient(client, mqtt5AsyncClient -> {
CompletableFuture<Mqtt5ConnAck> connectFuture = mqtt5AsyncClient.connect();
try {
connectFuture.get(Long.parseLong(mqttProtocol.getTimeout()), TimeUnit.MILLISECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(getErrorMessage(e.getMessage()));
}
});
try {
boolean received = latch.await(5, TimeUnit.SECONDS);
if (messageHolder.length() > 0) {
return messageHolder.toString();
} else if (!received) {
return "Message reception timed out after 5 seconds";
} else {
return "No valid message received";
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return e.getMessage();
} finally {
client.setCallback(null);
}
}
private String testSubscribe(MqttAsyncClient client, String topic) {
try {
IMqttToken subToken = client.subscribe(topic, 1);
subToken.waitForCompletion(5000);
return "";
} catch (MqttException e) {
logger.warn("MQTT subscribe test failed: {}", e.getMessage());
return e.getMessage();
}
}
private String testPublish(MqttAsyncClient client, String topic, String message) {
try {
MqttMessage mqttMessage = new MqttMessage(message.getBytes());
mqttMessage.setQos(1);
IMqttToken pubToken = client.publish(topic, mqttMessage);
pubToken.waitForCompletion(5000);
return "";
} catch (MqttException e) {
logger.warn("MQTT publish test failed: {}", e.getMessage());
return e.getMessage();
}
}
private String testUnSubscribe(MqttAsyncClient client, String topic) {
try {
IMqttToken unsubToken = client.unsubscribe(topic);
unsubToken.waitForCompletion(5000);
return "";
} catch (MqttException e) {
logger.warn("MQTT unsubscribe test failed: {}", e.getMessage());
return e.getMessage();
}
testDescribeAndPublish5(client, mqttProtocol, data);
convertToMetricsData(builder, metrics, responseTime, data);
client.disconnect();
}
/**
* Convert collected data to MetricsData
* collecting data of MQTT 3.1.1
*/
private void collectWithVersion3(Metrics metrics, Builder builder) {
MqttProtocol mqttProtocol = metrics.getMqtt();
Map<Object, String> data = new HashMap<>();
Mqtt3AsyncClient client = buildMqtt3Client(mqttProtocol);
long responseTime = connectClient(client, mqtt3AsyncClient -> {
CompletableFuture<Mqtt3ConnAck> connectFuture = mqtt3AsyncClient.connect();
try {
connectFuture.get(Long.parseLong(mqttProtocol.getTimeout()), TimeUnit.MILLISECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(getErrorMessage(e.getMessage()));
}
});
testDescribeAndPublish3(client, mqttProtocol, data);
convertToMetricsData(builder, metrics, responseTime, data);
client.disconnect();
}
private void testDescribeAndPublish3(Mqtt3AsyncClient client, MqttProtocol mqttProtocol, Map<Object, String> data) {
data.put("canDescribe", test(() -> {
client.subscribeWith().topicFilter(mqttProtocol.getTopic()).qos(MqttQos.AT_LEAST_ONCE).send();
client.unsubscribeWith().topicFilter(mqttProtocol.getTopic()).send();
}, "subscribe").toString());
data.put("canPublish", !mqttProtocol.testPublish() ? Boolean.FALSE.toString() : test(() -> {
client.publishWith().topic(mqttProtocol.getTopic())
.payload(mqttProtocol.getTestMessage().getBytes(StandardCharsets.UTF_8))
.qos(MqttQos.AT_LEAST_ONCE).send();
data.put("canPublish", Boolean.TRUE.toString());
}, "publish").toString());
}
private void testDescribeAndPublish5(Mqtt5AsyncClient client, MqttProtocol mqttProtocol, Map<Object, String> data) {
data.put("canDescribe", test(() -> {
client.subscribeWith().topicFilter(mqttProtocol.getTopic()).qos(MqttQos.AT_LEAST_ONCE).send();
client.unsubscribeWith().topicFilter(mqttProtocol.getTopic()).send();
}, "subscribe").toString());
data.put("canPublish", !mqttProtocol.testPublish() ? Boolean.FALSE.toString() : test(() -> {
client.publishWith().topic(mqttProtocol.getTopic())
.payload(mqttProtocol.getTestMessage().getBytes(StandardCharsets.UTF_8))
.qos(MqttQos.AT_LEAST_ONCE).send();
data.put("canPublish", Boolean.TRUE.toString());
}, "publish").toString());
}
private Mqtt5AsyncClient buildMqtt5Client(MqttProtocol mqttProtocol) {
Mqtt5ClientBuilder mqtt5ClientBuilder = Mqtt5Client.builder()
.serverHost(mqttProtocol.getHost())
.identifier(mqttProtocol.getClientId())
.serverPort(Integer.parseInt(mqttProtocol.getPort()));
if (mqttProtocol.hasAuth()) {
mqtt5ClientBuilder.simpleAuth().username(mqttProtocol.getUsername())
.password(mqttProtocol.getPassword().getBytes(StandardCharsets.UTF_8))
.applySimpleAuth();
}
return mqtt5ClientBuilder.buildAsync();
}
private Mqtt3AsyncClient buildMqtt3Client(MqttProtocol mqttProtocol) {
Mqtt3ClientBuilder mqtt3ClientBuilder = Mqtt3Client.builder()
.serverHost(mqttProtocol.getHost())
.identifier(mqttProtocol.getClientId())
.serverPort(Integer.parseInt(mqttProtocol.getPort()));
if (mqttProtocol.hasAuth()) {
mqtt3ClientBuilder.simpleAuth().username(mqttProtocol.getUsername())
.password(mqttProtocol.getPassword().getBytes(StandardCharsets.UTF_8))
.applySimpleAuth();
}
return mqtt3ClientBuilder.buildAsync();
}
public <T> long connectClient(T client, Consumer<T> connect) {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
connect.accept(client);
stopWatch.stop();
return stopWatch.getTotalTimeMillis();
}
private void convertToMetricsData(Builder builder, Metrics metrics, long responseTime, Map<Object, String> data) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : metrics.getAliasFields()) {
@@ -288,4 +207,25 @@ public class MqttCollectImpl extends AbstractCollect {
builder.addValueRow(valueRowBuilder.build());
}
private Boolean test(Runnable runnable, String operationName) {
try {
runnable.run();
return true;
} catch (Exception e) {
logger.error("{} fail", operationName, e);
}
return false;
}
private String getErrorMessage(String errorMessage) {
if (StringUtils.isBlank(errorMessage)) {
return "connect failed";
}
String[] split = errorMessage.split(":");
if (split.length > 1) {
return Arrays.stream(split).skip(1).collect(Collectors.joining(":"));
}
return errorMessage;
}
}
@@ -1,186 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.mqtt;
import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol;
import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openssl.PEMKeyPair;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManagerFactory;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.StringReader;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.Security;
import java.security.cert.Certificate;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Collection;
/**
* Support MQTT SSL Factory
*/
public class MqttSslFactory {
/**
* Get MSL Socket Factory
*/
public static SSLSocketFactory getMslSocketFactory(MqttProtocol mqttProtocol, boolean insecureSkipVerify) {
try {
Security.addProvider(new BouncyCastleProvider());
KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
Certificate[] chain = null;
if (mqttProtocol.getClientCert() != null && !mqttProtocol.getClientCert().isEmpty()) {
String formatClientCert = CertificateFormatter.formatCertificateChain(mqttProtocol.getClientCert());
try (InputStream certIn = new ByteArrayInputStream(formatClientCert.getBytes())) {
CertificateFactory cf = CertificateFactory.getInstance("X.509");
Collection<? extends Certificate> certs = cf.generateCertificates(certIn);
chain = certs.toArray(new Certificate[0]);
}
}
PrivateKey privateKey;
if (mqttProtocol.getClientKey() != null && !mqttProtocol.getClientKey().isEmpty()) {
String formatClientKey = CertificateFormatter.formatPrivateKey(mqttProtocol.getClientKey());
try (PEMParser pemParser = new PEMParser(new StringReader(formatClientKey))) {
JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC");
Object object = pemParser.readObject();
if (object instanceof PEMKeyPair) {
privateKey = converter.getPrivateKey(((PEMKeyPair) object).getPrivateKeyInfo());
} else if (object instanceof PrivateKeyInfo) {
privateKey = converter.getPrivateKey((PrivateKeyInfo) object);
} else {
throw new IllegalArgumentException("Unsupported private key type");
}
ks.setKeyEntry("private-key", privateKey, "".toCharArray(), chain);
}
}
TrustManager[] trustManagers;
if (insecureSkipVerify) {
trustManagers = createInsecureTrustManager();
} else {
String formatCaCert = CertificateFormatter.formatCertificateChain(mqttProtocol.getCaCert());
KeyStore trustStore = createMergedTrustStore(formatCaCert);
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
trustManagers = tmf.getTrustManagers();
}
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(ks, "".toCharArray());
SSLContext context = SSLContext.getInstance(mqttProtocol.getTlsVersion());
context.init(kmf.getKeyManagers(), trustManagers, null);
return context.getSocketFactory();
} catch (Exception e) {
throw new RuntimeException("Fails to SSL initialize: " + e.getMessage(), e);
}
}
/**
* Get SSL Socket Factory
*/
public static SSLSocketFactory getSslSocketFactory(MqttProtocol mqttProtocol, boolean insecureSkipVerify) {
try {
Security.addProvider(new BouncyCastleProvider());
TrustManager[] trustManagers;
if (insecureSkipVerify) {
trustManagers = createInsecureTrustManager();
} else {
String formatCaCert = CertificateFormatter.formatCertificateChain(mqttProtocol.getCaCert());
KeyStore trustStore = createMergedTrustStore(formatCaCert);
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
trustManagers = tmf.getTrustManagers();
}
SSLContext sslContext = SSLContext.getInstance(mqttProtocol.getTlsVersion());
sslContext.init(null, trustManagers, null);
return sslContext.getSocketFactory();
} catch (Exception e) {
throw new RuntimeException("Fails to SSL initialize: " + e.getMessage(), e);
}
}
private static TrustManager[] createInsecureTrustManager() {
return new TrustManager[]{
new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
};
}
private static KeyStore createMergedTrustStore(String caCertPem) throws Exception {
KeyStore mergedKs = KeyStore.getInstance(KeyStore.getDefaultType());
mergedKs.load(null, null);
TrustManagerFactory systemTmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
systemTmf.init((KeyStore) null);
X509TrustManager systemTm = (X509TrustManager) systemTmf.getTrustManagers()[0];
int systemIndex = 1;
for (X509Certificate cert : systemTm.getAcceptedIssuers()) {
mergedKs.setCertificateEntry("system-ca-" + systemIndex++, cert);
}
if (caCertPem != null && !caCertPem.isEmpty()) {
try (InputStream caIn = new ByteArrayInputStream(caCertPem.getBytes())) {
CertificateFactory cf = CertificateFactory.getInstance("X.509");
Collection<? extends Certificate> customCerts = cf.generateCertificates(caIn);
int customIndex = 1;
for (Certificate cert : customCerts) {
mergedKs.setCertificateEntry("custom-ca-" + customIndex++, cert);
}
}
}
return mergedKs;
}
}
@@ -0,0 +1,308 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.prometheus;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.PrometheusProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.net.ssl.SSLException;
import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
@Slf4j
public class PrometheusProxyCollectImpl implements PrometheusCollect {
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
public static final String RAW_TEXT_CONTENT_FIELD_NAME = "raw_text_content";
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
PrometheusProtocol prometheusProtocol = metrics.getPrometheus();
HttpUriRequest request;
try {
validateParams(metrics);
} catch (Exception e) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(e.getMessage());
return Collections.singletonList(builder.build());
}
HttpContext httpContext = createHttpContext(prometheusProtocol);
request = createHttpRequest(prometheusProtocol);
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
log.debug("Prometheus proxy collect, response status: {}", statusCode);
if (!defaultSuccessStatusCodes.contains(statusCode)) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return Collections.singletonList(builder.build());
}
String rawTextContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
builder.clearFields();
builder.clearValues();
CollectRep.Field rawDataField = CollectRep.Field.newBuilder()
.setName(RAW_TEXT_CONTENT_FIELD_NAME)
.setType(CommonConstants.TYPE_STRING)
.build();
builder.addField(rawDataField);
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(rawTextContent);
builder.addValueRow(valueRowBuilder.build());
builder.setCode(CollectRep.Code.SUCCESS);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error("Prometheus proxy collect error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e1);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (UnknownHostException e2) {
String errorMsg = CommonUtil.getMessageFromThrowable(e2);
log.info("Prometheus proxy collect unknown host: {}. Host: {}", errorMsg, prometheusProtocol.getHost(), e2);
builder.setCode(CollectRep.Code.UN_REACHABLE);
builder.setMsg("unknown host:" + errorMsg);
} catch (InterruptedIOException | ConnectException | SSLException e3) {
String errorMsg = CommonUtil.getMessageFromThrowable(e3);
log.info("Prometheus proxy collect connect error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e3);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (IOException e4) {
String errorMsg = CommonUtil.getMessageFromThrowable(e4);
log.info("Prometheus proxy collect IO error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e4);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error("Prometheus proxy collect unknown error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_PROMETHEUS;
}
private void validateParams(Metrics metrics) throws Exception {
if (metrics == null || metrics.getPrometheus() == null) {
throw new Exception("Prometheus collect must has prometheus params");
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (!StringUtils.hasText(protocol.getHost())
|| !StringUtils.hasText(protocol.getPort())) {
throw new Exception("Prometheus collect must has host and port params");
}
if (protocol.getPath() == null
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
/**
* create httpContext
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol prometheus protocol
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol) {
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
AuthCache authCache = new BasicAuthCache();
HttpHost targetHost = new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort()));
authCache.put(targetHost, new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return null;
}
/**
* create http request
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol http params
* @return http uri request
*/
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
for (Map.Entry<String, String> param : params.entrySet()) {
if (StringUtils.hasText(param.getValue())) {
requestBuilder.addParameter(param.getKey(), param.getValue());
}
}
}
requestBuilder.addHeader(HttpHeaders.CONNECTION, NetworkConstants.KEEP_ALIVE);
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
// headers The custom request header is overwritten here
Map<String, String> headers = protocol.getHeaders();
if (headers != null && !headers.isEmpty()) {
for (Map.Entry<String, String> header : headers.entrySet()) {
if (StringUtils.hasText(header.getValue())) {
requestBuilder.addHeader(CollectUtil.replaceUriSpecialChar(header.getKey()),
CollectUtil.replaceUriSpecialChar(header.getValue()));
}
}
}
if (headers == null || headers.keySet().stream().noneMatch(HttpHeaders.ACCEPT::equalsIgnoreCase)) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE + ";version=0.0.4,*/*;q=0.1");
}
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
if (StringUtils.hasText(authorization.getBearerTokenToken())) {
String value = DispatchConstants.BEARER + " " + authorization.getBearerTokenToken();
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
}
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
}
}
}
if (StringUtils.hasLength(protocol.getPayload())) {
requestBuilder.setEntity(new StringEntity(protocol.getPayload(), StandardCharsets.UTF_8));
if (headers == null || headers.keySet().stream().noneMatch(HttpHeaders.CONTENT_TYPE::equalsIgnoreCase)) {
requestBuilder.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN_VALUE);
}
}
String uriPath = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uriPath);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
} else {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout());
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
} else {
RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
}
/**
* get collect instance
* @return instance
*/
public static PrometheusProxyCollectImpl getInstance() {
return PrometheusProxyCollectImpl.SingleInstance.INSTANCE;
}
/**
* static instance
*/
private static class SingleInstance {
private static final PrometheusProxyCollectImpl INSTANCE = new PrometheusProxyCollectImpl();
}
}
@@ -185,10 +185,6 @@ public class OnlineParser {
stringBuilder.append((char) i);
i = getChar(inputStream);
}
// Skip \r character to handle Windows line endings
if (i == '\r') {
i = getChar(inputStream);
}
return new CharChecker(i);
}
@@ -197,10 +193,6 @@ public class OnlineParser {
while (i >= '0' && i <= '9') {
i = getChar(inputStream);
}
// Skip \r character to handle Windows line endings
if (i == '\r') {
i = getChar(inputStream);
}
return new CharChecker(i);
}
@@ -17,6 +17,11 @@
package org.apache.hertzbeat.collector.collect.database;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
@@ -24,14 +29,6 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test case for {@link JdbcCommonCollect}
*/
@@ -131,81 +128,4 @@ class JdbcCommonCollectTest {
String protocol = jdbcCommonCollect.supportProtocol();
assertEquals(DispatchConstants.PROTOCOL_JDBC, protocol);
}
@Test
void testUrlPassThrough() {
String[] testUrls = {
"jdbc:mysql://localhost:3306/test?allowPublicKeyRetrieval=true&useSSL=false",
"jdbc:mysql://localhost:3306/test?usessl=false&verifyServerCertificate=true",
"jdbc:mysql://localhost:3306/test?serverTimezone=UTC&autoReconnect=false"
};
JdbcCommonCollect jdbcCollect = new JdbcCommonCollect();
for (String originalUrl : testUrls) {
try {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.host("localhost")
.port("3306")
.platform("mysql")
.username("root")
.password("root")
.database("test")
.url(originalUrl)
.build();
// Use reflection to call constructDatabaseUrl method
Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class);
constructMethod.setAccessible(true);
String processedUrl = (String) constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306");
// Verify that the processed URL is the same as the original URL
assertEquals(originalUrl, processedUrl,
"URL should be passed through without modification: " + originalUrl);
} catch (Exception e) {
System.out.println("URL rejected by security validation: " + originalUrl + ", reason: " + e.getMessage());
}
}
}
@Test
void testConstructDatabaseUrlSecurityInterception() {
JdbcCommonCollect jdbcCollect = new JdbcCommonCollect();
String[] maliciousUrls = {
// URL length limit test
"jdbc:mysql://localhost:3306/test?" + "a".repeat(2050) + "=value",
// url format check
"jdbca:mysql://localhost:3306/test?allowLoadLocalInfile=true",
// backlist check
"jdbc:mysql://localhost:3306/test?allowLoadLocalInfile=true",
// universal detection of JDBC injection and deserialization attacks
"jdbc:mysql://localhost:3306/test?jndi:ldap://duansg.com/exploit",
// universal detection of bypass
"jdbc:mysql://localhost:3306/test?param=create\\trigger",
// database platform specific bypass detection
"jdbc:mysql://localhost:3306/test?allow\\nload\\nlocal\\ninfile=true"
};
// Test malicious URLs - should throw exceptions
for (String maliciousUrl : maliciousUrls) {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.host("localhost")
.port("3306")
.platform("mysql")
.username("root")
.password("root")
.database("test")
.url(maliciousUrl)
.build();
assertThrows(Exception.class, () -> {
try {
Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class);
constructMethod.setAccessible(true);
constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306");
} catch (InvocationTargetException e) {
throw e.getCause();
}
}, "Malicious URL should be blocked: " + maliciousUrl);
}
}
}
@@ -17,7 +17,14 @@
package org.apache.hertzbeat.collector.collect.http;
import com.google.common.collect.Lists;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
@@ -25,18 +32,6 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test case for {@link HttpCollectImpl}
*/
@@ -78,30 +73,6 @@ class HttpCollectImplTest {
assert "http".equals(protocol);
}
@Test
void parseResponseByWebsite() {
HttpProtocol http = HttpProtocol.builder().build();
http.setMethod("GET");
http.setHost("http://127.0.0.1");
http.setUrl("/");
http.setPort("8428");
http.setParseType("website");
http.setEnableUrlEncoding("true");
Metrics metrics = Metrics.builder()
.http(http)
.aliasFields(Lists.newArrayList("responseTime", "keyword", "statusCode"))
.build();
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
httpCollectImpl.collect(builder, metrics);
assertNotNull(builder.getValuesList());
for (CollectRep.ValueRow row : builder.getValuesList()) {
assertNotNull(row.getColumns(0));
assertEquals(row.getColumns(1), "0");
assertEquals(row.getColumns(2), "200");
}
}
@Test
void parseResponseByXmlPath() throws Exception {
// Create a sample XML response
@@ -189,356 +160,4 @@ class HttpCollectImplTest {
assertEquals("0.0", secondRow.getColumns(2), "Second server CPU should be 0.0");
assertEquals("0", secondRow.getColumns(3), "Second server memory should be 0");
}
@Test
void parseResponseByJsonPath() throws Exception {
String jsonResponse = "{"
+ " \"name\": \"jvm.memory.used\","
+ " \"description\": \"The amount of used memory\","
+ " \"baseUnit\": \"bytes\","
+ " \"measurements\": ["
+ " {"
+ " \"statistic\": \"VALUE\","
+ " \"value\": 90282296"
+ " }"
+ " ],"
+ " \"availableTags\": ["
+ " {"
+ " \"tag\": \"area\","
+ " \"values\": ["
+ " \"heap\","
+ " \"nonheap\""
+ " ]"
+ " },"
+ " {"
+ " \"tag\": \"id\","
+ " \"values\": ["
+ " \"G1 Survivor Space\","
+ " \"G1 Eden Space\""
+ " ]"
+ " }"
+ " ]"
+ "}";
HttpProtocol http = HttpProtocol.builder()
.parseType(DispatchConstants.PARSE_JSON_PATH)
.parseScript("$.availableTags[?(@.tag == \"id\")].values[*]")
.build();
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByJsonPath",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class,
Long.class);
parseMethod.setAccessible(true);
// Call the method
parseMethod.invoke(httpCollectImpl, jsonResponse, Lists.newArrayList("id"), http, builder, 100L);
// Verify the results
assertEquals(2, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("G1 Survivor Space", firstRow.getColumns(0));
CollectRep.ValueRow secondRow = capturedRows.get(1);
assertEquals("G1 Eden Space", secondRow.getColumns(0));
// number
String numberJson = "{"
+ " \"name\": \"system.cpu.usage\","
+ " \"description\": \"The \\\"recent cpu usage\\\" of the system the application is running in\","
+ " \"measurements\": ["
+ " {"
+ " \"statistic\": \"VALUE\","
+ " \"value\": 0.268751364291017"
+ " }"
+ " ],"
+ " \"availableTags\": []"
+ "}";
http = HttpProtocol.builder()
.parseType(DispatchConstants.PARSE_JSON_PATH)
.parseScript("$.measurements[?(@.statistic == \"VALUE\")].value")
.build();
capturedRows.clear();
builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
};
parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByJsonPath",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class,
Long.class);
parseMethod.setAccessible(true);
// Call the method
parseMethod.invoke(httpCollectImpl, numberJson, Lists.newArrayList("usage"), http, builder, 100L);
// Verify the results
assertEquals(1, capturedRows.size());
firstRow = capturedRows.get(0);
assertEquals("0.268751364291017", firstRow.getColumns(0));
}
@Test
void testParsePromQlLabelValue() throws Exception {
// Create Prometheus format test data
String prometheusData = """
{
"status": "success",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"__name__": "taos_cluster_info_first_ep",
"cluster_id": "590779086215866783",
"instance": "host.docker.internal:6043",
"job": "tdengine",
"value": "localhost:6030"
},
"value": [
1756233177.048,
"1"
]
}
]
}
}""";
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
@Override
public String getMetrics() {
return "jvm_memory_used_bytes";
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByPromQl",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, prometheusData, Lists.newArrayList("cluster_id", "value"), new HttpProtocol(), builder);
// Verify the results
assertEquals(1, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("590779086215866783", firstRow.getColumns(0));
assertEquals("localhost:6030", firstRow.getColumns(1));
}
@Test
void testParsePromQlMetricValue() throws Exception {
// Create Prometheus format test data
String prometheusData = """
{
"status": "success",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"__name__": "taos_cluster_info_first_ep",
"cluster_id": "590779086215866783",
"instance": "host.docker.internal:6043",
"job": "tdengine",
"value": "localhost:6030"
},
"value": [
1756233177.048,
"1"
]
}
]
}
}""";
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
@Override
public String getMetrics() {
return "jvm_memory_used_bytes";
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByPromQl",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, prometheusData, Lists.newArrayList("cluster_id", "value", "metric_value"), new HttpProtocol(), builder);
// Verify the results
assertEquals(1, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("590779086215866783", firstRow.getColumns(0));
assertEquals("localhost:6030", firstRow.getColumns(1));
assertEquals("1", firstRow.getColumns(2));
}
@Test
void testParsePromQlValue() throws Exception {
// Create Prometheus format test data
String prometheusData = """
{
"status": "success",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"__name__": "taos_cluster_info_first_ep",
"cluster_id": "590779086215866783",
"instance": "host.docker.internal:6043",
"job": "tdengine"
},
"value": [
1756233177.048,
"1"
]
}
]
}
}""";
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
@Override
public String getMetrics() {
return "jvm_memory_used_bytes";
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByPromQl",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, prometheusData, Lists.newArrayList("cluster_id", "value"), new HttpProtocol(), builder);
// Verify the results
assertEquals(1, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("590779086215866783", firstRow.getColumns(0));
assertEquals("1", firstRow.getColumns(1));
}
@Test
void testParsePrometheusLabelValue() throws Exception {
// Create Prometheus format test data
String prometheusData = """
# HELP jvm_memory_used_bytes The amount of used memory in bytes
# TYPE jvm_memory_used_bytes gauge
jvm_memory_used_bytes{area="heap",value="G1 Survivor Space"} 1048576
""";
InputStream inputStream = new ByteArrayInputStream(prometheusData.getBytes(StandardCharsets.UTF_8));
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
@Override
public String getMetrics() {
return "jvm_memory_used_bytes";
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByPrometheusExporter",
InputStream.class,
List.class,
CollectRep.MetricsData.Builder.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, inputStream, Lists.newArrayList("area", "value"), builder);
// Verify the results
assertEquals(1, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("heap", firstRow.getColumns(0));
assertEquals("G1 Survivor Space", firstRow.getColumns(1));
}
@Test
void testParsePrometheus() throws Exception {
// Create Prometheus format test data
String prometheusData = """
# HELP jvm_memory_used_bytes The amount of used memory
# TYPE jvm_memory_used_bytes gauge
jvm_memory_used_bytes{area="heap",id="G1 Eden Space"} 1.63577856E8
jvm_memory_used_bytes{area="heap",id="G1 Old Gen"} 2.7874304E7
jvm_memory_used_bytes{area="heap",id="G1 Survivor Space"} 512032.0
jvm_memory_used_bytes{area="nonheap",id="CodeCache"} 1.460288E7
jvm_memory_used_bytes{area="nonheap",id="Compressed Class Space"} 5844504.0
jvm_memory_used_bytes{area="nonheap",id="Metaspace"} 4.1576344E7
""";
InputStream inputStream = new ByteArrayInputStream(prometheusData.getBytes(StandardCharsets.UTF_8));
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
@Override
public String getMetrics() {
return "jvm_memory_used_bytes";
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByPrometheusExporter",
InputStream.class,
List.class,
CollectRep.MetricsData.Builder.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, inputStream, Lists.newArrayList("area", "id"), builder);
// Verify the results
assertEquals(6, capturedRows.size());
CollectRep.ValueRow firstRow = capturedRows.get(0);
assertEquals("heap", firstRow.getColumns(0));
assertEquals("G1 Eden Space", firstRow.getColumns(1));
capturedRows.forEach(t -> assertEquals(2, t.getColumnsList().size()));
}
}
@@ -17,91 +17,108 @@
package org.apache.hertzbeat.collector.collect.mqtt;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.hivemq.client.mqtt.MqttVersion;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test case for {@link MqttCollectImpl}
*/
class MqttCollectTest {
public class MqttCollectTest {
private MqttCollectImpl mqttCollect;
private Metrics metrics;
private MqttProtocol.MqttProtocolBuilder mqttBuilder;
private CollectRep.MetricsData.Builder builder;
@BeforeEach
void setup() {
public void setup() {
mqttCollect = new MqttCollectImpl();
metrics = new Metrics();
// Initialize base MQTT parameters for test cases
mqttBuilder = MqttProtocol.builder()
.host("example.com")
.port("1883")
.protocol("mqtt")
.timeout("5000")
.keepalive("60");
}
// Region: preCheck validation tests
@Test
// Verify preCheck throws exception when host is missing
void preCheckShouldThrowWhenHostMissing() {
metrics.setMqtt(mqttBuilder.host("").build());
assertThrows(IllegalArgumentException.class, () -> mqttCollect.preCheck(metrics));
MqttProtocol mqtt = MqttProtocol.builder().build();
metrics = Metrics.builder()
.mqtt(mqtt)
.build();
builder = CollectRep.MetricsData.newBuilder();
}
@Test
// Verify preCheck throws exception when port is missing
void preCheckShouldThrowWhenPortMissing() {
metrics.setMqtt(mqttBuilder.port("").build());
assertThrows(IllegalArgumentException.class, () -> mqttCollect.preCheck(metrics));
void preCheck() {
// host is empty
assertThrows(IllegalArgumentException.class, () -> {
mqttCollect.preCheck(metrics);
});
// port is empty
assertThrows(IllegalArgumentException.class, () -> {
MqttProtocol mqtt = MqttProtocol.builder().build();
mqtt.setHost("example.com");
metrics.setMqtt(mqtt);
mqttCollect.preCheck(metrics);
});
// protocol version is empty
assertThrows(IllegalArgumentException.class, () -> {
MqttProtocol mqtt = MqttProtocol.builder().build();
mqtt.setHost("example.com");
mqtt.setPort("1883");
metrics.setMqtt(mqtt);
mqttCollect.preCheck(metrics);
});
// everything is ok
assertDoesNotThrow(() -> {
MqttProtocol mqtt = MqttProtocol.builder().build();
mqtt.setHost("example.com");
mqtt.setPort("1883");
metrics.setMqtt(mqtt);
mqtt.setProtocolVersion("3.1.1");
mqttCollect.preCheck(metrics);
});
}
@Test
// Verify preCheck throws exception when MQTTS mutual auth is enabled but CA cert is missing
void preCheckShouldThrowWhenMqttsMutualAuthMissingCerts() {
metrics.setMqtt(mqttBuilder
.protocol("mqtts")
.enableMutualAuth("true")
.caCert("")
.clientCert("client.crt")
.clientKey("client.key")
.build());
assertThrows(IllegalArgumentException.class, () -> mqttCollect.preCheck(metrics));
void supportProtocol() {
Assertions.assertEquals(DispatchConstants.PROTOCOL_MQTT, mqttCollect.supportProtocol());
}
@Test
// Verify preCheck succeeds with valid standard MQTT parameters
void preCheckShouldSucceedWithValidMqttParams() {
metrics.setMqtt(mqttBuilder.build());
assertDoesNotThrow(() -> mqttCollect.preCheck(metrics));
}
void collect() {
// with version 3.1.1
assertDoesNotThrow(() -> {
MqttProtocol mqtt = MqttProtocol.builder().build();
mqtt.setHost("example.com");
mqtt.setPort("1883");
mqtt.setClientId("clientid");
mqtt.setTimeout("1");
mqtt.setProtocolVersion(MqttVersion.MQTT_3_1_1.name());
@Test
// Verify preCheck succeeds with valid MQTTS parameters including mutual authentication
void preCheckShouldSucceedWithValidMqttsMutualAuth() {
metrics.setMqtt(mqttBuilder
.protocol("mqtts")
.enableMutualAuth("true")
.caCert("ca.pem")
.clientCert("client.crt")
.clientKey("client.key")
.build());
assertDoesNotThrow(() -> mqttCollect.preCheck(metrics));
}
// End region
metrics.setMqtt(mqtt);
metrics.setAliasFields(new ArrayList<>());
@Test
// Verify supportProtocol method returns correct MQTT constant
void supportProtocolShouldReturnMqttConstant() {
assertEquals(DispatchConstants.PROTOCOL_MQTT, mqttCollect.supportProtocol());
mqttCollect.collect(builder, metrics);
});
assertDoesNotThrow(() -> {
MqttProtocol mqtt = MqttProtocol.builder().build();
mqtt.setHost("example.com");
mqtt.setPort("1883");
mqtt.setClientId("clientid");
mqtt.setTimeout("1");
mqtt.setProtocolVersion(MqttVersion.MQTT_5_0.name());
metrics.setMqtt(mqtt);
metrics.setAliasFields(new ArrayList<>());
mqttCollect.collect(builder, metrics);
});
}
}
@@ -105,186 +105,4 @@ class OnlineParserTest {
});
});
}
@Test
void testParseMetricsWithCrLf() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\r\n"
+ "# TYPE go_gc_duration_seconds summary\r\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0\r\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139\r\n";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\r\n"
+ "# TYPE go_gc_duration_seconds summary\r\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0 1234567890\r\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139 1234567890\r\n";
inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
@Test
void testParseMetricsWithLf() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n"
+ "# TYPE go_gc_duration_seconds summary\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139\n";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n"
+ "# TYPE go_gc_duration_seconds summary\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0 1234567890\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139 1234567890\n";
inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
@Test
void testParseMetricsWithoutFinalNewline() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\r\n"
+ "# TYPE go_gc_duration_seconds summary\r\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0\r\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n"
+ "# TYPE go_gc_duration_seconds summary\n"
+ "jvm_gc_pause_seconds_count{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 1.0\n"
+ "jvm_gc_pause_seconds_sum{action=\"end of major GC\",cause=\"Metadata GC Threshold\",} 0.139";
inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
@Test
void testParseMetricsWithEmptyLabelsAndCrLf() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\r\n"
+ "# TYPE go_gc_duration_seconds summary\r\n"
+ "jvm_gc_pause_seconds_count 1.0\r\n"
+ "jvm_gc_pause_seconds_sum{} 0.139";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
@Test
void testParseMetricsWithEmptyLabelsAndLf() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n"
+ "# TYPE go_gc_duration_seconds summary\n"
+ "jvm_gc_pause_seconds_count 1.0\n"
+ "jvm_gc_pause_seconds_sum{} 0.139";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
@Test
void testParseMetricsWithMixedLineEndings() throws Exception {
String str = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\r\n"
+ "# TYPE go_gc_duration_seconds summary\n"
+ "jvm_gc_pause_seconds_count 1.0\n"
+ "jvm_gc_pause_seconds_sum{} 0.139\r\n";
InputStream inputStream = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.values().size());
MetricFamily metricFamily = metricFamilyMap.get("jvm_gc_pause_seconds_count");
assertEquals("jvm_gc_pause_seconds_count", metricFamily.getName());
assertEquals(1.0, metricFamily.getMetricList().get(0).getValue());
MetricFamily metricFamily1 = metricFamilyMap.get("jvm_gc_pause_seconds_sum");
assertEquals("jvm_gc_pause_seconds_sum", metricFamily1.getName());
assertEquals(0.139, metricFamily1.getMetricList().get(0).getValue());
}
}
@@ -70,11 +70,14 @@
<version>${hertzbeat.version}</version>
</dependency>
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
@@ -84,15 +87,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- metrics -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
</dependencies>
<build>
@@ -345,4 +340,4 @@
</build>
</profile>
</profiles>
</project>
</project>
@@ -1,133 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.dispatch;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.metrics.HertzBeatMetricsCollector;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
*
*/
@Slf4j
@Component
public class CollectTaskTimeoutMonitor {
/**
* Collection task timeout value
*/
private static final long DURATION_TIME = 240_000L;
/**
* Metrics task and start time mapping map
*/
private final Map<String, MetricsTime> metricsTimeoutMonitorMap = new ConcurrentHashMap<>(16);
@Autowired
private HertzBeatMetricsCollector metricsCollector;
private CommonDispatcher commonDispatcher;
public void start(CommonDispatcher commonDispatcher) {
this.commonDispatcher = commonDispatcher;
// monitoring metrics collection task execution timeout
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("metrics-task-timeout-monitor-%d")
.setDaemon(true)
.build();
ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory);
scheduledExecutor.scheduleWithFixedDelay(this::monitorCollectTaskTimeout, 2, 20, TimeUnit.SECONDS);
}
public void putMetrics(String key, MetricsTime value) {
this.metricsTimeoutMonitorMap.put(key, value);
}
public MetricsTime removeMetrics(String key) {
return this.metricsTimeoutMonitorMap.remove(key);
}
private void monitorCollectTaskTimeout() {
try {
// Detect whether the collection unit of each metrics has timed out for 4 minutes,
// and if it times out, it will be discarded and an exception will be returned.
long deadline = System.currentTimeMillis() - DURATION_TIME;
for (Map.Entry<String, MetricsTime> entry : metricsTimeoutMonitorMap.entrySet()) {
MetricsTime metricsTime = entry.getValue();
if (metricsTime.getStartTime() < deadline) {
// Metrics collection timeout
MetricsTime removedMetricsTime = metricsTimeoutMonitorMap.remove(entry.getKey());
if (removedMetricsTime == null) {
continue;
}
WheelTimerTask timerJob = (WheelTimerTask) metricsTime.getTimeout().task();
Job job = timerJob.getJob();
// timeout metrics
if (metricsCollector != null) {
long duration = System.currentTimeMillis() - removedMetricsTime.getStartTime();
metricsCollector.recordCollectMetrics(job, duration, "timeout");
}
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
.setId(job.getMonitorId())
.setTenantId(job.getTenantId())
.setApp(job.getApp())
.setMetrics(metricsTime.getMetrics().getName())
.setPriority(metricsTime.getMetrics().getPriority())
.setTime(System.currentTimeMillis())
.setCode(CollectRep.Code.TIMEOUT)
.setMsg("collect timeout")
.build();
log.error("[Collect Timeout]: \n{}", metricsData);
if (metricsData.getPriority() == CommonConstants.AVAILABLE_METRICS) {
//todo 使用chain bootstrap
// commonDispatcher.dispatchCollectData(metricsTime.timeout, metricsTime.getMetrics(), metricsData);
}
}
}
} catch (Exception e) {
log.error("[Task Timeout Monitor]-{}.", e.getMessage(), e);
}
}
/**
* Metrics times.
*/
@Data
@AllArgsConstructor
public static class MetricsTime {
private long startTime;
private Metrics metrics;
private Timeout timeout;
}
}
@@ -17,23 +17,12 @@
package org.apache.hertzbeat.collector.dispatch;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gson.Gson;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.constants.ContextKey;
import org.apache.hertzbeat.collector.context.impl.DefaultContext;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.handler.ChainBootstrap;
import org.apache.hertzbeat.collector.handler.CollectMetricsDataDataStream;
import org.apache.hertzbeat.collector.listener.CalculateFieldsListener;
import org.apache.hertzbeat.collector.listener.MetricsDataDeliveryListener;
import org.apache.hertzbeat.collector.listener.RemoveTimeoutMonitorListener;
import org.apache.hertzbeat.collector.listener.RerunDataStream;
import org.apache.hertzbeat.collector.listener.ResponseJobDataListener;
import org.apache.hertzbeat.collector.listener.ValidateResponseListener;
import org.apache.hertzbeat.collector.metrics.HertzBeatMetricsCollector;
import org.apache.hertzbeat.collector.handler.impl.BatchExecuteTaskChain;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.collector.CollectorMetaData;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
@@ -44,21 +33,19 @@ import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
* Collection task and response data scheduler
@@ -67,6 +54,10 @@ import java.util.stream.Collectors;
@Slf4j
public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatch {
/**
* Collection task timeout value
*/
private static final long DURATION_TIME = 240_000L;
/**
* Trigger sub task max num
*/
@@ -88,41 +79,33 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
* collection data exporter
*/
private final CommonDataQueue commonDataQueue;
/**
* Metrics task and start time mapping map
*/
private final Map<String, MetricsTime> metricsTimeoutMonitorMap;
private final List<UnitConvert> unitConvertList;
private final WorkerPool workerPool;
private final CollectorMetaData metaData;
private final CollectTaskTimeoutMonitor collectTaskTimeoutMonitor;
@Autowired
private HertzBeatMetricsCollector metricsCollector;
private final String collectorIdentity;
public CommonDispatcher(MetricsCollectorQueue jobRequestQueue,
TimerDispatch timerDispatch,
CommonDataQueue commonDataQueue,
WorkerPool workerPool,
CollectJobService collectJobService,
List<UnitConvert> unitConvertList,
CollectTaskTimeoutMonitor collectTaskTimeoutMonitor) {
List<UnitConvert> unitConvertList) {
this.commonDataQueue = commonDataQueue;
this.jobRequestQueue = jobRequestQueue;
this.timerDispatch = timerDispatch;
this.unitConvertList = unitConvertList;
this.workerPool = workerPool;
this.metaData = CollectorMetaData.builder()
.identity(collectJobService.getCollectorIdentity())
.mode(collectJobService.getCollectorMode())
.startTime(new Date())
.build();
this.collectTaskTimeoutMonitor = collectTaskTimeoutMonitor;
// this.start();
this.collectTaskTimeoutMonitor.start(this);
this.collectorIdentity = collectJobService.getCollectorIdentity();
this.metricsTimeoutMonitorMap = new ConcurrentHashMap<>(16);
this.start();
}
@Deprecated
public void start() {
try {
// Pull the collection task from the task queue and put it into the thread pool for execution
@@ -150,141 +133,110 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
}
log.info("Thread Interrupted, Shutdown the [metrics-task-dispatcher]");
});
// monitoring metrics collection task execution timeout
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("metrics-task-timeout-monitor-%d")
.setDaemon(true)
.build();
ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory);
scheduledExecutor.scheduleWithFixedDelay(this::monitorCollectTaskTimeout, 2, 20, TimeUnit.SECONDS);
} catch (Exception e) {
log.error("Common Dispatcher error: {}.", e.getMessage(), e);
}
}
private void monitorCollectTaskTimeout() {
try {
// Detect whether the collection unit of each metrics has timed out for 4 minutes,
// and if it times out, it will be discarded and an exception will be returned.
long deadline = System.currentTimeMillis() - DURATION_TIME;
for (Map.Entry<String, MetricsTime> entry : metricsTimeoutMonitorMap.entrySet()) {
MetricsTime metricsTime = entry.getValue();
if (metricsTime.getStartTime() < deadline) {
// Metrics collection timeout
WheelTimerTask timerJob = (WheelTimerTask) metricsTime.getTimeout().task();
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
.setId(timerJob.getJob().getMonitorId())
.setTenantId(timerJob.getJob().getTenantId())
.setApp(timerJob.getJob().getApp())
.setMetrics(metricsTime.getMetrics().getName())
.setPriority(metricsTime.getMetrics().getPriority())
.setTime(System.currentTimeMillis())
.setCode(CollectRep.Code.TIMEOUT).setMsg("collect timeout").build();
log.error("[Collect Timeout]: \n{}", metricsData);
if (metricsData.getPriority() == 0) {
dispatchCollectData(metricsTime.timeout, metricsTime.getMetrics(), metricsData);
}
metricsTimeoutMonitorMap.remove(entry.getKey());
}
}
} catch (Exception e) {
log.error("[Task Timeout Monitor]-{}.", e.getMessage(), e);
}
}
@Override
public void dispatchMetricsTask(Timeout timeout) {
// Divide the collection task of a single application into corresponding collection tasks of the metrics under it.
// Divide the collection task of a single application into corresponding collection tasks of the metrics according to the metrics under it.
// Put each collect task into the thread pool for scheduling
WheelTimerTask timerTask = (WheelTimerTask) timeout.task();
Job job = timerTask.getJob();
ChainBootstrap bootstrap = constructMetricsCollectTaskChain(job);
//todo context需要划分作用域
bootstrap.addContext(ContextKey.META_DATA, metaData)
.addContext(ContextKey.JOB, job)
.addContext(ContextKey.TIMEOUT, timeout)
.addListener(new CalculateFieldsListener(unitConvertList))
.addListener(new ValidateResponseListener())
.onEachDataStreamComplete(new RemoveTimeoutMonitorListener(collectTaskTimeoutMonitor));
if (job.isCyclic()) {
bootstrap.withWorkerPool(workerPool)
.addListener(new MetricsDataDeliveryListener(commonDataQueue))
.onComplete(new RerunDataStream(timerDispatch));
} else {
bootstrap.addListener(new ResponseJobDataListener(timerDispatch));
}
bootstrap.start();
}
private ChainBootstrap constructMetricsCollectTaskChain(Job job) {
long now = System.currentTimeMillis();
Map<Byte, List<Metrics>> currentCollectMetrics = job.getMetrics().stream()
.filter(metrics -> (now >= metrics.getCollectTime() + metrics.getInterval() * 1000L))
.peek(metric -> {
metric.setCollectTime(now);
// Determine whether to configure aliasFields If not, configure the default
if ((metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) && metric.getFields() != null) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
// Set the default metrics execution priority, if not filled, the default last priority
if (metric.getPriority() == null) {
metric.setPriority(Byte.MAX_VALUE);
}
})
.collect(Collectors.groupingBy(Metrics::getPriority));
// the current collect metrics can not empty, if empty, add a default availability metrics
// due the metric collect is trigger by the previous metric collect
if (currentCollectMetrics.isEmpty()) {
Optional<Metrics> defaultMetricOption = job.getMetrics().stream()
.filter(metric -> metric.getPriority() == CommonConstants.AVAILABLE_METRICS).findFirst();
if (defaultMetricOption.isPresent()) {
Metrics defaultMetric = defaultMetricOption.get();
defaultMetric.setCollectTime(now);
currentCollectMetrics.put(CommonConstants.AVAILABLE_METRICS, Collections.singletonList(defaultMetric));
job.constructPriorMetrics();
Set<Metrics> metricsSet = job.getNextCollectMetrics(null, true);
metricsSet.forEach(metrics -> {
MetricsCollect metricsCollect = new MetricsCollect(metrics, timeout, this,
collectorIdentity, unitConvertList);
jobRequestQueue.addJob(metricsCollect);
if (metrics.getPrometheus() != null) {
metricsTimeoutMonitorMap.put(String.valueOf(job.getId()),
new MetricsTime(System.currentTimeMillis(), metrics, timeout));
} else {
log.error("metrics must has one priority 0 metrics at least.");
metricsTimeoutMonitorMap.put(job.getId() + "-" + metrics.getName(),
new MetricsTime(System.currentTimeMillis(), metrics, timeout));
}
}
ChainBootstrap chainBootstrap = ChainBootstrap.withContext(DefaultContext.newInstance())
.withChain(new BatchExecuteTaskChain<Metrics>());
// order by priority
currentCollectMetrics.keySet().stream()
.sorted()
.forEach(priority -> {
if (job.isCyclic() || isOneTimeJobAndIsAvailableMetrics(job, priority)) {
List<Metrics> metricsList = currentCollectMetrics.get(priority);
CollectMetricsDataDataStream collectHandler = CollectMetricsDataDataStream.builder()
.collectTaskTimeoutMonitor(collectTaskTimeoutMonitor)
.build();
collectHandler.setSourceDataList(metricsList);
chainBootstrap.addDataStream(collectHandler);
}
});
return chainBootstrap;
});
}
private boolean isOneTimeJobAndIsAvailableMetrics(Job job, byte priority) {
return (!job.isCyclic()) && priority == CommonConstants.AVAILABLE_METRICS;
}
@Deprecated
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
WheelTimerTask timerJob = (WheelTimerTask) timeout.task();
Job job = timerJob.getJob();
String monitorKey = job.getId() + "-" + metrics.getName();
if (metrics.isHasSubTask()) {
monitorKey = monitorKey + "-sub-" + metrics.getSubTaskId();
}
CollectTaskTimeoutMonitor.MetricsTime metricsTime = this.collectTaskTimeoutMonitor.removeMetrics(monitorKey);
// job completed metrics
if (metricsTime != null && metricsCollector != null) {
long duration = System.currentTimeMillis() - metricsTime.getStartTime();
String status = metricsData.getCode() == CollectRep.Code.SUCCESS ? "success" : "fail";
metricsCollector.recordCollectMetrics(job, duration, status);
}
if (metrics.isHasSubTask()) {
metricsTimeoutMonitorMap.remove(job.getId() + "-" + metrics.getName() + "-sub-" + metrics.getSubTaskId());
boolean isLastTask = metrics.consumeSubTaskResponse(metricsData);
if (isLastTask) {
metricsData = metrics.getSubTaskDataRef().get().build();
} else {
return;
}
} else {
metricsTimeoutMonitorMap.remove(job.getId() + "-" + metrics.getName());
}
Set<Metrics> metricsSet = job.getNextCollectMetrics(metrics, false);
if (job.isCyclic()) {
cyclicJobDebugLog(job, metricsData);
if (log.isDebugEnabled()) {
log.debug("Cyclic Job: {} - {} - {}", job.getMonitorId(), job.getApp(), metricsData.getMetrics());
for (CollectRep.ValueRow valueRow : metricsData.getValues()) {
for (CollectRep.Field field : metricsData.getFields()) {
log.debug("Field-->{},Value-->{}", field.getName(), valueRow.getColumns(metricsData.getFields().indexOf(field)));
}
}
}
// If metricsSet is null, it means that the execution is completed or whether the priority of the collection metrics is 0, that is, the availability collection metrics.
// If the availability collection fails, the next metrics scheduling will be cancelled and the next round of scheduling will be entered directly.
boolean isAvailableCollectFailed = metricsSet != null && !metricsSet.isEmpty()
&& metrics.getPriority() == CommonConstants.AVAILABLE_METRICS && metricsData.getCode() != CollectRep.Code.SUCCESS;
&& metrics.getPriority() == (byte) 0 && metricsData.getCode() != CollectRep.Code.SUCCESS;
if (metricsSet == null || isAvailableCollectFailed || job.isSd()) {
// The collection and execution task of this job are completed.
// The periodic task pushes the task to the time wheel again.
// First, determine the execution time of the task and the task collection interval.
if (!timeout.isCancelled()) {
long spendTime = System.currentTimeMillis() - job.getDispatchTime();
long interval = job.getInterval() - spendTime / 1000L;
long interval = job.getInterval() - spendTime / 1000;
interval = interval <= 0 ? 0 : interval;
timerDispatch.cyclicJob(timerJob, interval, TimeUnit.SECONDS);
}
}
else if (!metricsSet.isEmpty()) {
} else if (!metricsSet.isEmpty()) {
// The execution of the current level metrics is completed, and the execution of the next level metrics starts
// use pre collect metrics data to replace next metrics config params
List<Map<String, Configmap>> configmapList = CollectUtil.getConfigmapFromPreCollectData(metricsData);
@@ -295,15 +247,12 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
Set<String> cryPlaceholderFields = CollectUtil.matchCryPlaceholderField(GSON.toJsonTree(metricItem));
if (cryPlaceholderFields.isEmpty()) {
MetricsCollect metricsCollect = new MetricsCollect(metricItem, timeout, this,
metaData.getIdentity(), unitConvertList);
collectorIdentity, unitConvertList);
jobRequestQueue.addJob(metricsCollect);
this.collectTaskTimeoutMonitor.putMetrics(job.getId() + "-" + metricItem.getName(),
new CollectTaskTimeoutMonitor.MetricsTime(System.currentTimeMillis(), metricItem, timeout));
metricsTimeoutMonitorMap.put(job.getId() + "-" + metricItem.getName(),
new MetricsTime(System.currentTimeMillis(), metricItem, timeout));
continue;
}
boolean isSubTask = configmapList.stream().anyMatch(map -> map.keySet().stream().anyMatch(cryPlaceholderFields::contains));
int subTaskNum = isSubTask ? Math.min(configmapList.size(), MAX_SUB_TASK_NUM) : 1;
AtomicInteger subTaskNumAtomic = new AtomicInteger(subTaskNum);
@@ -319,30 +268,90 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
metric.setSubTaskId(index);
metric.setSubTaskDataRef(metricsDataReference);
MetricsCollect metricsCollect = new MetricsCollect(metric, timeout, this,
metaData.getIdentity(), unitConvertList);
collectorIdentity, unitConvertList);
jobRequestQueue.addJob(metricsCollect);
this.collectTaskTimeoutMonitor.putMetrics(job.getId() + "-" + metric.getName() + "-sub-" + index,
new CollectTaskTimeoutMonitor.MetricsTime(System.currentTimeMillis(), metric, timeout));
metricsTimeoutMonitorMap.put(job.getId() + "-" + metric.getName() + "-sub-" + index,
new MetricsTime(System.currentTimeMillis(), metric, timeout));
}
}
} else {
// The list of metrics at the current execution level has not been fully executed.
// It needs to wait for the execution of other metrics task of the same level to complete the execution and enter the next level for execution.
}
// If it is an asynchronous periodic cyclic task, directly response the collected data
if (job.isSd()) {
CollectRep.MetricsData sdMetricsData = CollectRep.MetricsData.newBuilder(metricsData).build();
commonDataQueue.sendServiceDiscoveryData(sdMetricsData);
}
commonDataQueue.sendMetricsData(metricsData);
} else {
// If it is a temporary one-time task, you need to wait for the collected data of all metrics task to be packaged and returned.
// Insert the current metrics data into the job for unified assembly
job.addCollectMetricsData(metricsData);
if (log.isDebugEnabled()) {
log.debug("One-time Job: {}", metricsData.getMetrics());
for (CollectRep.ValueRow valueRow : metricsData.getValues()) {
for (CollectRep.Field field : metricsData.getFields()) {
log.debug("Field-->{},Value-->{}", field.getName(), valueRow.getColumns(metricsData.getFields().indexOf(field)));
}
}
}
if (job.isSd() || metricsSet == null) {
// The collection and execution of all metrics of this job are completed
// and the result listener is notified of the combination of all metrics data
timerDispatch.responseSyncJobData(job.getId(), job.getResponseDataTemp());
} else if (!metricsSet.isEmpty()) {
// The execution of the current level metrics is completed, and the execution of the next level metrics starts
metricsSet.forEach(metricItem -> {
MetricsCollect metricsCollect = new MetricsCollect(metricItem, timeout, this,
collectorIdentity, unitConvertList);
jobRequestQueue.addJob(metricsCollect);
metricsTimeoutMonitorMap.put(job.getId() + "-" + metricItem.getName(),
new MetricsTime(System.currentTimeMillis(), metricItem, timeout));
});
} else {
// The list of metrics task at the current execution level has not been fully executed.
// It needs to wait for the execution of other metrics task of the same level to complete the execution and enter the next level for execution.
}
}
}
private void cyclicJobDebugLog(Job job, CollectRep.MetricsData metricsData) {
if (log.isDebugEnabled()) {
log.debug("Cyclic Job: {} - {} - {}", job.getMonitorId(), job.getApp(), metricsData.getMetrics());
metricsDataDebugLog(metricsData);
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
WheelTimerTask timerJob = (WheelTimerTask) timeout.task();
Job job = timerJob.getJob();
metricsTimeoutMonitorMap.remove(String.valueOf(job.getId()));
if (job.isCyclic()) {
// The collection and execution of all task of this job are completed.
// The periodic task pushes the task to the time wheel again.
// First, determine the execution time of the task and the task collection interval.
if (!timeout.isCancelled()) {
long spendTime = System.currentTimeMillis() - job.getDispatchTime();
long interval = job.getInterval() - spendTime / 1000;
interval = interval <= 0 ? 0 : interval;
timerDispatch.cyclicJob(timerJob, interval, TimeUnit.SECONDS);
}
// it is an asynchronous periodic cyclic task, directly response the collected data
metricsDataList.forEach(commonDataQueue::sendMetricsData);
} else {
// The collection and execution of all metrics of this job are completed
// and the result listener is notified of the combination of all metrics data
timerDispatch.responseSyncJobData(job.getId(), metricsDataList);
}
}
private void metricsDataDebugLog(CollectRep.MetricsData metricsData) {
for (CollectRep.ValueRow valueRow : metricsData.getValues()) {
for (CollectRep.Field field : metricsData.getFields()) {
log.debug("Field-->{},Value-->{}", field.getName(), valueRow.getColumns(metricsData.getFields().indexOf(field)));
}
}
/**
* Metrics times.
*/
@Data
@AllArgsConstructor
protected static class MetricsTime {
private long startTime;
private Metrics metrics;
private Timeout timeout;
}
}
}
@@ -129,6 +129,7 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
this.newTime = System.currentTimeMillis();
this.timeout = timeout;
this.metrics = metrics;
this.collectorIdentity = collectorIdentity;
WheelTimerTask timerJob = (WheelTimerTask) timeout.task();
Job job = timerJob.getJob();
this.id = job.getMonitorId();
@@ -152,9 +153,324 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
@Override
public void run() {
// collectDataDispatch.dispatchCollectData(timeout, metrics, metricsData);
this.startTime = System.currentTimeMillis();
setNewThreadName(id, app, startTime, metrics);
CollectRep.MetricsData.Builder response = CollectRep.MetricsData.newBuilder();
response.setApp(app).setId(id).setTenantId(tenantId)
.setLabels(labels).setAnnotations(annotations).addMetadataAll(metadata);
// for prometheus auto or proxy mode
if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(metrics.getProtocol())) {
List<CollectRep.MetricsData> metricsData;
// TODO: Refactor Prometheus metrics collection logic.
// The current implementation for proxy mode and auto mode needs review and potential simplification.
// Consider a more unified approach or clarify the conditions for each mode.
/*
// TODO USE PROXY MODE
if (prometheusProxyMode) {
List<CollectRep.MetricsData> proxyData = PrometheusProxyCollectImpl.getInstance().collect(response, metrics);
List<CollectRep.MetricsData> autoData = PrometheusAutoCollectImpl.getInstance().collect(response, metrics);
metricsData = new LinkedList<>();
if (proxyData != null) {
metricsData.addAll(proxyData);
}
if (autoData != null) {
metricsData.addAll(autoData);
}
} else {
metricsData = PrometheusAutoCollectImpl.getInstance().collect(response, metrics);
}
*/
metricsData = PrometheusAutoCollectImpl.getInstance().collect(response, metrics);
validateResponse(metricsData == null ? null : metricsData.stream().findFirst().orElse(null));
collectDataDispatch.dispatchCollectData(timeout, metrics, metricsData);
return;
}
response.setMetrics(metrics.getName());
// According to the metrics collection protocol, application type, etc.,
// dispatch to the real application metrics collection implementation class
AbstractCollect abstractCollect = CollectStrategyFactory.invoke(metrics.getProtocol());
if (abstractCollect == null) {
log.error("[Dispatcher] - not support this: app: {}, metrics: {}, protocol: {}.",
app, metrics.getName(), metrics.getProtocol());
response.setCode(CollectRep.Code.FAIL);
response.setMsg("not support " + app + ", "
+ metrics.getName() + ", " + metrics.getProtocol());
} else {
try {
abstractCollect.preCheck(metrics);
abstractCollect.collect(response, metrics);
} catch (Exception e) {
String msg = e.getMessage();
if (msg == null && e.getCause() != null) {
msg = e.getCause().getMessage();
}
if (e instanceof IllegalArgumentException) {
log.error("[Metrics PreCheck]: {}.", msg, e);
} else {
log.error("[Metrics Collect]: {}.", msg, e);
}
response.setCode(CollectRep.Code.FAIL);
if (msg != null) {
response.setMsg(msg);
}
}
}
// Alias attribute expression replacement calculation
if (fastFailed()) {
return;
}
calculateFields(metrics, response);
CollectRep.MetricsData metricsData = validateResponse(response);
collectDataDispatch.dispatchCollectData(timeout, metrics, metricsData);
}
/**
* Calculate the real metrics value according to the calculates and aliasFields configuration
*
* @param metrics Metrics configuration
* @param collectData Data collection
*/
public void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder collectData) {
collectData.setPriority(metrics.getPriority());
List<CollectRep.Field> fieldList = new LinkedList<>();
for (Metrics.Field field : metrics.getFields()) {
CollectRep.Field.Builder fieldBuilder = CollectRep.Field.newBuilder();
fieldBuilder.setName(field.getField()).setType(field.getType()).setLabel(field.isLabel());
if (field.getUnit() != null) {
fieldBuilder.setUnit(field.getUnit());
}
fieldList.add(fieldBuilder.build());
}
collectData.addAllFields(fieldList);
List<CollectRep.ValueRow> aliasRowList = collectData.getValuesList();
if (aliasRowList == null || aliasRowList.isEmpty()) {
return;
}
collectData.clearValues();
// Preprocess calculates first
if (metrics.getCalculates() == null) {
metrics.setCalculates(Collections.emptyList());
}
// eg: database_pages=Database pages unconventional mapping
Map<String, String> fieldAliasMap = new HashMap<>(8);
Map<String, JexlExpression> fieldExpressionMap = metrics.getCalculates()
.stream()
.map(cal -> transformCal(cal, fieldAliasMap))
.filter(Objects::nonNull)
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue));
if (metrics.getUnits() == null) {
metrics.setUnits(Collections.emptyList());
}
Map<String, Pair<String, String>> fieldUnitMap = metrics.getUnits()
.stream()
.map(this::transformUnit)
.filter(Objects::nonNull)
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair<String, String>) arr[1], (oldValue, newValue) -> newValue));
List<Metrics.Field> fields = metrics.getFields();
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
Map<String, String> aliasFieldValueMap = new HashMap<>(8);
Map<String, Object> fieldValueMap = new HashMap<>(8);
Map<String, Object> stringTypefieldValueMap = new HashMap<>(8);
Map<String, String> aliasFieldUnitMap = new HashMap<>(8);
CollectRep.ValueRow.Builder realValueRowBuilder = CollectRep.ValueRow.newBuilder();
for (CollectRep.ValueRow aliasRow : aliasRowList) {
for (int aliasIndex = 0; aliasIndex < aliasFields.size(); aliasIndex++) {
String aliasFieldValue = aliasRow.getColumns(aliasIndex);
String aliasField = aliasFields.get(aliasIndex);
if (!CommonConstants.NULL_VALUE.equals(aliasFieldValue)) {
aliasFieldValueMap.put(aliasField, aliasFieldValue);
// whether the alias field is a number
CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil
.extractDoubleAndUnitFromStr(aliasFieldValue);
if (doubleAndUnit != null && doubleAndUnit.getValue() != null) {
fieldValueMap.put(aliasField, doubleAndUnit.getValue());
if (doubleAndUnit.getUnit() != null) {
aliasFieldUnitMap.put(aliasField, doubleAndUnit.getUnit());
}
} else {
fieldValueMap.put(aliasField, aliasFieldValue);
}
stringTypefieldValueMap.put(aliasField, aliasFieldValue);
} else {
fieldValueMap.put(aliasField, null);
stringTypefieldValueMap.put(aliasField, null);
}
}
for (Metrics.Field field : fields) {
String realField = field.getField();
JexlExpression expression = fieldExpressionMap.get(realField);
String value = null;
String aliasFieldUnit = null;
if (expression != null) {
try {
Map<String, Object> context;
if (CommonConstants.TYPE_STRING == field.getType()) {
context = stringTypefieldValueMap;
} else {
for (Map.Entry<String, String> unitEntry : aliasFieldUnitMap.entrySet()) {
if (expression.getSourceText().contains(unitEntry.getKey())) {
aliasFieldUnit = unitEntry.getValue();
break;
}
}
context = fieldValueMap;
}
// Also executed when valueList is empty, covering pure string assignment expressions
Object objValue = JexlExpressionRunner.evaluate(expression, context);
if (objValue != null) {
value = String.valueOf(objValue);
}
} catch (Exception e) {
log.warn("[calculates execute warning, use original value.] {}", e.getMessage());
value = Optional.ofNullable(fieldValueMap.get(expression.getSourceText()))
.map(String::valueOf)
.orElse(null);
}
} else {
// does not exist then map the alias value
String aliasField = fieldAliasMap.get(realField);
if (aliasField != null) {
value = aliasFieldValueMap.get(aliasField);
} else {
value = aliasFieldValueMap.get(realField);
}
if (value != null) {
final byte fieldType = field.getType();
if (fieldType == CommonConstants.TYPE_NUMBER) {
CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil
.extractDoubleAndUnitFromStr(value);
final Double tempValue = doubleAndUnit == null ? null : doubleAndUnit.getValue();
value = tempValue == null ? null : String.valueOf(tempValue);
aliasFieldUnit = doubleAndUnit == null ? null : doubleAndUnit.getUnit();
} else if (fieldType == CommonConstants.TYPE_TIME) {
final int tempValue;
value = (tempValue = CommonUtil.parseTimeStrToSecond(value)) == -1 ? null : String.valueOf(tempValue);
}
}
}
Pair<String, String> unitPair = fieldUnitMap.get(realField);
if (aliasFieldUnit != null) {
if (unitPair != null) {
unitPair.setLeft(aliasFieldUnit);
} else if (field.getUnit() != null && !aliasFieldUnit.equalsIgnoreCase(field.getUnit())) {
unitPair = Pair.of(aliasFieldUnit, field.getUnit());
}
}
if (value != null && unitPair != null) {
for (UnitConvert unitConvert : unitConvertList) {
if (unitConvert.checkUnit(unitPair.getLeft()) && unitConvert.checkUnit(unitPair.getRight())) {
value = unitConvert.convert(value, unitPair.getLeft(), unitPair.getRight());
}
}
}
// Handle metrics values that may have units such as 34%, 34Mb, and limit values to 4 decimal places
if (CommonConstants.TYPE_NUMBER == field.getType()) {
value = CommonUtil.parseDoubleStr(value, field.getUnit());
}
if (value == null) {
value = CommonConstants.NULL_VALUE;
}
realValueRowBuilder.addColumn(value);
}
aliasFieldValueMap.clear();
fieldValueMap.clear();
aliasFieldUnitMap.clear();
stringTypefieldValueMap.clear();
CollectRep.ValueRow realValueRow = realValueRowBuilder.build();
realValueRowBuilder.clear();
// apply filter calculation to the real value row
if (!CollectionUtils.isEmpty(metrics.getFilters())) {
Map<String, Object> contextMap = new HashMap<>(8);
for (int i = 0; i < fields.size(); i++) {
Metrics.Field field = fields.get(i);
String value = realValueRow.getColumns(i);
contextMap.put(field.getField(), value);
}
boolean isMatch = false;
for (String filterExpr : metrics.getFilters()) {
try {
JexlExpression expression = JexlExpressionRunner.compile(filterExpr);
if ((Boolean) JexlExpressionRunner.evaluate(expression, contextMap)) {
isMatch = true;
break;
}
} catch (Exception e) {
log.warn("[metrics data row filters execute warning] {}.", e.getMessage());
}
}
if (!isMatch) {
// ignore this data row
continue;
}
}
collectData.addValueRow(realValueRow);
}
}
/**
* @param cal cal
* @param fieldAliasMap field alias map
* @return expr
*/
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap) {
int splitIndex = cal.indexOf("=");
String field = cal.substring(0, splitIndex).trim();
String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#");
JexlExpression expression;
try {
expression = JexlExpressionRunner.compile(expressionStr);
} catch (Exception e) {
fieldAliasMap.put(field, expressionStr);
return null;
}
return new Object[]{field, expression};
}
/**
* transform unit
*
* @param unit unit
* @return units
*/
private Object[] transformUnit(String unit) {
int equalIndex = unit.indexOf("=");
int arrowIndex = unit.indexOf("->");
if (equalIndex < 0 || arrowIndex < 0) {
return null;
}
String field = unit.substring(0, equalIndex).trim();
String originUnit = unit.substring(equalIndex + 1, arrowIndex).trim();
String newUnit = unit.substring(arrowIndex + 2).trim();
return new Object[]{field, Pair.of(originUnit, newUnit)};
}
private boolean fastFailed() {
return this.timeout == null || this.timeout.isCancelled();
}
private CollectRep.MetricsData validateResponse(CollectRep.MetricsData.Builder builder) {
long endTime = System.currentTimeMillis();
builder.setTime(endTime);
long runningTime = endTime - startTime;
long allTime = endTime - newTime;
if (startTime - newTime >= WARN_DISPATCH_TIME) {
log.warn("[Collector Dispatch Warn, Dispatch Use {}ms.", startTime - newTime);
}
if (builder.getCode() != CollectRep.Code.SUCCESS) {
log.info("[Collect Failed, Run {}ms, All {}ms] Reason: {}", runningTime, allTime, builder.getMsg());
} else {
log.info("[Collect Success, Run {}ms, All {}ms].", runningTime, allTime);
}
return builder.build();
}
private void validateResponse(CollectRep.MetricsData metricsData) {
if (metricsData == null) {
@@ -1,127 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.handler;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.constants.ContextKey;
import org.apache.hertzbeat.collector.constants.ContextStatus;
import org.apache.hertzbeat.collector.context.Context;
import org.apache.hertzbeat.collector.dispatch.CollectTaskTimeoutMonitor;
import org.apache.hertzbeat.collector.handler.impl.AbstractBatchDataStream;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
@Data
@Slf4j
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = true)
public class CollectMetricsDataDataStream extends AbstractBatchDataStream<Metrics, CollectRep.MetricsData.Builder> {
private CollectTaskTimeoutMonitor collectTaskTimeoutMonitor;
@Override
public CollectRep.MetricsData.Builder executeWithResponse(Context context, Metrics data) {
// preset start info
context.put(ContextKey.METRICS, data);
Job job = context.get(ContextKey.JOB);
long startTime = context.get(ContextKey.METRICS_COLLECT_START_TIME);
setNewThreadName(job.getMonitorId(), job.getApp(), startTime, data);
Timeout timeout = context.get(ContextKey.TIMEOUT);
String key = data.getPrometheus() != null ? String.valueOf(job.getId()) : job.getId() + "-" + data.getName();
context.put(ContextKey.METRICS_KEY, key);
this.collectTaskTimeoutMonitor.putMetrics(key, new CollectTaskTimeoutMonitor.MetricsTime(startTime, data, timeout));
CollectRep.MetricsData.Builder fetchedData = this.fetchData(job, data);
if (fetchedData.getCode() != CollectRep.Code.SUCCESS && CommonConstants.AVAILABLE_METRICS == data.getPriority()) {
context.setStatus(ContextStatus.TRUNCATE_HANDLER);
}
return fetchedData;
}
private void setNewThreadName(long monitorId, String app, long startTime, Metrics metrics) {
String builder = monitorId + "-" + app + "-" + metrics.getName() + "-" + String.valueOf(startTime).substring(9);
Thread.currentThread().setName(builder);
}
private CollectRep.MetricsData.Builder fetchData(Job job, Metrics metrics) {
CollectRep.MetricsData.Builder response = CollectRep.MetricsData.newBuilder();
response.setApp(job.getApp())
.setId(job.getMonitorId())
.setTenantId(job.getTenantId())
.setLabels(job.getLabels())
.setAnnotations(job.getAnnotations())
.addMetadataAll(job.getMetadata());
//todo transcribe Prometheus to different chain
// for prometheus auto or proxy mode
// if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(metrics.getProtocol())) {
// List<CollectRep.MetricsData> metricsData = PrometheusAutoCollectImpl.getInstance().collect(response, metrics);
// validateResponse(metricsData == null ? null : metricsData.stream().findFirst().orElse(null));
// collectDataDispatch.dispatchCollectData(timeout, metrics, metricsData);
// return null;
// }
response.setMetrics(metrics.getName());
// According to the metrics collection protocol, application type, etc.,
// dispatch to the real application metrics collection implementation class
AbstractCollect abstractCollect = CollectStrategyFactory.invoke(metrics.getProtocol());
if (abstractCollect == null) {
log.error("[Dispatcher] - not support this: app: {}, metrics: {}, protocol: {}.", job.getApp(), metrics.getName(), metrics.getProtocol());
response.setCode(CollectRep.Code.FAIL);
response.setMsg("not support " + job.getApp() + ", " + metrics.getName() + ", " + metrics.getProtocol());
return response;
}
try {
abstractCollect.preCheck(metrics);
abstractCollect.collect(response, metrics);
} catch (Exception e) {
String msg = e.getMessage();
if (msg == null && e.getCause() != null) {
msg = e.getCause().getMessage();
}
if (e instanceof IllegalArgumentException) {
log.error("[Metrics PreCheck]: {}.", msg, e);
} else {
log.error("[Metrics Collect]: {}.", msg, e);
}
response.setCode(CollectRep.Code.FAIL);
if (msg != null) {
response.setMsg(msg);
}
}
return response;
}
}
@@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.handler;
import org.apache.hertzbeat.collector.context.Context;
import org.apache.hertzbeat.collector.handler.impl.AbstractBatchDataStream;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
/**
*
*/
public class DynamicSubTaskCollectMetricsDataDataStream extends AbstractBatchDataStream<Metrics, CollectRep.MetricsData.Builder> {
@Override
public CollectRep.MetricsData.Builder executeWithResponse(Context context, Metrics data) {
//todo 动态拆分
return null;
}
}
@@ -1,289 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.listener;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.jexl3.JexlExpression;
import org.apache.hertzbeat.collector.constants.ContextKey;
import org.apache.hertzbeat.collector.context.Context;
import org.apache.hertzbeat.collector.dispatch.unit.UnitConvert;
import org.apache.hertzbeat.collector.handler.ContextBoundListener;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JexlExpressionRunner;
import org.apache.hertzbeat.common.util.Pair;
import org.springframework.util.CollectionUtils;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
/**
*
*/
@Slf4j
@AllArgsConstructor
public class CalculateFieldsListener implements ContextBoundListener<CollectRep.MetricsData.Builder> {
private List<UnitConvert> unitConvertList;
@Override
public void execute(Context context, CollectRep.MetricsData.Builder data) {
Metrics metrics = context.get(ContextKey.METRICS);
this.calculateFields(metrics, unitConvertList, data);
}
/**
* Calculate the real metrics value according to the calculates and aliasFields configuration
*
* @param metrics Metrics configuration
* @param collectData Data collection
*/
private void calculateFields(Metrics metrics, List<UnitConvert> unitConvertList, CollectRep.MetricsData.Builder collectData) {
collectData.setPriority(metrics.getPriority());
List<CollectRep.Field> fieldList = new LinkedList<>();
for (Metrics.Field field : metrics.getFields()) {
CollectRep.Field.Builder fieldBuilder = CollectRep.Field.newBuilder();
fieldBuilder.setName(field.getField()).setType(field.getType()).setLabel(field.isLabel());
if (field.getUnit() != null) {
fieldBuilder.setUnit(field.getUnit());
}
fieldList.add(fieldBuilder.build());
}
collectData.addAllFields(fieldList);
List<CollectRep.ValueRow> aliasRowList = collectData.getValuesList();
if (aliasRowList == null || aliasRowList.isEmpty()) {
return;
}
collectData.clearValues();
// Preprocess calculates first
if (metrics.getCalculates() == null) {
metrics.setCalculates(Collections.emptyList());
}
// eg: database_pages=Database pages unconventional mapping
Map<String, String> fieldAliasMap = new HashMap<>(8);
Map<String, JexlExpression> fieldExpressionMap = metrics.getCalculates()
.stream()
.map(cal -> transformCal(cal, fieldAliasMap))
.filter(Objects::nonNull)
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue));
if (metrics.getUnits() == null) {
metrics.setUnits(Collections.emptyList());
}
Map<String, Pair<String, String>> fieldUnitMap = metrics.getUnits()
.stream()
.map(this::transformUnit)
.filter(Objects::nonNull)
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair<String, String>) arr[1], (oldValue, newValue) -> newValue));
List<Metrics.Field> fields = metrics.getFields();
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
Map<String, String> aliasFieldValueMap = new HashMap<>(8);
Map<String, Object> fieldValueMap = new HashMap<>(8);
Map<String, Object> stringTypefieldValueMap = new HashMap<>(8);
Map<String, String> aliasFieldUnitMap = new HashMap<>(8);
CollectRep.ValueRow.Builder realValueRowBuilder = CollectRep.ValueRow.newBuilder();
for (CollectRep.ValueRow aliasRow : aliasRowList) {
for (int aliasIndex = 0; aliasIndex < aliasFields.size(); aliasIndex++) {
String aliasFieldValue = aliasRow.getColumns(aliasIndex);
String aliasField = aliasFields.get(aliasIndex);
if (!CommonConstants.NULL_VALUE.equals(aliasFieldValue)) {
aliasFieldValueMap.put(aliasField, aliasFieldValue);
// whether the alias field is a number
CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil
.extractDoubleAndUnitFromStr(aliasFieldValue);
if (doubleAndUnit != null && doubleAndUnit.getValue() != null) {
fieldValueMap.put(aliasField, doubleAndUnit.getValue());
if (doubleAndUnit.getUnit() != null) {
aliasFieldUnitMap.put(aliasField, doubleAndUnit.getUnit());
}
} else {
fieldValueMap.put(aliasField, aliasFieldValue);
}
stringTypefieldValueMap.put(aliasField, aliasFieldValue);
} else {
fieldValueMap.put(aliasField, null);
stringTypefieldValueMap.put(aliasField, null);
}
}
for (Metrics.Field field : fields) {
String realField = field.getField();
JexlExpression expression = fieldExpressionMap.get(realField);
String value = null;
String aliasFieldUnit = null;
if (expression != null) {
try {
Map<String, Object> context;
if (CommonConstants.TYPE_STRING == field.getType()) {
context = stringTypefieldValueMap;
} else {
for (Map.Entry<String, String> unitEntry : aliasFieldUnitMap.entrySet()) {
if (expression.getSourceText().contains(unitEntry.getKey())) {
aliasFieldUnit = unitEntry.getValue();
break;
}
}
context = fieldValueMap;
}
// Also executed when valueList is empty, covering pure string assignment expressions
Object objValue = JexlExpressionRunner.evaluate(expression, context);
if (objValue != null) {
value = String.valueOf(objValue);
}
} catch (Exception e) {
log.warn("[calculates execute warning, use original value.] {}", e.getMessage());
value = Optional.ofNullable(fieldValueMap.get(expression.getSourceText()))
.map(String::valueOf)
.orElse(null);
}
} else {
// does not exist then map the alias value
String aliasField = fieldAliasMap.get(realField);
if (aliasField != null) {
value = aliasFieldValueMap.get(aliasField);
} else {
value = aliasFieldValueMap.get(realField);
}
if (value != null) {
final byte fieldType = field.getType();
if (fieldType == CommonConstants.TYPE_NUMBER) {
CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil
.extractDoubleAndUnitFromStr(value);
final Double tempValue = doubleAndUnit == null ? null : doubleAndUnit.getValue();
value = tempValue == null ? null : String.valueOf(tempValue);
aliasFieldUnit = doubleAndUnit == null ? null : doubleAndUnit.getUnit();
} else if (fieldType == CommonConstants.TYPE_TIME) {
final int tempValue;
value = (tempValue = CommonUtil.parseTimeStrToSecond(value)) == -1 ? null : String.valueOf(tempValue);
}
}
}
Pair<String, String> unitPair = fieldUnitMap.get(realField);
if (aliasFieldUnit != null) {
if (unitPair != null) {
unitPair.setLeft(aliasFieldUnit);
} else if (field.getUnit() != null && !aliasFieldUnit.equalsIgnoreCase(field.getUnit())) {
unitPair = Pair.of(aliasFieldUnit, field.getUnit());
}
}
if (value != null && unitPair != null) {
for (UnitConvert unitConvert : unitConvertList) {
if (unitConvert.checkUnit(unitPair.getLeft()) && unitConvert.checkUnit(unitPair.getRight())) {
value = unitConvert.convert(value, unitPair.getLeft(), unitPair.getRight());
}
}
}
// Handle metrics values that may have units such as 34%, 34Mb, and limit values to 4 decimal places
if (CommonConstants.TYPE_NUMBER == field.getType()) {
value = CommonUtil.parseDoubleStr(value, field.getUnit());
}
if (value == null) {
value = CommonConstants.NULL_VALUE;
}
realValueRowBuilder.addColumn(value);
}
aliasFieldValueMap.clear();
fieldValueMap.clear();
aliasFieldUnitMap.clear();
stringTypefieldValueMap.clear();
CollectRep.ValueRow realValueRow = realValueRowBuilder.build();
realValueRowBuilder.clear();
// apply filter calculation to the real value row
if (!CollectionUtils.isEmpty(metrics.getFilters())) {
Map<String, Object> contextMap = new HashMap<>(8);
for (int i = 0; i < fields.size(); i++) {
Metrics.Field field = fields.get(i);
String value = realValueRow.getColumns(i);
contextMap.put(field.getField(), value);
}
boolean isMatch = false;
for (String filterExpr : metrics.getFilters()) {
try {
JexlExpression expression = JexlExpressionRunner.compile(filterExpr);
if ((Boolean) JexlExpressionRunner.evaluate(expression, contextMap)) {
isMatch = true;
break;
}
} catch (Exception e) {
log.warn("[metrics data row filters execute warning] {}.", e.getMessage());
}
}
if (!isMatch) {
// ignore this data row
continue;
}
}
collectData.addValueRow(realValueRow);
}
}
/**
* @param cal cal
* @param fieldAliasMap field alias map
* @return expr
*/
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap) {
int splitIndex = cal.indexOf("=");
if (splitIndex < 0) {
return null;
}
String field = cal.substring(0, splitIndex).trim();
String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#");
JexlExpression expression;
try {
expression = JexlExpressionRunner.compile(expressionStr);
} catch (Exception e) {
fieldAliasMap.put(field, expressionStr);
return null;
}
return new Object[]{field, expression};
}
/**
* transform unit
*
* @param unit unit
* @return units
*/
private Object[] transformUnit(String unit) {
int equalIndex = unit.indexOf("=");
int arrowIndex = unit.indexOf("->");
if (equalIndex < 0 || arrowIndex < 0) {
return null;
}
String field = unit.substring(0, equalIndex).trim();
String originUnit = unit.substring(equalIndex + 1, arrowIndex).trim();
String newUnit = unit.substring(arrowIndex + 2).trim();
return new Object[]{field, Pair.of(originUnit, newUnit)};
}
}
@@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.listener;
import org.apache.hertzbeat.collector.constants.ContextKey;
import org.apache.hertzbeat.collector.constants.ContextStatus;
import org.apache.hertzbeat.collector.context.Context;
import org.apache.hertzbeat.collector.handler.ContextBoundListener;
import org.apache.hertzbeat.common.timer.Timeout;
/**
*
*/
public class CommonMetricsDataListener<T> implements ContextBoundListener<T> {
@Override
public void execute(Context context, T data) {
Timeout timeout = context.get(ContextKey.TIMEOUT);
if (timeout == null || timeout.isCancelled()) {
context.setStatus(ContextStatus.STOP);
}
}
}

Some files were not shown because too many files have changed in this diff Show More