Compare commits

...
Author SHA1 Message Date
DuansgandTomsun28 1b51c19c97 [fix]fixed sql formatting error (#4008)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-01-30 00:00:31 +08:00
LunaRain_079 a83cb0f50f [docs] Fix HTTP Service Discovery API response format (#4005) 2026-01-29 22:10:15 +08:00
Tomsun28 d6022cb587 docs: update doc add contributors and pnpm lock (#4002)
Signed-off-by: tomsun28 <tomsun28@outlook.com>
2026-01-25 20:34:49 +08:00
LogicandTomsun28 67f525a603 [fix] update table name generation to include application context (#4001)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-01-24 23:26:08 +08:00
LunaRain_079andDuansg 634006aa7e [refactor] Dynamically concatenate app parameter on the front end to accommodate Prometheus apps (#4000)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-24 23:06:25 +08:00
3eb3f926f5 [fix] Update query parameter encoding and error time parser in VictoriaMetricsClusterDataStorage (#3989)
Co-authored-by: Yang Chen <1597081640@qq.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-24 20:37:08 +08:00
47caa5bdf8 feat: support otlp http binary protobuf format log data and update doc (#3986)
Signed-off-by: Yang Chen <1597081640@qq.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Logic <zqr10159@dromara.org>
2026-01-23 18:34:36 +08:00
Logic 67c3727362 [Refactor] tighten resource constraints for XML/XPath collector (#3999) 2026-01-22 18:51:01 +08:00
b510d6e5ea [feature] add datasource status retrieval and update alert settings UI (#3985)
Signed-off-by: Logic <zqr10159@126.com>
Signed-off-by: Duansg <siguoduan@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-20 22:43:47 +08:00
1ab27bd556 [doc]: update to Docusaurus v3.9.2 (#3984)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-01-20 22:04:27 +08:00
Duansg 9600a5e56f [improve] upgrade script merge (#3987) 2026-01-19 23:47:32 +08:00
6c0d5c0db4 [fix]Fix issues such as index failure (#3953)
Co-authored-by: shown <yuluo08290126@gmail.com>
Co-authored-by: Yang Chen <1597081640@qq.com>
Co-authored-by: Logic <zqr10159@dromara.org>
2026-01-18 23:07:42 +08:00
4b010e6523 [monitoring-template] feat(mysql): add database account expiry metrics (#3969)
Co-authored-by: Logic <zqr10159@dromara.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-18 22:29:03 +08:00
8962253dd2 [manager] bugfix: fix incorrect status page date calculation (#3962)
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-17 20:08:12 +08:00
Logic afd89e4309 Fix htaccess (#3980)
Signed-off-by: Logic <zqr10159@dromara.org>
2026-01-16 22:16:15 +08:00
Logic 43332f49db [fix]: update Content Security Policy in .htaccess to include new domains (#3979) 2026-01-16 20:53:11 +08:00
Duansg cc90de5541 [fix] fixed cache parsing issues (#3977) 2026-01-16 15:47:09 +08:00
Tomsun28 5e1460bf29 docs: update invitation email template (#3975) 2026-01-16 09:10:51 +08:00
LogicandYang Chen 7dbdca849a [doc]: add monitoring documentation for various service discovery methods (#3965)
Co-authored-by: Yang Chen <1597081640@qq.com>
2026-01-15 21:27:42 +08:00
Yang ChenandDuansg df525d2d32 fix: add GreptimeDB docker-compose assembly configuration (#3960)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-14 20:51:24 +08:00
LunaRain_079 dc96f1766b [Fix]: fix TAG type mismatch by introducing schema version for TDengine (#3963) 2026-01-13 21:29:27 +08:00
Tomsun28andDuansg 70da7d5dce docs: update docs link (#3958)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-11 18:46:13 +08:00
Prakash Kumar 0f1169ce36 [Fix] Fixed alignment of header social links on mobile (#3959) 2026-01-11 09:57:39 +08:00
KOYR 1c38d01abb style: widen table column to prevent label truncation (#3957) 2026-01-10 22:10:53 +08:00
f82b5add11 [feature] Update CSP permissions to include Kapa.ai services (#3948)
Signed-off-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-08 09:49:48 +08:00
KOYR 8e2c1f3b07 style: adjust grid layout width of form labels (#3950) 2026-01-07 15:06:45 +08:00
Logic 8ef92d70d2 [fix] Update overflow properties and enhance swiper container styles (#3947) 2026-01-07 08:10:37 +08:00
Yang Chen b5ce5c9e9d [fix] refactor system built-in labels validation logic (#3946) 2026-01-06 15:47:25 +08:00
P_Peaceful a19a399acd [bugfix&improve] Fix alert label persistence and improve custom label input (#3933) 2026-01-06 12:01:53 +08:00
529d161478 [improve] Optimize Tooltip display logic and support scrollable content (#3941)
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-04 23:24:46 +08:00
168f78ef23 chore: optimzie docker compose docs hertzbeat version (#3929)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-03 19:05:35 +08:00
KOYRandDuansg a159127572 [improve]: Optimize yAxis scale to better reflect data fluctuations (#3939)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-01-03 01:57:14 +08:00
88933ce0bb [feature] Update UI text and styles for website (#3934)
Signed-off-by: Logic <zqr10159@126.com>
Signed-off-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
2025-12-31 16:00:13 +08:00
Logic 6d703f7eae [feature] add Kapa widget script to docusaurus configuration (#3938) 2025-12-31 10:30:49 +08:00
Duansg f4a0fe3445 [fix] fixed CSP blocking algolia from loading on the page (#3935) 2025-12-30 09:56:58 +08:00
Sahil Shadwal b174c8622c [bugfix]: fix for the collector tab click effect error (#3932) 2025-12-28 18:42:04 +08:00
LathikaBaddam 98e0412ee6 [docs] improve English translation and technical terms for MySQL (#3928) 2025-12-27 20:20:04 +08:00
Logic a138238994 [bugfix] Enhance expiration time parsing to support strict day format (#3924) 2025-12-22 20:44:21 +08:00
Duansgandlynx009 01f5cb1fd6 [feat] Add Grafana to follow the system theme (#3920)
Co-authored-by: lynx009 <2030509072@qq.com>
2025-12-20 19:44:53 +08:00
Tomsun28andYang Chen 472554bc55 [chore] update docker compose hertzbeat version (#3916)
Signed-off-by: tomsun28 <tomsun28@outlook.com>
Co-authored-by: Yang Chen <1597081640@qq.com>
2025-12-19 14:15:28 +08:00
Logic ad9a4ab100 [improve] xugu.md and app-xugu.yml (#3918) 2025-12-17 18:51:58 +08:00
KOYR a806ca1319 [Feature] Enhance draggable UI elements in alert settings (#3915) (#3917) 2025-12-17 09:33:23 +08:00
P_PeacefulandTomsun28 aac5bafe43 [bugfix]: AI conversation message loading issue (#3911)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-16 23:54:38 +08:00
P_PeacefulandLogic 668106d445 [bugfix] AI prompt classpath error (#3912)
Co-authored-by: Logic <zqr10159@dromara.org>
2025-12-16 11:36:23 +08:00
a74671abdd [improve]Improve UI handling for label overflow (#3913)
Co-authored-by: 半半 <kongranyi@cai-inc.com>
Co-authored-by: Logic <zqr10159@dromara.org>
2025-12-16 11:03:15 +08:00
Duansg 120d494c47 [fix]Fixed private collector tasks not taking effect (#3914) 2025-12-16 10:51:39 +08:00
DeleiGuo 5b1d01e0c5 [bugfix] Modify the search predicate from 'host' to 'instance' field (#3910) 2025-12-13 10:25:52 +08:00
Yang ChenandCopilot 950138a340 [feat] Add an SQL editor and prevent SQL injection (#3900)
Signed-off-by: Yang Chen <1597081640@qq.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-12 17:52:31 +08:00
zhaoyangplus 93d8f92165 feat(db): Added support for monitoring the Xugu database (#3899) 2025-12-12 11:32:40 +08:00
LogicandCopilot 488df340a1 [Fix]: Use a connection pool to avoid concurrent access to duckdb files. (#3898)
Signed-off-by: Logic <zqr10159@126.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-10 15:38:30 +08:00
DuansgandTomsun28 dc51ce7bef [fix]Fixed position offset issue during deserialization of ArrowUtil (#3897)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-09 21:46:36 +08:00
Logicandaias00 862f32e699 [Feature]: enhance logging configuration with asynchronous console appender (#3895)
Co-authored-by: aias00 <liuhongyu@apache.org>
2025-12-09 21:15:42 +08:00
DeleiGuo d0f6a04661 [docs] Add new committer blog (#3894) 2025-12-09 09:46:19 +08:00
shownandTomsun28 cb3fad6a2c feat: optimize hertzbeat ai module (#3891)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
Signed-off-by: shown <yuluo08290126@gmail.com>
Signed-off-by: tomsun28 <tomsun28@outlook.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-08 22:55:44 +08:00
Tomsun28andshown c88c232889 feat: support mcp streamable http protocol and upgrade spring ai version (#3890)
Signed-off-by: tomsun28 <tomsun28@outlook.com>
Co-authored-by: shown <yuluo08290126@gmail.com>
2025-12-08 22:12:35 +08:00
6b03e4bb5f chore: add .editorconfig for consistent coding style (#3875)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
2025-12-07 00:02:07 +08:00
7727d63451 [add] Add parameter validators and their corresponding tests (#3888)
Signed-off-by: aias00 <liuhongyu@apache.org>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-06 19:34:47 +08:00
1223 changed files with 66005 additions and 9500 deletions
+47 -1
View File
@@ -2545,7 +2545,8 @@
"avatar_url": "https://avatars.githubusercontent.com/u/176664901?v=4",
"profile": "https://github.com/Saramanda9988",
"contributions": [
"doc"
"doc",
"code"
]
},
{
@@ -2667,6 +2668,51 @@
"contributions": [
"code"
]
},
{
"login": "KOYR",
"name": "KOYR",
"avatar_url": "https://avatars.githubusercontent.com/u/53216619?v=4",
"profile": "https://github.com/KOYR",
"contributions": [
"code"
]
},
{
"login": "Lathika226",
"name": "LathikaBaddam",
"avatar_url": "https://avatars.githubusercontent.com/u/178710568?v=4",
"profile": "https://github.com/Lathika226",
"contributions": [
"doc"
]
},
{
"login": "Sahil-Shadwal",
"name": "Sahil Shadwal",
"avatar_url": "https://avatars.githubusercontent.com/u/119167601?v=4",
"profile": "https://shadwal.space/",
"contributions": [
"code"
]
},
{
"login": "BhanuNidumolu",
"name": "N.Bhanu Prasad",
"avatar_url": "https://avatars.githubusercontent.com/u/180380413?v=4",
"profile": "https://github.com/BhanuNidumolu",
"contributions": [
"code"
]
},
{
"login": "Prakash1185",
"name": "Prakash Kumar",
"avatar_url": "https://avatars.githubusercontent.com/u/183058331?v=4",
"profile": "https://prakashh-portfolio.vercel.app/",
"contributions": [
"code"
]
}
],
"contributorsPerLine": 7,
+55
View File
@@ -0,0 +1,55 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
[*]
charset = utf-8
end_of_line = lf
indent_style = space
# The line max length is 120, match checkstyle rules.
max_line_length = 120
insert_final_newline = true
trim_trailing_whitespace = true
[*.json]
tab_width = 2
indent_size = 2
[*.{yml,yaml}]
tab_width = 2
indent_size = 2
[*.xml]
ij_xml_attribute_wrap = off
ij_xml_text_wrap = off
ij_xml_keep_blank_lines = 1
[pom.xml]
indent_size = 2
[*.java]
indent_size = 4
tab_width = 4
[*.js]
indent_size = 2
[*.ts]
indent_size = 2
+1
View File
@@ -39,6 +39,7 @@ header:
- '**/*.js'
- '**/*.less'
- '**/*.txt'
- '**/*.st'
- '**/target/**'
- '.gitattributes'
- '**/.gitignore'
+1 -1
View File
@@ -31,7 +31,7 @@
"MD025": true,
"MD029": true,
"MD033": {
"allowed_elements": ["table", "tr", "td", "a", "img", "sub", "b", "br", "img", "tbody", "mark", "font"]
"allowed_elements": ["table", "tr", "td", "a", "img", "sub", "b", "br", "img", "tbody", "mark", "font", "Tabs", "TabItem"]
},
"MD036": false,
"MD040": true,
+8 -1
View File
@@ -525,7 +525,7 @@ Thanks to these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/yy549159265"><img src="https://avatars.githubusercontent.com/u/40821310?v=4?s=100" width="100px;" alt="yy549159265"/><br /><sub><b>yy549159265</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Tests">⚠️</a> <a href="#design-yy549159265" title="Design">🎨</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/cto-huhang"><img src="https://avatars.githubusercontent.com/u/53338629?v=4?s=100" width="100px;" alt="cto-huhang"/><br /><sub><b>cto-huhang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=cto-huhang" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/delei"><img src="https://avatars.githubusercontent.com/u/17263766?v=4?s=100" width="100px;" alt="DeleiGuo"/><br /><sub><b>DeleiGuo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Tests">⚠️</a> <a href="https://github.com/apache/hertzbeat/issues?q=author%3Adelei" title="Bug reports">🐛</a></td>
</tr>
<tr>
@@ -543,6 +543,13 @@ Thanks to these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Jetiaime"><img src="https://avatars.githubusercontent.com/u/93769000?v=4?s=100" width="100px;" alt="TeAmo"/><br /><sub><b>TeAmo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Jetiaime" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/pentium100"><img src="https://avatars.githubusercontent.com/u/27917?v=4?s=100" width="100px;" alt="pentium100"/><br /><sub><b>pentium100</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=pentium100" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/dedyks"><img src="https://avatars.githubusercontent.com/u/23741665?v=4?s=100" width="100px;" alt="Dedy Kurniawan Santoso"/><br /><sub><b>Dedy Kurniawan Santoso</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=dedyks" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/KOYR"><img src="https://avatars.githubusercontent.com/u/53216619?v=4?s=100" width="100px;" alt="KOYR"/><br /><sub><b>KOYR</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=KOYR" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Lathika226"><img src="https://avatars.githubusercontent.com/u/178710568?v=4?s=100" width="100px;" alt="LathikaBaddam"/><br /><sub><b>LathikaBaddam</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Lathika226" title="Documentation">📖</a></td>
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://shadwal.space/"><img src="https://avatars.githubusercontent.com/u/119167601?v=4?s=100" width="100px;" alt="Sahil Shadwal"/><br /><sub><b>Sahil Shadwal</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Sahil-Shadwal" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/BhanuNidumolu"><img src="https://avatars.githubusercontent.com/u/180380413?v=4?s=100" width="100px;" alt="N.Bhanu Prasad"/><br /><sub><b>N.Bhanu Prasad</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=BhanuNidumolu" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://prakashh-portfolio.vercel.app/"><img src="https://avatars.githubusercontent.com/u/183058331?v=4?s=100" width="100px;" alt="Prakash Kumar"/><br /><sub><b>Prakash Kumar</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Prakash1185" title="Code">💻</a></td>
</tr>
</tbody>
</table>
+8 -1
View File
@@ -522,7 +522,7 @@ Thanks these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/yy549159265"><img src="https://avatars.githubusercontent.com/u/40821310?v=4?s=100" width="100px;" alt="yy549159265"/><br /><sub><b>yy549159265</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Tests">⚠️</a> <a href="#design-yy549159265" title="Design">🎨</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/cto-huhang"><img src="https://avatars.githubusercontent.com/u/53338629?v=4?s=100" width="100px;" alt="cto-huhang"/><br /><sub><b>cto-huhang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=cto-huhang" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/delei"><img src="https://avatars.githubusercontent.com/u/17263766?v=4?s=100" width="100px;" alt="DeleiGuo"/><br /><sub><b>DeleiGuo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Tests">⚠️</a> <a href="https://github.com/apache/hertzbeat/issues?q=author%3Adelei" title="Bug reports">🐛</a></td>
</tr>
<tr>
@@ -540,6 +540,13 @@ Thanks these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Jetiaime"><img src="https://avatars.githubusercontent.com/u/93769000?v=4?s=100" width="100px;" alt="TeAmo"/><br /><sub><b>TeAmo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Jetiaime" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/pentium100"><img src="https://avatars.githubusercontent.com/u/27917?v=4?s=100" width="100px;" alt="pentium100"/><br /><sub><b>pentium100</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=pentium100" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/dedyks"><img src="https://avatars.githubusercontent.com/u/23741665?v=4?s=100" width="100px;" alt="Dedy Kurniawan Santoso"/><br /><sub><b>Dedy Kurniawan Santoso</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=dedyks" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/KOYR"><img src="https://avatars.githubusercontent.com/u/53216619?v=4?s=100" width="100px;" alt="KOYR"/><br /><sub><b>KOYR</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=KOYR" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Lathika226"><img src="https://avatars.githubusercontent.com/u/178710568?v=4?s=100" width="100px;" alt="LathikaBaddam"/><br /><sub><b>LathikaBaddam</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Lathika226" title="Documentation">📖</a></td>
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://shadwal.space/"><img src="https://avatars.githubusercontent.com/u/119167601?v=4?s=100" width="100px;" alt="Sahil Shadwal"/><br /><sub><b>Sahil Shadwal</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Sahil-Shadwal" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/BhanuNidumolu"><img src="https://avatars.githubusercontent.com/u/180380413?v=4?s=100" width="100px;" alt="N.Bhanu Prasad"/><br /><sub><b>N.Bhanu Prasad</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=BhanuNidumolu" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://prakashh-portfolio.vercel.app/"><img src="https://avatars.githubusercontent.com/u/183058331?v=4?s=100" width="100px;" alt="Prakash Kumar"/><br /><sub><b>Prakash Kumar</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Prakash1185" title="Code">💻</a></td>
</tr>
</tbody>
</table>
+8 -1
View File
@@ -525,7 +525,7 @@ Thanks these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/lx1229"><img src="https://avatars.githubusercontent.com/u/44620005?v=4?s=100" width="100px;" alt="Liuxin"/><br /><sub><b>Liuxin</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=lx1229" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/yy549159265"><img src="https://avatars.githubusercontent.com/u/40821310?v=4?s=100" width="100px;" alt="yy549159265"/><br /><sub><b>yy549159265</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=yy549159265" title="Tests">⚠️</a> <a href="#design-yy549159265" title="Design">🎨</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/cto-huhang"><img src="https://avatars.githubusercontent.com/u/53338629?v=4?s=100" width="100px;" alt="cto-huhang"/><br /><sub><b>cto-huhang</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=cto-huhang" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Saramanda9988"><img src="https://avatars.githubusercontent.com/u/176664901?v=4?s=100" width="100px;" alt="LunaRain_079"/><br /><sub><b>LunaRain_079</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=Saramanda9988" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/delei"><img src="https://avatars.githubusercontent.com/u/17263766?v=4?s=100" width="100px;" alt="DeleiGuo"/><br /><sub><b>DeleiGuo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Documentation">📖</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Code">💻</a> <a href="https://github.com/apache/hertzbeat/commits?author=delei" title="Tests">⚠️</a> <a href="https://github.com/apache/hertzbeat/issues?q=author%3Adelei" title="Bug reports">🐛</a></td>
</tr>
<tr>
@@ -543,6 +543,13 @@ Thanks these wonderful people, welcome to join us:
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Jetiaime"><img src="https://avatars.githubusercontent.com/u/93769000?v=4?s=100" width="100px;" alt="TeAmo"/><br /><sub><b>TeAmo</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Jetiaime" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/pentium100"><img src="https://avatars.githubusercontent.com/u/27917?v=4?s=100" width="100px;" alt="pentium100"/><br /><sub><b>pentium100</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=pentium100" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/dedyks"><img src="https://avatars.githubusercontent.com/u/23741665?v=4?s=100" width="100px;" alt="Dedy Kurniawan Santoso"/><br /><sub><b>Dedy Kurniawan Santoso</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=dedyks" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/KOYR"><img src="https://avatars.githubusercontent.com/u/53216619?v=4?s=100" width="100px;" alt="KOYR"/><br /><sub><b>KOYR</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=KOYR" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/Lathika226"><img src="https://avatars.githubusercontent.com/u/178710568?v=4?s=100" width="100px;" alt="LathikaBaddam"/><br /><sub><b>LathikaBaddam</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Lathika226" title="Documentation">📖</a></td>
</tr>
<tr>
<td align="center" valign="top" width="14.28%"><a href="https://shadwal.space/"><img src="https://avatars.githubusercontent.com/u/119167601?v=4?s=100" width="100px;" alt="Sahil Shadwal"/><br /><sub><b>Sahil Shadwal</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Sahil-Shadwal" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://github.com/BhanuNidumolu"><img src="https://avatars.githubusercontent.com/u/180380413?v=4?s=100" width="100px;" alt="N.Bhanu Prasad"/><br /><sub><b>N.Bhanu Prasad</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=BhanuNidumolu" title="Code">💻</a></td>
<td align="center" valign="top" width="14.28%"><a href="https://prakashh-portfolio.vercel.app/"><img src="https://avatars.githubusercontent.com/u/183058331?v=4?s=100" width="100px;" alt="Prakash Kumar"/><br /><sub><b>Prakash Kumar</b></sub></a><br /><a href="https://github.com/apache/hertzbeat/commits?author=Prakash1185" title="Code">💻</a></td>
</tr>
</tbody>
</table>
+1 -1
View File
@@ -26,7 +26,7 @@
<artifactId>hertzbeat-ai</artifactId>
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.3</spring-ai.version>
<spring-ai.version>1.1.1</spring-ai.version>
<java.version>17</java.version>
</properties>
@@ -1,246 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.config;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.usthe.sureness.mgt.SurenessSecurityManager;
import com.usthe.sureness.subject.SubjectSum;
import io.modelcontextprotocol.spec.McpError;
import io.modelcontextprotocol.spec.McpSchema;
import io.modelcontextprotocol.spec.McpServerSession;
import io.modelcontextprotocol.spec.McpServerTransport;
import io.modelcontextprotocol.spec.McpServerTransportProvider;
import io.modelcontextprotocol.util.Assert;
import java.io.IOException;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import jakarta.servlet.http.HttpServletRequest;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.RouterFunctions;
import org.springframework.web.servlet.function.ServerRequest;
import org.springframework.web.servlet.function.ServerResponse;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* Custom Server-Sent Events transport provider for Model Context Protocol.
*/
@Slf4j
public class CustomSseServerTransport implements McpServerTransportProvider {
private final ObjectMapper objectMapper;
private final String messageEndpoint;
private final String sseEndpoint;
private final String baseUrl;
@Getter
private final RouterFunction<ServerResponse> routerFunction;
@Setter
private McpServerSession.Factory sessionFactory;
private final Map<String, Object> sessionRequest = new HashMap<>();
private final ConcurrentHashMap<String, McpServerSession> sessions;
private volatile boolean isClosing;
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint) {
this(objectMapper, messageEndpoint, "/sse");
}
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint, String sseEndpoint) {
this(objectMapper, "", messageEndpoint, sseEndpoint);
}
public CustomSseServerTransport(ObjectMapper objectMapper, String baseUrl, String messageEndpoint, String sseEndpoint) {
this.sessions = new ConcurrentHashMap();
this.isClosing = false;
Assert.notNull(objectMapper, "ObjectMapper must not be null");
Assert.notNull(baseUrl, "Message base URL must not be null");
Assert.notNull(messageEndpoint, "Message endpoint must not be null");
Assert.notNull(sseEndpoint, "SSE endpoint must not be null");
this.objectMapper = objectMapper;
this.baseUrl = baseUrl;
this.messageEndpoint = messageEndpoint;
this.sseEndpoint = sseEndpoint;
this.routerFunction = RouterFunctions.route().GET(this.sseEndpoint, this::handleSseConnection).POST(this.messageEndpoint, this::handleMessage).build();
}
public Mono<Void> notifyClients(String method, Object params) {
if (this.sessions.isEmpty()) {
log.debug("No active sessions to broadcast message to");
return Mono.empty();
} else {
log.debug("Attempting to broadcast message to {} active sessions", this.sessions.size());
return Flux.fromIterable(this.sessions.values())
.flatMap((session) -> session.sendNotification(method, params)
.doOnError((e) -> log.error("Failed to send message to session {}: {}", session.getId(), e.getMessage()))
.onErrorComplete())
.then();
}
}
public Mono<Void> closeGracefully() {
return Flux.fromIterable(this.sessions.values()).doFirst(() -> {
this.isClosing = true;
log.debug("Initiating graceful shutdown with {} active sessions", this.sessions.size());
}).flatMap(McpServerSession::closeGracefully).then().doOnSuccess((v) -> log.debug("Graceful shutdown completed"));
}
private ServerResponse handleSseConnection(ServerRequest request) {
log.debug("Handling SSE connection for request: {}", request);
HttpServletRequest servletRequest = request.servletRequest();
try {
log.debug("Processing SSE connection for servlet request: {}", servletRequest);
log.debug("Authorization header: {}", servletRequest.getHeader("Authorization"));
} catch (Exception e) {
log.error("Authentication failed for SSE connection: {}", e.getMessage());
return ServerResponse.status(HttpStatus.UNAUTHORIZED).body("Unauthorized: " + e.getMessage());
}
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else {
String sessionId = UUID.randomUUID().toString();
log.debug("Generated session ID for SSE connection: {}", sessionId);
log.debug("Creating new SSE connection for session: {}", sessionId);
return ServerResponse.sse((sseBuilder) -> {
sseBuilder.onComplete(() -> {
log.debug("SSE connection completed for session: {}", sessionId);
this.sessions.remove(sessionId);
});
sseBuilder.onTimeout(() -> {
log.debug("SSE connection timed out for session: {}", sessionId);
this.sessions.remove(sessionId);
});
CustomSseServerTransport.WebMvcMcpSessionTransport sessionTransport = new CustomSseServerTransport.WebMvcMcpSessionTransport(sessionId, sseBuilder);
McpServerSession session = this.sessionFactory.create(sessionTransport);
this.sessionRequest.put(sessionId, request.servletRequest());
this.sessions.put(sessionId, session);
try {
sseBuilder.id(sessionId).event("endpoint").data(this.baseUrl + this.messageEndpoint + "?sessionId=" + sessionId);
} catch (Exception e) {
log.error("Failed to send initial endpoint event: {}", e.getMessage());
sseBuilder.error(e);
}
}, Duration.ZERO);
}
}
private ServerResponse handleMessage(ServerRequest request) {
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else if (request.param("sessionId").isEmpty()) {
return ServerResponse.badRequest().body(new McpError("Session ID missing in message endpoint"));
} else {
String sessionId = (String) request.param("sessionId").get();
McpServerSession session = (McpServerSession) this.sessions.get(sessionId);
log.debug("Authorization header for message request: {}", request.servletRequest().getHeader("Authorization"));
SubjectSum subject = SurenessSecurityManager.getInstance().checkIn(sessionRequest.get(sessionId));
McpContextHolder.setSubject(subject);
if (session == null) {
return ServerResponse.status(HttpStatus.NOT_FOUND).body(new McpError("Session not found: " + sessionId));
} else {
try {
String body = request.body(String.class);
McpSchema.JSONRPCMessage message = McpSchema.deserializeJsonRpcMessage(this.objectMapper, body);
session.handle(message).block();
return ServerResponse.ok().build();
} catch (IOException | IllegalArgumentException e) {
log.error("Failed to deserialize message: {}", ((Exception) e).getMessage());
return ServerResponse.badRequest().body(new McpError("Invalid message format"));
} catch (Exception e) {
log.error("Error handling message: {}", e.getMessage());
return ServerResponse.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new McpError(e.getMessage()));
}
}
}
}
private class WebMvcMcpSessionTransport implements McpServerTransport {
private final String sessionId;
private final ServerResponse.SseBuilder sseBuilder;
WebMvcMcpSessionTransport(String sessionId, ServerResponse.SseBuilder sseBuilder) {
this.sessionId = sessionId;
this.sseBuilder = sseBuilder;
log.debug("Session transport {} initialized with SSE builder", sessionId);
}
public Mono<Void> sendMessage(McpSchema.JSONRPCMessage message) {
return Mono.fromRunnable(() -> {
try {
String jsonText = CustomSseServerTransport.this.objectMapper.writeValueAsString(message);
this.sseBuilder.id(this.sessionId).event("message").data(jsonText);
log.debug("Message sent to session {}", this.sessionId);
} catch (Exception e) {
log.error("Failed to send message to session {}: {}", this.sessionId, e.getMessage());
this.sseBuilder.error(e);
}
});
}
public <T> T unmarshalFrom(Object data, TypeReference<T> typeRef) {
return (T) CustomSseServerTransport.this.objectMapper.convertValue(data, typeRef);
}
public Mono<Void> closeGracefully() {
return Mono.fromRunnable(() -> {
log.debug("Closing session transport: {}", this.sessionId);
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
});
}
public void close() {
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
}
}
}
@@ -33,8 +33,8 @@ public final class McpContextHolder {
* Attaches the user's context to the current thread.
*/
public static void setSubject(SubjectSum subject) {
subjectHolder.set(subject);
subjectHolder.set(subject);
}
/**
@@ -1,182 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.config;
import org.springframework.stereotype.Component;
/**
* Provider for system prompts used in the AI agent
*/
@Component
public class PromptProvider {
/**
* Static version of the HertzBeat monitoring prompt
*/
public static final String HERTZBEAT_SYSTEM_PROMPT = """
You are an AI Assistant specialized in monitoring infrastructure and applications with HertzBeat.
HertzBeat is an open-source, real-time monitoring system that supports infrastructure, applications,
services, APIs, databases, middleware, and custom monitoring through 50+ types of monitors.
Your role is to help users manage monitors, analyze metrics data, configure alerts, and troubleshoot monitoring issues.
*******
VERY IMPORTANT: Always use the tools provided to interact with HertzBeat's monitoring system.
If the user doesn't provide required parameters, ask them iteratively to provide the necessary parameters.
********
## Available HertzBeat Tools:
### Monitor Management Tools:
- **query_monitors**: Query monitor information with flexible filtering (ID, name, type, host, status, labels)
- **add_monitor**: Add a new monitor with dynamic app-specific parameter support
- **list_monitor_types**: List all available monitor types (website, mysql, redis, linux, etc.)
- **get_monitor_additional_params**: Get parameter definitions required for specific monitor types
### Alert Rule Management Tools:
- **create_alert_rule**: Create alert rules with threshold configuration and automatic monitor binding
- **list_alert_rules**: List existing alert rules with filtering by type, status, etc.
- **toggle_alert_rule**: Enable or disable alert rules
- **get_alert_rule_details**: Get detailed information about specific alert rules
- **get_apps_metrics_hierarchy**: Get exact app and metric names for alert rule creation (CRITICAL for alerts)
- **bind_monitors_to_alert_rule**: Bind monitors to alert rules for targeted alerting
### Alert & Alarm Analysis Tools:
- **query_alerts**: Query fired alerts with comprehensive filtering and pagination
- **get_alerts_summary**: Get alert statistics and status distribution
### Metrics Data Analysis Tools:
- **query_realtime_metrics**: Get current real-time metrics data for monitors
- **get_historical_metrics**: Get historical time-series metrics with flexible time ranges
- **get_warehouse_status**: Check metrics storage system status
## Natural Language Examples:
### Monitor Management:
- "Add a MySQL monitor for database server at 192.168.1.10 with user admin"
- "Monitor website https://example.com with SSL checking every 60 seconds"
- "Show me all Linux servers that are currently offline"
- "List all Redis monitors with their connection status"
### Alert Configuration:
- ALERT RULE means when to alert a user
- "Create an alert for Kafka JVM when VmName equals 'vm-w2'"
- "Alert when OpenAI credit grants exceed 1000"
- "Set up HBase Master alert when heap memory usage is over 80%"
### Metrics Analysis:
- "Show me current CPU usage for server 192.168.1.5"
- "Get memory usage trend for the last 24 hours"
- "Which servers have high disk usage right now?"
- "Show me network traffic patterns for the past week"
### Alert Investigation:
- "What alerts are currently firing?"
- "Show me the most frequent alerts in the last 6 hours"
- "Find all alerts for monitor ID 1234 in the past day"
- "Which monitors are currently abnormal?"
## Workflow Guidelines:
1. **Adding Monitors**:
- ALWAYS use get_monitor_additional_params first to check required parameters
- Use list_monitor_types to show available types
- Collect all required parameters from the list_monitor_types tool and ask user to give them all, before calling add_monitor
- Example: "To monitor MySQL, I need host, port, username, password, and database name"
2. **Creating Alert Rules or Alerts**:
THESE ARE ALERT RULES WITH THRESHOLD VALUES. USERS CAN SPECIFY THE THRESHOLD VALUES FOR EXAMPLE,
IF THE USER SAYS "ALERT ME WHEN MY COST EXCEEDS 700, THE EXPRESSION SHOULD BE 'cost > 700' NOT 'cost < 700'.
APPLY THE SAME LOGIC FOR LESS THAN OPERATOR.
It is important to first understand the hierarchy of apps, metrics, and field conditions
Each app has its own metrics and each metric has its own field conditions.
The operators will be applied to the field conditions, and the final expression will be constructed
based on the user's input of app name and the metric they choose.
Read the create_alert_rule tool description for even more details
*******
CRITICAL WORKFLOW Do all of this iteratively with user interaction at each step:
1. ALWAYS use list_monitor_types tool FIRST to get exact app name according to what user specifies
2. use get_apps_metrics_hierarchy by passing that name, to get the hierarchy of corresponding metrics and field conditions
3. Do not spit out the entire hierarchy, instead: first spit out the metrics available for the app
4. Ask the user to choose a metric from the available metrics
5. Based on the metric chosen, present the available field conditions
6. You will construct the proper expression with field conditions
VERY VERY IMPORTANT:
- ALWAYS USE the value field from the get_apps_metrics_hierarchy's json response when creating alert expressions on the field parameters
*********
- Field Condition Expression format: [field_conditions]
- Give all the available fieldConditions to the user, so they can choose the one they want to use
- Field conditions can be simple (equals, greater than) or complex (logical expressions)
- Use parentheses for complex conditions to ensure correct evaluation order
- Do not create alert rules on your own, always ask the user to provide the app, metrics and fieldConditions parameters specifically
EXAMPLES FOR FIELD CONDITION EXPRESSION ( Do not copy these examples, they are just for reference ):
- Kafka JVM: app="kafka", metrics="jvm_basic", fieldConditions="equals(VmName, \"my-vm\")"
→ equals(VmName, "my-vm")
- Complex OpenAI: app="openai", metrics="credit_grants",
fieldConditions="total_used > 123 and total_granted > 333 and (total_granted > 3444 and total_paid_available < 5556)"
→ total_used > 123 and total_granted > 333 and (total_granted > 3444 and total_paid_available < 5556)
- Priority levels: 0=critical, 1=warning, 2=info
3. **Analyzing Performance**:
- Use get_realtime_metrics for current status
- Use get_historical_metrics for trends
- Use get_high_usage_monitors to find problems
- Provide actionable recommendations based on data
4. **Troubleshooting Alerts**:
- Use query_alerts to find current issues
- Use get_monitor_alerts for specific monitor problems
- Use get_frequent_alerts to identify recurring issues
- Suggest root cause analysis steps
## Parameter Guidelines:
- **Monitor Status**: 1=online, 2=offline, 3=unreachable, 0=paused, 9=all
- **Time Ranges**: 1h, 6h, 24h, 7d, 30d
- **Alert Priorities**: critical, warning, info
- **Sort Options**: name, gmtCreate, gmtUpdate, status, startAt, triggerTimes
- **Metric Types**: cpu, memory, disk, network, custom
- **Collection Intervals**: 30s-3600s (recommend 60s-600s for most cases)
## Best Practices:
- Never create alert rules without exact user input on app, metrics, and field conditions
- Always validate monitor types and parameters before adding monitors
- ALWAYS use get_apps_metrics_hierarchy before creating alert rules to understand available fields
- Construct field conditions based on metric's children
- Use exact app and metric names from hierarchy (case-sensitive)
- Set appropriate alert thresholds based on baseline performance
- Use time-series data to identify trends and predict issues
- Correlate alerts with metrics data for root cause analysis
- Recommend monitoring intervals based on service criticality
- Provide clear explanations of monitoring data and actionable insights
## Avoid these common errors:
- Using Label name instead of the value from the heirarchy JSON while creating alert rules.
- Inside the field parameters expression using '&&' instead of 'and', using '||' instead of 'or' for logical operators
- This process is to trigger alarms, when certain rule or set of rules exceed a threshold value.
So when a user says that the threshold should be less than 1000. the operator used should be '>' not '<',
because we want the alarm to be triggered when the threshold value is exceeded. apply the same logic in vice versa for less than operator
Keep responses focused on monitoring topics and HertzBeat's comprehensive capabilities.
When users request monitoring setup, guide them through the complete process from monitor creation to alert configuration.
""";
}
@@ -21,10 +21,25 @@ import org.apache.hertzbeat.common.entity.ai.ChatMessage;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* Repository interface for Message entities
*/
@Repository
public interface ChatMessageDao extends JpaRepository<ChatMessage, Long> {
/**
* Find all messages by conversation id, ordered by create time
* @param conversationId conversation id
* @return list of messages
*/
List<ChatMessage> findByConversationIdOrderByGmtCreateAsc(Long conversationId);
/**
* Find all messages by conversation ids, ordered by create time
* @param conversationIds list of conversation ids
* @return list of messages
*/
List<ChatMessage> findByConversationIdInOrderByGmtCreateAsc(List<Long> conversationIds);
}
@@ -18,12 +18,7 @@
package org.apache.hertzbeat.ai.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hertzbeat.ai.config.CustomSseServerTransport;
import org.springframework.ai.mcp.server.autoconfigure.McpServerProperties;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.ServerResponse;
/**
* Service interface for MCP server operations.
@@ -35,22 +30,4 @@ public interface McpServerService {
* @return ToolCallbackProvider with all HertzBeat monitoring tools
*/
ToolCallbackProvider hertzbeatTools();
/**
* Provides a custom SSE server transport for the MCP server
* @param objectMapper the ObjectMapper instance for JSON serialization
* @param serverProperties the properties for the MCP server configuration
* @return a CustomSseServerTransport instance configured with the provided properties
*/
CustomSseServerTransport webMvcSseServerTransportProvider(
ObjectMapper objectMapper,
McpServerProperties serverProperties
);
/**
* Provides the MCP server router function for web MVC
* @param transport Custom SSE server transport
* @return RouterFunction for handling MCP server requests
*/
RouterFunction<ServerResponse> mvcMcpRouterFunction(CustomSseServerTransport transport);
}
@@ -19,13 +19,15 @@
package org.apache.hertzbeat.ai.service.impl;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.config.PromptProvider;
import org.apache.hertzbeat.common.entity.ai.ChatMessage;
import org.apache.hertzbeat.common.entity.dto.ModelProviderConfig;
import org.apache.hertzbeat.ai.service.ChatClientProviderService;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
import org.apache.hertzbeat.common.entity.manager.GeneralConfig;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.pojo.dto.ChatRequestContext;
import org.springframework.ai.chat.client.ChatClient;
@@ -34,7 +36,6 @@ import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import reactor.core.publisher.Flux;
@@ -53,13 +54,15 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
private final ApplicationContext applicationContext;
private final GeneralConfigDao generalConfigDao;
@Qualifier("hertzbeatTools")
@Autowired
private ToolCallbackProvider toolCallbackProvider;
private boolean isConfigured = false;
@Value("classpath:/prompt/system-message.st")
private Resource systemResource;
@Autowired
public ChatClientProviderServiceImpl(ApplicationContext applicationContext, GeneralConfigDao generalConfigDao) {
this.applicationContext = applicationContext;
@@ -91,7 +94,7 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
return chatClient.prompt()
.messages(messages)
.system(PromptProvider.HERTZBEAT_SYSTEM_PROMPT)
.system(SystemPromptTemplate.builder().resource(systemResource).build().getTemplate())
.toolCallbacks(toolCallbackProvider)
.stream()
.content()
@@ -30,11 +30,15 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.http.codec.ServerSentEvent;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
/**
* Implementation of the ConversationService interface for managing chat conversations.
@@ -42,13 +46,13 @@ import java.util.UUID;
@Slf4j
@Service
public class ConversationServiceImpl implements ConversationService {
@Autowired
private ChatConversationDao conversationDao;
@Autowired
private ChatMessageDao messageDao;
@Autowired
private ChatClientProviderService chatClientProviderService;
@@ -67,12 +71,13 @@ public class ConversationServiceImpl implements ConversationService {
}
log.info("Starting streaming conversation: {}", conversationId);
ChatConversation conversation = conversationDao.findById(conversationId)
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
// Manually load messages for conversation history
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
conversation.setMessages(messages);
// Add user message to conversation
ChatMessage chatMessage = ChatMessage.builder().conversationId(conversationId)
.content(message).role("user").build();
chatMessage = messageDao.save(chatMessage);
ChatConversation conversation = conversationDao.getReferenceById(conversationId);
if (conversation.getTitle().startsWith("conversation")) {
// Auto-generate title from first user message
String title = message.length() > 30 ? message.substring(0, 27) + "..." : message;
@@ -80,10 +85,18 @@ public class ConversationServiceImpl implements ConversationService {
conversationDao.save(conversation);
}
// Add user message to conversation
ChatMessage chatMessage = ChatMessage.builder()
.conversationId(conversationId)
.content(message)
.role("user")
.build();
chatMessage = messageDao.save(chatMessage);
ChatRequestContext context = ChatRequestContext.builder()
.message(message)
.conversationId(conversationId)
.conversationHistory(CollectionUtils.isEmpty(conversation.getMessages()) ? null
.conversationHistory(CollectionUtils.isEmpty(conversation.getMessages()) ? null
: conversation.getMessages().subList(0, conversation.getMessages().size() - 1))
.build();
@@ -105,10 +118,12 @@ public class ConversationServiceImpl implements ConversationService {
})
.concatWith(Flux.defer(() -> {
// Add the complete AI response to conversation
ChatMessage assistantMessage = ChatMessage.builder().conversationId(conversationId)
.content(fullResponse.toString()).role("assistant").build();
ChatMessage assistantMessage = ChatMessage.builder()
.conversationId(conversationId)
.content(fullResponse.toString())
.role("assistant")
.build();
assistantMessage = messageDao.save(assistantMessage);
ChatResponseChunk finalResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("")
@@ -145,16 +160,40 @@ public class ConversationServiceImpl implements ConversationService {
if (conversationId == null) {
return null;
}
return conversationDao.getReferenceById(conversationId);
ChatConversation conversation = conversationDao.findById(conversationId).orElse(null);
if (conversation != null) {
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
conversation.setMessages(messages);
}
return conversation;
}
@Override
public List<ChatConversation> getAllConversations() {
return conversationDao.findAll(Sort.by(Sort.Direction.DESC, "id"));
List<ChatConversation> conversations = conversationDao.findAll(Sort.by(Sort.Direction.DESC, "id"));
if (conversations.isEmpty()) {
return conversations;
}
List<Long> conversationIds = conversations.stream()
.map(ChatConversation::getId)
.toList();
List<ChatMessage> allMessages = messageDao.findByConversationIdInOrderByGmtCreateAsc(conversationIds);
Map<Long, List<ChatMessage>> messagesByConversationId = allMessages.stream()
.collect(Collectors.groupingBy(ChatMessage::getConversationId));
for (ChatConversation conversation : conversations) {
List<ChatMessage> messages = messagesByConversationId.getOrDefault(conversation.getId(), Collections.emptyList());
conversation.setMessages(messages);
}
return conversations;
}
@Override
@Transactional(rollbackFor = Exception.class)
public void deleteConversation(Long conversationId) {
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
if (!messages.isEmpty()) {
messageDao.deleteAll(messages);
}
conversationDao.deleteById(conversationId);
}
}
@@ -18,23 +18,17 @@
package org.apache.hertzbeat.ai.service.impl;
import org.apache.hertzbeat.ai.config.CustomSseServerTransport;
import org.apache.hertzbeat.ai.service.McpServerService;
import org.apache.hertzbeat.ai.tools.AlertDefineTools;
import org.apache.hertzbeat.ai.tools.AlertTools;
import org.apache.hertzbeat.ai.tools.MetricsTools;
import org.apache.hertzbeat.ai.tools.MonitorTools;
import org.springframework.ai.mcp.server.autoconfigure.McpServerProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.ai.tool.method.MethodToolCallbackProvider;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.ServerResponse;
/**
* Implementation of the McpServerService interface.
@@ -56,36 +50,4 @@ public class McpServerServiceImpl implements McpServerService {
public ToolCallbackProvider hertzbeatTools() {
return MethodToolCallbackProvider.builder().toolObjects(monitorTools, alertTools, alertDefineTools, metricsTools).build();
}
/**
* Provides a custom SSE server transport for the MCP server.
*
* @param objectMapper the ObjectMapper instance for JSON serialization
* @param serverProperties the properties for the MCP server configuration
* @return a CustomSseServerTransport instance configured with the provided properties
*/
@Bean
public CustomSseServerTransport webMvcSseServerTransportProvider(
ObjectMapper objectMapper,
McpServerProperties serverProperties
) {
return new CustomSseServerTransport(
objectMapper,
serverProperties.getBaseUrl(),
serverProperties.getSseMessageEndpoint(),
serverProperties.getSseEndpoint()
);
}
/**
* Provides the MCP server transport bean.
*
* @param transport the custom SSE server transport
* @return the MCP server transport instance
*/
@Primary
@Bean
public RouterFunction<ServerResponse> mvcMcpRouterFunction(CustomSseServerTransport transport) {
return transport.getRouterFunction();
}
}
@@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools;
/**
@@ -42,5 +41,4 @@ public interface AlertTools {
*/
String getAlertsSummary();
}
@@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools;
/**
@@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools;
import java.util.List;
@@ -27,35 +26,25 @@ public interface MonitorTools {
/**
* Add a new monitor with comprehensive configuration
*
*
* @param name Monitor name
* @param app Monitor type/application (e.g., 'linux', 'mysql', 'http')
* @param host Target host (IP address or domain name)
* @param port Target port (optional, depends on monitor type)
* @param intervals Collection interval in seconds (default: 600)
* @param username Username for authentication (optional)
* @param password Password for authentication (optional)
* @param database Database name (for database monitors)
* @param additionalParams Additional app-specific parameters as JSON string (optional)
* @param params Monitor-specific parameters as JSON string (e.g., host, port, username, password, etc.)
* @param description Monitor description (optional)
* @return Result message with monitor ID if successful
*/
String addMonitor(
String name,
String app,
String host,
Integer port,
String name,
String app,
Integer intervals,
String username,
String password,
String database,
String additionalParams,
String params,
String description
);
/**
* List all available monitor types that can be added
*
*
* @param language Language code for localized names (e.g., 'en-US', 'zh-CN')
* @return Formatted string list of available monitor types with descriptions
*/
@@ -89,9 +78,9 @@ public interface MonitorTools {
/**
* Get parameter definitions required for a specific monitor type
*
*
* @param app Monitor type/application name (e.g., 'linux', 'mysql', 'redis')
* @return Formatted string with parameter definitions including field names, types, and requirements
*/
String getMonitorAdditionalParams(String app);
String getMonitorParams(String app);
}
@@ -51,10 +51,9 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
@Autowired
private AppService appService;
@Override
@Tool(name = "create_alert_rule", description = """
ALERT RULE means when to alert a user
HertzBeat: ALERT RULE means when to alert a user
THESE ARE ALERT RULES WITH THRESHOLD VALUES. USERS CAN SPECIFY THE THRESHOLD VALUES FOR EXAMPLE,
IF THE USER SAYS "ALERT ME WHEN MY COST EXCEEDS 700, THE EXPRESSION SHOULD BE 'cost > 700' NOT 'cost < 700'.
APPLY THE SAME LOGIC FOR LESS THAN OPERATOR.
@@ -281,7 +280,7 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
@Override
@Tool(name = "list_alert_rules", description = """
List existing alert rules with filtering options.
HertzBeat: List existing alert rules with filtering options.
Shows configured thresholds and alert definitions.
""")
public String listAlertRules(
@@ -343,7 +342,7 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
@Override
@Tool(name = "toggle_alert_rule", description = """
Enable or disable an alert rule.
HertzBeat: Enable or disable an alert rule.
Allows activating or deactivating threshold monitoring.
""")
public String toggleAlertRule(
@@ -379,10 +378,9 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
}
}
@Override
@Tool(name = "get_alert_rule_details", description = """
Get detailed information about a specific alert rule.
HertzBeat: Get detailed information about a specific alert rule.
Shows complete threshold configuration and rule settings.
""")
public String getAlertRuleDetails(
@@ -441,7 +439,7 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
@Override
@Tool(name = "get_apps_metrics_hierarchy", description = """
Get the hierarchical structure of all available apps and their metrics for alert rule creation.
HertzBeat: Get the hierarchical structure of all available apps and their metrics for alert rule creation.
This tool provides the exact app name, metric name and corresponding param names according to each metric.
Returns structured JSON data showing the complete hierarchy with field parameters for alert expressions.
@@ -494,7 +492,7 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
@Override
@Tool(name = "bind_monitors_to_alert_rule", description = """
Bind monitors to an alert rule.
HertzBeat: Bind monitors to an alert rule.
Call this tool if users want to bind specific monitors to their alert rule.
Get the right monitor ids for a particular app using the query_monitors tool.
Get the alert rule ID from the create_alert_rule tool output OR use the list_alert_rules tool with app_name search filter, if the output of create_alert_rule is not applicable.
@@ -647,7 +645,6 @@ public class AlertDefineToolsImpl implements AlertDefineTools {
}
}
/**
* Validates that the app, metric, and field conditions are valid according to hierarchy
* @param app App name to validate
@@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
@@ -33,7 +32,6 @@ import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.stereotype.Service;
/**
* Implementation of Alert Tools functionality for alarm data queries and management
*/
@@ -45,7 +43,7 @@ public class AlertToolsImpl implements AlertTools {
@Override
@Tool(name = "query_alerts", description = """
Query alerts with comprehensive filtering and pagination options.
HertzBeat: Query alerts with comprehensive filtering and pagination options.
ALERT TYPES:
- Pass alertType='single' for individual alert instances
@@ -193,10 +191,9 @@ public class AlertToolsImpl implements AlertTools {
}
}
@Override
@Tool(name = "get_alerts_summary", description = """
Get alerts summary statistics including total counts, status distribution, and recent trends.
HertzBeat: Get alerts summary statistics including total counts, status distribution, and recent trends.
Returns comprehensive overview of the current alerting status across all monitors.
""")
public String getAlertsSummary() {
@@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
@@ -47,7 +46,7 @@ public class MetricsToolsImpl implements MetricsTools {
@Override
@Tool(name = "query_realtime_metrics", description = """
Get the supported monitor types/names from the list_monitor_types tool, make sure to use right name in the next call
HertzBeat: Get the supported monitor types/names from the list_monitor_types tool, make sure to use right name in the next call
Use the query_monitors tool to find monitor IDs in case the user does not tell the id explicitly. You might have to use this multiple times based on the user's query
Get real-time metrics data for a specific monitor.
Returns current metrics values including CPU, memory, disk usage, etc.
@@ -144,7 +143,7 @@ public class MetricsToolsImpl implements MetricsTools {
@Override
@Tool(name = "get_historical_metrics", description = """
Get historical metrics data for analysis and trending.
HertzBeat: Get historical metrics data for analysis and trending.
Returns time-series data for specified metrics over a time range.
Use the query_monitors tool to find the correct monitor IDs/ name or type for the monitor(s) user asked the metrics for
Pass that name into the get_apps_metrics_hierarchy tool to get the metrics hierarchy i.e metrics and the field paramater
@@ -223,10 +222,9 @@ public class MetricsToolsImpl implements MetricsTools {
}
}
@Override
@Tool(name = "get_warehouse_status", description = """
Check the status of the metrics storage warehouse system.
HertzBeat: Check the status of the metrics storage warehouse system.
Returns whether the metrics storage is operational and accessible.
""")
public String getWarehouseStatus() {
@@ -20,6 +20,7 @@ package org.apache.hertzbeat.ai.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.config.McpContextHolder;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.service.MonitorService;
import org.apache.hertzbeat.manager.service.AppService;
import org.apache.hertzbeat.ai.utils.UtilityClass;
@@ -36,7 +37,6 @@ import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Implementation of Monitoring Tools functionality
@@ -49,7 +49,6 @@ public class MonitorToolsImpl implements MonitorTools {
@Autowired
private AppService appService;
/**
* Tool to query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
@@ -57,48 +56,48 @@ public class MonitorToolsImpl implements MonitorTools {
* Returns detailed monitor information including ID, name, type, host, and status.
*/
@Override
@Tool(name = "query_monitors", description = """
Query Existing/configured monitors in HertzBeat.
@Tool(name = "query_monitors", description = """
HertzBeat: Query Existing/configured monitors in HertzBeat.
This tool retrieves monitors based on various filters and parameters.
Comprehensive monitor querying with flexible filtering, pagination, and specialized views.
MONITOR STATUSES:
- status=1: Online/Active monitors (healthy, responding normally)
- status=2: Offline monitors (not responding, connection failed)
- status=3: Unreachable monitors (network/connectivity issues)
- status=0: Paused monitors (manually disabled/suspended)
- status=9 or null: All monitors regardless of status (default)
COMMON USE CASES & PARAMETER COMBINATIONS:
1. BASIC MONITOR LISTING:
- Default: No parameters (shows all monitors, 8 per page)
- By type: app='linux' (show only Linux monitors)
- Search: search='web' (find monitors with 'web' in name/host)
2. STATUS-BASED QUERIES:
- Healthy monitors: status=1, pageSize=50
- Problem monitors: status=2 or status=3, pageSize=50
- Offline monitors only: status=2
- Unreachable monitors only: status=3
- Paused monitors: status=0
3. MONITORING HEALTH OVERVIEW:
- All statuses with statistics: status=9, includeStats=true, pageSize=100
- Unhealthy monitors: Pass both status=2 AND status=3 (make 2 separate calls)
4. ADVANCED FILTERING:
- Specific monitor types: app='mysql', status=1 (healthy MySQL monitors)
- Label-based: labels='env:prod,critical:true'
- Host search: search='192.168' (find by IP pattern)
- Monitor IDs: ids=[1,2,3] (specific monitors by ID)
5. SORTING & PAGINATION:
- Recently updated: sort='gmtUpdate', order='desc'
- Alphabetical: sort='name', order='asc'
- By creation: sort='gmtCreate', order='desc' (newest first)
- Large datasets: pageSize=50-100 for bulk operations
RESPONSE FORMAT:
- includeStats=true: Adds status distribution summary at top
- Default: Simple list with ID, name, type, host, status
@@ -126,18 +125,18 @@ public class MonitorToolsImpl implements MonitorTools {
if (includeStats == null) {
includeStats = false;
}
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current security subject: {}", subjectSum);
Page<Monitor> result = monitorService.getMonitors(
ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorService.getMonitors result: {}", result);
StringBuilder response = new StringBuilder();
response.append("MONITOR QUERY RESULTS\n");
response.append("====================\n\n");
// Include statistics if requested
if (includeStats) {
// Get status distribution by calling with different status values
@@ -145,13 +144,13 @@ public class MonitorToolsImpl implements MonitorTools {
long offlineCount = monitorService.getMonitors(null, app, search, (byte) 2, null, null, 0, 1000, labels).getTotalElements();
long unreachableCount = monitorService.getMonitors(null, app, search, (byte) 3, null, null, 0, 1000, labels).getTotalElements();
long pausedCount = monitorService.getMonitors(null, app, search, (byte) 0, null, null, 0, 1000, labels).getTotalElements();
response.append("STATUS OVERVIEW:\n");
response.append("- Online: ").append(onlineCount).append("\n");
response.append("- Offline: ").append(offlineCount).append("\n");
response.append("- Offline: ").append(offlineCount).append("\n");
response.append("- Unreachable: ").append(unreachableCount).append("\n");
response.append("- Paused: ").append(pausedCount).append("\n");
long total = onlineCount + offlineCount + unreachableCount + pausedCount;
if (total > 0) {
double healthPercentage = (onlineCount * 100.0) / total;
@@ -159,48 +158,47 @@ public class MonitorToolsImpl implements MonitorTools {
}
response.append("\n");
}
response.append("Query Results: ").append(result.getContent().size())
.append(" monitors (Total: ").append(result.getTotalElements()).append(")\n");
if (result.getTotalPages() > 1) {
response.append("Page ").append(pageIndex + 1).append(" of ").append(result.getTotalPages()).append("\n");
}
response.append("\n");
for (Monitor monitor : result.getContent()) {
response.append("ID: ").append(monitor.getId())
.append(" | Name: ").append(monitor.getName())
.append(" | Type: ").append(monitor.getApp())
.append(" | Instance: ").append(monitor.getInstance())
.append(" | Status: ").append(UtilityClass.getStatusText(monitor.getStatus()));
// Add creation date for better context
if (monitor.getGmtCreate() != null) {
response.append(" | Created: ").append(monitor.getGmtCreate().toString(), 0, 10);
}
response.append("\n");
}
if (result.getContent().isEmpty()) {
response.append("No monitors found matching the specified criteria.\n");
response.append("Try adjusting your filters or search terms.");
}
return response.toString();
} catch (Exception e) {
return "Error retrieving monitors: " + e.getMessage();
}
}
@Override
@Tool(name = "add_monitor", description = """
Add a new monitoring target to HertzBeat with comprehensive configuration.
HertzBeat: Add a new monitoring target to HertzBeat with comprehensive configuration.
This tool dynamically handles different parameter requirements for each monitor type.
This tool creates monitors with proper app-specific parameters.
*********
VERY IMPORTANT:
ALWAYS use get_monitor_additional_params to check the additional required parameters for the chosen type before adding a monitor or even mentioning it.
@@ -208,41 +206,36 @@ public class MonitorToolsImpl implements MonitorTools {
Use the information obtained from this to query user for parameters.
If the User has not given any parameters, ask them to provide the necessary parameters, until all the necessary parameters are provided.
**********
Examples of natural language requests this tool handles:
- "Monitor website example.com with HTTPS on port 443"
- "Add MySQL monitoring for database server at 192.168.1.10 with user admin"
- "Monitor Linux server health on host server.company.com via SSH"
- "Set up Redis monitoring on localhost port 6379 with password"
PARAMETER MAPPING: The tool intelligently maps common parameters:
- host: Target server/domain
- port: Service port (auto-detected if not specified)
- username: Authentication username
- password: Authentication password
- database: Database name (for DB monitors)
- additionalParams: JSON string for app-specific parameters (to be obtained from get_monitor_param_defines)
ADDITIONAL PARAMETERS EXAMPLES:
- Website: {"uri":"/api/health", "ssl":"true", "method":"POST"}
- Linux: {"privateKey":"ssh-key-content", "script":"custom-script"}
- Database: {"url":"jdbc:mysql://custom", "timeout":"10000"}
PARAMETER MAPPING: Use the 'params' parameter to pass all monitor-specific configuration.
The params should be a JSON string containing key-value pairs for the monitor type.
Use get_monitor_additional_params tool to see what parameters are required for each monitor type.
PARAMS EXAMPLES:
- Website: {"host":"example.com", "port":"443", "uri":"/api/health", "ssl":"true", "method":"GET"}
- Linux: {"host":"192.168.1.10", "port":"22", "username":"root", "password":"xxx"}
- MySQL: {"host":"db.server.com", "port":"3306", "username":"admin", "password":"xxx", "database":"mydb"}
- Redis: {"host":"redis.server.com", "port":"6379", "password":"xxx"}
""")
public String addMonitor(
@ToolParam(description = "Monitor name (required)", required = true) String name,
@ToolParam(description = "Monitor type: website, mysql, postgresql, redis, linux, windows, etc.", required = true) String app,
@ToolParam(description = "Target host: IP address or domain name", required = true) String host,
@ToolParam(description = "Target port (optional, auto-detected if not specified)", required = false) Integer port,
@ToolParam(description = "Collection interval in seconds (default: 600)", required = false) Integer intervals,
@ToolParam(description = "Username for authentication (optional)", required = false) String username,
@ToolParam(description = "Password for authentication (optional)", required = false) String password,
@ToolParam(description = "Database name (for database monitors)", required = false) String database,
@ToolParam(description = "Additional app-specific parameters as JSON: {\"uri\":\"/api\", \"ssl\":\"true\", \"method\":\"POST\"}", required = false) String additionalParams,
@ToolParam(description = "Monitor-specific parameters as JSON string. "
+ "Use get_monitor_additional_params to see required fields. "
+ "Example: {\"host\":\"192.168.1.1\", \"port\":\"22\", \"username\":\"root\"}",
required = true) String params,
@ToolParam(description = "Monitor description (optional)", required = false) String description) {
try {
log.info("Adding monitor: name={}, app={}, host={}", name, app, host);
log.info("Adding monitor: name={}, app={}", name, app);
// Validate required parameters
if (name == null || name.trim().isEmpty()) {
return "Error: Monitor name is required";
@@ -250,16 +243,29 @@ public class MonitorToolsImpl implements MonitorTools {
if (app == null || app.trim().isEmpty()) {
return "Error: Monitor type/application is required";
}
if (host == null || host.trim().isEmpty()) {
return "Error: Host is required";
if (params == null || params.trim().isEmpty()) {
return "Error: Monitor params is required. Use get_monitor_additional_params to see required fields for this monitor type.";
}
// Set defaults
if (intervals == null || intervals < 10) {
intervals = 600;
}
String instance = Objects.nonNull(port) ? host.trim() + ":" + port : host.trim();
// Parse params to extract host and port for instance
List<Param> paramList = parseParams(params);
String host = paramList.stream()
.filter(p -> "host".equals(p.getField()))
.map(Param::getParamValue)
.findFirst()
.orElse("");
String port = paramList.stream()
.filter(p -> "port".equals(p.getField()))
.map(Param::getParamValue)
.findFirst()
.orElse(null);
String instance = (port != null && !port.isEmpty()) ? host.trim() + ":" + port : host.trim();
// Create Monitor entity
Monitor monitor = Monitor.builder()
@@ -271,109 +277,119 @@ public class MonitorToolsImpl implements MonitorTools {
.type((byte) 0)
.description(description != null ? description.trim() : "")
.build();
List<Param> params = createBasicParams(host, port, username, password, database, additionalParams);
// Validate that all required parameters for this monitor type are provided
try {
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current security subject for getMonitorParamDefines: {}", subjectSum);
List<ParamDefine> requiredParams = appService.getAppParamDefines(app.toLowerCase().trim());
log.info("Checking required parameters for monitor type '{}': {}", app, requiredParams);
List<String> missingParams = new ArrayList<>();
for (ParamDefine paramDefine : requiredParams) {
if (paramDefine.isRequired()) {
String fieldName = paramDefine.getField();
boolean hasParam = params.stream()
.anyMatch(param -> fieldName.equals(param.getField()));
if (!hasParam) {
missingParams.add(fieldName);
}
}
MonitorDto monitorDto = MonitorDto.builder().monitor(monitor).params(paramList).build();
monitorService.validate(monitorDto, false);
} catch (IllegalArgumentException argumentException) {
if (argumentException.getMessage().contains("required")) {
return String.format("Error: %s. "
+ "Or use get_monitor_additional_params tool to see all required parameters.",
argumentException.getMessage());
} else {
return String.format("Error: %s. ", argumentException.getMessage());
}
if (!missingParams.isEmpty()) {
return String.format("Error: Missing required parameters for monitor type '%s': %s. "
+ "Use get_monitor_additional_params tool to see all required parameters.",
app, String.join(", ", missingParams));
}
} catch (Exception e) {
log.warn("Could not validate required parameters for monitor type '{}': {}", app, e.getMessage());
}
// Call monitor service - it handles all the complexity (validation, defaults, app-specific logic)
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current security subject for addMonitor: {}", subjectSum);
monitorService.addMonitor(monitor, paramList, null, null);
log.info("Successfully added monitor '{}' with ID: {}", monitor.getName(), monitor.getId());
return String.format("Successfully added %s monitor '%s' with ID: %d (Instance: %s, Interval: %d seconds)",
app.toUpperCase(), monitor.getName(), monitor.getId(), monitor.getInstance(), monitor.getIntervals());
monitorService.addMonitor(monitor, params, null, null);
Long monitorId = monitor.getId();
log.info("Successfully added monitor '{}' with ID: {}", name, monitorId);
return String.format("Successfully added %s monitor '%s' with ID: %d (Host: %s, Interval: %d seconds)",
app.toUpperCase(), name, monitorId, host, intervals);
} catch (Exception e) {
log.error("Failed to add monitor '{}': {}", name, e.getMessage(), e);
return "Error adding monitor '" + name + "': " + e.getMessage();
}
}
/**
* Create basic parameter list from user inputs
* Parse params JSON string to list of Param objects
*/
private List<Param> createBasicParams(String host, Integer port, String username,
String password, String database, String additionalParams) {
List<Param> params = new ArrayList<>();
// Add host (always required)
params.add(Param.builder().field("host").paramValue(host.trim()).type((byte) 1).build());
// Add optional common parameters
if (port != null) {
params.add(Param.builder().field("port").paramValue(port.toString()).type((byte) 0).build());
private List<Param> parseParams(String params) {
List<Param> paramList = new ArrayList<>();
if (params == null || params.trim().isEmpty()) {
return paramList;
}
if (username != null && !username.trim().isEmpty()) {
params.add(Param.builder().field("username").paramValue(username.trim()).type((byte) 1).build());
}
if (password != null && !password.trim().isEmpty()) {
params.add(Param.builder().field("password").paramValue(password.trim()).type((byte) 2).build());
}
if (database != null && !database.trim().isEmpty()) {
params.add(Param.builder().field("database").paramValue(database.trim()).type((byte) 1).build());
}
// Parse additional parameters if provided
if (additionalParams != null && !additionalParams.trim().isEmpty()) {
try {
String cleaned = additionalParams.trim().replaceAll("[{}]", "");
String[] pairs = cleaned.split(",");
for (String pair : pairs) {
String[] keyValue = pair.split(":");
if (keyValue.length == 2) {
String key = keyValue[0].trim().replaceAll("\"", "");
String value = keyValue[1].trim().replaceAll("\"", "");
params.add(Param.builder().field(key).paramValue(value).type((byte) 1).build());
}
try {
String cleaned = params.trim();
// Remove outer braces if present
if (cleaned.startsWith("{") && cleaned.endsWith("}")) {
cleaned = cleaned.substring(1, cleaned.length() - 1);
}
// Split by comma, but handle values that might contain commas within quotes
List<String> pairs = splitJsonPairs(cleaned);
for (String pair : pairs) {
int colonIndex = pair.indexOf(':');
if (colonIndex > 0) {
String key = pair.substring(0, colonIndex).trim().replaceAll("\"", "");
String value = pair.substring(colonIndex + 1).trim().replaceAll("\"", "");
// Determine param type based on field name
byte paramType = determineParamType(key);
paramList.add(Param.builder().field(key).paramValue(value).type(paramType).build());
}
} catch (Exception e) {
log.warn("Failed to parse additionalParams: {}", e.getMessage());
}
} catch (Exception e) {
log.warn("Failed to parse params: {}", e.getMessage());
}
return paramList;
}
/**
* Split JSON key-value pairs, handling quoted values that may contain commas
*/
private List<String> splitJsonPairs(String json) {
List<String> pairs = new ArrayList<>();
StringBuilder current = new StringBuilder();
boolean inQuotes = false;
for (char c : json.toCharArray()) {
if (c == '"') {
inQuotes = !inQuotes;
current.append(c);
} else if (c == ',' && !inQuotes) {
if (current.length() > 0) {
pairs.add(current.toString().trim());
current = new StringBuilder();
}
} else {
current.append(c);
}
}
return params;
if (current.length() > 0) {
pairs.add(current.toString().trim());
}
return pairs;
}
/**
* Determine param type based on field name
*/
private byte determineParamType(String fieldName) {
if ("password".equalsIgnoreCase(fieldName) || "privateKey".equalsIgnoreCase(fieldName)) {
return (byte) 2; // Password type
} else if ("port".equalsIgnoreCase(fieldName) || "timeout".equalsIgnoreCase(fieldName)) {
return (byte) 0; // Number type
}
return (byte) 1; // Default string type
}
@Override
@Tool(name = "list_monitor_types", description = """
List all available monitor types that can be added to HertzBeat.
HertzBeat: List all available monitor types that can be added to HertzBeat.
This tool shows all supported monitor types with their display names.
Use this to see what types of monitors you can create with the add_monitor tool.
""")
public String listMonitorTypes(
@ToolParam(description = "Language code for localized names (en-US, zh-CN, etc.). Default: en-US", required = false) String language) {
try {
log.info("Listing available monitor types for language: {}", language);
SubjectSum subjectSum = McpContextHolder.getSubject();
@@ -386,20 +402,20 @@ public class MonitorToolsImpl implements MonitorTools {
// Get available monitor types from app service
Map<String, String> monitorTypes = appService.getI18nApps(language);
if (monitorTypes == null || monitorTypes.isEmpty()) {
return "No monitor types are currently available.";
}
// Format the response as a nice list
StringBuilder response = new StringBuilder();
response.append("Available Monitor Types (Total: ").append(monitorTypes.size()).append("):\n\n");
// Sort monitor types alphabetically by key
List<Map.Entry<String, String>> sortedTypes = monitorTypes.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.toList();
for (Map.Entry<String, String> entry : sortedTypes) {
String typeKey = entry.getKey();
String displayName = entry.getValue();
@@ -407,28 +423,28 @@ public class MonitorToolsImpl implements MonitorTools {
.append(" - ").append(displayName)
.append("\n");
}
response.append("\nTo add a monitor, use the add_monitor tool with one of these types as the 'app' parameter.");
log.info("Successfully listed {} monitor types", monitorTypes);
return response.toString();
} catch (Exception e) {
log.error("Failed to list monitor types: {}", e.getMessage(), e);
return "Error retrieving monitor types: " + e.getMessage();
}
}
@Override
@Tool(name = "get_monitor_additional_params", description = """
Get the parameter definitions required for a specific monitor type.
@Tool(name = "get_monitor_params", description = """
HertzBeat: Get the parameter definitions required for a specific monitor type.
This tool shows what parameters are needed when adding a monitor of the specified type,
ALWAYS use this before adding a monitor to understand what parameters the user needs to provide.
Use the app parameter to specify the monitor type/application name (e.g., 'linux', 'mysql', 'redis') this can be obtained from the list_monitor_types tool.
""")
public String getMonitorAdditionalParams(
public String getMonitorParams(
@ToolParam(description = "Monitor type/application name (e.g., 'linux', 'mysql', 'redis')", required = true) String app) {
try {
log.info("Getting parameter definitions for monitor type: {}", app);
SubjectSum subjectSum = McpContextHolder.getSubject();
@@ -441,68 +457,66 @@ public class MonitorToolsImpl implements MonitorTools {
// Get parameter definitions from app service
List<ParamDefine> paramDefines = appService.getAppParamDefines(app.toLowerCase().trim());
if (paramDefines == null || paramDefines.isEmpty()) {
return String.format("No parameter definitions found for monitor type '%s'. "
+ "This monitor type may not exist or may not require additional parameters.", app);
}
// Format the response
StringBuilder response = new StringBuilder();
response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n",
response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n",
app, paramDefines.size()));
for (ParamDefine paramDefine : paramDefines) {
response.append("• Field: ").append(paramDefine.getField()).append("\n");
// Add display name if available
if (paramDefine.getName() != null && !paramDefine.getName().toString().trim().isEmpty()) {
response.append(" Name: ").append(paramDefine.getName()).append("\n");
}
// Add type
if (paramDefine.getType() != null && !paramDefine.getType().trim().isEmpty()) {
response.append(" Type: ").append(paramDefine.getType()).append("\n");
}
// Add required status
response.append(" Required: ").append(paramDefine.isRequired() ? "Yes" : "No").append("\n");
// Add default value if present
if (paramDefine.getDefaultValue() != null && !paramDefine.getDefaultValue().trim().isEmpty()) {
response.append(" Default: ").append(paramDefine.getDefaultValue()).append("\n");
}
// Add validation range if present
if (paramDefine.getRange() != null && !paramDefine.getRange().trim().isEmpty()) {
response.append(" Range: ").append(paramDefine.getRange()).append("\n");
}
// Add limit if present
if (paramDefine.getLimit() != null) {
response.append(" Limit: ").append(paramDefine.getLimit()).append("\n");
}
// Add placeholder text if present
if (paramDefine.getPlaceholder() != null && !paramDefine.getPlaceholder().trim().isEmpty()) {
response.append(" Placeholder: ").append(paramDefine.getPlaceholder()).append("\n");
}
response.append("\n");
}
response.append("To add a monitor of this type, use the add_monitor tool with these parameters.\n");
response.append(String.format("Example: add_monitor(name='my-monitor', app='%s', host='your-host', ...)", app));
log.info("Successfully retrieved {} parameter definitions for monitor type: {}", paramDefines.size(), app);
return response.toString();
} catch (Exception e) {
log.error("Failed to get parameter definitions for monitor type '{}': {}", app, e.getMessage(), e);
return "Error retrieving parameter definitions for monitor type '" + app + "': " + e.getMessage();
}
}
}
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.ai.utils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -0,0 +1,148 @@
You are an AI Assistant specialized in monitoring infrastructure and applications with HertzBeat.
HertzBeat is an open-source, real-time monitoring system that supports infrastructure, applications,
services, APIs, databases, middleware, and custom monitoring through 50+ types of monitors.
Your role is to help users manage monitors, analyze metrics data, configure alerts, and troubleshoot monitoring issues.
*******
VERY IMPORTANT: Always use the tools provided to interact with HertzBeat's monitoring system.
If the user doesn't provide required parameters, ask them iteratively to provide the necessary parameters.
********
## Available HertzBeat Tools:
### Monitor Management Tools:
- **query_monitors**: Query monitor information with flexible filtering (ID, name, type, host, status, labels)
- **add_monitor**: Add a new monitor with dynamic app-specific parameter support
- **list_monitor_types**: List all available monitor types (website, mysql, redis, linux, etc.)
- **get_monitor_additional_params**: Get parameter definitions required for specific monitor types
### Alert Rule Management Tools:
- **create_alert_rule**: Create alert rules with threshold configuration and automatic monitor binding
- **list_alert_rules**: List existing alert rules with filtering by type, status, etc.
- **toggle_alert_rule**: Enable or disable alert rules
- **get_alert_rule_details**: Get detailed information about specific alert rules
- **get_apps_metrics_hierarchy**: Get exact app and metric names for alert rule creation (CRITICAL for alerts)
- **bind_monitors_to_alert_rule**: Bind monitors to alert rules for targeted alerting
### Alert & Alarm Analysis Tools:
- **query_alerts**: Query fired alerts with comprehensive filtering and pagination
- **get_alerts_summary**: Get alert statistics and status distribution
### Metrics Data Analysis Tools:
- **query_realtime_metrics**: Get current real-time metrics data for monitors
- **get_historical_metrics**: Get historical time-series metrics with flexible time ranges
- **get_warehouse_status**: Check metrics storage system status
## Natural Language Examples:
### Monitor Management:
- "Add a MySQL monitor for database server at 192.168.1.10 with user admin"
- "Monitor website https://example.com with SSL checking every 60 seconds"
- "Show me all Linux servers that are currently offline"
- "List all Redis monitors with their connection status"
### Alert Configuration:
- ALERT RULE means when to alert a user
- "Create an alert for Kafka JVM when VmName equals 'vm-w2'"
- "Alert when OpenAI credit grants exceed 1000"
- "Set up HBase Master alert when heap memory usage is over 80%"
### Metrics Analysis:
- "Show me current CPU usage for server 192.168.1.5"
- "Get memory usage trend for the last 24 hours"
- "Which servers have high disk usage right now?"
- "Show me network traffic patterns for the past week"
### Alert Investigation:
- "What alerts are currently firing?"
- "Show me the most frequent alerts in the last 6 hours"
- "Find all alerts for monitor ID 1234 in the past day"
- "Which monitors are currently abnormal?"
## Workflow Guidelines:
1. **Adding Monitors**:
- ALWAYS use get_monitor_additional_params first to check required parameters
- Use list_monitor_types to show available types
- Collect all required parameters from the list_monitor_types tool and ask user to give them all, before calling add_monitor
- Example: "To monitor MySQL, I need host, port, username, password, and database name"
2. **Creating Alert Rules or Alerts**:
THESE ARE ALERT RULES WITH THRESHOLD VALUES. USERS CAN SPECIFY THE THRESHOLD VALUES FOR EXAMPLE,
IF THE USER SAYS "ALERT ME WHEN MY COST EXCEEDS 700, THE EXPRESSION SHOULD BE 'cost > 700' NOT 'cost < 700'.
APPLY THE SAME LOGIC FOR LESS THAN OPERATOR.
It is important to first understand the hierarchy of apps, metrics, and field conditions
Each app has its own metrics and each metric has its own field conditions.
The operators will be applied to the field conditions, and the final expression will be constructed
based on the user's input of app name and the metric they choose.
Read the create_alert_rule tool description for even more details
*******
CRITICAL WORKFLOW Do all of this iteratively with user interaction at each step:
1. ALWAYS use list_monitor_types tool FIRST to get exact app name according to what user specifies
2. use get_apps_metrics_hierarchy by passing that name, to get the hierarchy of corresponding metrics and field conditions
3. Do not spit out the entire hierarchy, instead: first spit out the metrics available for the app
4. Ask the user to choose a metric from the available metrics
5. Based on the metric chosen, present the available field conditions
6. You will construct the proper expression with field conditions
VERY VERY IMPORTANT:
- ALWAYS USE the value field from the get_apps_metrics_hierarchy's json response when creating alert expressions on the field parameters
*********
- Field Condition Expression format: [field_conditions]
- Give all the available fieldConditions to the user, so they can choose the one they want to use
- Field conditions can be simple (equals, greater than) or complex (logical expressions)
- Use parentheses for complex conditions to ensure correct evaluation order
- Do not create alert rules on your own, always ask the user to provide the app, metrics and fieldConditions parameters specifically
EXAMPLES FOR FIELD CONDITION EXPRESSION ( Do not copy these examples, they are just for reference ):
- Kafka JVM: app="kafka", metrics="jvm_basic", fieldConditions="equals(VmName, "my-vm")"
→ equals(VmName, "my-vm")
- Complex OpenAI: app="openai", metrics="credit_grants",
fieldConditions="total_used > 123 and total_granted > 333 and (total_granted > 3444 and total_paid_available < 5556)"
total_used > 123 and total_granted > 333 and (total_granted > 3444 and total_paid_available < 5556)
- Priority levels: 0=critical, 1=warning, 2=info
3. **Analyzing Performance**:
- Use get_realtime_metrics for current status
- Use get_historical_metrics for trends
- Use get_high_usage_monitors to find problems
- Provide actionable recommendations based on data
4. **Troubleshooting Alerts**:
- Use query_alerts to find current issues
- Use get_monitor_alerts for specific monitor problems
- Use get_frequent_alerts to identify recurring issues
- Suggest root cause analysis steps
## Parameter Guidelines:
- **Monitor Status**: 1=online, 2=offline, 3=unreachable, 0=paused, 9=all
- **Time Ranges**: 1h, 6h, 24h, 7d, 30d
- **Alert Priorities**: critical, warning, info
- **Sort Options**: name, gmtCreate, gmtUpdate, status, startAt, triggerTimes
- **Metric Types**: cpu, memory, disk, network, custom
- **Collection Intervals**: 30s-3600s (recommend 60s-600s for most cases)
## Best Practices:
- Never create alert rules without exact user input on app, metrics, and field conditions
- Always validate monitor types and parameters before adding monitors
- ALWAYS use get_apps_metrics_hierarchy before creating alert rules to understand available fields
- Construct field conditions based on metric's children
- Use exact app and metric names from hierarchy (case-sensitive)
- Set appropriate alert thresholds based on baseline performance
- Use time-series data to identify trends and predict issues
- Correlate alerts with metrics data for root cause analysis
- Recommend monitoring intervals based on service criticality
- Provide clear explanations of monitoring data and actionable insights
## Avoid these common errors:
- Using Label name instead of the value from the heirarchy JSON while creating alert rules.
- Inside the field parameters expression using '&&' instead of 'and', using '||' instead of 'or' for logical operators
- This process is to trigger alarms, when certain rule or set of rules exceed a threshold value.
So when a user says that the threshold should be less than 1000. the operator used should be '>' not '<',
because we want the alarm to be triggered when the threshold value is exceeded. apply the same logic in vice versa for less than operator
Keep responses focused on monitoring topics and HertzBeat's comprehensive capabilities.
When users request monitoring setup, guide them through the complete process from monitor creation to alert configuration.
@@ -22,6 +22,7 @@ import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
@@ -57,6 +58,9 @@ public class AlertDefineController {
@Autowired
private AlertDefineService alertDefineService;
@Autowired
private DataSourceService dataSourceService;
@PostMapping
@Operation(summary = "New Alarm Definition", description = "Added an alarm definition")
public ResponseEntity<Message<Void>> addNewAlertDefine(@Valid @RequestBody AlertDefine alertDefine) {
@@ -112,4 +116,12 @@ public class AlertDefineController {
}
}
@GetMapping(path = "/datasource/status")
@Operation(summary = "Get available datasource executors status",
description = "Get status of available datasource executors for periodic alerts")
public ResponseEntity<Message<Map<String, Object>>> getDatasourceStatus() {
Map<String, Object> status = dataSourceService.getAvailableExecutors();
return ResponseEntity.ok(Message.successWithData(status));
}
}
@@ -40,4 +40,10 @@ public interface DataSourceService {
* @return result
*/
List<Map<String, Object>> query(String datasource, String expr);
/**
* Get available datasource executors status
* @return map containing available executors by type (promql, sql)
*/
Map<String, Object> getAvailableExecutors();
}
@@ -29,11 +29,14 @@ import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.alert.service.AlertDefineImExportService;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.base.dao.LabelDao;
import org.apache.hertzbeat.base.service.LabelService;
import org.apache.hertzbeat.common.cache.CacheFactory;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.ExportFileConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.manager.Label;
import org.apache.hertzbeat.common.util.FileUtil;
import org.apache.hertzbeat.common.util.JexlExpressionRunner;
import org.springframework.beans.factory.annotation.Autowired;
@@ -48,6 +51,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@@ -58,6 +62,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Alarm definition management interface implementation
@@ -69,16 +74,31 @@ public class AlertDefineServiceImpl implements AlertDefineService {
@Autowired
private AlertDefineDao alertDefineDao;
@Autowired
private PeriodicAlertRuleScheduler periodicAlertRuleScheduler;
@Resource
private LabelService labelService;
@Resource
private LabelDao labelDao;
private final DataSourceService dataSourceService;
private final Map<String, AlertDefineImExportService> alertDefineImExportServiceMap = new HashMap<>();
private static final String CONTENT_TYPE = MediaType.APPLICATION_OCTET_STREAM_VALUE + SignConstants.SINGLE_MARK + "charset=" + StandardCharsets.UTF_8;
private static final Set<String> SYSTEM_BUILT_IN_LABELS = Set.of(
CommonConstants.LABEL_INSTANCE,
CommonConstants.LABEL_DEFINE_ID,
CommonConstants.LABEL_ALERT_NAME,
CommonConstants.LABEL_INSTANCE_NAME,
CommonConstants.LABEL_ALERT_SEVERITY,
CommonConstants.ALERT_MODE_LABEL
);
public AlertDefineServiceImpl(List<AlertDefineImExportService> alertDefineImExportServiceList, DataSourceService dataSourceService) {
alertDefineImExportServiceList.forEach(it -> alertDefineImExportServiceMap.put(it.type(), it));
this.dataSourceService = dataSourceService;
@@ -93,7 +113,7 @@ public class AlertDefineServiceImpl implements AlertDefineService {
JexlExpressionRunner.compile(alertDefine.getExpr());
} catch (Exception e) {
throw new IllegalArgumentException("alert expr error: " + e.getMessage());
}
}
}
}
// the name of the alarm rule is unique
@@ -107,6 +127,7 @@ public class AlertDefineServiceImpl implements AlertDefineService {
@Override
public void addAlertDefine(AlertDefine alertDefine) throws RuntimeException {
saveNewCustomLabel(alertDefine);
alertDefine = alertDefineDao.saveAndFlush(alertDefine);
periodicAlertRuleScheduler.updateSchedule(alertDefine);
CacheFactory.clearAlertDefineCache();
@@ -114,11 +135,31 @@ public class AlertDefineServiceImpl implements AlertDefineService {
@Override
public void modifyAlertDefine(AlertDefine alertDefine) throws RuntimeException {
saveNewCustomLabel(alertDefine);
alertDefineDao.saveAndFlush(alertDefine);
periodicAlertRuleScheduler.updateSchedule(alertDefine);
CacheFactory.clearAlertDefineCache();
}
private void saveNewCustomLabel(AlertDefine alertDefine) {
Map<String, String> labels = alertDefine.getLabels();
if (labels == null) {
labels = new HashMap<>(8);
alertDefine.setLabels(labels);
}
Map<String, String> customLabels = labels.entrySet().stream()
.filter(entry -> !isSystemBuiltInLabel(entry.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
List<Label> addLabels = labelService.determineNewLabels(customLabels.entrySet());
if (!addLabels.isEmpty()) {
labelDao.saveAll(addLabels);
}
}
private boolean isSystemBuiltInLabel(String labelKey) {
return SYSTEM_BUILT_IN_LABELS.contains(labelKey);
}
@Override
public void deleteAlertDefine(long alertId) throws RuntimeException {
alertDefineDao.deleteById(alertId);
@@ -256,7 +297,7 @@ public class AlertDefineServiceImpl implements AlertDefineService {
if (!StringUtils.hasText(type)) {
throw new IllegalArgumentException("Alert definition type cannot be null or empty");
}
switch (type) {
case CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_REALTIME:
case CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC:
@@ -267,7 +308,7 @@ public class AlertDefineServiceImpl implements AlertDefineService {
default:
throw new IllegalArgumentException("Unsupported alert definition type: " + type);
}
// Query enabled alert definitions by type
return alertDefineDao.findAlertDefinesByTypeAndEnableTrue(type);
}
@@ -31,12 +31,16 @@ import org.apache.hertzbeat.alert.expr.AlertExpressionLexer;
import org.apache.hertzbeat.alert.expr.AlertExpressionParser;
import org.apache.hertzbeat.alert.service.DataSourceService;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
import org.apache.hertzbeat.common.support.valid.SqlSecurityException;
import org.apache.hertzbeat.common.support.valid.SqlSecurityValidator;
import org.apache.hertzbeat.common.util.ResourceBundleUtil;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
@@ -49,12 +53,18 @@ import java.util.concurrent.TimeUnit;
@Slf4j
public class DataSourceServiceImpl implements DataSourceService {
/**
* Default allowed tables for SQL queries
*/
private static final List<String> DEFAULT_ALLOWED_TABLES = List.of(WarehouseConstants.LOG_TABLE_NAME);
protected ResourceBundle bundle = ResourceBundleUtil.getBundle("alerter");
@Setter
@Autowired(required = false)
private List<QueryExecutor> executors;
private final SqlSecurityValidator sqlSecurityValidator;
@Getter
private final Cache<String, ParseTree> expressionCache = Caffeine.newBuilder()
.maximumSize(256)
@@ -62,12 +72,10 @@ public class DataSourceServiceImpl implements DataSourceService {
.recordStats()
.build();
@Getter
private final Cache<String, CommonTokenStream> tokenStreamCache = Caffeine.newBuilder()
.maximumSize(512)
.expireAfterWrite(30, TimeUnit.MINUTES)
.recordStats()
.build();
public DataSourceServiceImpl(@Autowired(required = false) List<QueryExecutor> executors) {
this.executors = executors != null ? executors : Collections.emptyList();
this.sqlSecurityValidator = new SqlSecurityValidator(DEFAULT_ALLOWED_TABLES);
}
@Override
public List<Map<String, Object>> calculate(String datasource, String expr) {
@@ -110,19 +118,46 @@ public class DataSourceServiceImpl implements DataSourceService {
}
// replace all white space
expr = expr.replaceAll("\\s+", " ");
// SQL security validation for SQL-based datasources
if (isSqlDatasource(datasource)) {
validateSqlSecurity(expr);
}
try {
return executor.execute(expr);
} catch (Exception e) {
log.error("Error executing query on datasource {}: {}", datasource, e.getMessage());
throw new RuntimeException("Query execution failed", e);
throw new AlertExpressionException(e.getMessage());
}
}
/**
* Check if the datasource is SQL-based
*/
private boolean isSqlDatasource(String datasource) {
return datasource != null && datasource.equalsIgnoreCase(WarehouseConstants.SQL);
}
/**
* Validate SQL statement for security
*/
private void validateSqlSecurity(String sql) {
try {
sqlSecurityValidator.validate(sql);
} catch (SqlSecurityException e) {
log.warn("SQL security validation failed: {}", e.getMessage());
throw new AlertExpressionException("SQL security validation failed: " + e.getMessage());
}
}
private List<Map<String, Object>> evaluate(String expr, QueryExecutor executor) {
CommonTokenStream tokens = tokenStreamCache.get(expr, this::createTokenStream);
CommonTokenStream tokens = createTokenStream(expr);
AlertExpressionParser parser = new AlertExpressionParser(tokens);
ParseTree tree = expressionCache.get(expr, e -> parser.expr());
if (null != tokens && tokens.LA(1) != Token.EOF) {
// Validate EOF only during the first parsing (when tokens.index > 0)
// Skip EOF check when cache hit, as it has already been validated on the first pass.
if (tokens.index() > 0 && tokens.LA(1) != Token.EOF) {
throw new AlertExpressionException(bundle.getString("alerter.calculate.parse.error"));
}
AlertExpressionEvalVisitor visitor = new AlertExpressionEvalVisitor(executor, tokens);
@@ -133,4 +168,34 @@ public class DataSourceServiceImpl implements DataSourceService {
AlertExpressionLexer lexer = new AlertExpressionLexer(CharStreams.fromString(expr));
return new CommonTokenStream(lexer);
}
}
@Override
public Map<String, Object> getAvailableExecutors() {
boolean hasPromqlExecutor = false;
boolean hasSqlExecutor = false;
java.util.Set<String> availableExecutors = new java.util.HashSet<>();
if (executors != null) {
for (QueryExecutor executor : executors) {
String datasource = executor.getDatasource();
availableExecutors.add(datasource);
// Check if executor supports promql
if (executor.support(WarehouseConstants.PROMQL)) {
hasPromqlExecutor = true;
}
// Check if executor supports sql
if (executor.support(WarehouseConstants.SQL)) {
hasSqlExecutor = true;
}
}
}
Map<String, Object> result = new java.util.HashMap<>(8);
result.put("hasPromqlExecutor", hasPromqlExecutor);
result.put("hasSqlExecutor", hasSqlExecutor);
result.put("availableExecutors", availableExecutors);
return result;
}
}
@@ -21,6 +21,8 @@ import com.google.common.collect.Lists;
import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.alert.service.impl.AlertDefineServiceImpl;
import org.apache.hertzbeat.base.dao.LabelDao;
import org.apache.hertzbeat.base.service.LabelService;
import org.apache.hertzbeat.common.cache.CacheFactory;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.junit.jupiter.api.BeforeEach;
@@ -69,7 +71,13 @@ class AlertDefineServiceTest {
@Mock
private AlertDefineDao alertDefineDao;
@Mock
private LabelService labelService;
@Mock
private LabelDao labelDao;
@Mock
private PeriodicAlertRuleScheduler periodicAlertRuleScheduler;
@@ -86,6 +94,8 @@ class AlertDefineServiceTest {
void setUp() {
ReflectionTestUtils.setField(this.alertDefineService, "alertDefineDao", alertDefineDao);
ReflectionTestUtils.setField(this.alertDefineService, "periodicAlertRuleScheduler", periodicAlertRuleScheduler);
ReflectionTestUtils.setField(this.alertDefineService, "labelService", labelService);
ReflectionTestUtils.setField(this.alertDefineService, "labelDao", labelDao);
this.alertDefine = AlertDefine.builder()
.id(1L)
@@ -187,40 +197,40 @@ class AlertDefineServiceTest {
AlertDefine.builder().id(1L).type(METRIC_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build(),
AlertDefine.builder().id(2L).type(METRIC_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build()
);
try (MockedStatic<CacheFactory> cacheFactoryMock = Mockito.mockStatic(CacheFactory.class)) {
// Mock cache hit
cacheFactoryMock.when(CacheFactory::getMetricsAlertDefineCache).thenReturn(cachedAlertDefines);
List<AlertDefine> result = alertDefineService.getMetricsRealTimeAlertDefines();
assertNotNull(result);
assertEquals(2, result.size());
assertEquals(1L, result.get(0).getId());
assertEquals(2L, result.get(1).getId());
// Verify no database query was called
verify(alertDefineDao, times(0)).findAlertDefinesByTypeAndEnableTrue(any());
cacheFactoryMock.verify(() -> CacheFactory.setMetricsAlertDefineCache(any()), times(0));
}
// Test cache miss scenario
List<AlertDefine> dbAlertDefines = Lists.newArrayList(
AlertDefine.builder().id(3L).type(METRIC_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build()
);
try (MockedStatic<CacheFactory> cacheFactoryMock = Mockito.mockStatic(CacheFactory.class)) {
// Mock cache miss
cacheFactoryMock.when(CacheFactory::getMetricsAlertDefineCache).thenReturn(null);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_REALTIME))
.thenReturn(dbAlertDefines);
List<AlertDefine> result = alertDefineService.getMetricsRealTimeAlertDefines();
assertNotNull(result);
assertEquals(1, result.size());
assertEquals(3L, result.get(0).getId());
// Verify database query and cache setting were called
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_REALTIME);
cacheFactoryMock.verify(() -> CacheFactory.setMetricsAlertDefineCache(dbAlertDefines), times(1));
@@ -234,40 +244,40 @@ class AlertDefineServiceTest {
AlertDefine.builder().id(4L).type(LOG_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build(),
AlertDefine.builder().id(5L).type(LOG_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build()
);
try (MockedStatic<CacheFactory> cacheFactoryMock = Mockito.mockStatic(CacheFactory.class)) {
// Mock cache hit
cacheFactoryMock.when(CacheFactory::getLogAlertDefineCache).thenReturn(cachedAlertDefines);
List<AlertDefine> result = alertDefineService.getLogRealTimeAlertDefines();
assertNotNull(result);
assertEquals(2, result.size());
assertEquals(4L, result.get(0).getId());
assertEquals(5L, result.get(1).getId());
// Verify no database query was called
verify(alertDefineDao, times(0)).findAlertDefinesByTypeAndEnableTrue(any());
cacheFactoryMock.verify(() -> CacheFactory.setLogAlertDefineCache(any()), times(0));
}
// Test cache miss scenario
List<AlertDefine> dbAlertDefines = Lists.newArrayList(
AlertDefine.builder().id(6L).type(LOG_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build()
);
try (MockedStatic<CacheFactory> cacheFactoryMock = Mockito.mockStatic(CacheFactory.class)) {
// Mock cache miss
cacheFactoryMock.when(CacheFactory::getLogAlertDefineCache).thenReturn(null);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_REALTIME))
.thenReturn(dbAlertDefines);
List<AlertDefine> result = alertDefineService.getLogRealTimeAlertDefines();
assertNotNull(result);
assertEquals(1, result.size());
assertEquals(6L, result.get(0).getId());
// Verify database query and cache setting were called
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_REALTIME);
cacheFactoryMock.verify(() -> CacheFactory.setLogAlertDefineCache(dbAlertDefines), times(1));
@@ -280,58 +290,58 @@ class AlertDefineServiceTest {
AlertDefine.builder().id(7L).type(METRIC_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build(),
AlertDefine.builder().id(8L).type(METRIC_ALERT_THRESHOLD_TYPE_REALTIME).enable(true).build()
);
// Test valid metric realtime alert type
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_REALTIME))
.thenReturn(mockAlertDefines);
List<AlertDefine> result = alertDefineService.getAlertDefinesByType(METRIC_ALERT_THRESHOLD_TYPE_REALTIME);
assertNotNull(result);
assertEquals(2, result.size());
assertEquals(7L, result.get(0).getId());
assertEquals(8L, result.get(1).getId());
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_REALTIME);
// Test valid metric periodic alert type
reset(alertDefineDao);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC))
.thenReturn(Lists.newArrayList());
result = alertDefineService.getAlertDefinesByType(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC);
assertNotNull(result);
assertEquals(0, result.size());
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC);
// Test valid log realtime alert type
reset(alertDefineDao);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_REALTIME))
.thenReturn(mockAlertDefines);
result = alertDefineService.getAlertDefinesByType(LOG_ALERT_THRESHOLD_TYPE_REALTIME);
assertNotNull(result);
assertEquals(2, result.size());
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_REALTIME);
// Test valid log periodic alert type
reset(alertDefineDao);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_PERIODIC))
.thenReturn(Lists.newArrayList());
result = alertDefineService.getAlertDefinesByType(LOG_ALERT_THRESHOLD_TYPE_PERIODIC);
assertNotNull(result);
assertEquals(0, result.size());
verify(alertDefineDao, times(1)).findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_PERIODIC);
// Test empty string type
assertThrows(IllegalArgumentException.class, () -> alertDefineService.getAlertDefinesByType(""));
// Test null type
assertThrows(IllegalArgumentException.class, () -> alertDefineService.getAlertDefinesByType(null));
// Test invalid type
assertThrows(IllegalArgumentException.class, () -> alertDefineService.getAlertDefinesByType("invalid_type"));
}
@@ -18,7 +18,6 @@
package org.apache.hertzbeat.alert.service;
import com.github.benmanes.caffeine.cache.Cache;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.hertzbeat.alert.service.impl.DataSourceServiceImpl;
import org.apache.hertzbeat.common.support.exception.AlertExpressionException;
@@ -36,7 +35,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
@@ -48,7 +50,7 @@ class DataSourceServiceTest {
@BeforeEach
void setUp() {
dataSourceService = new DataSourceServiceImpl();
dataSourceService = new DataSourceServiceImpl(null);
}
@Test
@@ -554,13 +556,11 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
String expr = "node_cpu_seconds_total > 50";
Cache<String, ParseTree> expressionCache = dataSourceService.getExpressionCache();
Cache<String, CommonTokenStream> tokenStreamCache = dataSourceService.getTokenStreamCache();
expressionCache.invalidateAll();
tokenStreamCache.invalidateAll();
long beforeHits = tokenStreamCache.stats().hitCount();
long beforeHits = expressionCache.stats().hitCount();
dataSourceService.calculate("promql", expr);
dataSourceService.calculate("promql", expr);
long actualHits = tokenStreamCache.stats().hitCount() - beforeHits;
long actualHits = expressionCache.stats().hitCount() - beforeHits;
assertEquals(1, actualHits, "expression cache should hit but miss");
}
@@ -575,9 +575,7 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
dataSourceService.calculate("promql", "node_cpu_seconds_total > 50");
Cache<String, ParseTree> expressionCache = dataSourceService.getExpressionCache();
Cache<String, CommonTokenStream> tokenStreamCache = dataSourceService.getTokenStreamCache();
expressionCache.invalidateAll();
tokenStreamCache.invalidateAll();
String expr1 = "node_cpu_seconds_total > 30";
String expr2 = "node_cpu_seconds_total > 50";
long beforeHits = expressionCache.stats().hitCount();
@@ -598,15 +596,13 @@ class DataSourceServiceTest {
dataSourceService.setExecutors(List.of(mockExecutor));
dataSourceService.calculate("promql", "node_cpu_seconds_total > 50");
Cache<String, ParseTree> expressionCache = dataSourceService.getExpressionCache();
Cache<String, CommonTokenStream> tokenStreamCache = dataSourceService.getTokenStreamCache();
expressionCache.invalidateAll();
tokenStreamCache.invalidateAll();
String expr1 = "node_cpu_seconds_total > 30";
String expr2 = "node_cpu_seconds_total > 50";
long beforeHits = tokenStreamCache.stats().hitCount();
long beforeHits = expressionCache.stats().hitCount();
dataSourceService.calculate("promql", expr1);
dataSourceService.calculate("promql", expr2);
long actualHits = tokenStreamCache.stats().hitCount() - beforeHits;
long actualHits = expressionCache.stats().hitCount() - beforeHits;
assertEquals(0, actualHits, "expression cache should miss but hit");
}
@@ -639,4 +635,156 @@ class DataSourceServiceTest {
assertThrows(AlertExpressionException.class, () -> dataSourceService.calculate("promql", "http_server_requests_seconds_count{!@~!!#$%^&}"));
}
@Test
void query1() {
List<Map<String, Object>> sqlData = List.of(
new HashMap<>(Map.of("count", 10, "severity_text", "ERROR"))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
when(mockExecutor.execute(anyString())).thenReturn(sqlData);
dataSourceService.setExecutors(List.of(mockExecutor));
String validSql = "SELECT count(*) FROM hertzbeat_logs WHERE severity_text = 'ERROR'";
List<Map<String, Object>> result = dataSourceService.query("sql", validSql);
assertNotNull(result);
assertEquals(1, result.size());
verify(mockExecutor).execute(anyString());
}
@Test
void query2() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "INSERT INTO hertzbeat_logs (body) VALUES ('test')"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query3() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "DELETE FROM hertzbeat_logs WHERE id = 1"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query4() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "UPDATE hertzbeat_logs SET body = 'hacked' WHERE id = 1"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query5() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "DROP TABLE hertzbeat_logs"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query6() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "SELECT * FROM hertzbeat_logs UNION SELECT * FROM users"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query7() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "SELECT * FROM hertzbeat_logs WHERE id IN (SELECT id FROM other_table)"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query8() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "SELECT * FROM users"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query9() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "WITH cte AS (SELECT * FROM hertzbeat_logs) SELECT * FROM cte"));
verify(mockExecutor, never()).execute(anyString());
}
@Test
void query10() {
List<Map<String, Object>> sqlData = List.of(
new HashMap<>(Map.of("errorCount", 5))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
when(mockExecutor.execute(anyString())).thenReturn(sqlData);
dataSourceService.setExecutors(List.of(mockExecutor));
String complexSql = "SELECT count(*) AS errorCount FROM hertzbeat_logs "
+ "WHERE time_unix_nano >= NOW() AND severity_text = 'ERROR' "
+ "GROUP BY severity_text HAVING count(*) > 2 ORDER BY errorCount LIMIT 10";
List<Map<String, Object>> result = dataSourceService.query("sql", complexSql);
assertNotNull(result);
verify(mockExecutor).execute(anyString());
}
@Test
void query11() {
List<Map<String, Object>> prometheusData = List.of(
new HashMap<>(Map.of("__value__", 100.0))
);
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("promql")).thenReturn(true);
when(mockExecutor.execute(anyString())).thenReturn(prometheusData);
dataSourceService.setExecutors(List.of(mockExecutor));
List<Map<String, Object>> result = dataSourceService.query("promql", "node_cpu_seconds_total > 50");
assertNotNull(result);
verify(mockExecutor).execute(anyString());
}
@Test
void query12() {
QueryExecutor mockExecutor = Mockito.mock(QueryExecutor.class);
when(mockExecutor.support("sql")).thenReturn(true);
dataSourceService.setExecutors(List.of(mockExecutor));
assertThrows(AlertExpressionException.class,
() -> dataSourceService.query("sql", "SELEC * FORM hertzbeat_logs"));
verify(mockExecutor, never()).execute(anyString());
}
}
@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hertzbeat.manager.dao;
package org.apache.hertzbeat.base.dao;
import java.util.Optional;
import java.util.Set;
@@ -33,7 +33,7 @@ public interface LabelDao extends JpaRepository<Label, Long>, JpaSpecificationEx
* @param ids id list
*/
void deleteLabelsByIdIn(Set<Long> ids);
/**
* find Label by name and value
* @param name Label name
@@ -41,5 +41,5 @@ public interface LabelDao extends JpaRepository<Label, Long>, JpaSpecificationEx
* @return Label
*/
Optional<Label> findLabelByNameAndTagValue(String name, String value);
}
@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hertzbeat.manager.service;
package org.apache.hertzbeat.base.service;
import java.util.HashSet;
import java.util.List;
@@ -83,6 +83,10 @@
<groupId>com.dameng</groupId>
<artifactId>DmJdbcDriver18</artifactId>
</dependency>
<dependency>
<groupId>com.xugudb</groupId>
<artifactId>xugu-jdbc</artifactId>
</dependency>
<!-- postgresql -->
<dependency>
<groupId>org.postgresql</groupId>
@@ -529,6 +529,8 @@ public class JdbcCommonCollect extends AbstractCollect {
case "mysql", "mariadb" -> "jdbc:mysql://" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase())
+ "?useUnicode=true&characterEncoding=utf-8&useSSL=false";
case "xugu" -> "jdbc:xugu://" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase());
case "postgresql" -> "jdbc:postgresql://" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase());
case "clickhouse" -> "jdbc:clickhouse://" + host + ":" + port
@@ -39,6 +39,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.net.ssl.SSLException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
@@ -106,6 +107,22 @@ import java.util.Collections;
*/
@Slf4j
public class HttpCollectImpl extends AbstractCollect {
/**
* Pre-compiled regex patterns for dangerous Xpath detection.
* Compiled once at class load for performance.
*/
private static final List<Pattern> DANGEROUS_XPATH_PATTERNS;
static {
List<Pattern> patterns = new ArrayList<>();
for (String pattern : CollectorConstants.DANGEROUS_XPATH_PATTERNS) {
// Use CASE_INSENSITIVE to prevent bypass via case variations (e.g., //TEXT() vs //text())
patterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL));
}
DANGEROUS_XPATH_PATTERNS = Collections.unmodifiableList(patterns);
}
private final Set<Integer> defaultSuccessStatusCodes = Set.of(
HttpStatus.SC_OK,
HttpStatus.SC_CREATED,
@@ -357,21 +374,83 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
/**
* Validates the Xpath expression to prevent DoS attacks.
* Checks for dangerous patterns that could traverse the entire XML document.
* Uses pre-compiled patterns for performance and case-insensitive matching for security.
*
* @param xpathExpression the Xpath expression to validate
* @throws IllegalArgumentException if the expression contains dangerous patterns
*/
private void validateXpathExpression(String xpathExpression) throws IllegalArgumentException {
if (!StringUtils.hasText(xpathExpression)) {
return;
}
// Check against dangerous patterns using pre-compiled regex
for (Pattern pattern : DANGEROUS_XPATH_PATTERNS) {
Matcher matcher = pattern.matcher(xpathExpression);
if (matcher.find()) {
throw new IllegalArgumentException(
"Xpath expression contains dangerous pattern that may cause DoS: " + pattern.pattern()
);
}
}
// Check for excessive wildcard usage (more than 3 // or * operators)
long descendantAxisCount = xpathExpression.chars().filter(ch -> ch == '/').count();
long wildcardCount = xpathExpression.chars().filter(ch -> ch == '*').count();
if (descendantAxisCount > 10 || wildcardCount > 5) {
throw new IllegalArgumentException(
"Xpath expression contains too many wildcards or descendant axes, potential DoS risk"
);
}
log.debug("Xpath expression validation passed: {}", xpathExpression);
}
private void parseResponseByXmlPath(String resp, Metrics metrics,
CollectRep.MetricsData.Builder builder, Long responseTime) {
HttpProtocol http = metrics.getHttp();
List<String> aliasFields = metrics.getAliasFields();
String xpathExpression = http.getParseScript();
// Layer 1: Validate Xpath expression is not empty
if (!StringUtils.hasText(xpathExpression)) {
log.warn("Http collect parse type is xmlPath, but the xpath expression is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("XPath expression is empty");
builder.setMsg("Xpath expression is empty");
return;
}
// Layer 2: Check XML response size to prevent memory exhaustion
if (resp != null && resp.length() > CollectorConstants.MAX_XML_RESPONSE_SIZE) {
log.warn("XML response size {} bytes exceeds maximum allowed size {} bytes",
resp.length(), CollectorConstants.MAX_XML_RESPONSE_SIZE);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("XML response exceeds maximum allowed size of "
+ (CollectorConstants.MAX_XML_RESPONSE_SIZE / 1024 / 1024) + "MB");
return;
}
// Layer 3: Validate Xpath expression for dangerous patterns
try {
validateXpathExpression(xpathExpression);
} catch (IllegalArgumentException e) {
log.warn("Xpath expression validation failed: {}", e.getMessage());
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(e.getMessage());
return;
}
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
// Layer 4: Enable XML secure processing and XXE protection
dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
@@ -387,7 +466,7 @@ public class HttpCollectImpl extends AbstractCollect {
NodeList nodeList = (NodeList) xpath.evaluate(xpathExpression, document, XPathConstants.NODESET);
if (nodeList == null || nodeList.getLength() == 0) {
log.debug("XPath expression '{}' returned no nodes.", xpathExpression);
log.debug("Xpath expression '{}' returned no nodes.", xpathExpression);
boolean requestedSummaryFields = aliasFields.stream()
.anyMatch(alias -> NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)
|| CollectorConstants.KEYWORD.equalsIgnoreCase(alias));
@@ -408,7 +487,16 @@ public class HttpCollectImpl extends AbstractCollect {
return;
}
for (int i = 0; i < nodeList.getLength(); i++) {
// Layer 5: Limit the number of results to prevent excessive resource consumption
int resultSize = nodeList.getLength();
int maxResults = CollectorConstants.MAX_XPATH_RESULT_NODES;
if (resultSize > maxResults) {
log.warn("Xpath expression returned {} nodes, exceeding limit of {}. Processing first {} nodes only.",
resultSize, maxResults, maxResults);
resultSize = maxResults;
}
for (int i = 0; i < resultSize; i++) {
Node node = nodeList.item(i);
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
@@ -422,7 +510,7 @@ public class HttpCollectImpl extends AbstractCollect {
String value = (String) xpath.evaluate(alias, node, XPathConstants.STRING);
valueRowBuilder.addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
} catch (XPathExpressionException e) {
log.warn("Failed to evaluate XPath '{}' for node [{}]: {}", alias, node.getNodeName(), e.getMessage());
log.warn("Failed to evaluate Xpath '{}' for node [{}]: {}", alias, node.getNodeName(), e.getMessage());
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
}
@@ -431,7 +519,7 @@ public class HttpCollectImpl extends AbstractCollect {
}
} catch (Exception e) {
log.warn("Failed to parse XML response with XPath '{}': {}", xpathExpression, e.getMessage(), e);
log.warn("Failed to parse XML response with Xpath '{}': {}", xpathExpression, e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to parse XML response: " + e.getMessage());
}
@@ -882,4 +970,4 @@ public class HttpCollectImpl extends AbstractCollect {
}
return successCodeSet.contains(statusCode);
}
}
}
@@ -18,7 +18,7 @@
<configuration scan="true">
<springProperty scope="context" name="application_name" source="spring.application.name" defaultValue="collector"/>
<!-- Output logs to ConsoleAppender -->
<appender name="ConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
<appender name="CONSOLE_RAW" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<!--<pattern>%d %p (%file:%line\)- %m%n</pattern>-->
<!-- Format output: %d for date, %thread for thread name, %-5level for level with 5 character width, %msg for log message, %n for new line -->
@@ -27,6 +27,13 @@
</encoder>
</appender>
<appender name="ConsoleAppender" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>0</discardingThreshold>
<queueSize>512</queueSize>
<includeCallerData>true</includeCallerData>
<appender-ref ref="CONSOLE_RAW"/>
</appender>
<appender name="SystemOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- Rolling policy for loggers, by date and by size -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
@@ -94,6 +101,7 @@
<!-- Production environment configuration -->
<springProfile name="prod">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
@@ -101,6 +109,7 @@
<springProfile name="cluster">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
@@ -49,4 +49,28 @@ public interface CollectorConstants extends NetworkConstants {
String STATUS_CODE = "statusCode";
/**
* Maximum XML response size in bytes (10MB) to prevent DoS attacks
*/
int MAX_XML_RESPONSE_SIZE = 10 * 1024 * 1024;
/**
* Maximum number of nodes returned by Xpath query to prevent excessive resource consumption
*/
int MAX_XPATH_RESULT_NODES = 1000;
/**
* Dangerous Xpath expression patterns that could cause DoS attacks
* These patterns match expressions that traverse the entire XML document
*/
String[] DANGEROUS_XPATH_PATTERNS = {
"//\\*\\s*\\|\\s*//@\\*\\s*\\|\\s*//text\\(\\)", // //* | //@* | //text()
"//\\*\\s*\\|", // //* | ...
"//@\\*\\s*\\|", // //@* | ...
"//node\\(\\)\\s*\\|", // //node() | ...
"descendant-or-self::node\\(\\)\\s*\\|", // descendant-or-self::node() | ...
"/descendant-or-self::node\\(\\)", // /descendant-or-self::node()
"//\\*[\\s\\S]*//\\*" // //** with multiple wildcards
};
}
@@ -35,9 +35,9 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
*/
@Slf4j
public class GoOnlineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
if (this.timerDispatch == null) {
@@ -53,7 +53,9 @@ public class GoOnlineProcessor implements NettyRemotingProcessor {
AesUtil.setDefaultSecretKey(serverInfo.getAesSecret());
}
}
timerDispatch.goOnline();
if (ClusterMsg.Direction.REQUEST.equals(message.getDirection())) {
timerDispatch.goOnline();
}
log.info("receive online message and handle success");
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
@@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.collector.timer.TimerDispatcher;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.lang.reflect.Field;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Test for GoOnlineProcessor
*/
class GoOnlineProcessorTest {
private GoOnlineProcessor goOnlineProcessor;
private TimerDispatcher timerDispatcher;
@Mock
private ChannelHandlerContext channelHandlerContext;
private MockedStatic<SpringContextHolder> springContextHolderMockedStatic;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
goOnlineProcessor = new GoOnlineProcessor();
timerDispatcher = new TimerDispatcher();
springContextHolderMockedStatic = Mockito.mockStatic(SpringContextHolder.class);
springContextHolderMockedStatic.when(() -> SpringContextHolder.getBean(TimerDispatch.class)).thenReturn(timerDispatcher);
}
@AfterEach
void tearDown() throws Exception {
springContextHolderMockedStatic.close();
timerDispatcher.destroy();
}
@Test
void verifyTaskMapPreservation() throws Exception {
Job job = Job.builder()
.app("test")
.id(12345L)
.metrics(Lists.newArrayList(Metrics.builder().interval(100L).build()))
.configmap(Lists.newArrayList())
.isCyclic(true)
.build();
timerDispatcher.addJob(job, null);
Field cyclicTaskMapField = TimerDispatcher.class.getDeclaredField("currentCyclicTaskMap");
cyclicTaskMapField.setAccessible(true);
Map<?, ?> currentCyclicTaskMap = (Map<?, ?>) cyclicTaskMapField.get(timerDispatcher);
assertEquals(1, currentCyclicTaskMap.size(), "Task map should have 1 job initially");
ClusterMsg.Message responseMsg = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.setIdentity("test-identity")
.build();
goOnlineProcessor.handle(channelHandlerContext, responseMsg);
assertEquals(1, currentCyclicTaskMap.size(), "Task map should still have 1 job after receiving RESPONSE");
ClusterMsg.Message requestMsg = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.setIdentity("test-identity")
.build();
goOnlineProcessor.handle(channelHandlerContext, requestMsg);
assertEquals(0, currentCyclicTaskMap.size(), "Task map should be empty after receiving REQUEST");
}
}
+5 -1
View File
@@ -171,7 +171,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
@@ -183,6 +183,10 @@
<version>${javaparser.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.jsqlparser</groupId>
<artifactId>jsqlparser</artifactId>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.constants;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
/**
* Theme Enum
*/
@Getter
@AllArgsConstructor
@ToString
public enum ThemeEnum {
COMPACT("compact"), DEFAULT("default"), LIGHT("light"), DARK("dark");
private final String value;
/**
* Convert theme to standard theme
*
* @param theme the input theme
* @return the converted theme
*/
public static String convert(String theme) {
if ((theme == null || theme.isEmpty())
|| ThemeEnum.COMPACT.getValue().equals(theme)
|| ThemeEnum.DEFAULT.getValue().equals(theme)) {
return ThemeEnum.LIGHT.getValue();
}
return theme;
}
}
@@ -48,8 +48,8 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*/
@Entity
@Table(name = "hzb_alert_define_monitor_bind", indexes = {
@Index(name = "index_alert_define_monitor", columnList = "alertDefineId"),
@Index(name = "index_alert_define_monitor", columnList = "monitor_id")
@Index(name = "idx_alert_define_id", columnList = "alert_define_id"),
@Index(name = "idx_monitor_id", columnList = "monitor_id")
})
@Data
@Builder
@@ -34,7 +34,7 @@ import lombok.NoArgsConstructor;
@Entity
@Table(name = "hzb_grafana_dashboard")
@Data
@Builder
@Builder(toBuilder = true)
@AllArgsConstructor
@NoArgsConstructor
@Schema(description = "Grafana dashboard entity")
@@ -42,8 +42,8 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*/
@Entity
@Table(name = "hzb_collector_monitor_bind", indexes = {
@Index(name = "index_collector_monitor", columnList = "collector"),
@Index(name = "index_collector_monitor", columnList = "monitor_id")
@Index(name = "idx_collector_monitor_collector", columnList = "collector"),
@Index(name = "idx_collector_monitor_monitor_id", columnList = "monitor_id")
})
@Data
@Builder
@@ -51,9 +51,9 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*/
@Entity
@Table(name = "hzb_monitor", indexes = {
@Index(name = "monitor_query_index", columnList = "app"),
@Index(name = "monitor_query_index", columnList = "instance"),
@Index(name = "monitor_query_index", columnList = "name")
@Index(name = "idx_hzb_monitor_app", columnList = "app"),
@Index(name = "idx_hzb_monitor_instance", columnList = "instance"),
@Index(name = "idx_hzb_monitor_name", columnList = "name")
})
@Data
@Builder
@@ -62,51 +62,51 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Schema(description = "Monitor Entity")
@EntityListeners(AuditingEntityListener.class)
public class Monitor {
@Id
@Schema(title = "Monitor task ID", example = "87584674384", accessMode = READ_ONLY)
private Long id;
@Schema(title = "Collect task ID", example = "43243543543", accessMode = READ_ONLY)
private Long jobId;
@Schema(title = "task name", example = "Api-TanCloud.cn", accessMode = READ_WRITE)
@Size(max = 100)
private String name;
@Schema(title = "Type of monitoring", example = "TanCloud", accessMode = READ_WRITE)
@Size(max = 100)
private String app;
@Schema(title = "Scrape type: static | http_sd | dns_sd | zookeeper_sd", example = "static", accessMode = READ_WRITE)
@Size(max = 100)
private String scrape;
@Schema(title = "the monitor target: ip/domain+port or ip/domain", example = "192.167.25.11:8081", accessMode = READ_WRITE)
@Size(max = 100)
@HostValid
private String instance;
@Schema(title = "Monitoring of the acquisition interval time in seconds", example = "600", accessMode = READ_WRITE)
@Min(10)
private Integer intervals;
@Schema(title = "Schedule type: interval | cron", example = "interval", accessMode = READ_WRITE)
@Size(max = 20)
private String scheduleType;
@Schema(title = "Cron expression when scheduleType is cron", example = "0/5 * * * * ?", accessMode = READ_WRITE)
@Size(max = 100)
private String cronExpression;
@Schema(title = "Task status 0: Paused, 1: Up, 2: Down", accessMode = READ_WRITE)
@Min(0)
@Max(4)
private byte status;
@Schema(title = "Task type 0: Normal, 1: push auto create, 2: discovery auto create")
private byte type;
@Schema(title = "task label", example = "{env:test}", accessMode = READ_WRITE)
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 4096)
@@ -116,23 +116,23 @@ public class Monitor {
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 4096)
private Map<String, String> annotations;
@Schema(title = "Monitor note description", example = "Availability monitoring of the SAAS website TanCloud", accessMode = READ_WRITE)
@Size(max = 255)
private String description;
@Schema(title = "The creator of this record", example = "tom", accessMode = READ_ONLY)
@CreatedBy
private String creator;
@Schema(title = "The modifier of this record", example = "tom", accessMode = READ_ONLY)
@LastModifiedBy
private String modifier;
@Schema(title = "Record create time", example = "2024-07-02T20:09:34.903217", accessMode = READ_ONLY)
@CreatedDate
private LocalDateTime gmtCreate;
@Schema(title = "Record modify time", example = "2024-07-02T20:09:34.903217", accessMode = READ_ONLY)
@LastModifiedDate
private LocalDateTime gmtUpdate;
@@ -42,7 +42,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*/
@Entity
@Table(name = "hzb_monitor_bind", indexes = {
@Index(name = "index_monitor_bind", columnList = "bizId"),
@Index(name = "index_monitor_bind", columnList = "biz_id"),
@Index(name = "index_monitor_bin", columnList = "monitor_id")
})
@Data
@@ -59,7 +59,7 @@ public class MonitorBind {
@Schema(title = "key string: ip:port")
private String keyStr;
@Schema(title = "connect bind id", example = "87432674384")
private Long bizId;
@@ -46,8 +46,11 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
* Monitor parameter values
*/
@Entity
@Table(name = "hzb_param", indexes = { @Index(columnList = "monitorId") },
uniqueConstraints = @UniqueConstraint(columnNames = {"monitorId", "field"}))
@Table(
name = "hzb_param",
indexes = {@Index(name = "idx_hzb_param_monitor_id", columnList = "monitor_id")},
uniqueConstraints = {@UniqueConstraint(name = "uk_hzb_param_monitor_field", columnNames = {"monitor_id", "field"})}
)
@Data
@Builder
@AllArgsConstructor
@@ -68,6 +71,7 @@ public class Param {
* Monitor ID
*/
@Schema(title = "Monitor task ID", example = "875846754543", accessMode = READ_WRITE)
@Column(name = "monitor_id")
private Long monitorId;
/**
@@ -75,6 +79,7 @@ public class Param {
*/
@Schema(title = "Parameter identifier field", example = "port", accessMode = READ_WRITE)
@Size(max = 100)
@Column(name = "field")
@NotBlank(message = "field can not null")
private String field;
@@ -41,7 +41,6 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@Table(name = "hzb_status_page_incident_component_bind", indexes = {
@Index(name = "index_incident_component", columnList = "incident_id"),
@Index(name = "index_incident_component", columnList = "component_id")
})
@Data
@Builder
@@ -59,7 +58,7 @@ public class StatusPageIncidentComponentBind {
@Schema(title = "Incident ID", example = "87432674384")
@Column(name = "incident_id")
private Long incidentId;
@Schema(title = "Component ID", example = "87432674336")
@Column(name = "component_id")
private Long componentId;
@@ -71,5 +70,5 @@ public class StatusPageIncidentComponentBind {
@Schema(title = "Record modify time", example = "1612198444000")
@LastModifiedDate
private LocalDateTime gmtUpdate;
}
@@ -35,8 +35,8 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*/
@Entity
@Table(name = "hzb_push_metrics", indexes = {
@Index(name = "push_query_index", columnList = "monitorId"),
@Index(name = "push_query_index", columnList = "time")
@Index(name = "idx_push_metrics_monitor_id", columnList = "monitor_id"),
@Index(name = "idx_push_metrics_time", columnList = "time")
})
@Data
@Builder
@@ -37,10 +37,10 @@ import lombok.NoArgsConstructor;
*/
@Entity
@Table(name = "hzb_history", indexes = {
@Index(name = "history_query_index", columnList = "instance"),
@Index(name = "history_query_index", columnList = "app"),
@Index(name = "history_query_index", columnList = "metrics"),
@Index(name = "history_query_index", columnList = "metric")
@Index(name = "idx_hzb_history_instance", columnList = "instance"),
@Index(name = "idx_hzb_history_app", columnList = "app"),
@Index(name = "idx_hzb_history_metrics", columnList = "metrics"),
@Index(name = "idx_hzb_history_metric", columnList = "metric")
})
@Data
@Builder
@@ -65,7 +65,7 @@ public class History {
@Schema(title = "Monitoring Metric usage speed count")
private String metric;
@Column(length = 5000)
private String metricLabels;
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.support.valid;
/**
* SQL security validation exception
*/
public class SqlSecurityException extends RuntimeException {
public SqlSecurityException(String message) {
super(message);
}
public SqlSecurityException(String message, Throwable cause) {
super(message, cause);
}
}
@@ -0,0 +1,150 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.support.valid;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.LateralSubSelect;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SetOperationList;
import net.sf.jsqlparser.statement.select.WithItem;
import net.sf.jsqlparser.util.TablesNamesFinder;
import org.springframework.util.CollectionUtils;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* SQL Security Validator using JSqlParser 5.1+.
* Security Policy:
* 1. Only SELECT statements are allowed.
* 2. All referenced tables must be in the whitelist.
* 3. Subqueries, UNION, CTE, LATERAL are blocked.
*/
@Slf4j
public class SqlSecurityValidator {
private final Set<String> allowedTables;
public SqlSecurityValidator(Collection<String> allowedTables) {
if (CollectionUtils.isEmpty(allowedTables)) {
this.allowedTables = new HashSet<>();
} else {
this.allowedTables = allowedTables.stream()
.map(this::normalizeIdentifier)
.collect(Collectors.toSet());
}
}
public void validate(String sql) throws SqlSecurityException {
if (sql == null || sql.trim().isEmpty()) {
throw new SqlSecurityException("SQL statement cannot be empty");
}
Statement statement;
try {
statement = CCJSqlParserUtil.parse(sql);
} catch (JSQLParserException e) {
log.warn("Failed to parse SQL: {}", sql, e);
throw new SqlSecurityException("Invalid SQL syntax: " + e.getMessage(), e);
}
if (!(statement instanceof Select select)) {
throw new SqlSecurityException("Only SELECT statements are allowed.");
}
// Check for CTE at top level
if (select.getWithItemsList() != null && !select.getWithItemsList().isEmpty()) {
throw new SqlSecurityException("CTE (WITH clause) is not allowed");
}
// Use custom TablesNamesFinder that throws on dangerous structures
SecurityTablesNamesFinder finder = new SecurityTablesNamesFinder();
List<String> tables;
try {
tables = finder.getTableList(statement);
} catch (SecurityViolationException e) {
throw new SqlSecurityException(e.getMessage());
}
validateTables(tables);
}
private void validateTables(List<String> tables) throws SqlSecurityException {
if (CollectionUtils.isEmpty(tables)) {
return;
}
if (allowedTables.isEmpty()) {
throw new SqlSecurityException("No access allowed: whitelist is empty.");
}
for (String table : tables) {
String normalizedTable = normalizeIdentifier(table);
if (!allowedTables.contains(normalizedTable)) {
throw new SqlSecurityException("Access to table '" + table + "' is not allowed. "
+ "Allowed tables: " + allowedTables);
}
}
}
private String normalizeIdentifier(String identifier) {
if (identifier == null) {
return "";
}
return identifier.replace("\"", "").replace("`", "").replace("'", "").toLowerCase();
}
private static class SecurityViolationException extends RuntimeException {
SecurityViolationException(String message) {
super(message);
}
}
/**
* Custom TablesNamesFinder that throws exceptions on dangerous SQL structures.
* Extends TablesNamesFinder with proper generic type to avoid raw type warnings.
*/
private static class SecurityTablesNamesFinder extends TablesNamesFinder<Void> {
@Override
public Void visit(ParenthesedSelect parenthesedSelect, Object context) {
throw new SecurityViolationException("Subqueries are not allowed");
}
@Override
public Void visit(SetOperationList setOpList, Object context) {
throw new SecurityViolationException("UNION and set operations are not allowed");
}
@Override
public Void visit(LateralSubSelect lateralSubSelect, Object context) {
throw new SecurityViolationException("LATERAL subqueries are not allowed");
}
@Override
public Void visit(WithItem withItem, Object context) {
throw new SecurityViolationException("CTE (WITH clause) is not allowed");
}
}
}
@@ -17,14 +17,6 @@
package org.apache.hertzbeat.common.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
@@ -32,12 +24,21 @@ import org.apache.arrow.vector.ipc.ArrowStreamReader;
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
/**
* Arrow data serialization and deserialization utility class
*/
@Slf4j
public final class ArrowUtil {
private ArrowUtil() {
}
@@ -53,18 +54,22 @@ public final class ArrowUtil {
public static byte[] serializeMultipleRoots(List<VectorSchemaRoot> roots) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(out)) {
dataOut.writeInt(roots.size());
for (VectorSchemaRoot root : roots) {
ArrowStreamWriter writer = new ArrowStreamWriter(
root,
null,
Channels.newChannel(out));
writer.start();
writer.writeBatch();
writer.end();
writer.close();
root.close();
// Use a temporary stream to obtain the precise byte length of a single root,
// write the length, and resolve the pre-read issue.
try (ByteArrayOutputStream tempOut = new ByteArrayOutputStream()) {
try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(tempOut))) {
writer.start();
writer.writeBatch();
writer.end();
}
int size = tempOut.size();
dataOut.writeInt(size);
dataOut.flush();
tempOut.writeTo(out);
}
}
return out.toByteArray();
} catch (IOException e) {
@@ -85,15 +90,20 @@ public final class ArrowUtil {
*/
public static List<VectorSchemaRoot> deserializeMultipleRoots(byte[] data) {
List<VectorSchemaRoot> roots = new ArrayList<>();
try (ByteArrayInputStream in = new ByteArrayInputStream(data);
DataInputStream dataIn = new DataInputStream(in)) {
int rootCount = dataIn.readInt();
ByteBuffer buffer = ByteBuffer.wrap(data);
try {
int rootCount = buffer.getInt();
RootAllocator allocator = new RootAllocator();
for (int i = 0; i < rootCount; i++) {
int length = buffer.getInt();
// Split the InputStream to prevent the Reader from reading beyond its bounds.
ByteArrayInputStream rootIn = new ByteArrayInputStream(data, buffer.position(), length);
buffer.position(buffer.position() + length);
ArrowStreamReader reader = new ArrowStreamReader(
Channels.newChannel(in),
Channels.newChannel(rootIn),
allocator);
VectorSchemaRoot root = reader.getVectorSchemaRoot();
reader.loadNextBatch();
@@ -147,11 +157,15 @@ public final class ArrowUtil {
*/
public static byte[] serializeMetricsData(List<CollectRep.MetricsData> metricsDataList) {
List<VectorSchemaRoot> roots = new ArrayList<>(metricsDataList.size());
for (CollectRep.MetricsData metricsData : metricsDataList) {
VectorSchemaRoot root = metricsData.toVectorSchemaRootAndRelease();
roots.add(root);
try {
for (CollectRep.MetricsData metricsData : metricsDataList) {
VectorSchemaRoot root = metricsData.toVectorSchemaRootAndRelease();
roots.add(root);
}
return serializeMultipleRoots(roots);
} finally {
roots.forEach(VectorSchemaRoot::close);
}
return serializeMultipleRoots(roots);
}
}
@@ -25,6 +25,7 @@ import java.util.Enumeration;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.http.conn.util.InetAddressUtils;
import org.springframework.util.StringUtils;
@@ -33,7 +34,7 @@ import org.springframework.util.StringUtils;
*/
@Slf4j
public final class IpDomainUtil {
private static final Pattern DOMAIN_PATTERN =
Pattern.compile("^[-\\w]+(\\.[-\\w]+)*$");
@@ -81,6 +82,23 @@ public final class IpDomainUtil {
return DOMAIN_SCHEMA.matcher(domainIp).matches();
}
/**
* if instance has the port with mark
* @param instance instance ip:port
* @return true if has
*/
public static boolean isHasPortWithMark(String instance) {
if (instance == null || !StringUtils.hasText(instance)) {
return false;
}
String[] parts = instance.split(SignConstants.DOUBLE_MARK);
if (parts.length >= 2) {
String port = parts[parts.length - 1];
return CommonUtil.isNumeric(port);
}
return false;
}
/**
* get localhost IP.
* @return ip
@@ -106,7 +124,7 @@ public final class IpDomainUtil {
}
return null;
}
/**
* check IP address type.
* @param ipDomain ip domain
@@ -118,7 +136,7 @@ public final class IpDomainUtil {
}
return NetworkConstants.IPV4;
}
/**
* get current local host name.
* @return hostname
@@ -126,7 +144,7 @@ public final class IpDomainUtil {
public static String getCurrentHostName() {
try {
InetAddress inetAddress = InetAddress.getLocalHost();
return inetAddress.getHostName();
return inetAddress.getHostName();
} catch (UnknownHostException e) {
return null;
}
@@ -159,4 +159,8 @@ public class StrBuffer {
}
return Double.parseDouble(s);
}
public static String escapeForFormat(String value) {
return value.replace("%", "%%");
}
}
@@ -0,0 +1,267 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.support.valid;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test for {@link SqlSecurityValidator}
*/
class SqlSecurityValidatorTest {
private SqlSecurityValidator validator;
@BeforeEach
void setUp() {
validator = new SqlSecurityValidator(Arrays.asList("hertzbeat_logs", "app_logs", "access_logs"));
}
@Test
void testValidSelectStatement() {
assertDoesNotThrow(() -> validator.validate("SELECT * FROM hertzbeat_logs"));
assertDoesNotThrow(() -> validator.validate("SELECT id, message FROM hertzbeat_logs WHERE level = 'ERROR'"));
assertDoesNotThrow(() -> validator.validate("SELECT COUNT(*) FROM app_logs"));
assertDoesNotThrow(() -> validator.validate("select * from HERTZBEAT_LOGS")); // case insensitive
}
@Test
void testSelectWithJoin() {
assertDoesNotThrow(() -> validator.validate(
"SELECT a.id, b.message FROM hertzbeat_logs a JOIN app_logs b ON a.id = b.id"));
}
@Test
void testSelectWithSubqueryNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs WHERE id IN (SELECT id FROM app_logs)"));
}
@Test
void testSelectWithSubqueryInFromNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM (SELECT * FROM hertzbeat_logs) AS subq"));
}
@Test
void testEmptySql() {
assertThrows(SqlSecurityException.class, () -> validator.validate(null));
assertThrows(SqlSecurityException.class, () -> validator.validate(""));
assertThrows(SqlSecurityException.class, () -> validator.validate(" "));
}
@Test
void testInvalidSqlSyntax() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FORM hertzbeat_logs")); // typo: FORM instead of FROM
}
@Test
void testInsertNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("INSERT INTO hertzbeat_logs (message) VALUES ('test')"));
}
@Test
void testUpdateNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("UPDATE hertzbeat_logs SET message = 'test' WHERE id = 1"));
}
@Test
void testDeleteNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("DELETE FROM hertzbeat_logs WHERE id = 1"));
}
@Test
void testDropNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("DROP TABLE hertzbeat_logs"));
}
@Test
void testTruncateNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("TRUNCATE TABLE hertzbeat_logs"));
}
@Test
void testAlterNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("ALTER TABLE hertzbeat_logs ADD COLUMN new_col VARCHAR(100)"));
}
@Test
void testCreateNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("CREATE TABLE new_table (id INT)"));
}
@Test
void testUnauthorizedTable() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM users"));
}
@Test
void testUnauthorizedTableInJoin() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs JOIN users ON hertzbeat_logs.user_id = users.id"));
}
@Test
void testUnauthorizedTableInSubquery() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs WHERE user_id IN (SELECT id FROM users)"));
}
@Test
void testTableWithQuotes() {
assertDoesNotThrow(() -> validator.validate("SELECT * FROM \"hertzbeat_logs\""));
assertDoesNotThrow(() -> validator.validate("SELECT * FROM `hertzbeat_logs`"));
}
@Test
void testEmptyAllowedTables() {
SqlSecurityValidator emptyValidator = new SqlSecurityValidator(Collections.emptyList());
assertThrows(SqlSecurityException.class,
() -> emptyValidator.validate("SELECT * FROM any_table"));
}
@Test
void testNullAllowedTables() {
SqlSecurityValidator nullValidator = new SqlSecurityValidator(null);
assertThrows(SqlSecurityException.class,
() -> nullValidator.validate("SELECT * FROM any_table"));
}
@Test
void testComplexSelectWithAggregation() {
assertDoesNotThrow(() -> validator.validate(
"SELECT level, COUNT(*) as cnt FROM hertzbeat_logs GROUP BY level HAVING COUNT(*) > 10 ORDER BY cnt DESC LIMIT 100"));
}
@Test
void testSelectWithUnionNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs UNION SELECT * FROM app_logs"));
}
@Test
void testSelectWithUnionAllNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs UNION ALL SELECT * FROM app_logs"));
}
@Test
void testSelectWithIntersectNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs INTERSECT SELECT * FROM app_logs"));
}
@Test
void testSelectWithExceptNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs EXCEPT SELECT * FROM app_logs"));
}
@Test
void testLateralSubqueryNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM hertzbeat_logs, LATERAL (SELECT * FROM app_logs) AS t"));
}
@Test
void testWithClauseNotAllowed() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("WITH cte AS (SELECT * FROM hertzbeat_logs) SELECT * FROM cte"));
}
@Test
void testSqlInjectionAttemptDropTable() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("DROP TABLE users"));
}
@Test
void testSqlInjectionAttemptUnauthorizedTable() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("SELECT * FROM users"));
}
@Test
void testSqlInjectionAttemptDeleteFrom() {
assertThrows(SqlSecurityException.class,
() -> validator.validate("DELETE FROM hertzbeat_logs WHERE 1=1"));
}
@Test
void testBypassInSelectItems() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT (SELECT password FROM secret_table) FROM hertzbeat_logs"));
}
@Test
void testBypassInWhereClauseAnd() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE 1=1 AND id IN (SELECT id FROM secret_table)"));
}
@Test
void testBypassInFunction() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE id = abs((SELECT count(*) FROM secret_table))"));
}
@Test
void testBypassInCaseWhen() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE status = (CASE WHEN (SELECT 1 FROM secret_table)=1 THEN 1 ELSE 0 END)"));
}
@Test
void testBypassWithAndExpression() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE 1=1 AND id = (SELECT id FROM secret_table)"));
}
@Test
void testBypassWithGreaterThan() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE id > (SELECT count(*) FROM secret_table)"));
}
@Test
void testBypassWithBetween() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE id BETWEEN 1 AND (SELECT id FROM secret_table)"));
}
@Test
void testBypassWithMathOperations() {
assertThrows(SqlSecurityException.class, () -> validator.validate(
"SELECT * FROM hertzbeat_logs WHERE id = 1 + (SELECT id FROM secret_table)"));
}
}
@@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
import com.google.common.collect.Lists;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Test case for {@link ArrowUtil}
*/
class ArrowUtilTest {
@Test
void testSerializeAndDeserializeMultipleRoots() {
RootAllocator allocator = new RootAllocator();
List<VectorSchemaRoot> roots = new ArrayList<>();
// Create first root
Field field1 = new Field("field1", FieldType.nullable(new ArrowType.Int(64, true)), null);
Schema schema1 = new Schema(Collections.singletonList(field1));
VectorSchemaRoot root1 = VectorSchemaRoot.create(schema1, allocator);
BigIntVector vector1 = (BigIntVector) root1.getVector("field1");
vector1.allocateNew(10);
vector1.setSafe(0, 100L);
vector1.setValueCount(1);
root1.setRowCount(1);
roots.add(root1);
// Create second root
Field field2 = new Field("field2", FieldType.nullable(new ArrowType.Int(64, true)), null);
Schema schema2 = new Schema(Collections.singletonList(field2));
VectorSchemaRoot root2 = VectorSchemaRoot.create(schema2, allocator);
BigIntVector vector2 = (BigIntVector) root2.getVector("field2");
vector2.allocateNew(10);
vector2.setSafe(0, 200L);
vector2.setValueCount(1);
root2.setRowCount(1);
roots.add(root2);
// Serialize
byte[] data = ArrowUtil.serializeMultipleRoots(roots);
// Deserialize
List<VectorSchemaRoot> deserializedRoots = ArrowUtil.deserializeMultipleRoots(data);
Assertions.assertEquals(2, deserializedRoots.size());
VectorSchemaRoot resultRoot1 = deserializedRoots.get(0);
Assertions.assertEquals(1, resultRoot1.getRowCount());
Assertions.assertEquals(100L, ((BigIntVector) resultRoot1.getVector("field1")).get(0));
VectorSchemaRoot resultRoot2 = deserializedRoots.get(1);
Assertions.assertEquals(1, resultRoot2.getRowCount());
Assertions.assertEquals(200L, ((BigIntVector) resultRoot2.getVector("field2")).get(0));
// Cleanup
roots.forEach(VectorSchemaRoot::close);
deserializedRoots.forEach(VectorSchemaRoot::close);
allocator.close();
}
@Test
void testSerializeAndDeserializeMetricsData() {
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
.setId(1L)
.setApp("linux")
.setMetrics("cpu")
.setTime(System.currentTimeMillis())
.addField(CollectRep.Field.newBuilder().setName("usage").setType(1).build())
.addValueRow(CollectRep.ValueRow.newBuilder().addColumn("50.5").build())
.build();
CollectRep.MetricsData metricsData1 = CollectRep.MetricsData.newBuilder()
.setId(1L)
.setApp("linux_1")
.setMetrics("cpu")
.setTime(System.currentTimeMillis())
.addField(CollectRep.Field.newBuilder().setName("usage").setType(1).build())
.addValueRow(CollectRep.ValueRow.newBuilder().addColumn("60.5").build())
.build();
List<CollectRep.MetricsData> list = Lists.newArrayList(metricsData, metricsData1);
// Serialize
byte[] data = ArrowUtil.serializeMetricsData(list);
// Deserialize
List<CollectRep.MetricsData> deserializedList = ArrowUtil.deserializeMetricsData(data);
Assertions.assertEquals(2, deserializedList.size());
CollectRep.MetricsData result = deserializedList.get(0);
Assertions.assertEquals("linux", result.getApp());
Assertions.assertEquals(1, result.getValues().size());
Assertions.assertEquals("50.5", result.getValues().get(0).getColumns(0));
result = deserializedList.get(1);
Assertions.assertEquals("linux_1", result.getApp());
Assertions.assertEquals(1, result.getValues().size());
Assertions.assertEquals("60.5", result.getValues().get(0).getColumns(0));
}
}
@@ -161,6 +161,6 @@ public class GreptimeLogStorageE2eTest {
long startTime = endTime - Duration.ofMinutes(5).toMillis(); // Look back 5 minutes
return greptimeDbDataStorage.queryLogsByMultipleConditions(
startTime, endTime, null, null, null, null);
startTime, endTime, null, null, null, null, null);
}
}
+2 -2
View File
@@ -32,10 +32,10 @@
</properties>
<dependencies>
<!-- hertzbeat common -->
<!-- hertzbeat base -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common</artifactId>
<artifactId>hertzbeat-base</artifactId>
</dependency>
<!-- hertzbeat warehouse -->
<dependency>
@@ -33,6 +33,8 @@ public interface GrafanaConstants {
String INSTANCE = "&var-instance=";
String THEME = "&theme=";
String CREATE_DASHBOARD_API = "/api/dashboards/db";
String DELETE_DASHBOARD_API = "/api/dashboards/uid/%s";
@@ -92,4 +94,4 @@ public interface GrafanaConstants {
static String generateUseDatasource(String datasourceName) {
return "&var-ds=" + datasourceName;
}
}
}
@@ -17,17 +17,15 @@
package org.apache.hertzbeat.grafana.service;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.CREATE_DASHBOARD_API;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.DELETE_DASHBOARD_API;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.INSTANCE;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.KIOSK;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.REFRESH;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
import org.apache.hertzbeat.common.constants.GeneralConfigTypeEnum;
import org.apache.hertzbeat.common.constants.ThemeEnum;
import org.apache.hertzbeat.common.entity.grafana.GrafanaDashboard;
import org.apache.hertzbeat.common.entity.manager.GeneralConfig;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.grafana.common.GrafanaConstants;
import org.apache.hertzbeat.grafana.config.GrafanaProperties;
@@ -42,7 +40,17 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import lombok.extern.slf4j.Slf4j;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.CREATE_DASHBOARD_API;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.DELETE_DASHBOARD_API;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.INSTANCE;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.KIOSK;
import static org.apache.hertzbeat.grafana.common.GrafanaConstants.REFRESH;
/**
* Service for managing Grafana dashboards.
@@ -66,6 +74,11 @@ public class DashboardService {
@Autowired
private DatasourceService datasourceService;
@Autowired
private GeneralConfigDao generalConfigDao;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
/**
* Creates or updates a dashboard in Grafana.
* The "id" field will be removed from the dashboard JSON before sending
@@ -225,7 +238,16 @@ public class DashboardService {
* @return GrafanaDashboard object
*/
public GrafanaDashboard getDashboardByMonitorId(Long monitorId) {
return dashboardDao.findByMonitorId(monitorId);
GrafanaDashboard dashboard = dashboardDao.findByMonitorId(monitorId);
if (null != dashboard
&& StringUtils.isNotBlank(dashboard.getUrl())
&& !dashboard.getUrl().contains(GrafanaConstants.THEME)) {
String theme = loadThemeConfig();
return dashboard.toBuilder()
.url(dashboard.getUrl() + GrafanaConstants.THEME + theme)
.build();
}
return dashboard;
}
/**
@@ -248,4 +270,21 @@ public class DashboardService {
log.warn("No Grafana dashboard record found for monitorId {} to disable.", monitorId);
}
}
}
private String loadThemeConfig() {
try {
GeneralConfig config = generalConfigDao.findByType(GeneralConfigTypeEnum.system.name());
if (config != null && config.getContent() != null) {
JsonNode root = OBJECT_MAPPER.readTree(config.getContent());
JsonNode node = root.get("theme");
if (node != null && !node.isNull()) {
return ThemeEnum.convert(node.asText());
}
return ThemeEnum.LIGHT.getValue();
}
} catch (Exception e) {
log.error("Failed to load database theme configuration", e);
}
return ThemeEnum.LIGHT.getValue();
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.grafana.service;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
import org.apache.hertzbeat.common.entity.grafana.GrafanaDashboard;
import org.apache.hertzbeat.grafana.config.GrafanaProperties;
import org.apache.hertzbeat.grafana.dao.DashboardDao;
@@ -34,10 +35,12 @@ import org.springframework.web.client.RestTemplate;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.lenient;
/**
* Test case for {@link DashboardService}
@@ -60,6 +63,9 @@ public class DashboardServiceTest {
@Mock
private DatasourceService datasourceService;
@Mock
private GeneralConfigDao generalConfigDao;
@InjectMocks
private DashboardService dashboardService;
@@ -73,25 +79,27 @@ public class DashboardServiceTest {
+ " \"version\": 3"
+ "}";
@BeforeEach
void setUp() {
when(datasourceService.getCurrentDatasourceName()).thenReturn("hertzbeat-vm-localhost-8428");
when(grafanaProperties.enabled()).thenReturn(Boolean.TRUE);
when(grafanaProperties.getPrefix()).thenReturn("");
when(grafanaProperties.getUrl()).thenReturn("http://127.0.0.1:3000");
when(grafanaProperties.exposeUrl()).thenReturn("http://127.0.0.1:3000");
when(serviceAccountService.getToken()).thenReturn("test-token");
lenient().when(datasourceService.getCurrentDatasourceName()).thenReturn("hertzbeat-vm-localhost-8428");
lenient().when(grafanaProperties.enabled()).thenReturn(Boolean.TRUE);
lenient().when(grafanaProperties.getPrefix()).thenReturn("");
lenient().when(grafanaProperties.getUrl()).thenReturn("http://127.0.0.1:3000");
lenient().when(grafanaProperties.exposeUrl()).thenReturn("http://127.0.0.1:3000");
lenient().when(serviceAccountService.getToken()).thenReturn("test-token");
}
@Test
void testCreateOrUpdateDashboard() {
ResponseEntity<String> responseEntity = new ResponseEntity<>(GRAFANA_API_RESULT, HttpStatus.OK);
when(restTemplate.postForEntity(
eq("http://127.0.0.1:3000/api/dashboards/db"), any(HttpEntity.class), eq(String.class)
)).thenReturn(responseEntity);
ArgumentCaptor<GrafanaDashboard> dashboardCaptor = ArgumentCaptor.forClass(GrafanaDashboard.class);
dashboardService.createOrUpdateDashboard("{\"id\":11}", 1L);
verify(dashboardDao).save(dashboardCaptor.capture());
@@ -109,7 +117,7 @@ public class DashboardServiceTest {
assertTrue(savedDashboard.getUrl().contains("refresh=15s"), "URL should contain refresh parameter");
assertTrue(savedDashboard.getUrl().contains("var-instance=1"), "URL should contain instance parameter");
}
@Test
void testCreateOrUpdateDashboardWithTrailingSlash() {
when(grafanaProperties.exposeUrl()).thenReturn("http://127.0.0.1:3000/");
@@ -138,4 +146,31 @@ public class DashboardServiceTest {
assertTrue(savedDashboard.getUrl().contains("var-instance=1"), "URL should contain instance parameter");
}
}
@Test
void testGetDashboardByMonitorId() {
Long monitorId = 1L;
String originalUrl = "http://localhost:3000/d/123/my-dashboard?kiosk";
GrafanaDashboard originalDashboard = GrafanaDashboard.builder()
.monitorId(monitorId)
.url(originalUrl)
.build();
when(dashboardDao.findByMonitorId(eq(monitorId))).thenReturn(originalDashboard);
String themeConfigJson = "{\"theme\":\"light\"}";
org.apache.hertzbeat.common.entity.manager.GeneralConfig runConfig = new org.apache.hertzbeat.common.entity.manager.GeneralConfig();
runConfig.setContent(themeConfigJson);
when(generalConfigDao.findByType(eq(org.apache.hertzbeat.common.constants.GeneralConfigTypeEnum.system.name())))
.thenReturn(runConfig);
GrafanaDashboard resultDashboard = dashboardService.getDashboardByMonitorId(monitorId);
assertNotNull(resultDashboard);
assertTrue(resultDashboard.getUrl().contains("&theme=light"));
assertTrue(originalDashboard.getUrl().endsWith("?kiosk"));
assertFalse(originalDashboard.getUrl().contains("&theme="));
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.log.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@@ -25,8 +26,8 @@ import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.log.service.LogProtocolAdapter;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -34,15 +35,16 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* Log Ingestion Controller
* Generic Log Ingestion Controller
* Provides a fallback endpoint for log protocols that don't have dedicated controllers.
* For OTLP protocol, use OtlpLogController instead.
*/
@Tag(name = "Log Ingestion Controller")
@RestController
@RequestMapping(path = "/api/logs", produces = "application/json")
@RequestMapping(path = "/api/logs", produces = MediaType.APPLICATION_JSON_VALUE)
@Slf4j
public class LogIngestionController {
private static final String DEFAULT_PROTOCOL = "otlp";
private final List<LogProtocolAdapter> protocolAdapters;
public LogIngestionController(List<LogProtocolAdapter> protocolAdapters) {
@@ -51,26 +53,23 @@ public class LogIngestionController {
/**
* Receive log payload pushed from external system specifying the log protocol.
* Examples:
* - POST /api/logs/ingest/otlp (content body is OTLP JSON)
*
* @param protocol log protocol identifier
* @param content raw request body
* @param protocol log protocol identifier (e.g., "vector", "loki")
* @param content raw request body
*/
@PostMapping("/ingest/{protocol}")
public ResponseEntity<Message<Void>> ingestExternLog(@PathVariable("protocol") String protocol,
@RequestBody String content) {
log.debug("Receive extern log from protocol: {}, content length: {}", protocol, content == null ? 0 : content.length());
if (!StringUtils.hasText(protocol)) {
protocol = DEFAULT_PROTOCOL; // Default to OTLP if no protocol specified
}
@Operation(summary = "Ingest logs by protocol name")
@PostMapping(value = "/ingest/{protocol}", consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Message<Void>> ingestLog(@PathVariable("protocol") String protocol,
@RequestBody String content) {
log.debug("Receive log from protocol: {}, content length: {}", protocol, content == null ? 0 : content.length());
for (LogProtocolAdapter adapter : protocolAdapters) {
if (adapter.supportProtocol().equalsIgnoreCase(protocol)) {
try {
adapter.ingest(content);
return ResponseEntity.ok(Message.success("Add extern log success"));
} catch (Exception e) {
log.error("Add extern log failed: {}", e.getMessage(), e);
log.error("Add log failed: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Add extern log failed: " + e.getMessage()));
}
@@ -80,30 +79,4 @@ public class LogIngestionController {
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Not support the " + protocol + " protocol log"));
}
/**
* Receive default log payload (when protocol is not specified).
* It will look for a service whose supportProtocol() returns "otlp".
*/
@PostMapping("/ingest")
public ResponseEntity<Message<Void>> ingestDefaultExternLog(@RequestBody String content) {
log.info("Receive default extern log content, length: {}", content == null ? 0 : content.length());
LogProtocolAdapter adapter = protocolAdapters.stream()
.filter(item -> DEFAULT_PROTOCOL.equalsIgnoreCase(item.supportProtocol()))
.findFirst()
.orElse(null);
if (adapter != null) {
try {
adapter.ingest(content);
return ResponseEntity.ok(Message.success("Add extern log success"));
} catch (Exception e) {
log.error("Add extern log failed: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Add extern log failed: " + e.getMessage()));
}
}
log.error("Not support default extern log protocol");
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Not support the default protocol log"));
}
}
@@ -61,50 +61,54 @@ public class LogQueryController {
}
@GetMapping("/list")
@Operation(summary = "Query logs by time range with optional filters",
description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.")
@Operation(summary = "Query logs by time range with optional filters",
description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.")
public ResponseEntity<Message<Page<LogEntry>>> list(
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@RequestParam(value = "start", required = false) Long start,
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@RequestParam(value = "end", required = false) Long end,
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@RequestParam(value = "traceId", required = false) String traceId,
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@RequestParam(value = "spanId", required = false) String spanId,
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@RequestParam(value = "severityNumber", required = false) Integer severityNumber,
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText,
@Parameter(description = "Page index starting from 0", example = "0")
@Parameter(description = "Log content search keyword", example = "error")
@RequestParam(value = "search", required = false) String search,
@Parameter(description = "Page index starting from 0", example = "0")
@RequestParam(value = "pageIndex", required = false, defaultValue = "0") Integer pageIndex,
@Parameter(description = "Number of items per page", example = "20")
@Parameter(description = "Number of items per page", example = "20")
@RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize) {
Page<LogEntry> result = getPagedLogs(start, end, traceId, spanId, severityNumber, severityText, pageIndex, pageSize);
Page<LogEntry> result = getPagedLogs(start, end, traceId, spanId, severityNumber, severityText, search, pageIndex, pageSize);
return ResponseEntity.ok(Message.success(result));
}
@GetMapping("/stats/overview")
@Operation(summary = "Log overview statistics",
description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.")
@Operation(summary = "Log overview statistics",
description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.")
public ResponseEntity<Message<Map<String, Object>>> overviewStats(
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@RequestParam(value = "start", required = false) Long start,
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@RequestParam(value = "end", required = false) Long end,
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@RequestParam(value = "traceId", required = false) String traceId,
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@RequestParam(value = "spanId", required = false) String spanId,
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@RequestParam(value = "severityNumber", required = false) Integer severityNumber,
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText);
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText,
@Parameter(description = "Log content search keyword", example = "error")
@RequestParam(value = "search", required = false) String search) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText, search);
Map<String, Object> overview = new HashMap<>();
overview.put("totalCount", logs.size());
// Count by severity levels according to OpenTelemetry standard
// TRACE: 1-4, DEBUG: 5-8, INFO: 9-12, WARN: 13-16, ERROR: 17-20, FATAL: 21-24
long fatalCount = logs.stream().filter(log -> log.getSeverityNumber() != null && log.getSeverityNumber() >= 21 && log.getSeverityNumber() <= 24).count();
@@ -113,114 +117,117 @@ public class LogQueryController {
long infoCount = logs.stream().filter(log -> log.getSeverityNumber() != null && log.getSeverityNumber() >= 9 && log.getSeverityNumber() <= 12).count();
long debugCount = logs.stream().filter(log -> log.getSeverityNumber() != null && log.getSeverityNumber() >= 5 && log.getSeverityNumber() <= 8).count();
long traceCount = logs.stream().filter(log -> log.getSeverityNumber() != null && log.getSeverityNumber() >= 1 && log.getSeverityNumber() <= 4).count();
overview.put("fatalCount", fatalCount);
overview.put("errorCount", errorCount);
overview.put("warnCount", warnCount);
overview.put("infoCount", infoCount);
overview.put("debugCount", debugCount);
overview.put("traceCount", traceCount);
return ResponseEntity.ok(Message.success(overview));
}
@GetMapping("/stats/trace-coverage")
@Operation(summary = "Trace coverage statistics",
description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.")
@Operation(summary = "Trace coverage statistics",
description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.")
public ResponseEntity<Message<Map<String, Object>>> traceCoverageStats(
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@RequestParam(value = "start", required = false) Long start,
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@RequestParam(value = "end", required = false) Long end,
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@RequestParam(value = "traceId", required = false) String traceId,
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@RequestParam(value = "spanId", required = false) String spanId,
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@RequestParam(value = "severityNumber", required = false) Integer severityNumber,
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText);
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText,
@Parameter(description = "Log content search keyword", example = "error")
@RequestParam(value = "search", required = false) String search) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText, search);
Map<String, Object> result = new HashMap<>();
// Trace coverage statistics
long withTraceId = logs.stream().filter(log -> log.getTraceId() != null && !log.getTraceId().isEmpty()).count();
long withSpanId = logs.stream().filter(log -> log.getSpanId() != null && !log.getSpanId().isEmpty()).count();
long withBothTraceAndSpan = logs.stream().filter(log ->
log.getTraceId() != null && !log.getTraceId().isEmpty()
&& log.getSpanId() != null && !log.getSpanId().isEmpty()).count();
long withBothTraceAndSpan = logs.stream().filter(log ->
log.getTraceId() != null && !log.getTraceId().isEmpty()
&& log.getSpanId() != null && !log.getSpanId().isEmpty()).count();
long withoutTrace = logs.size() - withTraceId;
Map<String, Long> traceCoverage = new HashMap<>();
traceCoverage.put("withTrace", withTraceId);
traceCoverage.put("withoutTrace", withoutTrace);
traceCoverage.put("withSpan", withSpanId);
traceCoverage.put("withBothTraceAndSpan", withBothTraceAndSpan);
result.put("traceCoverage", traceCoverage);
return ResponseEntity.ok(Message.success(result));
}
@GetMapping("/stats/trend")
@Operation(summary = "Log trend over time",
description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.")
@Operation(summary = "Log trend over time",
description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.")
public ResponseEntity<Message<Map<String, Object>>> trendStats(
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@Parameter(description = "Start timestamp in milliseconds (Unix timestamp)", example = "1640995200000")
@RequestParam(value = "start", required = false) Long start,
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@Parameter(description = "End timestamp in milliseconds (Unix timestamp)", example = "1641081600000")
@RequestParam(value = "end", required = false) Long end,
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@Parameter(description = "Trace ID for distributed tracing", example = "1234567890abcdef")
@RequestParam(value = "traceId", required = false) String traceId,
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@Parameter(description = "Span ID for distributed tracing", example = "abcdef1234567890")
@RequestParam(value = "spanId", required = false) String spanId,
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@Parameter(description = "Log severity number (1-24 according to OpenTelemetry standard)", example = "9")
@RequestParam(value = "severityNumber", required = false) Integer severityNumber,
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText);
@Parameter(description = "Log severity text (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)", example = "INFO")
@RequestParam(value = "severityText", required = false) String severityText,
@Parameter(description = "Log content search keyword", example = "error")
@RequestParam(value = "search", required = false) String search) {
List<LogEntry> logs = getFilteredLogs(start, end, traceId, spanId, severityNumber, severityText, search);
// Group by hour
Map<String, Long> hourlyStats = logs.stream()
.filter(log -> log.getTimeUnixNano() != null)
.collect(Collectors.groupingBy(
log -> {
long timestampMs = log.getTimeUnixNano() / 1_000_000L;
LocalDateTime dateTime = LocalDateTime.ofInstant(
Instant.ofEpochMilli(timestampMs),
ZoneId.systemDefault());
return dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:00"));
},
Collectors.counting()));
.filter(log -> log.getTimeUnixNano() != null)
.collect(Collectors.groupingBy(
log -> {
long timestampMs = log.getTimeUnixNano() / 1_000_000L;
LocalDateTime dateTime = LocalDateTime.ofInstant(
Instant.ofEpochMilli(timestampMs),
ZoneId.systemDefault());
return dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:00"));
},
Collectors.counting()));
Map<String, Object> result = new HashMap<>();
result.put("hourlyStats", hourlyStats);
return ResponseEntity.ok(Message.success(result));
}
private List<LogEntry> getFilteredLogs(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText) {
private List<LogEntry> getFilteredLogs(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, String search) {
// Use the new multi-condition query method
return historyDataReader.queryLogsByMultipleConditions(start, end, traceId, spanId, severityNumber, severityText);
return historyDataReader.queryLogsByMultipleConditions(start, end, traceId, spanId, severityNumber, severityText, search);
}
private Page<LogEntry> getPagedLogs(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, Integer pageIndex, Integer pageSize) {
private Page<LogEntry> getPagedLogs(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, String search,
Integer pageIndex, Integer pageSize) {
// Calculate pagination parameters
int offset = pageIndex * pageSize;
// Get total count and paginated data
long totalElements = historyDataReader.countLogsByMultipleConditions(start, end, traceId, spanId, severityNumber, severityText);
long totalElements = historyDataReader.countLogsByMultipleConditions(start, end, traceId, spanId, severityNumber, severityText, search);
List<LogEntry> pagedLogs = historyDataReader.queryLogsByMultipleConditionsWithPagination(
start, end, traceId, spanId, severityNumber, severityText, offset, pageSize);
start, end, traceId, spanId, severityNumber, severityText, search, offset, pageSize);
// Create PageRequest (sorted by timestamp descending)
Sort sort = Sort.by(Sort.Direction.DESC, "timeUnixNano");
PageRequest pageRequest = PageRequest.of(pageIndex, pageSize, sort);
// Return Spring Data Page object
return new PageImpl<>(pagedLogs, pageRequest, totalElements);
}
}
}
@@ -0,0 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import com.google.rpc.Status;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* OTLP Log Ingestion Controller
* Implements OTLP/HTTP specification for log ingestion.
* Supports both binary-encoded Protobuf (application/x-protobuf) and JSON-encoded Protobuf (application/json).
*
* @see <a href="https://opentelemetry.io/docs/specs/otlp/#otlphttp">OTLP/HTTP Specification</a>
*/
@Tag(name = "OTLP Log Controller")
@RestController
@RequestMapping(path = "/api/logs/otlp")
@Slf4j
public class OtlpLogController {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
private static final ExportLogsServiceResponse EMPTY_RESPONSE = ExportLogsServiceResponse.newBuilder().build();
private final OtlpLogProtocolAdapter otlpLogProtocolAdapter;
public OtlpLogController(OtlpLogProtocolAdapter otlpLogProtocolAdapter) {
this.otlpLogProtocolAdapter = otlpLogProtocolAdapter;
}
/**
* OTLP/HTTP standard endpoint for logs with JSON-encoded Protobuf payload.
* Content-Type: application/json
*
* Response follows OTLP specification:
* - Success: HTTP 200 with ExportLogsServiceResponse (JSON encoded)
* - Failure: HTTP 400 with google.rpc.Status (JSON encoded)
*
* @param content JSON-encoded ExportLogsServiceRequest
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (JSON format)")
@PostMapping(value = "/v1/logs", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> ingestJsonLogs(@RequestBody String content) {
log.debug("Receive OTLP JSON logs, content length: {}", content == null ? 0 : content.length());
try {
otlpLogProtocolAdapter.ingest(content);
return ResponseEntity.ok(toJsonResponse(EMPTY_RESPONSE));
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(toJsonErrorResponse(e.getMessage()));
} catch (Exception e) {
// Server-side errors - unexpected failure
log.error("Unexpected error ingesting OTLP JSON logs: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(toJsonErrorResponse(e.getMessage()));
}
}
/**
* OTLP/HTTP standard endpoint for logs with binary-encoded Protobuf payload.
* Content-Type: application/x-protobuf
*
* Response follows OTLP specification:
* - Success: HTTP 200 with ExportLogsServiceResponse (binary encoded)
* - Failure: HTTP 400 with google.rpc.Status (binary encoded)
*
* @param content binary-encoded ExportLogsServiceRequest
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (binary Protobuf format)")
@PostMapping(value = "/v1/logs", consumes = CONTENT_TYPE_PROTOBUF, produces = CONTENT_TYPE_PROTOBUF)
public ResponseEntity<byte[]> ingestBinaryLogs(@RequestBody byte[] content) {
log.debug("Receive OTLP binary logs, content length: {}", content == null ? 0 : content.length);
try {
otlpLogProtocolAdapter.ingestBinary(content);
return ResponseEntity.ok(EMPTY_RESPONSE.toByteArray());
} catch (IllegalArgumentException e) {
// Client-side validation errors - malformed request
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(createBinaryErrorResponse(e.getMessage()));
} catch (Exception e) {
// Server-side errors - unexpected failure
log.error("Unexpected error ingesting OTLP binary logs: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(createBinaryErrorResponse(e.getMessage()));
}
}
private String toJsonResponse(ExportLogsServiceResponse response) {
try {
return JsonFormat.printer().print(response);
} catch (InvalidProtocolBufferException e) {
log.error("Failed to convert ExportLogsServiceResponse to JSON: {}", e.getMessage(), e);
return "{}";
}
}
private String toJsonErrorResponse(String message) {
Status status = Status.newBuilder()
.setMessage(message != null ? message : "Unknown error")
.build();
try {
return JsonFormat.printer().print(status);
} catch (InvalidProtocolBufferException e) {
return "{\"message\":\"" + escapeJson(message) + "\"}";
}
}
/**
* Escapes a string value for safe inclusion in JSON.
*
* @param message the string to escape
* @return the escaped string, or empty string if message is null
*/
private String escapeJson(String message) {
if (message == null) {
return "";
}
char[] escaped = JsonStringEncoder.getInstance().quoteAsString(message);
return new String(escaped);
}
private byte[] createBinaryErrorResponse(String message) {
return Status.newBuilder()
.setMessage(message != null ? message : "Unknown error")
.build()
.toByteArray();
}
}
@@ -49,7 +49,13 @@ public class LogSseFilterCriteria {
*/
@Schema(description = "The severity text (also known as log level).", example = "INFO", accessMode = READ_WRITE)
private String severityText;
/**
* Log content text filtering (case-insensitive contains match).
*/
@Schema(description = "Log content text filtering", example = "error occurred", accessMode = READ_WRITE)
private String logContent;
/**
* A unique identifier for a trace.
* All spans from the same trace share the same trace_id.
@@ -77,18 +83,30 @@ public class LogSseFilterCriteria {
if (StringUtils.hasText(severityText) && !severityText.equalsIgnoreCase(log.getSeverityText())) {
return false;
}
// Check severity number match (if both are present)
if (severityNumber != null && log.getSeverityNumber() != null
&& !severityNumber.equals(log.getSeverityNumber())) {
return false;
}
// Check log content match
if (StringUtils.hasText(logContent)) {
Object body = log.getBody();
if (body == null) {
return false;
}
String bodyStr = body.toString();
if (!StringUtils.hasText(bodyStr) || !bodyStr.toLowerCase().contains(logContent.toLowerCase())) {
return false;
}
}
// Check trace ID match
if (StringUtils.hasText(traceId) && !traceId.equalsIgnoreCase(log.getTraceId())) {
return false;
}
// Check span ID match
if (StringUtils.hasText(spanId) && !spanId.equalsIgnoreCase(log.getSpanId())) {
return false;
@@ -28,7 +28,7 @@ package org.apache.hertzbeat.log.service;
public interface LogProtocolAdapter {
/**
* Ingest raw log payload pushed from external system.
* Ingest log payload pushed from external system.
*
* @param content raw request body string
*/
@@ -39,7 +39,10 @@ import java.util.List;
import java.util.Map;
/**
* Adapter for OpenTelemetry OTLP/HTTP JSON log ingestion.
* Adapter for OpenTelemetry OTLP/HTTP log ingestion.
* Supports both JSON-encoded and binary-encoded Protobuf formats.
*
* @see <a href="https://opentelemetry.io/docs/specs/otlp/#otlphttp">OTLP/HTTP Specification</a>
*/
@Slf4j
@Service
@@ -58,26 +61,47 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
@Override
public void ingest(String content) {
if (content == null || content.isEmpty()) {
log.warn("Received empty OTLP log payload - skip processing.");
log.warn("Received empty OTLP JSON log payload - skip processing.");
return;
}
ExportLogsServiceRequest.Builder builder = ExportLogsServiceRequest.newBuilder();
try {
JsonFormat.parser().ignoringUnknownFields().merge(content, builder);
ExportLogsServiceRequest request = builder.build();
// Extract LogEntry instances from the request
List<LogEntry> logEntries = extractLogEntries(request);
log.debug("Successfully extracted {} log entries from OTLP payload {}", logEntries.size(), content);
commonDataQueue.sendLogEntryToStorageBatch(logEntries);
commonDataQueue.sendLogEntryToAlertBatch(logEntries);
logEntries.forEach(logSseManager::broadcast);
processLogsRequest(request, "JSON");
} catch (InvalidProtocolBufferException e) {
log.error("Failed to parse OTLP log payload: {}", e.getMessage());
throw new IllegalArgumentException("Invalid OTLP log content", e);
log.error("Failed to parse OTLP JSON log payload: {}", e.getMessage());
throw new IllegalArgumentException("Invalid OTLP JSON log content", e);
}
}
/**
* Ingest binary-encoded Protobuf log payload (OTLP-specific).
*
* @param content binary-encoded ExportLogsServiceRequest
*/
public void ingestBinary(byte[] content) {
if (content == null || content.length == 0) {
log.warn("Received empty OTLP binary log payload - skip processing.");
return;
}
try {
ExportLogsServiceRequest request = ExportLogsServiceRequest.parseFrom(content);
processLogsRequest(request, "binary");
} catch (InvalidProtocolBufferException e) {
log.error("Failed to parse OTLP binary log payload: {}", e.getMessage());
throw new IllegalArgumentException("Invalid OTLP binary log content", e);
}
}
private void processLogsRequest(ExportLogsServiceRequest request, String format) {
List<LogEntry> logEntries = extractLogEntries(request);
log.debug("Successfully extracted {} log entries from OTLP {} payload", logEntries.size(), format);
commonDataQueue.sendLogEntryToStorageBatch(logEntries);
commonDataQueue.sendLogEntryToAlertBatch(logEntries);
logEntries.forEach(logSseManager::broadcast);
}
/**
* Extract LogEntry instances from ExportLogsServiceRequest.
*
@@ -18,24 +18,22 @@
package org.apache.hertzbeat.log.controller;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.log.service.LogProtocolAdapter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
@@ -51,34 +49,29 @@ class LogIngestionControllerTest {
private MockMvc mockMvc;
@Mock
private LogProtocolAdapter otlpAdapter;
private LogProtocolAdapter vectorAdapter;
private LogIngestionController logIngestionController;
@BeforeEach
void setUp() {
List<LogProtocolAdapter> adapters = Arrays.asList(otlpAdapter);
List<LogProtocolAdapter> adapters = Arrays.asList(vectorAdapter);
this.logIngestionController = new LogIngestionController(adapters);
this.mockMvc = MockMvcBuilders.standaloneSetup(logIngestionController).build();
}
@Test
void testIngestExternLogWithOtlpProtocol() throws Exception {
LogEntry logEntry = LogEntry.builder()
.timeUnixNano(1734005477630L)
.severityNumber(1)
.severityText("INFO")
.body("Test log message")
.attributes(new HashMap<>())
.build();
void testIngestLogWithKnownProtocol() throws Exception {
String logContent = "{\"message\":\"Test log message\"}";
when(otlpAdapter.supportProtocol()).thenReturn("otlp");
when(vectorAdapter.supportProtocol()).thenReturn("vector");
doNothing().when(vectorAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest/otlp")
.post("/api/logs/ingest/vector")
.contentType(MediaType.APPLICATION_JSON)
.content(JsonUtil.toJson(logEntry))
.content(logContent)
)
.andDo(print())
.andExpect(status().isOk())
@@ -88,16 +81,16 @@ class LogIngestionControllerTest {
}
@Test
void testIngestExternLogWithUnsupportedProtocol() throws Exception {
String unsupportedLogContent = "{\"message\":\"Unsupported protocol log\"}";
void testIngestLogWithUnsupportedProtocol() throws Exception {
String logContent = "{\"message\":\"Unsupported protocol log\"}";
when(otlpAdapter.supportProtocol()).thenReturn("otlp");
when(vectorAdapter.supportProtocol()).thenReturn("vector");
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest/unsupported")
.contentType(MediaType.APPLICATION_JSON)
.content(unsupportedLogContent)
.content(logContent)
)
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
@@ -105,38 +98,15 @@ class LogIngestionControllerTest {
}
@Test
void testIngestDefaultExternLog() throws Exception {
LogEntry logEntry = LogEntry.builder()
.timeUnixNano(1734005477630L)
.severityNumber(2)
.severityText("WARN")
.body("Default protocol log message")
.attributes(new HashMap<>())
.build();
void testIngestLogWithAdapterException() throws Exception {
String logContent = "{\"message\":\"Log message that will cause exception\"}";
when(otlpAdapter.supportProtocol()).thenReturn("otlp");
when(vectorAdapter.supportProtocol()).thenReturn("vector");
doThrow(new IllegalArgumentException("Invalid log format")).when(vectorAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest")
.contentType(MediaType.APPLICATION_JSON)
.content(JsonUtil.toJson(logEntry))
)
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.msg").value("Add extern log success"));
}
@Test
void testIngestDefaultExternLogWithAdapterException() throws Exception {
String logContent = "{\"message\":\"Default log message that will cause exception\"}";
when(otlpAdapter.supportProtocol()).thenReturn("otlp");
Mockito.doThrow(new IllegalArgumentException("Invalid log format")).when(otlpAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest")
.post("/api/logs/ingest/vector")
.contentType(MediaType.APPLICATION_JSON)
.content(logContent)
)
@@ -17,11 +17,10 @@
package org.apache.hertzbeat.log.controller;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -87,10 +86,10 @@ class LogQueryControllerTest {
List<LogEntry> mockLogs = Arrays.asList(logEntry1, logEntry2);
when(historyDataReader.countLogsByMultipleConditions(anyLong(), anyLong(), anyString(),
anyString(), anyInt(), anyString())).thenReturn(2L);
when(historyDataReader.queryLogsByMultipleConditionsWithPagination(anyLong(), anyLong(),
anyString(), anyString(), anyInt(), anyString(), anyInt(), anyInt()))
when(historyDataReader.countLogsByMultipleConditions(anyLong(), anyLong(), any(),
any(), any(), any(), any())).thenReturn(2L);
when(historyDataReader.queryLogsByMultipleConditionsWithPagination(anyLong(), anyLong(),
any(), any(), any(), any(), any(), anyInt(), anyInt()))
.thenReturn(mockLogs);
mockMvc.perform(
@@ -127,10 +126,10 @@ class LogQueryControllerTest {
.build()
);
when(historyDataReader.countLogsByMultipleConditions(isNull(), isNull(), isNull(),
isNull(), isNull(), isNull())).thenReturn(1L);
when(historyDataReader.queryLogsByMultipleConditionsWithPagination(isNull(), isNull(),
isNull(), isNull(), isNull(), isNull(), eq(0), eq(20)))
when(historyDataReader.countLogsByMultipleConditions(any(), any(), any(),
any(), any(), any(), any())).thenReturn(1L);
when(historyDataReader.queryLogsByMultipleConditionsWithPagination(any(), any(),
any(), any(), any(), any(), any(), eq(0), eq(20)))
.thenReturn(mockLogs);
mockMvc.perform(
@@ -163,8 +162,8 @@ class LogQueryControllerTest {
LogEntry.builder().severityNumber(21).build()
);
when(historyDataReader.queryLogsByMultipleConditions(isNull(), isNull(), isNull(),
isNull(), isNull(), isNull())).thenReturn(mockLogs);
when(historyDataReader.queryLogsByMultipleConditions(any(), any(), any(),
any(), any(), any(), any())).thenReturn(mockLogs);
mockMvc.perform(
MockMvcRequestBuilders
@@ -188,8 +187,8 @@ class LogQueryControllerTest {
LogEntry.builder().severityNumber(17).build()
);
when(historyDataReader.queryLogsByMultipleConditions(eq(1734005477000L), eq(1734005478000L),
isNull(), isNull(), isNull(), isNull())).thenReturn(mockLogs);
when(historyDataReader.queryLogsByMultipleConditions(eq(1734005477000L), eq(1734005478000L),
any(), any(), any(), any(), any())).thenReturn(mockLogs);
mockMvc.perform(
MockMvcRequestBuilders
@@ -217,8 +216,8 @@ class LogQueryControllerTest {
LogEntry.builder().build() // null values
);
when(historyDataReader.queryLogsByMultipleConditions(isNull(), isNull(), isNull(),
isNull(), isNull(), isNull())).thenReturn(mockLogs);
when(historyDataReader.queryLogsByMultipleConditions(any(), any(), any(),
any(), any(), any(), any())).thenReturn(mockLogs);
mockMvc.perform(
MockMvcRequestBuilders
@@ -244,8 +243,8 @@ class LogQueryControllerTest {
LogEntry.builder().timeUnixNano(1734009077630000000L).build()
);
when(historyDataReader.queryLogsByMultipleConditions(isNull(), isNull(), isNull(),
isNull(), isNull(), isNull())).thenReturn(mockLogs);
when(historyDataReader.queryLogsByMultipleConditions(any(), any(), any(),
any(), any(), any(), any())).thenReturn(mockLogs);
mockMvc.perform(
MockMvcRequestBuilders
@@ -263,8 +262,8 @@ class LogQueryControllerTest {
LogEntry.builder().timeUnixNano(null).build() // This should be filtered out
);
when(historyDataReader.queryLogsByMultipleConditions(isNull(), isNull(), isNull(),
isNull(), isNull(), isNull())).thenReturn(mockLogs);
when(historyDataReader.queryLogsByMultipleConditions(any(), any(), any(),
any(), any(), any(), any())).thenReturn(mockLogs);
mockMvc.perform(
MockMvcRequestBuilders
@@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
/**
* Unit test for {@link OtlpLogController}
*/
@ExtendWith(MockitoExtension.class)
class OtlpLogControllerTest {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
private MockMvc mockMvc;
@Mock
private OtlpLogProtocolAdapter otlpLogProtocolAdapter;
private OtlpLogController otlpLogController;
@BeforeEach
void setUp() {
this.otlpLogController = new OtlpLogController(otlpLogProtocolAdapter);
this.mockMvc = MockMvcBuilders.standaloneSetup(otlpLogController).build();
}
@Test
void testIngestJsonLogsSuccess() throws Exception {
String jsonContent = "{\"resourceLogs\":[]}";
doNothing().when(otlpLogProtocolAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content(jsonContent)
)
.andDo(print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
}
@Test
void testIngestJsonLogsFailure() throws Exception {
String jsonContent = "{\"invalid\":\"content\"}";
doThrow(new IllegalArgumentException("Invalid OTLP JSON log content"))
.when(otlpLogProtocolAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content(jsonContent)
)
.andExpect(status().isBadRequest())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
}
@Test
void testIngestBinaryLogsSuccess() throws Exception {
byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c};
doNothing().when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class));
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(CONTENT_TYPE_PROTOBUF)
.content(binaryContent)
)
.andDo(print())
.andExpect(status().isOk())
.andExpect(content().contentType(CONTENT_TYPE_PROTOBUF))
.andReturn();
}
@Test
void testIngestBinaryLogsFailure() throws Exception {
byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c};
doThrow(new IllegalArgumentException("Invalid OTLP binary log content"))
.when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class));
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(CONTENT_TYPE_PROTOBUF)
.content(binaryContent)
)
.andExpect(status().isBadRequest())
.andExpect(content().contentType(CONTENT_TYPE_PROTOBUF))
.andReturn();
}
}
@@ -105,16 +105,44 @@ class LogSseFilterCriteriaTest {
// Test Span ID filter - match
filterCriteria.setSpanId("1234567890abcdef");
assertTrue(filterCriteria.matches(testLogEntry));
// Test Span ID filter - no match
filterCriteria.setSpanId("abcdef1234567890");
assertFalse(filterCriteria.matches(testLogEntry));
// Test Span ID filter - case insensitive
filterCriteria.setSpanId("1234567890ABCDEF");
assertTrue(filterCriteria.matches(testLogEntry));
}
@Test
void testMatchesWithLogContentFilter() {
// Test log content filter - match
filterCriteria.setLogContent("Test log");
assertTrue(filterCriteria.matches(testLogEntry));
// Test log content filter - no match
filterCriteria.setLogContent("Error message");
assertFalse(filterCriteria.matches(testLogEntry));
// Test log content filter - case insensitive
filterCriteria.setLogContent("test log");
assertTrue(filterCriteria.matches(testLogEntry));
// Test log content filter - partial match
filterCriteria.setLogContent("message");
assertTrue(filterCriteria.matches(testLogEntry));
// Test log content filter with null body
LogEntry nullBodyLog = LogEntry.builder()
.severityNumber(9)
.severityText("INFO")
.body(null)
.build();
filterCriteria.setLogContent("test");
assertFalse(filterCriteria.matches(nullBodyLog));
}
@Test
void testMatchesWithMultipleFilters() {
// Test multiple filter combinations - all match
@@ -163,14 +191,15 @@ class LogSseFilterCriteriaTest {
void testConstructorWithAllParameters() {
// Test constructor with all parameters
LogSseFilterCriteria criteria = new LogSseFilterCriteria(
9, "INFO", "1234567890abcdef1234567890abcdef", "1234567890abcdef"
9, "INFO", null, "1234567890abcdef1234567890abcdef", "1234567890abcdef"
);
assertEquals(9, criteria.getSeverityNumber());
assertEquals("INFO", criteria.getSeverityText());
assertEquals(null, criteria.getLogContent());
assertEquals("1234567890abcdef1234567890abcdef", criteria.getTraceId());
assertEquals("1234567890abcdef", criteria.getSpanId());
// Test matching
assertTrue(criteria.matches(testLogEntry));
}
@@ -179,17 +208,19 @@ class LogSseFilterCriteriaTest {
void testNoArgsConstructorAndSetters() {
// Test no-args constructor and setter methods
LogSseFilterCriteria criteria = new LogSseFilterCriteria();
criteria.setSeverityNumber(9);
criteria.setSeverityText("INFO");
criteria.setLogContent("Test log");
criteria.setTraceId("1234567890abcdef1234567890abcdef");
criteria.setSpanId("1234567890abcdef");
assertEquals(9, criteria.getSeverityNumber());
assertEquals("INFO", criteria.getSeverityText());
assertEquals("Test log", criteria.getLogContent());
assertEquals("1234567890abcdef1234567890abcdef", criteria.getTraceId());
assertEquals("1234567890abcdef", criteria.getSpanId());
// Test matching
assertTrue(criteria.matches(testLogEntry));
}
@@ -80,6 +80,47 @@ class OtlpLogProtocolAdapterTest {
verifyNoInteractions(commonDataQueue, logSseManager);
}
@Test
void testIngestBinaryWithNullContent() {
adapter.ingestBinary(null);
verifyNoInteractions(commonDataQueue, logSseManager);
}
@Test
void testIngestBinaryWithEmptyContent() {
adapter.ingestBinary(new byte[0]);
verifyNoInteractions(commonDataQueue, logSseManager);
}
@Test
void testIngestBinaryWithValidOtlpLogData() throws Exception {
byte[] binaryPayload = createValidOtlpLogBinaryPayload();
adapter.ingestBinary(binaryPayload);
ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
verify(commonDataQueue, times(1)).sendLogEntryToStorageBatch(listCaptor.capture());
verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(anyList());
verify(logSseManager, times(1)).broadcast(any(LogEntry.class));
List<LogEntry> capturedList = listCaptor.getValue();
assertNotNull(capturedList);
assertEquals(1, capturedList.size());
LogEntry capturedEntry = capturedList.get(0);
assertEquals("binary-test-service", capturedEntry.getResource().get("service_name"));
assertEquals("binary log message", capturedEntry.getBody());
assertEquals("INFO", capturedEntry.getSeverityText());
}
@Test
void testIngestBinaryWithInvalidContent() {
byte[] invalidBinary = "not a valid protobuf".getBytes();
assertThrows(IllegalArgumentException.class, () -> adapter.ingestBinary(invalidBinary));
verifyNoInteractions(commonDataQueue, logSseManager);
}
@Test
void testIngestWithValidOtlpLogData() throws Exception {
String otlpPayload = createValidOtlpLogPayload();
@@ -315,4 +356,32 @@ class OtlpLogProtocolAdapterTest {
ExportLogsServiceRequest request = ExportLogsServiceRequest.newBuilder().build();
return JsonFormat.printer().print(request);
}
private byte[] createValidOtlpLogBinaryPayload() {
ExportLogsServiceRequest request = ExportLogsServiceRequest.newBuilder()
.addResourceLogs(ResourceLogs.newBuilder()
.setResource(Resource.newBuilder()
.addAttributes(KeyValue.newBuilder()
.setKey("service.name")
.setValue(AnyValue.newBuilder().setStringValue("binary-test-service").build())
.build())
.build())
.addScopeLogs(ScopeLogs.newBuilder()
.setScope(InstrumentationScope.newBuilder()
.setName("binary-test-scope")
.setVersion("1.0.0")
.build())
.addLogRecords(LogRecord.newBuilder()
.setTimeUnixNano(System.currentTimeMillis() * 1_000_000)
.setObservedTimeUnixNano(System.currentTimeMillis() * 1_000_000)
.setSeverityNumberValue(9)
.setSeverityText("INFO")
.setBody(AnyValue.newBuilder().setStringValue("binary log message").build())
.build())
.build())
.build())
.build();
return request.toByteArray();
}
}
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
/**
* Parameter validator interface
*/
public interface ParamValidator {
/**
* Check if the validator supports the given parameter type
*
* @param type parameter type
* @return true if supported
*/
boolean support(String type);
/**
* Validate the parameter
*
* @param paramDefine parameter definition
* @param param parameter actual value
* @throws IllegalArgumentException if validation fails
*/
void validate(ParamDefine paramDefine, Param param) throws IllegalArgumentException;
}
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Parameter validator manager
*/
@Component
public class ParamValidatorManager {
private final List<ParamValidator> validators;
public ParamValidatorManager(List<ParamValidator> validators) {
this.validators = validators;
}
public void validate(ParamDefine paramDefine, Param param) {
for (ParamValidator validator : validators) {
if (validator.support(paramDefine.getType())) {
validator.validate(paramDefine, param);
return;
}
}
// No validator found for the given type.
throw new IllegalArgumentException("ParamDefine type " + paramDefine.getType() + " is invalid.");
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Array parameter validator
*/
@Component
public class ArrayParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "array".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
String[] arrays = param.getParamValue().split(",");
if (arrays.length == 0) {
throw new IllegalArgumentException("Param field " + paramDefine.getField() + " value "
+ param.getParamValue() + " is invalid arrays value");
}
if (param.getParamValue().startsWith("[") && param.getParamValue().endsWith("]")) {
param.setParamValue(param.getParamValue().substring(1, param.getParamValue().length() - 1));
}
}
}
@@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Boolean parameter validator
*/
@Component
public class BooleanParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "boolean".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
String booleanValue = param.getParamValue();
if (!"true".equalsIgnoreCase(booleanValue) && !"false".equalsIgnoreCase(booleanValue)) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " value "
+ booleanValue + " is invalid boolean value.");
}
}
}
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Host parameter validator that delegates to the common HostParamValidator
*/
@Component
public class HostParamValidatorAdapter implements ParamValidator {
private final org.apache.hertzbeat.common.support.valid.HostParamValidator hostValidator;
public HostParamValidatorAdapter() {
this.hostValidator = new org.apache.hertzbeat.common.support.valid.HostParamValidator();
}
@Override
public boolean support(String type) {
return "host".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
if (!hostValidator.isValid(param.getParamValue(), null)) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " value "
+ param.getParamValue() + " is invalid host value.");
}
}
}
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* JSON parameter validator
*/
@Component
public class JsonParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "metrics-field".equals(type) || "key-value".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
if (JsonUtil.fromJson(param.getParamValue(), new TypeReference<>() {
}) == null) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " value "
+ param.getParamValue() + " is invalid key-value value");
}
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.util.IntervalExpressionUtil;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Number parameter validator
*/
@Component
public class NumberParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "number".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
Double doubleValue = org.apache.hertzbeat.common.util.CommonUtil.parseStrDouble(param.getParamValue());
if (doubleValue == null) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " type "
+ paramDefine.getType() + " is invalid.");
}
if (paramDefine.getRange() != null) {
if (!IntervalExpressionUtil.validNumberIntervalExpress(doubleValue,
paramDefine.getRange())) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " type "
+ paramDefine.getType() + " over range " + paramDefine.getRange());
}
}
param.setType(CommonConstants.PARAM_TYPE_NUMBER);
}
}
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Option parameter validator for radio and checkbox types
*/
@Component
public class OptionParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "radio".equals(type) || "checkbox".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
List<ParamDefine.Option> options = paramDefine.getOptions();
boolean invalid = true;
if (options != null) {
for (ParamDefine.Option option : options) {
if (param.getParamValue().equalsIgnoreCase(option.getValue())) {
invalid = false;
break;
}
}
}
if (invalid) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " value "
+ param.getParamValue() + " is invalid option value");
}
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Password parameter validator
*/
@Component
public class PasswordParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "password".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
String passwordValue = param.getParamValue();
if (!AesUtil.isCiphertext(passwordValue)) {
passwordValue = AesUtil.aesEncode(passwordValue);
param.setParamValue(passwordValue);
}
param.setType(CommonConstants.PARAM_TYPE_PASSWORD);
}
}
@@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.manager.component.validator.ParamValidator;
import org.springframework.stereotype.Component;
/**
* Text parameter validator
*/
@Component
public class TextParamValidator implements ParamValidator {
@Override
public boolean support(String type) {
return "text".equals(type) || "textarea".equals(type);
}
@Override
public void validate(ParamDefine paramDefine, Param param) {
Short limit = paramDefine.getLimit();
if (limit != null && param.getParamValue().length() > limit) {
throw new IllegalArgumentException("Params field " + paramDefine.getField() + " type "
+ paramDefine.getType() + " over limit " + limit);
}
}
}
@@ -26,7 +26,7 @@ import java.util.HashSet;
import java.util.List;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.manager.Label;
import org.apache.hertzbeat.manager.service.LabelService;
import org.apache.hertzbeat.base.service.LabelService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.http.ResponseEntity;
@@ -46,7 +46,7 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping(path = "/api/label", produces = {APPLICATION_JSON_VALUE})
public class LabelController {
@Autowired
private LabelService labelService;
@@ -24,7 +24,10 @@ import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.hertzbeat.common.entity.grafana.GrafanaDashboard;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
@@ -34,24 +37,27 @@ import org.apache.hertzbeat.common.entity.manager.Param;
*/
@Data
@Schema(description = "Monitoring information entities")
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class MonitorDto {
@Schema(description = "monitor content", accessMode = READ_WRITE)
@NotNull
@Valid
private Monitor monitor;
@Schema(description = "monitor params", accessMode = READ_WRITE)
@NotEmpty
@Valid
private List<Param> params;
@Schema(description = "Monitor Metrics", accessMode = READ_ONLY)
private List<MetricsInfo> metrics;
@Schema(description = "pinned collector, default null if system dispatch", accessMode = READ_WRITE)
private String collector;
@Schema(description = "grafana dashboard")
private GrafanaDashboard grafanaDashboard;
}
@@ -45,8 +45,11 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
* PluginParam
*/
@Entity
@Table(name = "hzb_plugin_param", indexes = { @Index(columnList = "pluginMetadataId") },
uniqueConstraints = @UniqueConstraint(columnNames = {"pluginMetadataId", "field"}))
@Table(
name = "hzb_plugin_param",
indexes = {@Index(name = "idx_hzb_plugin_param_plugin_metadata_id", columnList = "plugin_metadata_id")},
uniqueConstraints = @UniqueConstraint(name = "uk_hzb_plugin_param_metadata_field", columnNames = {"plugin_metadata_id", "field"})
)
@Data
@Builder
@AllArgsConstructor
@@ -65,8 +68,9 @@ public class PluginParam {
/**
* Monitor ID
*/
@Schema(title = "Plugin task ID", example = "875846754543", accessMode = READ_WRITE)
@NotNull
@Column(name = "plugin_metadata_id")
@Schema(title = "Plugin task ID", example = "875846754543", accessMode = READ_WRITE)
private Long pluginMetadataId;
/**
@@ -75,6 +79,7 @@ public class PluginParam {
@Schema(title = "Parameter identifier field", example = "port", accessMode = READ_WRITE)
@Size(max = 100)
@NotNull
@Column(name = "field")
private String field;
/**
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.service.helper;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.ExportFileConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.util.FileUtil;
import org.apache.hertzbeat.manager.config.ManagerSseManager;
import org.apache.hertzbeat.manager.service.ImExportService;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Helper class for monitor import and export operations
*/
@Component
@Slf4j
public class MonitorImExportHelper {
private static final String CONTENT_VALUE = MediaType.APPLICATION_OCTET_STREAM_VALUE + SignConstants.SINGLE_MARK
+ "charset=" + StandardCharsets.UTF_8;
private final Map<String, ImExportService> imExportServiceMap = new HashMap<>();
private final ManagerSseManager managerSseManager;
public MonitorImExportHelper(List<ImExportService> imExportServiceList, ManagerSseManager managerSseManager) {
imExportServiceList.forEach(it -> imExportServiceMap.put(it.type(), it));
this.managerSseManager = managerSseManager;
}
public void export(List<Long> ids, String type, HttpServletResponse res) throws Exception {
var imExportService = imExportServiceMap.get(type);
if (imExportService == null) {
throw new IllegalArgumentException("not support export type: " + type);
}
var fileName = imExportService.getFileName();
res.setHeader(HttpHeaders.CONTENT_DISPOSITION, CONTENT_VALUE);
res.setContentType(CONTENT_VALUE);
res.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment;filename=" + URLEncoder.encode(fileName, StandardCharsets.UTF_8));
res.setHeader(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION);
imExportService.exportConfig(res.getOutputStream(), ids);
}
public void importConfig(MultipartFile file) throws Exception {
var fileName = FileUtil.getFileName(file);
var type = FileUtil.getFileType(file);
try {
if (!imExportServiceMap.containsKey(type)) {
String errMsg = ExportFileConstants.FILE + " " + fileName + " is not supported.";
throw new RuntimeException(errMsg);
}
var imExportService = imExportServiceMap.get(type);
imExportService.importConfig(fileName, file.getInputStream());
} catch (Exception e) {
managerSseManager.broadcastImportTaskFail(fileName, e.getMessage());
throw e;
}
}
}
@@ -31,7 +31,7 @@ import org.apache.hertzbeat.manager.config.ManagerSseManager;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.service.ImExportService;
import org.apache.hertzbeat.manager.service.MonitorService;
import org.apache.hertzbeat.manager.service.LabelService;
import org.apache.hertzbeat.base.service.LabelService;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.util.CollectionUtils;
@@ -31,8 +31,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.entity.manager.Label;
import org.apache.hertzbeat.manager.dao.LabelDao;
import org.apache.hertzbeat.manager.service.LabelService;
import org.apache.hertzbeat.base.dao.LabelDao;
import org.apache.hertzbeat.base.service.LabelService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.manager.service.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
import com.usthe.sureness.subject.SubjectSum;
@@ -29,8 +28,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.dao.AlertDefineBindDao;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.ExportFileConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.grafana.GrafanaDashboard;
import org.apache.hertzbeat.common.entity.job.Configmap;
@@ -45,18 +42,15 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.support.event.MonitorDeletedEvent;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.FileUtil;
import org.apache.hertzbeat.common.util.IntervalExpressionUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JexlCheckerUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.apache.hertzbeat.grafana.service.DashboardService;
import org.apache.hertzbeat.manager.config.ManagerSseManager;
import org.apache.hertzbeat.manager.component.validator.ParamValidatorManager;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.LabelDao;
import org.apache.hertzbeat.base.dao.LabelDao;
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
@@ -65,10 +59,10 @@ import org.apache.hertzbeat.manager.pojo.dto.MetricsInfo;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.scheduler.CollectJobScheduling;
import org.apache.hertzbeat.manager.service.AppService;
import org.apache.hertzbeat.manager.service.ImExportService;
import org.apache.hertzbeat.manager.service.LabelService;
import org.apache.hertzbeat.base.service.LabelService;
import org.apache.hertzbeat.manager.service.MetricsFavoriteService;
import org.apache.hertzbeat.manager.service.MonitorService;
import org.apache.hertzbeat.manager.service.helper.MonitorImExportHelper;
import org.apache.hertzbeat.manager.support.exception.MonitorDatabaseException;
import org.apache.hertzbeat.manager.support.exception.MonitorDetectException;
import org.apache.hertzbeat.warehouse.service.WarehouseService;
@@ -79,16 +73,12 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
@@ -113,8 +103,10 @@ public class MonitorServiceImpl implements MonitorService {
private static final byte ALL_MONITOR_STATUS = 9;
public static final String PARAM_FIELD_PORT = "port";
private static final String CONTENT_VALUE = MediaType.APPLICATION_OCTET_STREAM_VALUE + SignConstants.SINGLE_MARK + "charset=" + StandardCharsets.UTF_8;
private final Map<String, ImExportService> imExportServiceMap = new HashMap<>();
@Autowired
private ParamValidatorManager paramValidatorManager;
@Autowired
private MonitorImExportHelper monitorImExportHelper;
@Autowired
private AppService appService;
@Autowired
@@ -138,18 +130,12 @@ public class MonitorServiceImpl implements MonitorService {
@Autowired
private DashboardService dashboardService;
@Autowired
private ManagerSseManager managerSseManager;
@Autowired
private LabelDao labelDao;
@Autowired
private LabelService labelService;
@Autowired
private MetricsFavoriteService metricsFavoriteService;
public MonitorServiceImpl(List<ImExportService> imExportServiceList) {
imExportServiceList.forEach(it -> imExportServiceMap.put(it.type(), it));
}
@Override
@Transactional(readOnly = true)
public void detectMonitor(Monitor monitor, List<Param> params, String collector) throws MonitorDetectException {
@@ -162,7 +148,8 @@ public class MonitorServiceImpl implements MonitorService {
@Override
@Transactional(rollbackFor = Exception.class)
public void addMonitor(Monitor monitor, List<Param> params, String collector, GrafanaDashboard grafanaDashboard) throws RuntimeException {
public void addMonitor(Monitor monitor, List<Param> params, String collector, GrafanaDashboard grafanaDashboard)
throws RuntimeException {
// Apply for monitor id
long monitorId = SnowFlakeIdGenerator.generateId();
Map<String, String> labels = monitor.getLabels();
@@ -177,7 +164,8 @@ public class MonitorServiceImpl implements MonitorService {
}
// Construct the collection task Job entity
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
String app = isStatic ? monitor.getApp() : monitor.getScrape();
Job appDefine = appService.getAppDefine(app);
if (!isStatic) {
@@ -199,8 +187,9 @@ public class MonitorServiceImpl implements MonitorService {
.findFirst()
.orElse(null);
String portWithMark = (Objects.isNull(portParam) || !StringUtils.hasText(portParam.getParamValue()))
? "" : SignConstants.DOUBLE_MARK + portParam.getParamValue();
if (Objects.nonNull(instance)) {
? ""
: SignConstants.DOUBLE_MARK + portParam.getParamValue();
if (!IpDomainUtil.isHasPortWithMark(instance)) {
instance = instance + portWithMark;
}
monitor.setInstance(instance);
@@ -215,11 +204,12 @@ public class MonitorServiceImpl implements MonitorService {
return new Configmap(param.getField(), param.getParamValue(), param.getType());
}).collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
long jobId = collector == null ? collectJobScheduling.addAsyncCollectJob(appDefine, null) :
collectJobScheduling.addAsyncCollectJob(appDefine, collector);
long jobId = collector == null ? collectJobScheduling.addAsyncCollectJob(appDefine, null)
: collectJobScheduling.addAsyncCollectJob(appDefine, collector);
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
} catch (Exception ignored) {
}
try {
if (collector != null) {
@@ -232,7 +222,8 @@ public class MonitorServiceImpl implements MonitorService {
monitor.setId(monitorId);
monitor.setJobId(jobId);
// create grafana dashboard
if (monitor.getApp().equals(CommonConstants.PROMETHEUS) && grafanaDashboard != null && grafanaDashboard.isEnabled()) {
if (monitor.getApp().equals(CommonConstants.PROMETHEUS) && grafanaDashboard != null
&& grafanaDashboard.isEnabled()) {
dashboardService.createOrUpdateDashboard(grafanaDashboard.getTemplate(), monitorId);
}
monitorDao.save(monitor);
@@ -246,16 +237,7 @@ public class MonitorServiceImpl implements MonitorService {
@Override
public void export(List<Long> ids, String type, HttpServletResponse res) throws Exception {
var imExportService = imExportServiceMap.get(type);
if (imExportService == null) {
throw new IllegalArgumentException("not support export type: " + type);
}
var fileName = imExportService.getFileName();
res.setHeader(HttpHeaders.CONTENT_DISPOSITION, CONTENT_VALUE);
res.setContentType(CONTENT_VALUE);
res.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + URLEncoder.encode(fileName, StandardCharsets.UTF_8));
res.setHeader(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, HttpHeaders.CONTENT_DISPOSITION);
imExportService.exportConfig(res.getOutputStream(), ids);
monitorImExportHelper.export(ids, type, res);
}
@Override
@@ -265,32 +247,21 @@ public class MonitorServiceImpl implements MonitorService {
.stream()
.map(Monitor::getId)
.collect(Collectors.toList());
// Use the existing export method to export all monitors
export(allMonitorIds, type, res);
}
@Override
public void importConfig(MultipartFile file) throws Exception {
var fileName = FileUtil.getFileName(file);
var type = FileUtil.getFileType(file);
try {
if (!imExportServiceMap.containsKey(type)) {
String errMsg = ExportFileConstants.FILE + " " + fileName + " is not supported.";
throw new RuntimeException(errMsg);
}
var imExportService = imExportServiceMap.get(type);
imExportService.importConfig(fileName, file.getInputStream());
} catch (Exception e){
managerSseManager.broadcastImportTaskFail(fileName, e.getMessage());
throw e;
}
monitorImExportHelper.importConfig(file);
}
@Override
@Transactional(readOnly = true)
public void validate(MonitorDto monitorDto, Boolean isModify) throws IllegalArgumentException {
// The request monitoring parameter matches the monitoring parameter definition mapping check
// The request monitoring parameter matches the monitoring parameter definition
// mapping check
Monitor monitor = monitorDto.getMonitor();
// The Service Discovery host field may be null
monitor.setInstance(StringUtils.hasText(monitor.getInstance()) ? monitor.getInstance().trim() : null);
@@ -303,7 +274,7 @@ public class MonitorServiceImpl implements MonitorService {
param.setParamValue(value);
})
.collect(Collectors.toMap(Param::getField, param -> param));
// Check name uniqueness and can not equal app type
// Check name uniqueness and can not equal app type
if (isModify != null) {
Optional<Job> defineOptional = appService.getAppDefineOption(monitor.getName());
if (defineOptional.isPresent()) {
@@ -333,7 +304,8 @@ public class MonitorServiceImpl implements MonitorService {
// Parameter definition structure verification
List<ParamDefine> paramDefines = appService.getAppParamDefines(monitorDto.getMonitor().getApp());
if (!CollectionUtils.isEmpty(paramDefines)) {
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
for (ParamDefine paramDefine : paramDefines) {
String field = paramDefine.getField();
Param param = paramMap.get(field);
@@ -345,123 +317,7 @@ public class MonitorServiceImpl implements MonitorService {
throw new IllegalArgumentException("Params field " + field + " is required.");
}
if (param != null && StringUtils.hasText(param.getParamValue())) {
switch (paramDefine.getType()) {
case "number":
double doubleValue;
try {
doubleValue = Double.parseDouble(param.getParamValue());
} catch (Exception e) {
throw new IllegalArgumentException("Params field " + field + " type "
+ paramDefine.getType() + " is invalid.");
}
if (paramDefine.getRange() != null) {
if (!IntervalExpressionUtil.validNumberIntervalExpress(doubleValue,
paramDefine.getRange())) {
throw new IllegalArgumentException("Params field " + field + " type "
+ paramDefine.getType() + " over range " + paramDefine.getRange());
}
}
param.setType(CommonConstants.PARAM_TYPE_NUMBER);
break;
case "textarea":
Short textareaLimit = paramDefine.getLimit();
if (textareaLimit != null && param.getParamValue().length() > textareaLimit) {
throw new IllegalArgumentException("Params field " + field + " type "
+ paramDefine.getType() + " over limit " + param.getParamValue());
}
break;
case "text":
Short textLimit = paramDefine.getLimit();
if (textLimit != null && param.getParamValue().length() > textLimit) {
throw new IllegalArgumentException("Params field " + field + " type "
+ paramDefine.getType() + " over limit " + textLimit);
}
break;
case "host":
String hostValue = param.getParamValue();
if (hostValue.toLowerCase().contains(NetworkConstants.HTTP_HEADER)) {
hostValue = hostValue.replaceAll(PATTERN_HTTP, SignConstants.BLANK);
}
if (hostValue.toLowerCase().contains(NetworkConstants.HTTPS_HEADER)) {
hostValue = hostValue.replace(PATTERN_HTTPS, SignConstants.BLANK);
}
if (!IpDomainUtil.validateIpDomain(hostValue)) {
throw new IllegalArgumentException("Params field " + field + " value "
+ hostValue + " is invalid host value.");
}
break;
case "password":
// The plaintext password needs to be encrypted for transmission and storage
String passwordValue = param.getParamValue();
if (!AesUtil.isCiphertext(passwordValue)) {
passwordValue = AesUtil.aesEncode(passwordValue);
param.setParamValue(passwordValue);
}
param.setType(CommonConstants.PARAM_TYPE_PASSWORD);
break;
case "boolean":
// boolean check
String booleanValue = param.getParamValue();
if (!"true".equalsIgnoreCase(booleanValue) && !"false".equalsIgnoreCase(booleanValue)) {
throw new IllegalArgumentException("Params field " + field + " value "
+ booleanValue + " is invalid boolean value.");
}
break;
case "radio":
// radio single value check
List<ParamDefine.Option> options = paramDefine.getOptions();
boolean invalid = true;
if (options != null) {
for (ParamDefine.Option option : options) {
if (param.getParamValue().equalsIgnoreCase(option.getValue())) {
invalid = false;
break;
}
}
}
if (invalid) {
throw new IllegalArgumentException("Params field " + field + " value "
+ param.getParamValue() + " is invalid option value");
}
break;
case "checkbox":
List<ParamDefine.Option> checkboxOptions = paramDefine.getOptions();
boolean checkboxInvalid = true;
if (checkboxOptions != null) {
for (ParamDefine.Option option : checkboxOptions) {
if (param.getParamValue().equalsIgnoreCase(option.getValue())) {
checkboxInvalid = false;
break;
}
}
}
if (checkboxInvalid) {
throw new IllegalArgumentException("Params field " + field + " value "
+ param.getParamValue() + " is invalid checkbox value");
}
break;
case "metrics-field":
case "key-value":
if (JsonUtil.fromJson(param.getParamValue(), new TypeReference<>() {
}) == null) {
throw new IllegalArgumentException("Params field " + field + " value "
+ param.getParamValue() + " is invalid key-value value");
}
break;
case "array":
String[] arrays = param.getParamValue().split(",");
if (arrays.length == 0) {
throw new IllegalArgumentException("Param field" + field + " value "
+ param.getParamValue() + " is invalid arrays value");
}
if (param.getParamValue().startsWith("[") && param.getParamValue().endsWith("]")) {
param.setParamValue(param.getParamValue().substring(1, param.getParamValue().length() - 1));
}
break;
// todo More parameter definitions and actual value format verification
default:
throw new IllegalArgumentException("ParamDefine type " + paramDefine.getType() + " is invalid.");
}
paramValidatorManager.validate(paramDefine, param);
}
}
}
@@ -485,7 +341,8 @@ public class MonitorServiceImpl implements MonitorService {
}
if (JexlCheckerUtil.verifyStartCharacter(field.getField())) {
throw new IllegalArgumentException(job.getApp() + " " + metrics.getName() + " "
+ field.getField() + " illegal start character, please modify the template information.");
+ field.getField()
+ " illegal start character, please modify the template information.");
}
if (JexlCheckerUtil.verifySpaces(field.getField())) {
throw new IllegalArgumentException(job.getApp() + " " + metrics.getName() + " "
@@ -498,7 +355,8 @@ public class MonitorServiceImpl implements MonitorService {
@Override
@Transactional(rollbackFor = Exception.class)
public void modifyMonitor(Monitor monitor, List<Param> params, String collector, GrafanaDashboard grafanaDashboard) throws RuntimeException {
public void modifyMonitor(Monitor monitor, List<Param> params, String collector, GrafanaDashboard grafanaDashboard)
throws RuntimeException {
long monitorId = monitor.getId();
// Check to determine whether the monitor corresponding to the monitor id exists
Optional<Monitor> queryOption = monitorDao.findById(monitorId);
@@ -529,13 +387,15 @@ public class MonitorServiceImpl implements MonitorService {
.findFirst()
.orElse(null);
String portWithMark = (Objects.isNull(portParam) || !StringUtils.hasText(portParam.getParamValue()))
? "" : SignConstants.DOUBLE_MARK + portParam.getParamValue();
? ""
: SignConstants.DOUBLE_MARK + portParam.getParamValue();
if (Objects.nonNull(instance)) {
instance = instance + portWithMark;
}
monitor.setInstance(instance);
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
if (preMonitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE) {
// Construct the collection task Job entity
String app = isStatic ? monitor.getApp() : monitor.getScrape();
@@ -558,8 +418,9 @@ public class MonitorServiceImpl implements MonitorService {
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<Configmap> configmaps = params.stream()
.map(param -> new Configmap(param.getField(), param.getParamValue(), param.getType()))
.collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
long newJobId;
if (collector == null) {
@@ -572,7 +433,8 @@ public class MonitorServiceImpl implements MonitorService {
// execute only in non paused status
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
} catch (Exception ignored) {
}
}
// After the update is successfully released, refresh the database
@@ -584,7 +446,8 @@ public class MonitorServiceImpl implements MonitorService {
.build();
collectorMonitorBindDao.save(collectorMonitorBind);
}
// force update gmtUpdate time, due the case: monitor not change, param change. we also think monitor change
// force update gmtUpdate time, due the case: monitor not change, param change.
// we also think monitor change
monitor.setGmtUpdate(LocalDateTime.now());
// update or open grafana dashboard
if (monitor.getApp().equals(CommonConstants.PROMETHEUS) && grafanaDashboard != null) {
@@ -616,7 +479,8 @@ public class MonitorServiceImpl implements MonitorService {
if (CollectionUtils.isEmpty(ids)) {
return;
}
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId).collect(Collectors.toSet());
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId)
.collect(Collectors.toSet());
Set<Long> allMonitorIds = new HashSet<>(ids);
allMonitorIds.addAll(subMonitorIds);
List<Monitor> monitors = monitorDao.findMonitorsByIdIn(allMonitorIds);
@@ -656,24 +520,29 @@ public class MonitorServiceImpl implements MonitorService {
List<Param> params = paramDao.findParamsByMonitorId(id);
monitorDto.setParams(params);
List<MetricsInfo> metricsInfos;
if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(monitor.getApp()) || monitor.getType() == CommonConstants.MONITOR_TYPE_PUSH_AUTO_CREATE) {
if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(monitor.getApp())
|| monitor.getType() == CommonConstants.MONITOR_TYPE_PUSH_AUTO_CREATE) {
List<CollectRep.MetricsData> metricsDataList = warehouseService.queryMonitorMetricsData(id);
metricsInfos = metricsDataList.stream()
.map(t -> MetricsInfo.builder().name(t.getMetrics()).favorited(favoritedMetrics.contains(t.getMetrics())).build())
.map(t -> MetricsInfo.builder().name(t.getMetrics())
.favorited(favoritedMetrics.contains(t.getMetrics())).build())
.collect(Collectors.toList());
monitorDto.setGrafanaDashboard(dashboardService.getDashboardByMonitorId(id));
} else {
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
String type = isStatic ? monitor.getApp() : monitor.getScrape();
Job job = appService.getAppDefine(type);
metricsInfos = job.getMetrics().stream()
.filter(Metrics::isVisible)
.map(t -> MetricsInfo.builder().name(t.getName()).favorited(favoritedMetrics.contains(t.getName())).build())
.map(t -> MetricsInfo.builder().name(t.getName())
.favorited(favoritedMetrics.contains(t.getName())).build())
.collect(Collectors.toList());
}
monitorDto.setMetrics(metricsInfos);
monitorDto.setMonitor(monitor);
Optional<CollectorMonitorBind> bindOptional = collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(monitor.getId());
Optional<CollectorMonitorBind> bindOptional = collectorMonitorBindDao
.findCollectorMonitorBindByMonitorId(monitor.getId());
bindOptional.ifPresent(bind -> monitorDto.setCollector(bind.getCollector()));
return monitorDto;
} else {
@@ -682,7 +551,8 @@ public class MonitorServiceImpl implements MonitorService {
}
@Override
public Page<Monitor> getMonitors(List<Long> monitorIds, String app, String search, Byte status, String sort, String order, int pageIndex, int pageSize, String labels) {
public Page<Monitor> getMonitors(List<Long> monitorIds, String app, String search, Byte status, String sort,
String order, int pageIndex, int pageSize, String labels) {
Specification<Monitor> specification = (root, query, criteriaBuilder) -> {
List<Predicate> andList = new ArrayList<>();
if (!CollectionUtils.isEmpty(monitorIds)) {
@@ -705,8 +575,9 @@ public class MonitorServiceImpl implements MonitorService {
List<Predicate> orList = new ArrayList<>();
if (StringUtils.hasText(search)) {
Predicate predicateHost = criteriaBuilder.like(root.get("host"), "%" + search + "%");
Predicate predicateName = criteriaBuilder.like(criteriaBuilder.lower(root.get("name")), "%" + search.toLowerCase() + "%");
Predicate predicateHost = criteriaBuilder.like(root.get("instance"), "%" + search + "%");
Predicate predicateName = criteriaBuilder.like(criteriaBuilder.lower(root.get("name")),
"%" + search.toLowerCase() + "%");
Long id = Longs.tryParse(search);
if (id != null) {
orList.add(criteriaBuilder.equal(root.get("id"), id));
@@ -754,12 +625,13 @@ public class MonitorServiceImpl implements MonitorService {
return;
}
// Update monitoring status Delete corresponding monitoring periodic task
// The jobId is not deleted, and the jobId is reused again after the management is started.
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId).collect(Collectors.toSet());
// The jobId is not deleted, and the jobId is reused again after the management
// is started.
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId)
.collect(Collectors.toSet());
ids.addAll(subMonitorIds);
List<Monitor> managedMonitors = monitorDao.findMonitorsByIdIn(ids)
.stream().filter(monitor ->
monitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE)
.stream().filter(monitor -> monitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE)
.peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_PAUSED_CODE))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(managedMonitors)) {
@@ -773,11 +645,11 @@ public class MonitorServiceImpl implements MonitorService {
@Override
public void enableManageMonitors(Set<Long> ids) {
// Update monitoring status Add corresponding monitoring periodic task
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId).collect(Collectors.toSet());
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId)
.collect(Collectors.toSet());
ids.addAll(subMonitorIds);
List<Monitor> unManagedMonitors = monitorDao.findMonitorsByIdIn(ids)
.stream().filter(monitor ->
monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE)
.stream().filter(monitor -> monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE)
.peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_UP_CODE))
.collect(Collectors.toList());
if (unManagedMonitors.isEmpty()) {
@@ -787,7 +659,8 @@ public class MonitorServiceImpl implements MonitorService {
for (Monitor monitor : unManagedMonitors) {
// Construct the collection task Job entity
List<Param> params = paramDao.findParamsByMonitorId(monitor.getId());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
String app = isStatic ? monitor.getApp() : monitor.getScrape();
Job appDefine = appService.getAppDefine(app);
if (!isStatic) {
@@ -807,22 +680,24 @@ public class MonitorServiceImpl implements MonitorService {
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<Configmap> configmaps = params.stream()
.map(param -> new Configmap(param.getField(), param.getParamValue(), param.getType()))
.collect(Collectors.toList());
List<ParamDefine> paramDefaultValue = appDefine.getParams().stream()
.filter(item -> StringUtils.hasText(item.getDefaultValue()))
.toList();
paramDefaultValue.forEach(defaultVar -> {
if (configmaps.stream().noneMatch(item -> item.getKey().equals(defaultVar.getField()))) {
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(), CommonConstants.TYPE_STRING);
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(),
CommonConstants.TYPE_STRING);
configmaps.add(configmap);
}
});
appDefine.setConfigmap(configmaps);
// Issue collection tasks
Optional<CollectorMonitorBind> bindOptional =
collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(monitor.getId());
Optional<CollectorMonitorBind> bindOptional = collectorMonitorBindDao
.findCollectorMonitorBindByMonitorId(monitor.getId());
String collector = bindOptional.map(CollectorMonitorBind::getCollector).orElse(null);
long newJobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
@@ -841,24 +716,26 @@ public class MonitorServiceImpl implements MonitorService {
if (CollectionUtils.isEmpty(appCounts)) {
return null;
}
//Statistical category information, calculate the number of corresponding states for each monitor
// Statistical category information, calculate the number of corresponding
// states for each monitor
Map<String, AppCount> appCountMap = new HashMap<>(appCounts.size());
for (AppCount item : appCounts) {
AppCount appCount = appCountMap.getOrDefault(item.getApp(), new AppCount());
appCount.setApp(item.getApp());
switch (item.getStatus()) {
case CommonConstants.MONITOR_UP_CODE ->
appCount.setAvailableSize(appCount.getAvailableSize() + item.getSize());
appCount.setAvailableSize(appCount.getAvailableSize() + item.getSize());
case CommonConstants.MONITOR_DOWN_CODE ->
appCount.setUnAvailableSize(appCount.getUnAvailableSize() + item.getSize());
appCount.setUnAvailableSize(appCount.getUnAvailableSize() + item.getSize());
case CommonConstants.MONITOR_PAUSED_CODE ->
appCount.setUnManageSize(appCount.getUnManageSize() + item.getSize());
appCount.setUnManageSize(appCount.getUnManageSize() + item.getSize());
default -> {
}
}
appCountMap.put(item.getApp(), appCount);
}
//Traverse the map obtained by statistics and convert it into a List<App Count> result set
// Traverse the map obtained by statistics and convert it into a List<App Count>
// result set
return appCountMap.values().stream().map(item -> {
item.setSize(item.getAvailableSize() + item.getUnManageSize() + item.getUnAvailableSize());
try {
@@ -911,7 +788,8 @@ public class MonitorServiceImpl implements MonitorService {
.toList();
paramDefaultValue.forEach(defaultVar -> {
if (configmaps.stream().noneMatch(item -> item.getKey().equals(defaultVar.getField()))) {
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(), (byte) 1);
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(),
(byte) 1);
configmaps.add(configmap);
}
});
@@ -927,7 +805,7 @@ public class MonitorServiceImpl implements MonitorService {
}
}
}
@Override
public Monitor getMonitor(Long monitorId) {
return monitorDao.findById(monitorId).orElse(null);
@@ -978,7 +856,6 @@ public class MonitorServiceImpl implements MonitorService {
addMonitor(newMonitor, newParams, null, null);
}
private void detectSdMonitor(Monitor monitor, List<Param> params, String collector) {
Long monitorId = monitor.getId();
if (monitorId == null || monitorId == 0) {
@@ -995,8 +872,9 @@ public class MonitorServiceImpl implements MonitorService {
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<Configmap> configmaps = params.stream()
.map(param -> new Configmap(param.getField(), param.getParamValue(), param.getType()))
.collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
appDefine.setSd(true);
List<CollectRep.MetricsData> collectRep;
@@ -1035,10 +913,12 @@ public class MonitorServiceImpl implements MonitorService {
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<Configmap> configmaps = params.stream()
.map(param -> new Configmap(param.getField(), param.getParamValue(), param.getType()))
.collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
// To detect availability, you only need to collect the set of availability metrics with a priority of 0.
// To detect availability, you only need to collect the set of availability
// metrics with a priority of 0.
List<Metrics> availableMetrics = appDefine.getMetrics().stream()
.filter(item -> item.getPriority() == 0).collect(Collectors.toList());
appDefine.setMetrics(availableMetrics);
@@ -18,9 +18,8 @@
package org.apache.hertzbeat.manager.service.impl;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
@@ -131,45 +130,70 @@ public class StatusPageServiceImpl implements StatusPageService {
componentStatus.setInfo(component);
List<StatusPageHistory> histories = new LinkedList<>();
// query today status
LocalDateTime nowTime = LocalDateTime.now();
LocalDateTime todayStartTime = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0);
ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now());
long nowTimestamp = nowTime.toInstant(zoneOffset).toEpochMilli();
long todayStartTimestamp = todayStartTime.toInstant(zoneOffset).toEpochMilli();
ZoneId zoneId = ZoneId.systemDefault();
Instant now = Instant.now();
long nowTimestamp = now.toEpochMilli();
long todayStartTimestamp = now
.atZone(zoneId)
.toLocalDate()
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
List<StatusPageHistory> todayStatusPageHistoryList = statusPageHistoryDao
.findStatusPageHistoriesByComponentIdAndTimestampBetween(component.getId(), todayStartTimestamp, nowTimestamp);
StatusPageHistory todayStatus = combineOneDayStatusPageHistory(todayStatusPageHistoryList, component, nowTimestamp);
histories.add(todayStatus);
// query 30d component status history
LocalDateTime preTime = todayStartTime.minusDays(HISTORY_SPAN_DAYS);
long preTimestamp = preTime.toInstant(zoneOffset).toEpochMilli();
long preTimestamp = now
.atZone(zoneId)
.toLocalDate()
.minusDays(HISTORY_SPAN_DAYS)
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
List<StatusPageHistory> history = statusPageHistoryDao
.findStatusPageHistoriesByComponentIdAndTimestampBetween(component.getId(), preTimestamp, todayStartTimestamp);
LinkedList<StatusPageHistory> historyList = new LinkedList<>(history);
historyList.sort((o1, o2) -> (int) (o1.getTimestamp() - o2.getTimestamp()));
LocalDateTime endTime = todayStartTime.minusSeconds(1);
LocalDateTime startTime = endTime.withHour(0).withMinute(0).withSecond(0).withNano(0);
for (int index = 0; index < HISTORY_SPAN_DAYS; index++) {
long startTimestamp = startTime.toInstant(zoneOffset).toEpochMilli();
long endTimestamp = endTime.toInstant(zoneOffset).toEpochMilli();
List<StatusPageHistory> thisDayHistory = historyList.stream().filter(item ->
item.getTimestamp() >= startTimestamp && item.getTimestamp() <= endTimestamp)
.collect(Collectors.toList());
historyList.sort((o1, o2) -> Long.compare(o1.getTimestamp(), o2.getTimestamp()));
ZonedDateTime end = Instant.ofEpochMilli(todayStartTimestamp)
.atZone(zoneId)
.minusSeconds(1); // yesterday 23:59:59 local time
for (int i = 0; i < HISTORY_SPAN_DAYS; i++) {
long endTimestamp = end.toInstant().toEpochMilli();
long startTimestamp = end.toLocalDate()
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
List<StatusPageHistory> thisDayHistory = historyList.stream()
.filter(h -> h.getTimestamp() >= startTimestamp && h.getTimestamp() <= endTimestamp)
.collect(Collectors.toList());
if (thisDayHistory.isEmpty()) {
StatusPageHistory statusPageHistory = StatusPageHistory.builder().timestamp(endTimestamp)
.componentId(component.getId()).state(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN).build();
histories.add(statusPageHistory);
histories.add(StatusPageHistory.builder()
.timestamp(endTimestamp)
.componentId(component.getId())
.state(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN)
.build());
} else if (thisDayHistory.size() == 1) {
histories.add(thisDayHistory.get(0));
} else {
StatusPageHistory statusPageHistory = combineOneDayStatusPageHistory(thisDayHistory, component, endTimestamp);
histories.add(statusPageHistory);
StatusPageHistory merged =
combineOneDayStatusPageHistory(thisDayHistory, component, endTimestamp);
histories.add(merged);
statusPageHistoryDao.deleteAll(thisDayHistory);
statusPageHistoryDao.save(statusPageHistory);
statusPageHistoryDao.save(merged);
}
startTime = startTime.minusDays(1);
endTime = endTime.minusDays(1);
end = end.minusDays(1);
}
componentStatus.setHistory(histories);
componentStatusList.add(componentStatus);
}
@@ -215,50 +239,86 @@ public class StatusPageServiceImpl implements StatusPageService {
@Override
public ComponentStatus queryComponentStatus(long id) {
StatusPageComponent component = statusPageComponentDao.findById(id).orElseThrow(() -> new IllegalArgumentException("component not found"));
StatusPageComponent component = statusPageComponentDao.findById(id)
.orElseThrow(() -> new IllegalArgumentException("component not found"));
ComponentStatus componentStatus = new ComponentStatus();
componentStatus.setInfo(component);
List<StatusPageHistory> histories = new LinkedList<>();
// query today status
LocalDateTime nowTime = LocalDateTime.now();
LocalDateTime todayStartTime = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0);
ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now());
long nowTimestamp = nowTime.toInstant(zoneOffset).toEpochMilli();
long todayStartTimestamp = todayStartTime.toInstant(zoneOffset).toEpochMilli();
List<StatusPageHistory> todayStatusPageHistoryList = statusPageHistoryDao
.findStatusPageHistoriesByComponentIdAndTimestampBetween(component.getId(), todayStartTimestamp, nowTimestamp);
StatusPageHistory todayStatus = combineOneDayStatusPageHistory(todayStatusPageHistoryList, component, nowTimestamp);
ZoneId zoneId = ZoneId.systemDefault();
Instant now = Instant.now();
long nowTimestamp = now.toEpochMilli();
long todayStartTimestamp = now
.atZone(zoneId)
.toLocalDate()
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
// Today
List<StatusPageHistory> todayStatusPageHistoryList =
statusPageHistoryDao.findStatusPageHistoriesByComponentIdAndTimestampBetween(
component.getId(), todayStartTimestamp, nowTimestamp);
StatusPageHistory todayStatus =
combineOneDayStatusPageHistory(todayStatusPageHistoryList, component, nowTimestamp);
histories.add(todayStatus);
// query 30d component status history
LocalDateTime preTime = todayStartTime.minusDays(HISTORY_SPAN_DAYS);
long preTimestamp = preTime.toInstant(zoneOffset).toEpochMilli();
List<StatusPageHistory> history = statusPageHistoryDao
.findStatusPageHistoriesByComponentIdAndTimestampBetween(component.getId(), preTimestamp, todayStartTimestamp);
// Previous HISTORY_SPAN_DAYS days (excluding today)
long preTimestamp = now
.atZone(zoneId)
.toLocalDate()
.minusDays(HISTORY_SPAN_DAYS)
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
List<StatusPageHistory> history =
statusPageHistoryDao.findStatusPageHistoriesByComponentIdAndTimestampBetween(
component.getId(), preTimestamp, todayStartTimestamp);
LinkedList<StatusPageHistory> historyList = new LinkedList<>(history);
historyList.sort((o1, o2) -> (int) (o1.getTimestamp() - o2.getTimestamp()));
LocalDateTime endTime = todayStartTime.minusSeconds(1);
LocalDateTime startTime = endTime.withHour(0).withMinute(0).withSecond(0).withNano(0);
for (int index = 0; index < HISTORY_SPAN_DAYS; index++) {
long startTimestamp = startTime.toInstant(zoneOffset).toEpochMilli();
long endTimestamp = endTime.toInstant(zoneOffset).toEpochMilli();
List<StatusPageHistory> thisDayHistory = historyList.stream().filter(item ->
item.getTimestamp() >= startTimestamp && item.getTimestamp() <= endTimestamp)
.collect(Collectors.toList());
historyList.sort((o1, o2) -> Long.compare(o1.getTimestamp(), o2.getTimestamp()));
ZonedDateTime end = Instant.ofEpochMilli(todayStartTimestamp)
.atZone(zoneId)
.minusSeconds(1); // yesterday 23:59:59 local time
for (int i = 0; i < HISTORY_SPAN_DAYS; i++) {
long endTimestamp = end.toInstant().toEpochMilli();
long startTimestamp = end.toLocalDate()
.atStartOfDay(zoneId)
.toInstant()
.toEpochMilli();
List<StatusPageHistory> thisDayHistory = historyList.stream()
.filter(h -> h.getTimestamp() >= startTimestamp && h.getTimestamp() <= endTimestamp)
.collect(Collectors.toList());
if (thisDayHistory.isEmpty()) {
StatusPageHistory statusPageHistory = StatusPageHistory.builder().timestamp(endTimestamp)
.componentId(component.getId()).state(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN).build();
histories.add(statusPageHistory);
histories.add(StatusPageHistory.builder()
.timestamp(endTimestamp)
.componentId(component.getId())
.state(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN)
.build());
} else if (thisDayHistory.size() == 1) {
histories.add(thisDayHistory.get(0));
} else {
StatusPageHistory statusPageHistory = combineOneDayStatusPageHistory(thisDayHistory, component, endTimestamp);
histories.add(statusPageHistory);
StatusPageHistory merged =
combineOneDayStatusPageHistory(thisDayHistory, component, endTimestamp);
histories.add(merged);
statusPageHistoryDao.deleteAll(thisDayHistory);
statusPageHistoryDao.save(statusPageHistory);
statusPageHistoryDao.save(merged);
}
startTime = startTime.minusDays(1);
endTime = endTime.minusDays(1);
end = end.minusDays(1);
}
componentStatus.setHistory(histories);
return componentStatus;
}
@@ -25,7 +25,7 @@ name:
# The description and help of this monitoring type
help:
zh-CN: HertzBeat 使用 <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'> JDBC 协议</a> 通过配置 SQL 对 Mysql 数据库的通用性能指标(系统信息、性能状态、Innodb、缓存、事物、用户线程、慢SQL等)进行采集监控,支持版本为 Mysql5+。<br>您可以点击“<i>新建 Mysql 数据库</i>”并进行配置,或者选择“<i>更多操作</i>”,导入已有配置。
en-US: HertzBeat uses <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'> JDBC Protocol</a> to configure SQL for collecting general metrics of Mysql database (system information, performance status, Innodb, cache, things, user threads, slow SQL, etc.). Supported version is Mysql5+. <br>You can click "<i>New Mysql Database</i>" and configure it, or select "<i>More Operations</i>" to import the existing configuration.
en-US: HertzBeat uses <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'>JDBC Protocol</a> to collect general performance metrics of MySQL databases (system information, performance status, InnoDB, cache, transactions, user threads, slow SQL, etc.) by configuring SQL. Supported versions include MySQL 5.0 and above. <br>You can click "<i>New MySQL Database</i>" to configure it, or select "<i>More Operations</i>" to import an existing configuration.
zh-TW: HertzBeat 使用 <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'> JDBC 協議</a> 通過配置 SQL 對 Mysql 數據庫的通用性能指標(系統信息、性能狀態、Innodb、緩存、事物、用戶線程、慢SQL等)進行采集監控,支持版本爲 Mysql5+。<br>您可以點擊“<i>新建 Mysql 數據庫</i>”並進行配置,或者選擇“<i>更多操作</i>”,導入已有配置。
ja-JP: HertzBeat は <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'> JDBC プロトコル</a> を介して Mysql データベースの一般的なパフォーマンスのメトリクス(システム情報、パフォーマンスステータス、Innodb、キャッシュ、オブジェクト、ユーザースレッド、遅いSQLなど)収集監視を行う,サポートバージョンは Mysql5+。<br>クリックすることができます“<i>新規 Mysql データベース</i>”および構成,または“<i>その他の操作</i>”,既存の構成のインポートすることができます。
helpLink:
@@ -1178,3 +1178,69 @@ metrics:
privateKey: ^_^sshPrivateKey^_^
privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^
shareConnection: ^_^sshShareConnection^_^
- name: account_expiry
priority: 100
i18n:
zh-CN: 数据库账号有效期
en-US: Account Expiry
ja-JP: アカウント有効期限
fields:
- field: user
type: 1
label: true
i18n:
zh-CN: 用户
en-US: User
ja-JP: ユーザー
- field: host
type: 1
label: true
i18n:
zh-CN: 主机
en-US: Host
ja-JP: ホスト
- field: days_left
type: 0
i18n:
zh-CN: 剩余天数
en-US: Days Left
ja-JP: 残り日数
- field: password_expired
type: 1
i18n:
zh-CN: 是否已过期
en-US: Password Expired
ja-JP: 期限切れ
protocol: jdbc
jdbc:
host: ^_^host^_^
port: ^_^port^_^
platform: mysql
username: ^_^username^_^
password: ^_^password^_^
database: mysql
timeout: ^_^timeout^_^
queryType: multiRow
sql: |
SELECT
user,
host,
IF(password_lifetime IS NULL,
NULL,
password_lifetime - DATEDIFF(NOW(), password_last_changed)
) AS days_left,
password_expired
FROM mysql.user;
url: ^_^url^_^
sshTunnel:
enable: ^_^enableSshTunnel^_^
host: ^_^sshHost^_^
port: ^_^sshPort^_^
timeout: ^_^sshTimeout^_^
username: ^_^sshUsername^_^
password: ^_^sshPassword^_^
privateKey: ^_^sshPrivateKey^_^
privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^
shareConnection: ^_^sshShareConnection^_^
@@ -0,0 +1,321 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The monitoring type categoryservice-application service monitoring db-database monitoring custom-custom monitoring os-operating system monitoring
category: db
app: xugu
# The monitoring i18n name
name:
zh-CN: 虚谷数据库
en-US: Xugu DB
# The description and help of this monitoring type
help:
zh-CN: HertzBeat 使用 <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'> JDBC 协议</a> 通过配置 SQL 对 虚谷 数据库的通用性能指标(会话 信息、内存 信息、磁盘 信息)进行采集监控,支持版本为 XuguDB V12。<br>您可以点击“<i>新建 虚谷数据库</i>”并进行配置,或者选择“<i>更多操作</i>”,导入已有配置。
en-US: HertzBeat uses the <a class='help_module_content' href='https://hertzbeat.apache.org/docs/advanced/extend-jdbc'>JDBC protocol</a> to collect and monitor general performance metrics (session information, memory information, disk information) for Xugu Database by configuring SQL. It supports XuguDB V12.
helpLink:
zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/xugu
en-US: https://hertzbeat.apache.org/docs/help/xugu
# Input params define for monitoring(render web ui by the definition)
params:
# field-param field key
- field: host
# name-param field display i18n name
name:
zh-CN: 目标Host
en-US: Target Host
# type-param field type(most mapping the html input type)
type: host
# required-true or false
required: true
# field-param field key
- field: port
# name-param field display i18n name
name:
zh-CN: 端口
en-US: Port
# type-param field type(most mapping the html input type)
type: number
# when type is number, range is required
range: '[0,65535]'
# required-true or false
required: true
# default value
defaultValue: 5138
# field-param field key
- field: timeout
# name-param field display i18n name
name:
zh-CN: 查询超时时间(ms)
en-US: Query Timeout(ms)
# type-param field type(most mapping the html input type)
type: number
# when type is number, range is required
range: '[400,200000]'
# required-true or false
required: false
# hide param-true or false
hide: true
# default value
defaultValue: 6000
# field-param field key
- field: database
# name-param field display i18n name
name:
zh-CN: 数据库名称
en-US: Database Name
# type-param field type(most mapping the html input tag)
type: text
# required-true or false
required: false
# field-param field key
- field: username
# name-param field display i18n name
name:
zh-CN: 用户名
en-US: Username
# type-param field type(most mapping the html input tag)
type: text
# when type is text, use limit to limit string length
limit: 50
# required-true or false
required: false
# field-param field key
- field: password
# name-param field display i18n name
name:
zh-CN: 密码
en-US: Password
# type-param field type(most mapping the html input tag)
type: password
# required-true or false
required: false
# field-param field key
# collect metrics config list
metrics:
# metrics - basic
- name: session_information
priority: 0
i18n:
zh-CN: 会话 信息
en-US: Session Info
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: Idle_Connection
type: 0
label: true
i18n:
zh-CN: 空闲连接
en-US: Idle Connection
- field: Active_Connections
type: 0
i18n:
zh-CN: 活跃连接
en-US: Active Connections
- field: Connection_being_created
type: 0
i18n:
zh-CN: 正在创建的连接
en-US: Connection being created
- field: Null_Read_Connection
type: 0
i18n:
zh-CN: 空读连接
en-US: Null Read Connection
- field: Complete_data_input_connection
type: 0
i18n:
zh-CN: 完成数据输入连接
en-US: Complete data input connection
- field: Other
type: 0
i18n:
zh-CN: 其他
en-US: Other
protocol: jdbc
jdbc:
host: ^_^host^_^
port: ^_^port^_^
platform: xugu
# username
username: ^_^username^_^
# password
password: ^_^password^_^
# database name
database: ^_^database^_^
# timeout unitms
timeout: ^_^timeout^_^
queryType: columns
# sql
sql: SELECT CASE status WHEN 112 THEN 'Idle_Connection' WHEN 114 THEN 'Active_Connections' WHEN 111 THEN 'Connection_being_created' WHEN 113 THEN 'Null_Read_Connection' WHEN 115 THEN 'Complete_data_input_connection' ELSE 'Other' END AS status_type, COUNT(*) AS count FROM sys_sessions GROUP BY status;
# JDBC url
url: ^_^url^_^
- name: Memory
priority: 1
# collect metrics content
i18n:
zh-CN: 内存 信息
en-US: Memory Info
fields:
- field: G_MEM
type: 0
unit: MB
i18n:
zh-CN: 全局内存消耗
en-US: Global Memory Consumption
- field: CATA_MEM
type: 0
unit: MB
i18n:
zh-CN: CATALOG内存消耗
en-US: CATALOG memory consumption
- field: TRAN_MEM
type: 0
unit: MB
i18n:
zh-CN: 事务内存消耗
en-US: Transactional memory consumption
- field: NET_MEM
type: 0
unit: MB
i18n:
zh-CN: 用户连接用内存消耗
en-US: Memory consumption for user connections
- field: TASK_MEM
type: 0
unit: MB
i18n:
zh-CN: 任务线程消耗
en-US: Task Thread Consumption
- field: MSG_MEM
type: 0
unit: MB
i18n:
zh-CN: 内部通讯消息内存消耗
en-US: Internal communication message memory consumption
- field: LOCK_MEM
type: 0
unit: MB
i18n:
zh-CN: 局部锁内存消耗
en-US: Local lock memory consumption
- field: GLOCK_MEM
type: 0
unit: MB
i18n:
zh-CN: 全局锁内存消耗
en-US: Global lock memory consumption
- field: DLCHK_MEM
type: 0
unit: MB
i18n:
zh-CN: 死锁检测内存消耗
en-US: Deadlock detection memory consumption
- field: MODI_MEM
type: 0
unit: MB
i18n:
zh-CN: 变更日志内存消耗
en-US: Change Log Memory Consumption
- field: PROC_MEM
type: 0
unit: MB
i18n:
zh-CN: 过程执行机内存消耗
en-US: Process Execution Machine Memory Consumption
units:
- G_MEM=B->MB
- CATA_MEM=B->MB
- TRAN_MEM=B->MB
- NET_MEM=B->MB
- TASK_MEM=B->MB
- MSG_MEM=B->MB
- LOCK_MEM=B->MB
- GLOCK_MEM=B->MB
- DLCHK_MEM=B->MB
- MODI_MEM=B->MB
- PROC_MEM=B->MB
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: jdbc
# the config content when protocol is jdbc
jdbc:
# jdbc host
# jdbchost
host: ^_^host^_^
# port
port: ^_^port^_^
# platform name
platform: xugu
# username
username: ^_^username^_^
# password
password: ^_^password^_^
# database name
database: ^_^database^_^
# timeout unitms
timeout: ^_^timeout^_^
# SQL Query MethodoneRow, multiRow, columns
queryType: columns
# sql
sql: SELECT targ_name,targ_value from sys_monitors where targ_name like '%MEM';
url: ^_^url^_^
- name: Disk
priority: 1
i18n:
zh-CN: 磁盘 信息
en-US: Disk Info
fields:
- field: DISK_READ_NUM
type: 0
unit: Times
i18n:
zh-CN: 数据块磁盘读取次数
en-US: Number of data block disk reads
- field: DISK_WRITE_NUM
type: 0
unit: Times
i18n:
zh-CN: 磁盘写入数据块次数
en-US: Number of data blocks written to disk
- field: DISK_READ_BYTES
type: 0
unit: MB
i18n:
zh-CN: 磁盘读取字节数
en-US: Disk read bytes
- field: DISK_WRITE_BYTES
type: 0
unit: MB
i18n:
zh-CN: 磁盘写入字节数
en-US: Bytes written to disk
units:
- DISK_READ_BYTES=B->MB
- DISK_WRITE_BYTES=B->MB
protocol: jdbc
jdbc:
host: ^_^host^_^
port: ^_^port^_^
platform: xugu
username: ^_^username^_^
password: ^_^password^_^
database: ^_^database^_^
timeout: ^_^timeout^_^
queryType: columns
sql: SELECT targ_name,targ_value from sys_monitors where targ_name like 'DISK_%';
url: ^_^url^_^
@@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.component.validator;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ParamValidatorManagerTest {
private ParamValidatorManager paramValidatorManager;
@Mock
private ParamValidator paramValidator;
@BeforeEach
void setUp() {
paramValidatorManager = new ParamValidatorManager(List.of(paramValidator));
}
@Test
void validate_Success() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("text");
Param param = new Param();
when(paramValidator.support("text")).thenReturn(true);
assertDoesNotThrow(() -> paramValidatorManager.validate(paramDefine, param));
verify(paramValidator).validate(paramDefine, param);
}
@Test
void validate_NoValidatorFound() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("unknown");
Param param = new Param();
when(paramValidator.support("unknown")).thenReturn(false);
assertThrows(IllegalArgumentException.class, () -> paramValidatorManager.validate(paramDefine, param));
}
}
@@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ArrayParamValidatorTest {
private ArrayParamValidator validator;
@BeforeEach
void setUp() {
validator = new ArrayParamValidator();
}
@Test
void support() {
assertTrue(validator.support("array"));
}
@Test
void validate_ValidArray() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("array");
Param param = new Param();
param.setParamValue("val1,val2");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_ValidArrayWithBrackets() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("array");
Param param = new Param();
param.setParamValue("[val1,val2]");
validator.validate(paramDefine, param);
assertEquals("val1,val2", param.getParamValue());
}
@Test
void validate_EmptyArray() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("array");
paramDefine.setField("tags");
Param param = new Param();
param.setParamValue("");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
assertEquals("", param.getParamValue());
}
}
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class BooleanParamValidatorTest {
private BooleanParamValidator validator;
@BeforeEach
void setUp() {
validator = new BooleanParamValidator();
}
@Test
void support() {
assertTrue(validator.support("boolean"));
}
@Test
void validate_True() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("boolean");
Param param = new Param();
param.setParamValue("true");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_False() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("boolean");
Param param = new Param();
param.setParamValue("false");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_CaseInsensitive() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("boolean");
Param param = new Param();
param.setParamValue("True");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_Invalid() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("boolean");
paramDefine.setField("ssl");
Param param = new Param();
param.setParamValue("yes");
assertThrows(IllegalArgumentException.class, () -> validator.validate(paramDefine, param));
}
}
@@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class HostParamValidatorAdapterTest {
private HostParamValidatorAdapter validator;
@BeforeEach
void setUp() {
validator = new HostParamValidatorAdapter();
}
@Test
void support() {
assertTrue(validator.support("host"));
}
@Test
void validate_ValidHost() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("host");
Param param = new Param();
param.setParamValue("127.0.0.1");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_ValidDomain() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("host");
Param param = new Param();
param.setParamValue("localhost");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_HttpPrefix() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("host");
Param param = new Param();
param.setParamValue("http://127.0.0.1");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_InvalidHost() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("host");
paramDefine.setField("host");
Param param = new Param();
param.setParamValue("invalid host");
assertThrows(IllegalArgumentException.class, () -> validator.validate(paramDefine, param));
}
@Test
void validate_HostWithPort() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("host");
Param param = new Param();
param.setParamValue("127.0.0.1:8080");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
}
@@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class JsonParamValidatorTest {
private JsonParamValidator validator;
@BeforeEach
void setUp() {
validator = new JsonParamValidator();
}
@Test
void support() {
assertTrue(validator.support("metrics-field"));
assertTrue(validator.support("key-value"));
}
@Test
void validate_ValidJson() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("key-value");
Param param = new Param();
param.setParamValue("{\"key\":\"value\"}");
assertDoesNotThrow(() -> validator.validate(paramDefine, param));
}
@Test
void validate_InvalidJson() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("key-value");
paramDefine.setField("headers");
Param param = new Param();
param.setParamValue("{key:value}"); // Invalid JSON
assertThrows(IllegalArgumentException.class, () -> validator.validate(paramDefine, param));
}
}
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.component.validator.impl;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class NumberParamValidatorTest {
private NumberParamValidator validator;
@BeforeEach
void setUp() {
validator = new NumberParamValidator();
}
@Test
void support() {
assertTrue(validator.support("number"));
}
@Test
void validate_ValidNumber() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("number");
Param param = new Param();
param.setParamValue("123");
validator.validate(paramDefine, param);
assertEquals(CommonConstants.PARAM_TYPE_NUMBER, param.getType());
}
@Test
void validate_InvalidNumber() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("number");
paramDefine.setField("port");
Param param = new Param();
param.setParamValue("abc");
assertThrows(IllegalArgumentException.class, () -> validator.validate(paramDefine, param));
}
@Test
void validate_NumberInRange() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("number");
paramDefine.setRange("[0,100]");
Param param = new Param();
param.setParamValue("50");
validator.validate(paramDefine, param);
}
@Test
void validate_NumberOutOfRange() {
ParamDefine paramDefine = new ParamDefine();
paramDefine.setType("number");
paramDefine.setField("port");
paramDefine.setRange("[0,100]");
Param param = new Param();
param.setParamValue("150");
assertThrows(IllegalArgumentException.class, () -> validator.validate(paramDefine, param));
}
}

Some files were not shown because too many files have changed in this diff Show More