Compare commits

...
Author SHA1 Message Date
Logic 6a53e1b58e fix(e2e): pin Vector image for log tests 2026-07-15 23:20:34 +08:00
aias00 2f1fa48bb3 Merge branch 'master' into feature/angular-three-signals 2026-07-15 07:16:35 -07:00
shown aee4fcc19b fix(ai): correct conversation message handling (#4208)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
2026-07-15 22:03:36 +08:00
Logic 34940db118 refactor: remove obsolete template marketplace (#4207) 2026-07-14 21:39:54 +08:00
NekoPunchandaias00 0d3f142849 fix: collector boot fails when jar manifest lacks Class-Path (#4190)
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-07-14 00:36:09 -07:00
37623cbc37 fix: pin collector docker image root path across versions (#4189)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-07-13 08:27:22 -07:00
Tomsun28andgithub-actions[bot] 2106bbb074 Update hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/observability/OtlpMetricsInventory.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
2026-07-13 22:46:33 +08:00
Tomsun28andgithub-actions[bot] 30076e6ea6 Update hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/observability/SignalPage.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
2026-07-13 22:46:24 +08:00
Tomsun28andgithub-actions[bot] cce7b7ac46 Update hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/observability/MetricSeries.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
2026-07-13 22:46:14 +08:00
shown 1980d9a611 feat: replace deprecated Spring AI Tool API (#4206)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
2026-07-13 22:39:23 +08:00
DuansgandClaude Fable 5 cc8206062f [fix] Fixed matching issues caused by date components and outdated time zone offsets (#4192)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-13 07:23:02 -07:00
Duansg 8b6d0d0056 [fix] carry job labels/annotations/metadata in timeout metrics data (#4205) 2026-07-13 04:00:00 -07:00
NekoPunchandDuansg 7d14ed80ba [fix] expose prestodb auth parameters and wire authorization into all metrics (#4188)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-07-12 23:46:00 -07:00
Duansg 2c376356ef Merge branch 'master' into feature/angular-three-signals 2026-07-13 14:13:42 +08:00
NekoPunch f69c0f7b10 docs: fix dead taosKeeper doc links (#4202) 2026-07-13 14:11:39 +08:00
Duansg 4cf968a3f4 [fix] Fix notice rule label matching against grouped alerts (#4191) 2026-07-12 08:34:45 -07:00
Duansg 9bd2796119 [fix] cancel the overwritten timeout in cyclicJob to fix HashedWheelTimeout memory leak (#4201) 2026-07-12 08:05:40 -07:00
shown 0734013a2a fix: avoid LLM generates sql has sql injection error (#4200)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
2026-07-12 22:45:19 +08:00
shown 6ec4958997 [fix] Prevent NPE when VictoriaMetrics compression is unset (#4199) 2026-07-12 17:25:02 +08:00
shown e57c728364 feat: fix ssh connection bug (#4198) 2026-07-12 16:56:05 +08:00
Logic 5d523ddce3 docs(pr): add Angular observability screenshots 2026-07-12 08:01:47 +08:00
Logic 1c0c881c58 test(log): isolate OTLP gRPC from storage e2e 2026-07-11 22:11:37 +08:00
Logic 46cce89d7e test(observability): allow licensed Greptime schema 2026-07-11 21:53:36 +08:00
Logic 45848ecdb5 Merge remote-tracking branch 'apache/feature/angular-three-signals' into feature/angular-three-signals 2026-07-11 21:49:21 +08:00
Logic 216e3f9d29 fix(ci): gate Greptime signal controllers 2026-07-11 21:47:22 +08:00
Duansgandgithub-actions[bot] 9f22a1cdd5 Update hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/observability/MetricPoint.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Duansg <siguoduan@gmail.com>
2026-07-11 20:57:30 +08:00
Duansg a899c7c518 Merge branch 'master' into feature/angular-three-signals 2026-07-11 20:55:46 +08:00
Duansg 265c3f1872 [fix] Fix startup_failure: pin third-party actions to ASF-allowlisted SHAs (#4193) 2026-07-11 17:03:45 +08:00
Logic a9222b2a78 test(observability): cover transition release contracts 2026-07-11 16:24:17 +08:00
Logic d5cb2e6660 feat(web): add Angular three-signal workbenches 2026-07-11 16:23:40 +08:00
Logic 1c03f830e5 feat(observability): add entity-free OTLP signal APIs 2026-07-11 16:23:08 +08:00
90f4c28b86 [collector]bugfix: close jdbc connection on statement creation failure (#4184)
Co-authored-by: hutiefang <hutiefang@qq.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Signed-off-by: hutiefang <hutiefang@qq.com>
2026-07-11 16:08:34 +08:00
LiuHanZhiandTomsun28 577ab10c5f docs: fix 13 typos in documentation and comments (#4187)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-08 22:31:26 +08:00
ef2672b06c [doc] fix typos and stale setup instructions across docs (#4186)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-08 22:05:27 +08:00
NekoPunchandDuansg c2c3095505 [monitoring-template] fix: correct malformed html tags in help text (#4185)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-07-08 21:54:47 +08:00
liutianyouandTomsun28 09a8bd809a [fix] upgrade the version of spring-ai. (#4176)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-04 18:43:16 +08:00
3c87b977ed [bugfix] Fix duplicate collector job timeouts (#4169)
Co-authored-by: hutiefang <hutiefang@qq.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-04 16:52:52 +08:00
hutiefang76andTomsun28 375ee5494e [collector]bugfix: honor XML response charset (#4167)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-04 14:27:37 +08:00
wilmerdooleyandTomsun28 a4dbf0b9bc [fix] persist custom monitor labels when writing to GreptimeDB (#4151)
Signed-off-by: wilmerdooley <wilmerdooley1@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-03 20:51:56 +08:00
2fd363dfdd [feat] add ntfy as a new alert notification channel (#4132)
Co-authored-by: lynx009 <2030509072@qq.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-03 00:30:57 +08:00
2389db7b3b [feat]:add db account monitoring metrics for postgresql. (#4131)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-07-02 23:44:56 +08:00
liutianyou 1b7f7f9733 [fix] fixed the issue of plugin upload failure. (#4179) 2026-06-29 11:13:42 +08:00
Duansgandaias00 ec5564b54f [fix] pre-validate monitor import batch to avoid orphan tasks and empty list (#4180)
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-06-29 09:54:43 +08:00
Duansgandaias00 87062df97d [fix] fix monitor import/export losing annotations and cron schedule … (#4166)
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-06-28 07:36:55 +08:00
Duansgandaias00 a54be93ff5 [fix] keep monitor selection across pages in monitor list (#4170)
Co-authored-by: aias00 <liuhongyu@apache.org>
2026-06-28 00:02:18 +08:00
Duansg 660bda3d0f [doc] update bulletin help docs for editing and metric-tree selection (#4177) 2026-06-27 23:59:54 +08:00
wilmerdooley 45416c475e fix: avoid blocking modify monitor by redundant detect call (#4154) 2026-06-22 12:47:16 +08:00
Duansg 31aa9b5168 [fix] fix monitor list showing previous page as unavailable when paging (#4161) 2026-06-18 18:37:52 +08:00
Duansg c785f6042d [fix] fixed an issue where cron jobs were not rescheduled (#4159) 2026-06-18 11:38:58 +08:00
Duansg bf6c451c94 [feat] support editing definitions and simplify metric selection (#4158) 2026-06-17 23:41:46 +08:00
d5cf683f4f [bugfix] Fix imported monitor instance being empty due to host/instan… (#4142)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: lynx009 <2030509072@qq.com>
2026-05-29 00:19:38 +08:00
6959441f84 fix(monitor): fix Redis Sentinel helpLink, sentinel metric types, add sentinel_timedout_scripts field (#4122)
Signed-off-by: abhyudayareddy <54602866+abhyudayareddy@users.noreply.github.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-05-28 23:48:15 +08:00
Duansg 42f2365e29 [fix] fix the dead link issue in ci (#4148) 2026-05-28 23:23:59 +08:00
Duansg 3496ebf497 [fix] fixed the JMX service URL validation issue (#4146) 2026-05-28 09:39:54 +08:00
70ef12d086 [monitoring-template] feat(sqlserver): add database account expiry metrics (#3737) (#4141)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-05-24 10:31:48 +08:00
Duansg 9d8924f9e4 [fix] fix Dockerfile directory creation error (#4143) 2026-05-22 15:42:06 +08:00
DuansgandTomsun28 ecb238804b [improve] Improve status page settings UI for better UX (#4139)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-05-22 14:30:34 +08:00
Duansgandzhusaidong 5cb6ee2eb8 [improve] Improve the styling of the monitoring list page (#4138)
Signed-off-by: zhusaidong <zhusaidong@gmail.com>
Co-authored-by: zhusaidong <zhusaidong@gmail.com>
2026-05-20 18:12:26 +08:00
675238b1dd [doc] update metrics storage docs to use duckdb instead of jpa (#4112)
Co-authored-by: shown <yuluo08290126@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-05-13 21:47:38 +08:00
zhusaidongandDuansg fcb5a9f08f bugfix Update delete dashboard response message to success (#4133)
Signed-off-by: zhusaidong <zhusaidong@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-05-07 14:22:19 +08:00
Duansg 4a2ddd2506 [improve] Improve LogRealTimeAlertE2eTest group alert flakiness (#4137) 2026-05-07 12:20:45 +08:00
Duansg 35cc62b844 [fix] Fix some issues related to ci (#4135) 2026-05-06 15:11:35 +08:00
Duansg 42eec4d559 [improve] Improvements to NVIDIA's monitoring history charts are not displayed (#4123) 2026-04-21 15:08:13 +08:00
aias00 e57d617db9 ci: reduce repeated setup and nightly rebuild work (#4119) 2026-04-16 18:27:57 +08:00
P_Peaceful 34dad832c3 [doc] add questdb init doc to sidebar and unify label (#4113) 2026-04-11 10:00:00 +08:00
P_PeacefulandTomsun28 50d7548748 [fix] support realtime metric and log edit tooltip (#4110)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-04-05 23:42:25 +08:00
P_Peaceful 87d55d8ad2 [doc] rename WeChat Public to WeChat Official Account (#4109) 2026-04-05 23:31:38 +08:00
P_Peaceful 4680ae547a [doc]: fix MD001 heading increment issues in home docs (#4106) 2026-04-02 23:57:43 +08:00
P_Peaceful c8209f18ad [doc] update markdownlint config to ignore MD060 (#4105) 2026-04-01 21:27:06 +08:00
shown c8021dc8d7 feat: add common protocol entity check (#4097)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
2026-04-01 08:15:57 +08:00
P_Peaceful e0145f6a4f [feature]: support victoria metrics cluster query executor (#4103) 2026-03-31 15:01:31 +08:00
pentium100andLogic 6d459957fb feat(nacos-sd): add auth and filtering parameters for Nacos service d… (#4099)
Co-authored-by: Logic <zqr10159@dromara.org>
2026-03-29 22:33:33 +08:00
b73f824e3c [Improve] Add common protoctl entity check (#4084)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-28 12:10:06 +08:00
2bb4f43494 [improve] optimize the function of creating monitoring using AI (#3922)
Signed-off-by: aias00 <liuhongyu@apache.org>
Co-authored-by: lynx009 <2030509072@qq.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: shown <yuluo08290126@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: manshi <lty496227@digital-engine.com>
2026-03-27 23:05:14 +08:00
P_Peaceful 695be421f2 [doc] improve e2e local run guide (#4095) 2026-03-27 16:25:00 +08:00
P_Peacefulandshown 75d6dee367 [ci]: update the dependency versions of GitHub Actions workflows (#4089)
Co-authored-by: shown <yuluo08290126@gmail.com>
2026-03-27 12:45:07 +08:00
Logic a819451011 docs: remove reddit and qq links from readmes (#4092) 2026-03-27 07:51:29 +08:00
P_Peaceful cdb6411b83 [improve] fix e2e test failure caused by api-testing reverse test behavior (#4091)
Signed-off-by: P_Peaceful <wenzhengcheng0223@163.com>
2026-03-25 23:24:07 +08:00
P_Peaceful cdb913c472 [CI] update Docker build-push-action version (#4088)
Signed-off-by: P_Peaceful <wenzhengcheng0223@163.com>
2026-03-24 23:11:47 +08:00
a4d2630320 [fix] fix cyclic silence time matching and cross-midnight handling (#4081)
Co-authored-by: yunfan24 <yunfan24@outlook.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-22 22:05:27 +08:00
DuansgandTomsun28 b9e07d81f9 [feat] add monitoring template for the LM Studio platform (#4082)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-22 17:58:02 +08:00
Duansg affccb7fb7 [fix] fixed issue with the token management style (#4083) 2026-03-22 11:23:13 +08:00
DuansgandTomsun28 3e6a7962f6 [feat] add API token management functionality (#4080)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-21 16:20:59 +08:00
922aa311e0 [feat]: add MySQL R2DBC query engine support (#4074)
Signed-off-by: Logic <zqr10159@dromara.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-21 16:12:17 +08:00
2016b18249 [docs]doc: fix typos and grammar in docs and AccountController (#4072)
Signed-off-by: Duansg <siguoduan@gmail.com>
Co-authored-by: shown <yuluo08290126@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-03-21 15:39:39 +08:00
zhluandDuansg 0297f0d4f8 [build]fix: remove duplicate version properties in root pom (#4073)
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-03-19 23:46:35 +08:00
5a98b56a5e [docs]doc: unify naming HertzBeat and agentless across docs and assets (#4075)
Co-authored-by: Logic <zqr10159@dromara.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-03-18 22:12:02 +08:00
Duansg bed2e4b3a0 [fix] fix zookeeper configuration not taking effect (#4077) 2026-03-18 21:19:02 +08:00
Duansg d1e28e7621 [improve] improve the hierarchical alignment of token issuance (#4076) 2026-03-18 09:06:07 +08:00
shownandDuansg 61c9b050b5 feat[common]: add core protoctl entity check (#4065)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2026-03-17 20:08:59 +08:00
LogicandTomsun28 f4858d5fcc [feat]: Native collector (#4066)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-17 09:30:08 +08:00
Duansg 07fd5b25c3 [improve] improve plugin upload validation (#4067) 2026-03-17 09:14:38 +08:00
DuansgandTomsun28 d16a95d8d6 [feat]Add monitoring support for Ollama LLM runtime (#4064)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-12 00:52:53 +08:00
shownandTomsun28 39567816e9 feat[common]: add protocol entity check (#4060)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2026-03-12 00:45:21 +08:00
Logic af56746de1 [feat]: migrate blocking executors to virtual threads (#4062) 2026-03-12 00:36:29 +08:00
shown d99e567494 feat: update nacos client to 3.1.1 (#4061)
Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>
2026-03-10 16:47:25 +08:00
2139 changed files with 37971 additions and 32557 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
{
"name": "Hertzbeat DevContainer",
"name": "HertzBeat DevContainer",
"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
"features": {
"ghcr.io/devcontainers/features/java:1": {
@@ -29,7 +29,7 @@
},
"portsAttributes": {
"4200": {
"label": "Hertzbeat Web",
"label": "HertzBeat Web",
"onAutoForward": "notify"
}
},
-1
View File
@@ -17,7 +17,6 @@
/home export-ignore
/hip export-ignore
/template-marketplace export-ignore
/.github export-ignore
/.idea export-ignore
/.devcontainer export-ignore
Binary file not shown.

After

Width:  |  Height:  |  Size: 194 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 315 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 282 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 262 KiB

+50 -8
View File
@@ -38,6 +38,10 @@ on:
- 'script/**'
- 'material/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
backend-build:
runs-on: ubuntu-latest
@@ -46,16 +50,58 @@ jobs:
- uses: ./script/ci/github-actions/setup-deps
- name: Build with Maven
run: mvnd clean -B package -Prelease -Dmaven.test.skip=false --file pom.xml
run: mvnd clean -B package -pl '!hertzbeat-e2e' -Prelease -Dmaven.test.skip=false --file pom.xml
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4.0.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
- name: Upload backend distribution
uses: actions/upload-artifact@v4
with:
name: backend-dist-${{ github.run_id }}
path: dist/
retention-days: 1
compression-level: 0
backend-maven-e2e:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./script/ci/github-actions/setup-deps
- name: Build backend Maven E2E modules
run: |
mvnd clean -B package \
-pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-log-e2e \
-am \
-Dmaven.test.skip=false \
--file pom.xml
- name: Upload Maven E2E coverage reports to Codecov
uses: codecov/codecov-action@v4.0.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
backend-image-e2e:
needs: backend-build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Download backend distribution
uses: actions/download-artifact@v4
with:
name: backend-dist-${{ github.run_id }}
path: dist/
- name: Build Image
uses: docker/build-push-action@v3
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./dist
file: ./script/docker/server/Dockerfile
@@ -64,12 +110,9 @@ jobs:
- name: Run E2E
run: |
sudo curl -L https://github.com/docker/compose/releases/download/v2.23.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
sudo chmod u+x /usr/local/bin/docker-compose
cd e2e
sudo docker-compose version
sudo docker-compose up --exit-code-from testing --remove-orphans
docker compose version
docker compose up --exit-code-from testing --remove-orphans
# upload application logs
- name: Upload logs & API test reports
@@ -80,4 +123,3 @@ jobs:
path: |
e2e/logs/
e2e/report/
@@ -0,0 +1,89 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Collector Native Release
run-name: Native collector release build (${{ github.ref_name }})
on:
workflow_dispatch:
jobs:
build-native-collector:
name: Native collector (${{ matrix.platform }})
permissions:
contents: read
timeout-minutes: 120
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- platform: linux-amd64
runner: ubuntu-24.04
archive_ext: tar.gz
- platform: linux-arm64
runner: ubuntu-24.04-arm
archive_ext: tar.gz
- platform: macos-amd64
runner: macos-13
archive_ext: tar.gz
- platform: macos-arm64
runner: macos-14
archive_ext: tar.gz
- platform: windows-amd64
runner: windows-latest
archive_ext: zip
steps:
- uses: actions/checkout@v4
- name: Set up GraalVM JDK 25
uses: graalvm/setup-graalvm@186d0493a2df5eb62df5ecc498883d18fd58c303 # v1.6.2
with:
distribution: graalvm-community
java-version: '25'
github-token: ${{ secrets.GITHUB_TOKEN }}
cache: maven
native-image-job-reports: 'true'
- name: Verify toolchain
shell: pwsh
run: |
java -version
native-image --version
mvn -version
- name: Build native collector package
run: mvn -B -pl hertzbeat-collector/hertzbeat-collector-collector -am -Pnative -DskipTests package
- name: Locate native collector package
id: package
shell: pwsh
run: |
$package = Get-ChildItem -Path "dist/apache-hertzbeat-collector-native-*-${{ matrix.platform }}-bin.${{ matrix.archive_ext }}" | Select-Object -First 1
if (-not $package) {
throw "Native collector package not found for ${{ matrix.platform }}"
}
"archive=$($package.FullName)" >> $env:GITHUB_OUTPUT
- name: Upload native collector package
uses: actions/upload-artifact@v4
with:
name: apache-hertzbeat-collector-native-${{ matrix.platform }}
path: ${{ steps.package.outputs.archive }}
retention-days: 14
+25 -7
View File
@@ -27,6 +27,10 @@ on:
paths:
- 'home/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
docs-build:
runs-on: ubuntu-latest
@@ -34,14 +38,27 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: pnpm/action-setup@v4
with:
version: 10
- uses: actions/setup-node@v4
with:
node-version: '20'
cache: pnpm
cache-dependency-path: home/pnpm-lock.yaml
- uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Check Markdown
uses: DavidAnson/markdownlint-cli2-action@992badcdf24e3b8eb7e87ff9287fe931bcb00c6e
uses: DavidAnson/markdownlint-cli2-action@8de2aa07cae85fd17c0b35642db70cf5495f1d25 # v24.0.0
with:
globs: './home/**/*.md'
globs: |
./home/README.md
./home/blog/**/*.md
./home/docs/**/*.md
./home/i18n/zh-cn/docusaurus-plugin-content-blog/**/*.md
./home/i18n/zh-cn/docusaurus-plugin-content-docs/current/**/*.md
- name: Check filenames
run: python ./script/ci/docs/check_file_name.py ./script/ci/docs/check_file_name.json
@@ -49,14 +66,15 @@ jobs:
- name: Dead Link Check
run: |
sudo npm install -g markdown-link-check@3.8.7
find ./home -name "*.md" > all_md_files.txt
find ./home/docs ./home/blog ./home/i18n/zh-cn/docusaurus-plugin-content-docs/current ./home/i18n/zh-cn/docusaurus-plugin-content-blog -name "*.md" > all_md_files.txt
printf '%s\n' ./home/README.md >> all_md_files.txt
grep -vFf ./script/ci/exclude_files.txt all_md_files.txt > to_check.txt
xargs -P 8 -a to_check.txt -I{} markdown-link-check -c ./script/ci/link_check.json -q "{}"
- name: NPM INSTALL
- name: PNPM INSTALL
working-directory: home
run: npm install
run: pnpm install
- name: NPM BUILD
- name: PNPM BUILD
working-directory: home
run: npm run build
run: pnpm run build
+6
View File
@@ -27,6 +27,10 @@ on:
# Allow manual workflow trigger
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
# Set permissions for deployment
permissions:
contents: write
@@ -48,6 +52,8 @@ jobs:
uses: actions/setup-node@v4
with:
node-version: '20'
cache: pnpm
cache-dependency-path: home/pnpm-lock.yaml
# Setup pnpm
- name: Setup pnpm
@@ -30,6 +30,10 @@ on:
paths:
- 'web-app/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-latest
@@ -38,6 +42,11 @@ jobs:
- uses: pnpm/action-setup@v4
with:
version: 10
- uses: actions/setup-node@v4
with:
node-version: '20'
cache: pnpm
cache-dependency-path: web-app/pnpm-lock.yaml
- name: Install
working-directory: web-app
run: pnpm install
+4
View File
@@ -26,6 +26,10 @@ on:
pull_request:
branches: [ master, dev ]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
check-license-header:
name: check-license-header
+1 -1
View File
@@ -44,7 +44,7 @@ jobs:
uses: actions/checkout@v4
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
with:
toolchain: ${{ env.RUST_VERSION }}
components: rustfmt, clippy
+9 -6
View File
@@ -27,6 +27,11 @@ on:
- cron: '0 1 * * *'
push:
branches: [ action* ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
check-license-header:
name: Monitor E2E Test
@@ -36,9 +41,9 @@ jobs:
- uses: actions/checkout@v4
- uses: ./script/ci/github-actions/setup-deps
- name: Build with Maven
run: mvn clean -B package -Prelease -Dmaven.test.skip=true --file pom.xml
run: mvnd -B clean package -Prelease -Dmaven.test.skip=true --file pom.xml
- name: Build Image
uses: docker/build-push-action@v3
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./dist
file: ./script/docker/server/Dockerfile
@@ -46,8 +51,6 @@ jobs:
tags: apache/hertzbeat:test
- name: Run K8s Monitor E2E Test
run: |
sudo curl -L https://github.com/docker/compose/releases/download/v2.23.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
sudo chmod u+x /usr/local/bin/docker-compose
cd e2e/k8s
sudo docker-compose version
sudo docker-compose up --exit-code-from testing --remove-orphans
docker compose version
docker compose up --exit-code-from testing --remove-orphans
+19 -13
View File
@@ -24,6 +24,10 @@ on:
push:
branches: [ action* ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-latest
@@ -34,39 +38,41 @@ jobs:
- uses: pnpm/action-setup@v4
with:
version: 10
- uses: actions/setup-node@v4
with:
node-version: '20'
cache: pnpm
cache-dependency-path: web-app/pnpm-lock.yaml
- name: Build the Frontend
run: |
cd web-app
pnpm install
pnpm package
- name: Build the Backend
run: |
mvn clean install
mvn clean package -Prelease -DskipTests
cd hertzbeat-collector
mvn clean package -Pcluster -DskipTests
mvnd -B clean package -Prelease,cluster -Dmaven.test.skip=false --file pom.xml
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a
- uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
- name: Log in to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and Push Server
uses: docker/build-push-action@v6
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./dist
file: ./script/docker/server/Dockerfile
platforms: linux/amd64,linux/arm64
push: true
tags: apache/hertzbeat:nightly
- name: Build and Push Collector
uses: docker/build-push-action@v6
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./dist
file: ./script/docker/collector/Dockerfile
+1 -1
View File
@@ -44,7 +44,7 @@ vscode:
ports:
- port: 4200
name: Hertzbeat
name: HertzBeat
onOpen: open-browser
- port: 1157
+9 -2
View File
@@ -15,7 +15,13 @@
// limitations under the License.
//
{
"globs": ["home/**/*.md"],
"globs": [
"home/README.md",
"home/blog/**/*.md",
"home/docs/**/*.md",
"home/i18n/zh-cn/docusaurus-plugin-content-blog/**/*.md",
"home/i18n/zh-cn/docusaurus-plugin-content-docs/current/**/*.md"
],
"config": {
"default": true,
"MD001": true,
@@ -38,7 +44,8 @@
"MD045": true,
"MD046": true,
"MD047": true,
"MD052": true
"MD052": true,
"MD060": false
},
"ignores": [
"**/node_modules/**",
+70 -70
View File
@@ -1,10 +1,10 @@
## Contributor Guide [中文贡献者指南](#贡献者指南)
## Contributor Guide [中文贡献者指南](#贡献者指南)
> We are committed to maintaining a happy community that helps each other, welcome every contributor to join us!
> We are committed to maintaining a happy community that helps each other, welcome every contributor to join us!
### Kinds of Contributions
### Kinds of Contributions
> In the HertzBeat community, there are many ways to contribute:
> In the HertzBeat community, there are many ways to contribute:
- 💻**Code**: Can help the community complete some tasks, write new features or fix some bugs;
@@ -20,19 +20,19 @@
- 💬**Suggestion**: You can also make some suggestions to the project or community to promote the healthy development of the community;
More see [Contribution Types](https://allcontributors.org/emoji-key)
More see [Contribution Types](https://allcontributors.org/en/reference/emoji-key/)
Even small corrections to typos are very welcome :)
Even small corrections to typos are very welcome :)
### Getting HertzBeat up and running
### Getting HertzBeat up and running
> To get HertzBeat code running on your development tools, and able to debug with breakpoints.
> This is a front-end and back-end separation project. To start the local code, the back-end [startup](hertzbeat-startup) and the front-end [web-app](web-app) must be started separately.
> This is a front-end and back-end separation project. To start the local code, the back-end [startup](hertzbeat-startup) and the front-end [web-app](web-app) must be started separately.
#### Backend start
1. Requires `maven3+`, `java21` and `lombok` environments
1. Requires `maven3+`, `java25` and `lombok` environments
2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml`
@@ -57,51 +57,51 @@ Even small corrections to typos are very welcome :)
6. Browser access to localhost:4200 to start, default account/password is *admin/hertzbeat*
### Find tasks
### Find tasks
Find the issue you are interested in! On our GitHub repo issue list, we often publish some issues with the label good first issue or status: volunteer wanted.
These issues welcome the help of contributors. Among them, good first issues tend to have low thresholds and are suitable for novices.
Find the issue you are interested in! On our GitHub repo issue list, we often publish some issues with the label good first issue or status: volunteer wanted.
These issues welcome the help of contributors. Among them, good first issues tend to have low thresholds and are suitable for novices.
Of course, if you have a good idea, you can also propose it directly on GitHub Discussion or contact with community.
Of course, if you have a good idea, you can also propose it directly on GitHub Discussion or contact with community.
### Submit Pull Request
### Submit Pull Request
1. First you need to fork your target [hertzbeat repository](https://github.com/apache/hertzbeat).
1. First you need to fork your target [hertzbeat repository](https://github.com/apache/hertzbeat).
2. Then download the code locally with git command:
```shell
git clone git@github.com:${YOUR_USERNAME}/hertzbeat.git #Recommended
git clone git@github.com:${YOUR_USERNAME}/hertzbeat.git #Recommended
```
3. After the download is complete, please refer to the getting started guide or README file of the target repository to initialize the project.
3. After the download is complete, please refer to the getting started guide or README file of the target repository to initialize the project.
4. Then, you can refer to the following command to submit the code:
```shell
git checkout -b a-feature-branch #Recommended
git checkout -b a-feature-branch #Recommended
```
5. Submit the coed as a commit, the commit message format specification required: [module name or type name]feature or bugfix or doc: custom message.
5. Submit the code as a commit, the commit message format specification required: [module name or type name]feature or bugfix or doc: custom message.
```shell
git add <modified file/path>
git commit -m '[docs]feature: necessary instructions' #Recommended
git add <modified file/path>
git commit -m '[docs]feature: necessary instructions' #Recommended
```
6. Push to the remote repository
6. Push to the remote repository
```shell
git push origin a-feature-branch
git push origin a-feature-branch
```
7. Then you can initiate a new PR (Pull Request) on GitHub.
7. Then you can initiate a new PR (Pull Request) on GitHub.
Please note that the title of the PR needs to conform to our spec, and write the necessary description in the PR to facilitate code review by Committers and other contributors.
Please note that the title of the PR needs to conform to our spec, and write the necessary description in the PR to facilitate code review by Committers and other contributors.
### Wait for the code to be merged
### Wait for the code to be merged
After submitting the PR, the Committee or the community's friends will review the code you submitted (Code Review), and will propose some modification suggestions or conduct some discussions. Please pay attention to your PR in time.
After submitting the PR, the Committee or the community's friends will review the code you submitted (Code Review), and will propose some modification suggestions or conduct some discussions. Please pay attention to your PR in time.
If subsequent changes are required, there is no need to initiate a new PR. After submitting a commit on the original branch and pushing it to the remote repository, the PR will be automatically updated.
If subsequent changes are required, there is no need to initiate a new PR. After submitting a commit on the original branch and pushing it to the remote repository, the PR will be automatically updated.
In addition, our project has a relatively standardized and strict CI inspection process. After submitting PR, CI will be triggered. Please pay attention to whether it passes the CI inspection.
In addition, our project has a relatively standardized and strict CI inspection process. After submitting PR, CI will be triggered. Please pay attention to whether it passes the CI inspection.
Finally, the Committers can merge the PR into the master branch.
Finally, the Committers can merge the PR into the master branch.
### After the code is merged
### After the code is merged
After the code has been merged, you can delete the development branch on both the local and remote repositories:
After the code has been merged, you can delete the development branch on both the local and remote repositories:
```shell
git branch -d a-dev-branch
@@ -112,18 +112,18 @@ On the master/main branch, you can do the following to sync the upstream reposit
```shell
git remote add upstream https://github.com/apache/hertzbeat.git #Bind the remote warehouse, if it has been executed, it does not need to be executed again
git checkout master
git checkout master
git pull upstream master
```
### HertzBeat Improvement Proposal (HIP)
If you have major new features(e.g., support metrics push gateway, support logs monitoring), you need to write a design document known as a HertzBeat Improvement Proposal (HIP). Before starting to write a HIP, make sure you follow the process [here](https://github.com/apache/hertzbeat/tree/master/hip).
### How to become a Committer?
### How to become a Committer?
With the above steps, you are a contributor to HertzBeat. Repeat the previous steps to stay active in the community, keep at, you can become a Committer!
With the above steps, you are a contributor to HertzBeat. Repeat the previous steps to stay active in the community, keep at, you can become a Committer!
### Join Discussion
### Join Discussion
[Join the Mailing Lists](https://lists.apache.org/list.html?dev@hertzbeat.apache.org) : Mail to `dev-subscribe@hertzbeat.apache.org` to subscribe mailing lists.
@@ -133,19 +133,19 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
<br>
<br>
------
<br>
## 贡献者指南
## 贡献者指南
> 非常欢迎参与项目贡献,我们致力于维护一个互相帮助的快乐社区。
> 非常欢迎参与项目贡献,我们致力于维护一个互相帮助的快乐社区。
### 贡献方式
### 贡献方式
> 在 HertzBeat 社区,贡献方式有很多:
> 在 HertzBeat 社区,贡献方式有很多:
- 💻**代码**:可以帮助社区完成一些任务、编写新的feature或者是修复一些bug;
@@ -159,20 +159,20 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
- 💡**布道**:可以帮助宣传或推广 HertzBeat 社区,在 meetup 或 summit 中演讲;
- 💬**建议**:也可以对项目或者社区提出一些建议,促进社区的良性发展;
- 💬**建议**:也可以对项目或者社区提出一些建议,促进社区的良性发展;
更多贡献方式参见 [Contribution Types](https://allcontributors.org/emoji-key)
更多贡献方式参见 [Contribution Types](https://allcontributors.org/en/reference/emoji-key/)
即便是小到错别字的修正我们也都非常欢迎 :)
即便是小到错别字的修正我们也都非常欢迎 :)
### 让 HertzBeat 运行起来
### 让 HertzBeat 运行起来
> 让 HertzBeat 的代码在您的开发工具上运行起来,并且能够断点调试。
> 让 HertzBeat 的代码在您的开发工具上运行起来,并且能够断点调试。
> 此为前后端分离项目,本地代码启动需将后端[hertzbeat-startup](hertzbeat-startup)和前端[web-app](web-app)分别启动生效。
#### 后端启动
1. 需要 `maven3+`, `java21` 和 `lombok` 环境
1. 需要 `maven3+`, `java25` 和 `lombok` 环境
2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml`
@@ -197,71 +197,71 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
6. 浏览器访问 localhost:4200 即可开始,默认账号密码 *admin/hertzbeat*
### 寻找任务
### 寻找任务
寻找您感兴趣的Issue!在我们的GitHub仓库和邮件列表中,我们经常会发布一些带有 good first issue 或者 status: volunteer wanted 标签的issue,这些issue都欢迎贡献者的帮助。
其中good first issue往往门槛较低、适合新手。
其中good first issue往往门槛较低、适合新手。
当然,如果您有好的想法,也可以直接在GitHub Discussion 中提出或者联系社区。
当然,如果您有好的想法,也可以直接在GitHub Discussion 中提出或者联系社区。
### 提交 Pull Request
1. 首先您需要 Fork 目标仓库 [hertzbeat repository](https://github.com/apache/hertzbeat).
2. 然后 用git命令 将代码下载到本地:
2. 然后 用git命令 将代码下载到本地:
```shell
git clone git@github.com:${YOUR_USERNAME}/hertzbeat.git #Recommended
git clone git@github.com:${YOUR_USERNAME}/hertzbeat.git #Recommended
```
3. 下载完成后,请参考目标仓库的入门指南或者 README 文件对项目进行初始化。
4. 接着,您可以参考如下命令进行代码的提交, 切换新的分支, 进行开发:
4. 接着,您可以参考如下命令进行代码的提交, 切换新的分支, 进行开发:
```shell
git checkout -b a-feature-branch #Recommended
git checkout -b a-feature-branch #Recommended
```
5. 提交 commit , commit 描述信息需要符合约定格式: [module name or type name]feature or bugfix or doc: custom message.
5. 提交 commit , commit 描述信息需要符合约定格式: [module name or type name]feature or bugfix or doc: custom message.
```shell
git add <modified file/path>
git commit -m '[docs]feature: necessary instructions' #Recommended
git add <modified file/path>
git commit -m '[docs]feature: necessary instructions' #Recommended
```
6. 推送到远程仓库
6. 推送到远程仓库
```shell
git push origin a-feature-branch
git push origin a-feature-branch
```
7. 然后您就可以在 GitHub 上发起新的 PR (Pull Request)。
请注意 PR 的标题需要符合我们的规范,并且在 PR 中写上必要的说明,来方便 Committer 和其他贡献者进行代码审查。
请注意 PR 的标题需要符合我们的规范,并且在 PR 中写上必要的说明,来方便 Committer 和其他贡献者进行代码审查。
### 等待PR代码被合并
### 等待PR代码被合并
在提交了 PR 后,Committer 或者社区的小伙伴们会对您提交的代码进行审查(Code Review),会提出一些修改建议,或者是进行一些讨论,请及时关注您的PR。
在提交了 PR 后,Committer 或者社区的小伙伴们会对您提交的代码进行审查(Code Review),会提出一些修改建议,或者是进行一些讨论,请及时关注您的PR。
若后续需要改动,不需要发起一个新的 PR,在原有的分支上提交 commit 并推送到远程仓库后,PR会自动更新。
另外,我们的项目有比较规范和严格的 CI 检查流程,在提交 PR 之后会触发 CI,请注意是否通过 CI 检查。
最后,Committer 可以将 PR 合并入 master 主分支。
最后,Committer 可以将 PR 合并入 master 主分支。
### 代码被合并后
### 代码被合并后
在代码被合并后,您就可以在本地和远程仓库删除这个开发分支了:
在代码被合并后,您就可以在本地和远程仓库删除这个开发分支了:
```shell
git branch -d a-dev-branch
git push origin --delete a-dev-branch
```
在主分支上,您可以执行以下操作来同步上游仓库:
在主分支上,您可以执行以下操作来同步上游仓库:
```shell
git remote add upstream https://github.com/apache/hertzbeat.git #Bind the remote warehouse, if it has been executed, it does not need to be executed again
git checkout master
git checkout master
git pull upstream master
```
### HertzBeat 改进提案 (HIP)
如果您有重大的新特性(例如,支持指标推送网关,日志监控等),您需要编写一个被称为 HertzBeat 改进提案(HertzBeat Improvement ProposalHIP)的设计文档。在开始编写 HIP 之前,请确保您遵循了[这里](https://github.com/apache/hertzbeat/tree/master/hip)的流程。
### 如何成为Committer
### 如何成为Committer
通过上述步骤,您就是 HertzBeat 的贡献者了。重复前面的步骤,在社区中保持活跃,坚持下去,您就能成为 Committer!
通过上述步骤,您就是 HertzBeat 的贡献者了。重复前面的步骤,在社区中保持活跃,坚持下去,您就能成为 Committer!
### 加入讨论交流
@@ -271,4 +271,4 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
### 模块
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
+36 -39
View File
@@ -10,7 +10,6 @@
</p>
[![Discord](https://img.shields.io/badge/Chat-Discord-7289DA?logo=discord)](https://discord.gg/Fb6M73htGr)
[![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
[![Twitter](https://img.shields.io/twitter/follow/hertzbeat1024?logo=twitter)](https://x.com/hertzbeat1024)
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8139/badge)](https://www.bestpractices.dev/projects/8139)
[![codecov](https://codecov.io/gh/apache/HertzBeat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/apache/hertzbeat)
@@ -21,13 +20,13 @@
[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/apache/hertzbeat)
[![zread](https://img.shields.io/badge/Ask_Zread-_.svg?style=flat&color=00b0aa&labelColor=000000&logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iMTYiIHZpZXdCb3g9IjAgMCAxNiAxNiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTQuOTYxNTYgMS42MDAxSDIuMjQxNTZDMS44ODgxIDEuNjAwMSAxLjYwMTU2IDEuODg2NjQgMS42MDE1NiAyLjI0MDFWNC45NjAxQzEuNjAxNTYgNS4zMTM1NiAxLjg4ODEgNS42MDAxIDIuMjQxNTYgNS42MDAxSDQuOTYxNTZDNS4zMTUwMiA1LjYwMDEgNS42MDE1NiA1LjMxMzU2IDUuNjAxNTYgNC45NjAxVjIuMjQwMUM1LjYwMTU2IDEuODg2NjQgNS4zMTUwMiAxLjYwMDEgNC45NjE1NiAxLjYwMDFaIiBmaWxsPSIjZmZmIi8%2BCjxwYXRoIGQ9Ik00Ljk2MTU2IDEwLjM5OTlIMi4yNDE1NkMxLjg4ODEgMTAuMzk5OSAxLjYwMTU2IDEwLjY4NjQgMS42MDE1NiAxMS4wMzk5VjEzLjc1OTlDMS42MDE1NiAxNC4xMTM0IDEuODg4MSAxNC4zOTk5IDIuMjQxNTYgMTQuMzk5OUg0Ljk2MTU2QzUuMzE1MDIgMTQuMzk5OSA1LjYwMTU2IDE0LjExMzQgNS42MDE1NiAxMy43NTk5VjExLjAzOTlDNS42MDE1NiAxMC42ODY0IDUuMzE1MDIgMTAuMzk5OSA0Ljk2MTU2IDEwLjM5OTlaIiBmaWxsPSIjZmZmIi8%2BCjxwYXRoIGQ9Ik0xMy43NTg0IDEuNjAwMUgxMS4wMzg0QzEwLjY4NSAxLjYwMDEgMTAuMzk4NCAxLjg4NjY0IDEwLjM5ODQgMi4yNDAxVjQuOTYwMUMxMC4zOTg0IDUuMzEzNTYgMTAuNjg1IDUuNjAwMSAxMS4wMzg0IDUuNjAwMUgxMy43NTg0QzE0LjExMTkgNS42MDAxIDE0LjM5ODQgNS4zMTM1NiAxNC4zOTg0IDQuOTYwMVYyLjI0MDFDMTQuMzk4NCAxLjg4NjY0IDE0LjExMTkgMS42MDAxIDEzLjc1ODQgMS42MDAxWiIgZmlsbD0iI2ZmZiIvPgo8cGF0aCBkPSJNNCAxMkwxMiA0TDQgMTJaIiBmaWxsPSIjZmZmIi8%2BCjxwYXRoIGQ9Ik00IDEyTDEyIDQiIHN0cm9rZT0iI2ZmZiIgc3Ryb2tlLXdpZHRoPSIxLjUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L3N2Zz4K&logoColor=ffffff)](https://zread.ai/apache/hertzbeat)
**Home: [hertzbeat.apache.org](https://hertzbeat.apache.org)**
**Home: [hertzbeat.apache.org](https://hertzbeat.apache.org)**
**Email: <a href="mailto:dev-subscribe@hertzbeat.apache.org">Mail to `dev-subscribe@hertzbeat.apache.org`</a> to subscribe mailing lists**
## 🎡 <font color="green">Introduction</font>
[Apache HertzBeat™](https://github.com/apache/hertzbeat) is an AI-powered next-generation open source real-time observability system. Unified metrics and logs collection, centralized alerting distribution, intelligent management and analysis. No Agent required, high performance cluster, provides powerful custom monitoring and status page building capabilities.
[Apache HertzBeat™](https://github.com/apache/hertzbeat) is an AI-powered next-generation open source real-time observability system. Unified metrics and logs collection, centralized alerting distribution, intelligent management and analysis. No Agent required, high performance cluster, provides powerful custom monitoring and status page building capabilities.
### Features
@@ -40,17 +39,17 @@
- High performance, supports horizontal expansion of multi-collector clusters, multi-isolated network monitoring and cloud-edge collaboration.
- Provides powerful status page building capabilities, easily communicate the real-time status of your service to users.
> HertzBeat's unified platform, AI intelligence, powerful customization, multi-type support, high performance, and easy expansion, aims to help users quickly and conveniently achieve observability requirements.
> HertzBeat's unified platform, AI intelligence, powerful customization, multi-type support, high performance, and easy expansion, aims to help users quickly and conveniently achieve observability requirements.
----
----
## 🥐 Architecture
## 🥐 Architecture
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
![HertzBeat](home/static/img/docs/hertzbeat-arch.png)
## ⛄ Supported
## ⛄ Supported
> We define all metrics collection types such as `mysql`, `jvm`, and `k8s` as `YML` templates, allowing users to import them to support corresponding types of monitoring.
> Welcome everyone to contribute your customized general metrics type YML template during use.
@@ -93,18 +92,18 @@
[HuaweiSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-huawei_switch.yml), [TpLinkSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-tplink_switch.yml),
[H3cSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-h3c_switch.yml)
- And More Your Custom Template.
- Notified Support `Discord` `Slack` `Telegram` `Email` `Dingtalk` `WeChat` `FeiShu` `Webhook` `SMS` `ServerChan`.
- Notified Support `Discord` `Slack` `Telegram` `Email` `DingTalk` `WeChat` `FeiShu` `Webhook` `SMS` `ServerChan`.
## 🐕 Quick Start
## 🐕 Quick Start
- If you wish to deploy HertzBeat locally, please refer to the following Deployment Documentation for instructions.
### 🍞 Install HertzBeat
### 🍞 Install HertzBeat
> HertzBeat supports installation through source code, docker or package, cpu support x86/arm64.
> HertzBeat supports installation through source code, docker or package, cpu support x86/arm64.
##### 1Install quickly via docker
##### 1Install quickly via docker
1. Just one command to get started
@@ -112,7 +111,7 @@
docker run -d -p 1157:1157 -p 1158:1158 --name hertzbeat apache/hertzbeat
```
2. Access `http://localhost:1157` to start, default account: `admin/hertzbeat`
2. Access `http://localhost:1157` to start, default account: `admin/hertzbeat`
3. Deploy collector clusters (Optional)
@@ -126,16 +125,16 @@
- `-e MANAGER_PORT=1158` : set the main hertzbeat server port, default 1158.
Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache.org/docs/start/docker-deploy)
Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache.org/docs/start/docker-deploy)
##### 2Install via package
##### 2Install via package
1. Download the release package `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional)
3. Run command `$ ./bin/startup.sh ` or `bin/startup.bat`
4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat`
1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional)
3. Run command `$ ./bin/startup.sh ` or `bin/startup.bat`
4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat`
5. Deploy collector clusters (Optional)
- Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download](https://hertzbeat.apache.org/docs/download)
- Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download](https://hertzbeat.apache.org/docs/download)
- Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port`
```yaml
collector:
@@ -148,25 +147,27 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache
manager-host: ${MANAGER_HOST:127.0.0.1}
manager-port: ${MANAGER_PORT:1158}
```
- Run command `$ ./bin/startup.sh ` or `bin/startup.bat`
- Access `http://localhost:1157` and you will see the registered new collector in dashboard
- If you do not provide JDBC drivers in `ext-lib`, MySQL, MariaDB, and OceanBase can use the built-in query engine and run on the native collector package as well. TiDB follows the same rule for its SQL query metric set.
- If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC after restart for MySQL, MariaDB, and OceanBase. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics are unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers.
- Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package.
- Access `http://localhost:1157` and you will see the registered new collector in dashboard
Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apache.org/docs/start/package-deploy)
Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apache.org/docs/start/package-deploy)
##### 3Start via source code
##### 3Start via source code
1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`.
2. Backendneed `maven3+`, `java21`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service.
3. Webneed `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup.
4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat`
1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`.
2. Backendneed `maven3+`, `java25`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service.
3. Webneed `nodejs` and `pnpm` environment, run `pnpm install` then `pnpm start` in `web-app` directory after backend startup.
4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat`
Detailed steps refer to [CONTRIBUTING](CONTRIBUTING.md)
Detailed steps refer to [CONTRIBUTING](CONTRIBUTING.md)
##### 4Install All(hertzbeat+postgresql+tsdb) via Docker-compose
##### 4Install All(hertzbeat+postgresql+tsdb) via Docker-compose
Install the postgresql/mysql database, victoria-metrics/iotdb/tdengine database and hertzbeat at one time through [docker-compose deployment script](script/docker-compose).
Detailed steps refer to [Install via Docker-Compose](script/docker-compose/README.md)
Detailed steps refer to [Install via Docker-Compose](script/docker-compose/README.md)
##### 5: Install All(hertzbeat+collector+postgresql+tsdb) via kubernetes helm charts
@@ -178,8 +179,8 @@ Detailed steps refer to [Artifact Hub](https://artifacthub.io/packages/helm/hert
## ✨ Contributors
Thanks to these wonderful people, welcome to join us:
[Contributor Guide](CONTRIBUTING.md)
Thanks to these wonderful people, welcome to join us:
[Contributor Guide](CONTRIBUTING.md)
<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
<!-- prettier-ignore-start -->
@@ -557,7 +558,7 @@ Thanks to these wonderful people, welcome to join us:
<!-- markdownlint-restore -->
<!-- prettier-ignore-end -->
<!-- ALL-CONTRIBUTORS-LIST:END -->
<!-- ALL-CONTRIBUTORS-LIST:END -->
## 💬 Join discussion
@@ -569,14 +570,10 @@ Thanks to these wonderful people, welcome to join us:
WeChat Group : Add friend `ahertzbeat` and invite to the group.
WeChat Public : Search ID `usthecom`.
[QQ Group](https://qm.qq.com/q/xxqecSC2cw) : Group num `1035688434`
WeChat Official Account : Search ID `usthecom`.
[Github Discussion](https://github.com/apache/hertzbeat/discussions)
[Reddit Community](https://www.reddit.com/r/hertzbeat/)
[Follow Us Twitter](https://x.com/hertzbeat1024)
[Subscribe YouTube](https://www.youtube.com/channel/UCri75zfWX0GHqJFPENEbLow)
+7 -10
View File
@@ -10,7 +10,6 @@
</p>
[![Discord](https://img.shields.io/badge/Chat-Discord-7289DA?logo=discord)](https://discord.gg/Fb6M73htGr)
[![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
[![Twitter](https://img.shields.io/twitter/follow/hertzbeat1024?logo=twitter)](https://x.com/hertzbeat1024)
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8139/badge)](https://www.bestpractices.dev/projects/8139)
[![codecov](https://codecov.io/gh/apache/HertzBeat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/apache/hertzbeat)
@@ -127,12 +126,12 @@
##### 方式二:通过安装包安装
1. 下载您系统环境对应的安装包 `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml` (可选)
3. 部署启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat`
4. 浏览器访问 `http://localhost:1157` 即可开始,默认账号密码 `admin/hertzbeat`
5. 部署采集器集群(可选)
- 下载采集器安装包 `hertzbeat-collector-xx.tar.gz` 到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download)
- 下载采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`(JVM 采集器)或与你目标平台匹配的 Native 采集器安装包,例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download)
- 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主 HertzBeat 服务的对外 IP,端口,当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port`
```yaml
collector:
@@ -145,7 +144,9 @@
manager-host: ${MANAGER_HOST:127.0.0.1}
manager-port: ${MANAGER_PORT:1158}
```
- 启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat`
- 如果没有在 `ext-lib` 中提供 JDBC 驱动,MySQL、MariaDB、OceanBase 可以直接使用内置查询引擎,也可以使用 Native 采集器安装包;TiDB 的 SQL 查询指标也遵循同样规则。
- 如果在 `ext-lib` 中放入了 `mysql-connector-j`,主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC;这一点现在适用于 MySQL、MariaDB、OceanBaseTiDB 的 SQL 查询指标也遵循同样规则,而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包,因为它们依赖外置 JDBC 驱动。
- JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动,Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动
- 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器
更多配置详细步骤参考 [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy)
@@ -153,8 +154,8 @@
##### 方式三:本地代码启动
1. 此为前后端分离项目,本地代码调试需要分别启动后端工程 `hertzbeat-startup` 和前端工程 `web-app`
2. 后端:需要 `maven3+`, `java21` 和 `lombok` 环境,修改 `YML` 配置信息,添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。
3. 前端:需要 `nodejs npm angular-cli`环境,待本地后端启动后,在 `web-app` 目录下启动 `ng serve --open`
2. 后端:需要 `maven3+`, `java25` 和 `lombok` 环境,修改 `YML` 配置信息,添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。
3. 前端:需要 `nodejs` 和 `pnpm` 环境,待本地后端启动后,在 `web-app` 目录下执行 `pnpm install` 再执行 `pnpm start`
4. 浏览器访问 `http://localhost:4200` 即可开始,默认账号密码 `admin/hertzbeat`
详细步骤参考 [参与贡献之本地代码启动](CONTRIBUTING.md)
@@ -568,12 +569,8 @@ Thanks these wonderful people, welcome to join us:
微信公众号 : 搜索 ID `usthecom`.
[QQ交流群](https://qm.qq.com/q/xxqecSC2cw) : 群号 `1035688434`
[Github Discussion](https://github.com/apache/hertzbeat/discussions)
[Reddit Community](https://www.reddit.com/r/hertzbeat/)
[Follow Us Twitter](https://x.com/hertzbeat1024)
[Subscribe YouTube](https://www.youtube.com/channel/UCri75zfWX0GHqJFPENEbLow)
+18 -21
View File
@@ -10,7 +10,6 @@
</p>
[![Discord](https://img.shields.io/badge/Chat-Discord-7289DA?logo=discord)](https://discord.gg/Fb6M73htGr)
[![Reddit](https://img.shields.io/badge/Reddit-Community-7289DA?logo=reddit)](https://www.reddit.com/r/hertzbeat/)
[![Twitter](https://img.shields.io/twitter/follow/hertzbeat1024?logo=twitter)](https://x.com/hertzbeat1024)
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8139/badge)](https://www.bestpractices.dev/projects/8139)
[![codecov](https://codecov.io/gh/apache/HertzBeat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/apache/hertzbeat)
@@ -91,7 +90,7 @@
[HuaweiSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-huawei_switch.yml), [TpLinkSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-tplink_switch.yml),
[H3cSwitch](https://raw.githubusercontent.com/apache/hertzbeat/master/hertzbeat-manager/src/main/resources/define/app-h3c_switch.yml)
- その他、カスタマイズされたモニターテンプレート。
- 通知利用可能 `Discord` `Slack` `Telegram` `Email` `Dingtalk` `WeChat` `FeiShu` `Webhook` `SMS` `ServerChan`
- 通知利用可能 `Discord` `Slack` `Telegram` `Email` `DingTalk` `WeChat` `FeiShu` `Webhook` `SMS` `ServerChan`
## 🐕 クイックスタート
@@ -108,7 +107,7 @@
docker run -d -p 1157:1157 -p 1158:1158 --name hertzbeat apache/hertzbeat
```
2. スタート:`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード:`admin/hertzbeat`。
2. スタート:`http://localhost:1157`にアクセスします。デフォルトのアカウントとパスワード:`admin/hertzbeat`。
3. コレクタークラスタのデプロイメント(オプション)
@@ -122,17 +121,17 @@
- `-e MANAGER_PORT=1158` :メインhertzbeatサーバポート。
詳細ステップ [通过 Docker 方式安装 HertzBeat](https://hertzbeat.apache.org/docs/start/docker-deploy)
詳細ステップ [Docker HertzBeat をインストール](https://hertzbeat.apache.org/docs/start/docker-deploy)
##### 方式2:インストールパッケージ
1. リリースパッケージをダウンロード `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
2. HertzBeatのymlファイルを設定 `hertzbeat/config/application.yml` (オプション
3. コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行
4. スタート:`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード`admin/hertzbeat`
1. リリースパッケージ `apache-hertzbeat-xx-bin.tar.gz` をダウンロードします [Download](https://hertzbeat.apache.org/docs/download)
2. HertzBeat の設定ファイル `hertzbeat/config/application.yml` を編集します(任意
3. コマンド `$ ./bin/startup.sh` または `bin/startup.bat` を実行します
4. ブラウザで `http://localhost:1157` にアクセスします。デフォルトのアカウントとパスワード`admin/hertzbeat` です
5. コレクタークラスタのデプロイメント(オプション)
- コレクターパッケージを別のホストにダウンロード `hertzbeat-collector-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download)
- コレクターのymlファイルを設定 `hertzbeat-collector/config/application.yml`
- 別ホストにコレクターのインストールパッケージ `apache-hertzbeat-collector-xx-bin.tar.gz`(JVM コレクター)または対象プラットフォーム向けの Native コレクターパッケージ(例: `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`)をダウンロードします [Download](https://hertzbeat.apache.org/docs/download)
- コレクターの設定ファイル `hertzbeat-collector/config/application.yml` を編集します
```yaml
collector:
dispatch:
@@ -148,25 +147,27 @@
- `mode: ${MODE:public}`:実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。
- `manager-host: ${MANAGER_HOST:127.0.0.1}`:メインhertzbeatサーバーのIP。
- `manager-port: ${MANAGER_PORT:1158}`:メインhertzbeatサーバポート。
- コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行
- `http://localhost:1157`にアクセスし、登録された新しいコレクターを見ることがでます
- `ext-lib` に JDBC ドライバーを置かない場合、MySQL、MariaDB、OceanBase は組み込みのクエリエンジンを使って Native コレクターパッケージでも監視できます。TiDB も SQL クエリのメトリクスセットについては同じルールです
- `ext-lib` に `mysql-connector-j` を置いた場合は、再起動後に組み込みサーバーコレクターまたは JVM コレクターが MySQL、MariaDB、OceanBase で自動的に JDBC を優先します。TiDB も SQL クエリのメトリクスセットについては同じルールで、HTTP メトリクスは影響を受けません。Oracle と DB2 は引き続き外部 JDBC ドライバーに依存するため、JVM コレクターパッケージを使用してください
- JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。
- メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。
詳細ステップ [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy)
詳細ステップ [インストールパッケージで HertzBeat をインストール](https://hertzbeat.apache.org/docs/start/package-deploy)
##### 方式3:ローカルの実行
1. ローカルの実行には、バックエンドのプロジェクト`hertzbeat-startup`とフロントエンドのプロジェクト`web-app`を起動する必要があります。
2. バックエンド:`maven3+`、`Java21``lombok`の環境必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し`hertzbeat-startup` を起動します。
3. フロントエンド:`nodejs npm angular-cli`の環境必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `ng serve --open` というコマンドを実行します。
2. バックエンド:`maven3+`、`Java25``lombok` の環境必要です。`YML` 設定を修正し、Java 仮想マシンパラメータに `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し`hertzbeat-startup` を起動します。
3. フロントエンド:`nodejs` と `pnpm` の環境必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `pnpm install` を実行し、続けて `pnpm start` を実行します。
4. スタート:`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード:`admin/hertzbeat`。
詳細ステップ [貢献ガイド](CONTRIBUTING.md)
##### 方式4:Docker-Compose
[Docker-Compose 部署脚本](script/docker-compose)でpostgresql/mysqlデータベース、victoria-metrics、iotdb、またはtdengine時系列データベースとHertzbeat一括デプロイ。
[Docker-Compose 部署脚本](script/docker-compose)でpostgresql/mysqlデータベース、victoria-metrics、iotdb、またはtdengine時系列データベースとHertzBeat一括デプロイ。
詳細ステップ [通过 Docker-Compose 安装 HertzBeat](script/docker-compose/README.md)
詳細ステップ [Docker-Compose HertzBeat をインストール](script/docker-compose/README.md)
##### 方式5:Kubernetes Helm Charts
@@ -571,12 +572,8 @@ WeChatグループ : `ahertzbeat` を検索.
WeChat公式アカウント : `usthecom`を検索.
[QQグループ](https://qm.qq.com/q/xxqecSC2cw) : グループ番号 `1035688434`
[Github Discussion](https://github.com/apache/hertzbeat/discussions)
[Reddit Community](https://www.reddit.com/r/hertzbeat/)
[Follow Us Twitter](https://x.com/hertzbeat1024)
[Subscribe YouTube](https://www.youtube.com/channel/UCri75zfWX0GHqJFPENEbLow)
+28 -7
View File
@@ -1,14 +1,35 @@
Please add the corresponding e2e (aka end-to-end) test cases if you add or update APIs.
## How to work
* Start and watch the [docker-compose](https://docs.docker.com/compose/) via [the script](script/start.sh)
* Run the e2e testing via [api-testing](https://github.com/LinuxSuRen/api-testing)
* It will run the test cases from top to bottom
* You can add the necessary asserts to it
* Start and watch the [Docker Compose](https://docs.docker.com/compose/) environment
* The Compose file is `e2e/docker-compose.yml`
* The `testing` container uses [api-testing](https://github.com/LinuxSuRen/api-testing)
* The main test definition file is `e2e/testsuite.yaml`
* Before starting the suite locally, build the `apache/hertzbeat:test` image from the repository root:
```bash
mvn clean -B package -Prelease -Dmaven.test.skip=true --file pom.xml
docker build -t apache/hertzbeat:test -f script/docker/server/Dockerfile ./dist
```
* Run the E2E tests via [api-testing](https://github.com/LinuxSuRen/api-testing)
* The test cases run from top to bottom
* You can add the necessary assertions there
* Test data files are under `e2e/data/`
* The test report is generated at `e2e/report/report.md`
* HertzBeat logs are written to `e2e/logs/`
## Run locally
Please follow these steps if you want to run the e2e testing locally.
Please follow these steps if you want to run the E2E tests locally.
> Please make sure you have installed docker-compose v2
> Please make sure that Docker Compose v2 is installed
* Change the directory to `e2e`, then execute `./script/start.sh`
* Build the local `apache/hertzbeat:test` image from the repository root
* Change the directory to `e2e`, then run:
```bash
docker compose up --exit-code-from testing --remove-orphans
```
* If your environment uses the standalone Compose binary, use `docker-compose` instead of `docker compose`
* After the run, check `e2e/report/report.md` and `e2e/logs/` for the test result and runtime logs
+1 -1
View File
@@ -18,7 +18,7 @@
version: '3.8'
services:
testing:
image: ghcr.io/linuxsuren/api-testing:v0.0.17
image: ghcr.io/linuxsuren/api-testing:v0.0.19
environment:
SERVER: http://hertzbeat:1157
container_name: e2e-testing
+50
View File
@@ -29,6 +29,7 @@ param:
monitorUDP: "{{randAlpha 6}}"
monitorWebsite: "{{randAlpha 6}}"
monitorFTP: "{{randAlpha 6}}"
apiTokenName: "e2e-token-{{randAlpha 6}}"
labelName: "{{randAlpha 3}}"
items:
- name: login
@@ -69,6 +70,8 @@ items:
- name: missing-auth-header
request:
api: /api/monitors
header:
Accept: application/json
expect:
statusCode: 401
- name: createSitemapMonitor
@@ -385,6 +388,53 @@ items:
header:
Authorization: Bearer {{.login.data.token}}
## API Token Management
- name: generateApiToken
request:
api: /api/account/token/generate?name={{.apiTokenName}}&expireSeconds=-1
method: POST
header:
Authorization: Bearer {{.login.data.token}}
expect:
bodyFieldsExpect:
code: "0"
- name: listApiTokens
request:
api: /api/account/token
header:
Authorization: Bearer {{.login.data.token}}
expect:
bodyFieldsExpect:
code: "0"
data.0.name: "{{.apiTokenName}}"
data.0.creator: "admin"
- name: listApiTokensByGeneratedToken
request:
api: /api/account/token
header:
Authorization: Bearer {{.generateApiToken.data.token}}
expect:
bodyFieldsExpect:
code: "0"
data.0.name: "{{.apiTokenName}}"
- name: useGeneratedTokenToCallApi
request:
api: /api/monitors?pageIndex=0&pageSize=8
header:
Authorization: Bearer {{.generateApiToken.data.token}}
expect:
bodyFieldsExpect:
code: "0"
- name: deleteApiToken
request:
api: /api/account/token/{{(index .listApiTokens.data 0).id | int64}}
method: DELETE
header:
Authorization: Bearer {{.login.data.token}}
expect:
bodyFieldsExpect:
code: "0"
## Config
- name: getConfigEmail
request:
+2 -2
View File
@@ -26,8 +26,8 @@
<artifactId>hertzbeat-ai</artifactId>
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.1.1</spring-ai.version>
<java.version>21</java.version>
<spring-ai.version>2.0.0</spring-ai.version>
<java.version>25</java.version>
</properties>
<dependencies>
@@ -27,7 +27,6 @@ import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.ai.openai.OpenAiChatOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ConfigurableApplicationContext;
@@ -101,21 +100,17 @@ public class LlmConfig {
}
}
OpenAiApi.Builder builder = new OpenAiApi.Builder();
builder.baseUrl(modelProviderConfig.getBaseUrl());
builder.apiKey(modelProviderConfig.getApiKey());
builder.completionsPath("/chat/completions");
// Create Chat Options
// Create Chat Options with baseUrl and apiKey
OpenAiChatOptions openAiChatOptions = OpenAiChatOptions.builder()
.baseUrl(modelProviderConfig.getBaseUrl())
.apiKey(modelProviderConfig.getApiKey())
.model(modelProviderConfig.getModel())
.temperature(0.3)
.build();
// Create Chat Model
OpenAiChatModel openAiChatModel = OpenAiChatModel.builder()
.openAiApi(builder.build())
.defaultOptions(openAiChatOptions)
.options(openAiChatOptions)
.build();
// Create and return ChatClient
@@ -26,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.config.McpContextHolder;
import org.apache.hertzbeat.ai.pojo.dto.ChatRequestContext;
import org.apache.hertzbeat.ai.pojo.dto.ChatResponseChunk;
import org.apache.hertzbeat.ai.pojo.dto.SecurityData;
import org.apache.hertzbeat.ai.service.ConversationService;
import org.apache.hertzbeat.common.entity.ai.ChatConversation;
import org.apache.hertzbeat.common.entity.dto.Message;
@@ -78,12 +79,12 @@ public class ChatController {
McpContextHolder.setSubject(subject);
if (context.getMessage() == null || context.getMessage().trim().isEmpty()) {
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(context.getConversationId())
.response("Error: Message cannot be empty")
.build();
.conversationId(context.getConversationId())
.response("Error: Message cannot be empty")
.build();
return Flux.just(ServerSentEvent.builder(errorResponse)
.event("error")
.build());
.event("error")
.build());
}
log.info("Received streaming chat request for conversation: {}", context.getConversationId());
@@ -92,12 +93,12 @@ public class ChatController {
} catch (Exception e) {
log.error("Error in stream chat endpoint: ", e);
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(context.getConversationId())
.response("An error occurred: " + e.getMessage())
.build();
.conversationId(context.getConversationId())
.response("An error occurred: " + e.getMessage())
.build();
return Flux.just(ServerSentEvent.builder(errorResponse)
.event("error")
.build());
.event("error")
.build());
}
}
@@ -134,7 +135,7 @@ public class ChatController {
@GetMapping(path = "/conversations/{conversationId}")
@Operation(summary = "Get conversation history", description = "Get detailed information and message history for a specific conversation")
public ResponseEntity<Message<ChatConversation>> getConversation(
@Parameter(description = "Conversation ID", example = "12345678") @PathVariable(value = "conversationId") Long conversationId) {
@Parameter(description = "Conversation ID", example = "12345678") @PathVariable(value = "conversationId") Long conversationId) {
ChatConversation conversation = conversationService.getConversation(conversationId);
return ResponseEntity.ok(Message.success(conversation));
}
@@ -148,8 +149,21 @@ public class ChatController {
@DeleteMapping(path = "/conversations/{conversationId}")
@Operation(summary = "Delete conversation", description = "Delete a specific conversation and all its messages")
public ResponseEntity<Message<Void>> deleteConversation(
@Parameter(description = "Conversation ID", example = "2345678") @PathVariable("conversationId") Long conversationId) {
@Parameter(description = "Conversation ID", example = "2345678") @PathVariable("conversationId") Long conversationId) {
conversationService.deleteConversation(conversationId);
return ResponseEntity.ok(Message.success());
}
/**
* Save data submitted by secure form
* @param securityData security data
* @return save result
*/
@PostMapping(path = "/security")
@Operation(summary = "save security data", description = "Save security data")
public ResponseEntity<Message<Boolean>> commitSecurityData(@Valid @RequestBody SecurityData securityData) {
return ResponseEntity.ok(Message.success(conversationService.saveSecurityData(securityData)));
}
}
@@ -15,26 +15,24 @@
* limitations under the License.
*/
package org.apache.hertzbeat.templatehub.model.DTO;
package org.apache.hertzbeat.ai.pojo.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Builder;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Refresh Token Response
* security data
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "flash token response")
public class RefreshTokenResponse {
@Schema(title = "Access Token")
private String token;
public class SecurityData {
@NotNull
@Schema(description = "Conversation ID", example = "123")
private Long conversationId;
@NotNull
@Schema(description = "security data", example = "{\"password\":\"xxxxx\"}")
private String securityData;
@Schema(title = "Refresh Token")
private String refreshToken;
}
@@ -19,6 +19,7 @@
package org.apache.hertzbeat.ai.service;
import org.apache.hertzbeat.ai.pojo.dto.ChatResponseChunk;
import org.apache.hertzbeat.ai.pojo.dto.SecurityData;
import org.apache.hertzbeat.common.entity.ai.ChatConversation;
import org.springframework.http.codec.ServerSentEvent;
import reactor.core.publisher.Flux;
@@ -33,7 +34,7 @@ public interface ConversationService {
/**
* Send a message and receive a streaming response
*
* @param message The user's message
* @param message The user's message
* @param conversationId Optional conversation ID for continuing a chat
* @return Flux of ServerSentEvent for streaming the response
*/
@@ -67,4 +68,13 @@ public interface ConversationService {
* @param conversationId Conversation ID to delete
*/
void deleteConversation(Long conversationId);
/**
* save security data for a conversation
*
* @param securityData securityData
* @return save result
*/
Boolean saveSecurityData(SecurityData securityData);
}
@@ -18,6 +18,10 @@
package org.apache.hertzbeat.ai.service.impl;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.sop.model.SopDefinition;
import org.apache.hertzbeat.ai.sop.model.SopParameter;
@@ -27,9 +31,12 @@ import org.apache.hertzbeat.common.entity.dto.ModelProviderConfig;
import org.apache.hertzbeat.ai.service.ChatClientProviderService;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
import org.apache.hertzbeat.common.entity.manager.GeneralConfig;
import org.apache.hertzbeat.common.support.event.AiProviderConfigChangeEvent;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.ai.chat.prompt.SystemPromptTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.event.EventListener;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.pojo.dto.ChatRequestContext;
@@ -44,14 +51,12 @@ import org.springframework.context.ApplicationContext;
import reactor.core.publisher.Flux;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
/**
* Implementation of the {@link ChatClientProviderService}.
* Provides functionality to interact with the ChatClient for handling chat
* messages.
* Implementation of the {@link ChatClientProviderService}. Provides functionality to interact with the ChatClient for
* handling chat messages.
*/
@Slf4j
@Service
@@ -64,21 +69,28 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
private final GeneralConfigDao generalConfigDao;
private ModelProviderConfig modelProviderConfig;
private final SkillRegistry skillRegistry;
@Autowired
@Qualifier("hertzbeatTools")
private ToolCallbackProvider toolCallbackProvider;
private boolean isConfigured = false;
@Value("classpath:/prompt/system-message.st")
private Resource systemResource;
@Value("classpath:/prompt/extra-message-protected.st")
private Resource extraResourceProtected;
@Autowired
public ChatClientProviderServiceImpl(ApplicationContext applicationContext,
GeneralConfigDao generalConfigDao,
@Lazy SkillRegistry skillRegistry) {
public ChatClientProviderServiceImpl(ApplicationContext applicationContext,
GeneralConfigDao generalConfigDao,
@Lazy SkillRegistry skillRegistry) {
this.applicationContext = applicationContext;
this.generalConfigDao = generalConfigDao;
this.skillRegistry = skillRegistry;
@@ -89,7 +101,7 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
try {
// Get the current (potentially refreshed) ChatClient instance
ChatClient chatClient = applicationContext.getBean("openAiChatClient", ChatClient.class);
List<Message> messages = new ArrayList<>();
// Add conversation history if available
@@ -112,13 +124,13 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
String systemPrompt = buildSystemPrompt(context.getConversationId());
return chatClient.prompt()
.messages(messages)
.system(systemPrompt)
.toolCallbacks(toolCallbackProvider)
.stream()
.content()
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", context.getConversationId()))
.doOnError(error -> log.error("Error in streaming chat: {}", error.getMessage(), error));
.messages(messages)
.system(systemPrompt)
.tools(toolCallbackProvider)
.stream()
.content()
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", context.getConversationId()))
.doOnError(error -> log.error("Error in streaming chat: {}", error.getMessage(), error));
} catch (Exception e) {
log.error("Error setting up streaming chat: {}", e.getMessage(), e);
@@ -133,30 +145,43 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
try {
String template = systemResource.getContentAsString(StandardCharsets.UTF_8);
String skillsList = generateSkillsList();
return template
.replace(SKILLS_PLACEHOLDER, skillsList)
.replace(CONVERSATION_ID_PLACEHOLDER, String.valueOf(conversationId));
template = template
.replace(SKILLS_PLACEHOLDER, skillsList)
.replace(CONVERSATION_ID_PLACEHOLDER, String.valueOf(conversationId));
// add extra prompt for protected model to guide it to use protected tools
if (Objects.equals(modelProviderConfig.getParticipationModel(), "PROTECTED")) {
Map<String, Object> metadata = new HashMap<>();
metadata.put("conversationId", conversationId);
return template + SystemPromptTemplate.builder().resource(extraResourceProtected).build()
.create(metadata)
.getContents();
} else {
return template;
}
} catch (IOException e) {
log.error("Failed to read system prompt template: {}", e.getMessage());
return "";
}
}
/**
* Generate a formatted list of available skills for the system prompt.
*/
private String generateSkillsList() {
List<SopDefinition> skills = skillRegistry.getAllSkills();
if (skills.isEmpty()) {
return "No skills currently available. Use listSkills tool to refresh.";
}
StringBuilder sb = new StringBuilder();
for (SopDefinition skill : skills) {
sb.append("- **").append(skill.getName()).append("**: ");
sb.append(skill.getDescription());
// Add parameter hints
if (skill.getParameters() != null && !skill.getParameters().isEmpty()) {
sb.append(" (requires: ");
@@ -171,16 +196,24 @@ public class ChatClientProviderServiceImpl implements ChatClientProviderService
}
sb.append("\n");
}
return sb.toString();
}
@EventListener(AiProviderConfigChangeEvent.class)
public void onAiProviderConfigChange(AiProviderConfigChangeEvent event) {
GeneralConfig providerConfig = generalConfigDao.findByType("provider");
this.modelProviderConfig = JsonUtil.fromJson(providerConfig.getContent(), ModelProviderConfig.class);
}
@Override
public boolean isConfigured() {
if (!isConfigured) {
GeneralConfig providerConfig = generalConfigDao.findByType("provider");
ModelProviderConfig modelProviderConfig = JsonUtil.fromJson(providerConfig.getContent(), ModelProviderConfig.class);
isConfigured = modelProviderConfig != null && modelProviderConfig.getApiKey() != null;
ModelProviderConfig modelProviderConfig = JsonUtil.fromJson(providerConfig.getContent(),
ModelProviderConfig.class);
isConfigured = modelProviderConfig != null && modelProviderConfig.getApiKey() != null;
this.modelProviderConfig = modelProviderConfig;
}
return isConfigured;
}
@@ -17,21 +17,23 @@
package org.apache.hertzbeat.ai.service.impl;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.dao.ChatConversationDao;
import org.apache.hertzbeat.ai.dao.ChatMessageDao;
import org.apache.hertzbeat.ai.pojo.dto.ChatRequestContext;
import org.apache.hertzbeat.ai.pojo.dto.ChatResponseChunk;
import org.apache.hertzbeat.ai.pojo.dto.SecurityData;
import org.apache.hertzbeat.ai.service.ChatClientProviderService;
import org.apache.hertzbeat.ai.service.ConversationService;
import org.apache.hertzbeat.common.entity.ai.ChatConversation;
import org.apache.hertzbeat.common.entity.ai.ChatMessage;
import org.apache.hertzbeat.common.util.AesUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.http.codec.ServerSentEvent;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import java.util.Collections;
@@ -62,17 +64,17 @@ public class ConversationServiceImpl implements ConversationService {
// Check if provider is properly configured
if (!chatClientProviderService.isConfigured()) {
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("Provider is not configured. Please configure your AI Provider.")
.build();
.conversationId(conversationId)
.response("Provider is not configured. Please configure your AI Provider.")
.build();
return Flux.just(ServerSentEvent.builder(errorResponse)
.event("error")
.build());
.event("error")
.build());
}
log.info("Starting streaming conversation: {}", conversationId);
ChatConversation conversation = conversationDao.findById(conversationId)
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
// Manually load messages for conversation history
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
@@ -94,58 +96,58 @@ public class ConversationServiceImpl implements ConversationService {
chatMessage = messageDao.save(chatMessage);
ChatRequestContext context = ChatRequestContext.builder()
.message(message)
.conversationId(conversationId)
.conversationHistory(CollectionUtils.isEmpty(conversation.getMessages()) ? null
: conversation.getMessages().subList(0, conversation.getMessages().size() - 1))
.build();
.message(message)
.conversationId(conversationId)
.conversationHistory(messages)
.build();
// Stream response from AI service
StringBuilder fullResponse = new StringBuilder();
ChatMessage finalChatMessage = chatMessage;
return chatClientProviderService.streamChat(context)
.map(chunk -> {
fullResponse.append(chunk);
ChatResponseChunk responseChunk = ChatResponseChunk.builder()
.conversationId(conversationId)
.userMessageId(finalChatMessage.getId())
.response(chunk)
.build();
.map(chunk -> {
fullResponse.append(chunk);
ChatResponseChunk responseChunk = ChatResponseChunk.builder()
.conversationId(conversationId)
.userMessageId(finalChatMessage.getId())
.response(chunk)
.build();
return ServerSentEvent.builder(responseChunk)
.event("message")
.build();
})
.concatWith(Flux.defer(() -> {
// Add the complete AI response to conversation
ChatMessage assistantMessage = ChatMessage.builder()
.conversationId(conversationId)
.content(fullResponse.toString())
.role("assistant")
.build();
assistantMessage = messageDao.save(assistantMessage);
ChatResponseChunk finalResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("")
.assistantMessageId(assistantMessage.getId())
.build();
return ServerSentEvent.builder(responseChunk)
.event("message")
.build();
})
.concatWith(Flux.defer(() -> {
// Add the complete AI response to conversation
ChatMessage assistantMessage = ChatMessage.builder()
.conversationId(conversationId)
.content(fullResponse.toString())
.role("assistant")
.build();
assistantMessage = messageDao.save(assistantMessage);
ChatResponseChunk finalResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("")
.assistantMessageId(assistantMessage.getId())
.build();
return Flux.just(ServerSentEvent.builder(finalResponse)
.event("complete")
.build());
}))
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", conversationId))
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", conversationId, error.getMessage(), error))
.onErrorResume(error -> {
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("An error occurred: " + error.getMessage())
.userMessageId(finalChatMessage.getId())
.build();
return Flux.just(ServerSentEvent.builder(errorResponse)
.event("error")
.build());
});
return Flux.just(ServerSentEvent.builder(finalResponse)
.event("complete")
.build());
}))
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", conversationId))
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", conversationId,
error.getMessage(), error))
.onErrorResume(error -> {
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.response("An error occurred: " + error.getMessage())
.userMessageId(finalChatMessage.getId())
.build();
return Flux.just(ServerSentEvent.builder(errorResponse)
.event("error")
.build());
});
}
@Override
@@ -175,13 +177,14 @@ public class ConversationServiceImpl implements ConversationService {
return conversations;
}
List<Long> conversationIds = conversations.stream()
.map(ChatConversation::getId)
.toList();
.map(ChatConversation::getId)
.toList();
List<ChatMessage> allMessages = messageDao.findByConversationIdInOrderByGmtCreateAsc(conversationIds);
Map<Long, List<ChatMessage>> messagesByConversationId = allMessages.stream()
.collect(Collectors.groupingBy(ChatMessage::getConversationId));
.collect(Collectors.groupingBy(ChatMessage::getConversationId));
for (ChatConversation conversation : conversations) {
List<ChatMessage> messages = messagesByConversationId.getOrDefault(conversation.getId(), Collections.emptyList());
List<ChatMessage> messages = messagesByConversationId.getOrDefault(conversation.getId(),
Collections.emptyList());
conversation.setMessages(messages);
}
return conversations;
@@ -196,4 +199,17 @@ public class ConversationServiceImpl implements ConversationService {
}
conversationDao.deleteById(conversationId);
}
@Override
public Boolean saveSecurityData(SecurityData securityData) {
Optional<ChatConversation> chatConversation = conversationDao.findById(securityData.getConversationId());
if (chatConversation.isPresent()) {
ChatConversation conversation = chatConversation.get();
conversation.setSecurityData(AesUtil.aesEncode(securityData.getSecurityData()));
conversationDao.save(conversation);
return true;
}
return false;
}
}
@@ -27,19 +27,39 @@ public interface MonitorTools {
/**
* Add a new monitor with comprehensive configuration
*
* @param name Monitor name
* @param app Monitor type/application (e.g., 'linux', 'mysql', 'http')
* @param intervals Collection interval in seconds (default: 600)
* @param params Monitor-specific parameters as JSON string (e.g., host, port, username, password, etc.)
* @param name Monitor name
* @param app Monitor type/application (e.g., 'linux', 'mysql', 'http')
* @param intervals Collection interval in seconds (default: 600)
* @param params Monitor-specific parameters as JSON string (e.g., host, port, username, password, etc.)
* @param description Monitor description (optional)
* @return Result message with monitor ID if successful
*/
String addMonitor(
String name,
String app,
Integer intervals,
String params,
String description
String name,
String app,
Integer intervals,
String params,
String description
);
/**
* Add a new monitor with comprehensive configuration, methods specifically for hertzbeat ai
*
* @param conversationId Conversation ID
* @param name Monitor name
* @param app Monitor type/application (e.g., 'linux', 'mysql', 'http')
* @param intervals Collection interval in seconds (default: 600)
* @param params Monitor-specific parameters as JSON string (e.g., host, port, username, password, etc.)
* @param description Monitor description (optional)
* @return Result message with monitor ID if successful
*/
String addMonitorProtected(
Long conversationId,
String name,
String app,
Integer intervals,
String params,
String description
);
/**
@@ -52,29 +72,30 @@ public interface MonitorTools {
/**
* Comprehensive monitor querying with flexible filtering, pagination, and specialized views
* @param ids Specific monitor IDs to retrieve (optional)
* @param app Monitor type filter (linux, mysql, http, etc.)
* @param status Monitor status (1=online, 2=offline, 3=unreachable, 0=paused, 9=all)
* @param search Search in monitor names or hosts (partial matching)
* @param labels Label filters, format: 'key1:value1,key2:value2'
* @param sort Sort field (name, gmtCreate, gmtUpdate, status, app)
* @param order Sort order (asc, desc)
* @param pageIndex Page number starting from 0
* @param pageSize Items per page (1-100 recommended)
*
* @param ids Specific monitor IDs to retrieve (optional)
* @param app Monitor type filter (linux, mysql, http, etc.)
* @param status Monitor status (1=online, 2=offline, 3=unreachable, 0=paused, 9=all)
* @param search Search in monitor names or hosts (partial matching)
* @param labels Label filters, format: 'key1:value1,key2:value2'
* @param sort Sort field (name, gmtCreate, gmtUpdate, status, app)
* @param order Sort order (asc, desc)
* @param pageIndex Page number starting from 0
* @param pageSize Items per page (1-100 recommended)
* @param includeStats Include status statistics summary
* @return Comprehensive monitor information with optional statistics
*/
String queryMonitors(
List<Long> ids,
String app,
Byte status,
String search,
String labels,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
Boolean includeStats);
List<Long> ids,
String app,
Byte status,
String search,
String labels,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
Boolean includeStats);
/**
* Get parameter definitions required for a specific monitor type
@@ -146,7 +146,7 @@ public class MetricsToolsImpl implements MetricsTools {
HertzBeat: Get historical metrics data for analysis and trending.
Returns time-series data for specified metrics over a time range.
Use the query_monitors tool to find the correct monitor IDs/ name or type for the monitor(s) user asked the metrics for
Pass that name into the get_apps_metrics_hierarchy tool to get the metrics hierarchy i.e metrics and the field paramater
Pass that name into the get_apps_metrics_hierarchy tool to get the metrics hierarchy i.e metrics and the field parameter
DO NOT USE THE LABEL FIELD ALWAYS USE THE VALUE FIELD FROM THE HIERARCHY JSON
Ask user to provide the filters for labels, history and interval aggregation
""")
@@ -18,9 +18,17 @@
package org.apache.hertzbeat.ai.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.ai.config.McpContextHolder;
import org.apache.hertzbeat.ai.dao.ChatConversationDao;
import org.apache.hertzbeat.common.entity.ai.ChatConversation;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo;
import org.apache.hertzbeat.manager.service.MonitorService;
import org.apache.hertzbeat.manager.service.AppService;
import org.apache.hertzbeat.ai.utils.UtilityClass;
@@ -32,7 +40,7 @@ import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import tools.jackson.core.type.TypeReference;
import java.util.ArrayList;
import java.util.List;
@@ -44,76 +52,79 @@ import java.util.Map;
@Slf4j
@Service
public class MonitorToolsImpl implements MonitorTools {
@Autowired
private MonitorService monitorService;
@Autowired
private AppService appService;
@Autowired
private ChatConversationDao conversationDao;
/**
* Tool to query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
* pagination.
* Returns detailed monitor information including ID, name, type, host, and status.
* Tool to query monitor information with flexible filtering and pagination. Supports filtering by monitor IDs,
* type, status, host, labels, sorting, and pagination. Returns detailed monitor information including ID, name,
* type, host, and status.
*/
@Override
@Tool(name = "query_monitors", description = """
HertzBeat: Query Existing/configured monitors in HertzBeat.
This tool retrieves monitors based on various filters and parameters.
Comprehensive monitor querying with flexible filtering, pagination, and specialized views.
HertzBeat: Query Existing/configured monitors in HertzBeat.
This tool retrieves monitors based on various filters and parameters.
Comprehensive monitor querying with flexible filtering, pagination, and specialized views.
MONITOR STATUSES:
- status=1: Online/Active monitors (healthy, responding normally)
- status=2: Offline monitors (not responding, connection failed)
- status=3: Unreachable monitors (network/connectivity issues)
- status=0: Paused monitors (manually disabled/suspended)
- status=9 or null: All monitors regardless of status (default)
MONITOR STATUSES:
- status=1: Online/Active monitors (healthy, responding normally)
- status=2: Offline monitors (not responding, connection failed)
- status=3: Unreachable monitors (network/connectivity issues)
- status=0: Paused monitors (manually disabled/suspended)
- status=9 or null: All monitors regardless of status (default)
COMMON USE CASES & PARAMETER COMBINATIONS:
COMMON USE CASES & PARAMETER COMBINATIONS:
1. BASIC MONITOR LISTING:
- Default: No parameters (shows all monitors, 8 per page)
- By type: app='linux' (show only Linux monitors)
- Search: search='web' (find monitors with 'web' in name/host)
1. BASIC MONITOR LISTING:
- Default: No parameters (shows all monitors, 8 per page)
- By type: app='linux' (show only Linux monitors)
- Search: search='web' (find monitors with 'web' in name/host)
2. STATUS-BASED QUERIES:
- Healthy monitors: status=1, pageSize=50
- Problem monitors: status=2 or status=3, pageSize=50
- Offline monitors only: status=2
- Unreachable monitors only: status=3
- Paused monitors: status=0
2. STATUS-BASED QUERIES:
- Healthy monitors: status=1, pageSize=50
- Problem monitors: status=2 or status=3, pageSize=50
- Offline monitors only: status=2
- Unreachable monitors only: status=3
- Paused monitors: status=0
3. MONITORING HEALTH OVERVIEW:
- All statuses with statistics: status=9, includeStats=true, pageSize=100
- Unhealthy monitors: Pass both status=2 AND status=3 (make 2 separate calls)
3. MONITORING HEALTH OVERVIEW:
- All statuses with statistics: status=9, includeStats=true, pageSize=100
- Unhealthy monitors: Pass both status=2 AND status=3 (make 2 separate calls)
4. ADVANCED FILTERING:
- Specific monitor types: app='mysql', status=1 (healthy MySQL monitors)
- Label-based: labels='env:prod,critical:true'
- Host search: search='192.168' (find by IP pattern)
- Monitor IDs: ids=[1,2,3] (specific monitors by ID)
4. ADVANCED FILTERING:
- Specific monitor types: app='mysql', status=1 (healthy MySQL monitors)
- Label-based: labels='env:prod,critical:true'
- Host search: search='192.168' (find by IP pattern)
- Monitor IDs: ids=[1,2,3] (specific monitors by ID)
5. SORTING & PAGINATION:
- Recently updated: sort='gmtUpdate', order='desc'
- Alphabetical: sort='name', order='asc'
- By creation: sort='gmtCreate', order='desc' (newest first)
- Large datasets: pageSize=50-100 for bulk operations
5. SORTING & PAGINATION:
- Recently updated: sort='gmtUpdate', order='desc'
- Alphabetical: sort='name', order='asc'
- By creation: sort='gmtCreate', order='desc' (newest first)
- Large datasets: pageSize=50-100 for bulk operations
RESPONSE FORMAT:
- includeStats=true: Adds status distribution summary at top
- Default: Simple list with ID, name, type, host, status
- Shows total count and pagination info
""")
RESPONSE FORMAT:
- includeStats=true: Adds status distribution summary at top
- Default: Simple list with ID, name, type, host, status
- Shows total count and pagination info
""")
public String queryMonitors(
@ToolParam(description = "Specific monitor IDs to retrieve (optional)", required = false) List<Long> ids,
@ToolParam(description = "Monitor type filter: 'linux', 'mysql', 'http', 'redis', etc. (optional)", required = false) String app,
@ToolParam(description = "Monitor status: 1=online, 2=offline, 3=unreachable, 0=paused, 9=all (default: 9)", required = false) Byte status,
@ToolParam(description = "Search in monitor names or hosts (partial matching)", required = false) String search,
@ToolParam(description = "Label filters, format: 'key1:value1,key2:value2'", required = false) String labels,
@ToolParam(description = "Sort field: 'name', 'gmtCreate', 'gmtUpdate', 'status', 'app' (default: gmtCreate)", required = false) String sort,
@ToolParam(description = "Sort order: 'asc' (ascending) or 'desc' (descending, default)", required = false) String order,
@ToolParam(description = "Page number starting from 0 (default: 0)", required = false) Integer pageIndex,
@ToolParam(description = "Items per page: 1-100 recommended (default: 20)", required = false) Integer pageSize,
@ToolParam(description = "Include status statistics summary (default: false)", required = false) Boolean includeStats) {
@ToolParam(description = "Specific monitor IDs to retrieve (optional)", required = false) List<Long> ids,
@ToolParam(description = "Monitor type filter: 'linux', 'mysql', 'http', 'redis', etc. (optional)", required = false) String app,
@ToolParam(description = "Monitor status: 1=online, 2=offline, 3=unreachable, 0=paused, 9=all (default: 9)", required = false) Byte status,
@ToolParam(description = "Search in monitor names or hosts (partial matching)", required = false) String search,
@ToolParam(description = "Label filters, format: 'key1:value1,key2:value2'", required = false) String labels,
@ToolParam(description = "Sort field: 'name', 'gmtCreate', 'gmtUpdate', 'status', 'app' (default: gmtCreate)", required = false) String sort,
@ToolParam(description = "Sort order: 'asc' (ascending) or 'desc' (descending, default)", required = false) String order,
@ToolParam(description = "Page number starting from 0 (default: 0)", required = false) Integer pageIndex,
@ToolParam(description = "Items per page: 1-100 recommended (default: 20)", required = false) Integer pageSize,
@ToolParam(description = "Include status statistics summary (default: false)", required = false) Boolean includeStats) {
try {
// Set defaults
if (pageSize == null || pageSize <= 0) {
@@ -130,7 +141,7 @@ public class MonitorToolsImpl implements MonitorTools {
log.debug("Current security subject: {}", subjectSum);
Page<Monitor> result = monitorService.getMonitors(
ids, app, search, status, sort, order, pageIndex, pageSize, labels);
ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorService.getMonitors result: {}", result);
StringBuilder response = new StringBuilder();
@@ -160,19 +171,20 @@ public class MonitorToolsImpl implements MonitorTools {
}
response.append("Query Results: ").append(result.getContent().size())
.append(" monitors (Total: ").append(result.getTotalElements()).append(")\n");
.append(" monitors (Total: ").append(result.getTotalElements()).append(")\n");
if (result.getTotalPages() > 1) {
response.append("Page ").append(pageIndex + 1).append(" of ").append(result.getTotalPages()).append("\n");
response.append("Page ").append(pageIndex + 1).append(" of ").append(result.getTotalPages())
.append("\n");
}
response.append("\n");
for (Monitor monitor : result.getContent()) {
response.append("ID: ").append(monitor.getId())
.append(" | Name: ").append(monitor.getName())
.append(" | Type: ").append(monitor.getApp())
.append(" | Instance: ").append(monitor.getInstance())
.append(" | Status: ").append(UtilityClass.getStatusText(monitor.getStatus()));
.append(" | Name: ").append(monitor.getName())
.append(" | Type: ").append(monitor.getApp())
.append(" | Instance: ").append(monitor.getInstance())
.append(" | Status: ").append(UtilityClass.getStatusText(monitor.getStatus()));
// Add creation date for better context
if (monitor.getGmtCreate() != null) {
@@ -224,15 +236,59 @@ public class MonitorToolsImpl implements MonitorTools {
- Redis: {"host":"redis.server.com", "port":"6379", "password":"xxx"}
""")
public String addMonitor(
@ToolParam(description = "Monitor name (required)", required = true) String name,
@ToolParam(description = "Monitor type: website, mysql, postgresql, redis, linux, windows, etc.", required = true) String app,
@ToolParam(description = "Collection interval in seconds (default: 600)", required = false) Integer intervals,
@ToolParam(description = "Monitor-specific parameters as JSON string. "
+ "Use get_monitor_additional_params to see required fields. "
+ "Example: {\"host\":\"192.168.1.1\", \"port\":\"22\", \"username\":\"root\"}",
required = true) String params,
@ToolParam(description = "Monitor description (optional)", required = false) String description) {
@ToolParam(description = "Monitor name (required)", required = true) String name,
@ToolParam(description = "Monitor type: website, mysql, postgresql, redis, linux, windows, etc.", required = true) String app,
@ToolParam(description = "Collection interval in seconds (default: 600)", required = false) Integer intervals,
@ToolParam(description = "Monitor-specific parameters as JSON string. "
+ "Use get_monitor_additional_params to see required fields. "
+ "Example: {\"host\":\"192.168.1.1\", \"port\":\"22\", \"username\":\"root\"}",
required = true) String params,
@ToolParam(description = "Monitor description (optional)", required = false) String description) {
return addMonitorProtected(null, name, app, intervals, params, description);
}
@Override
@Tool(name = "add_monitor_protected", description = """
HertzBeat: Add a new monitoring target to HertzBeat with comprehensive configuration.
This tool dynamically handles different parameter requirements for each monitor type.
This tool creates monitors with proper app-specific parameters.
*********
VERY IMPORTANT:
ALWAYS use get_monitor_additional_params to check the additional required parameters for the chosen type before adding a monitor or even mentioning it.
Use list_monitor_types tool to see available monitor type names to use here in the app parameter.
Use the information obtained from this to query user for parameters.
If the User has not given any parameters, ask them to provide the necessary parameters, until all the necessary parameters are provided.
If the conversation ID is unknown, please call the `add_monitor` method.
**********
Examples of natural language requests this tool handles:
- "Monitor website example.com with HTTPS on port 443"
- "Add MySQL monitoring for database server at 192.168.1.10 with user admin"
- "Monitor Linux server health on host server.company.com via SSH"
- "Set up Redis monitoring on localhost port 6379 with password"
PARAMETER MAPPING: Use the 'params' parameter to pass all monitor-specific configuration.
The params should be a JSON string containing key-value pairs for the monitor type.
Use get_monitor_additional_params tool to see what parameters are required for each monitor type.
PARAMS EXAMPLES:
- Website: {"host":"example.com", "port":"443", "uri":"/api/health", "ssl":"true", "method":"GET"}
- Linux: {"host":"192.168.1.10", "port":"22", "username":"root", "password":"xxx"}
- MySQL: {"host":"db.server.com", "port":"3306", "username":"admin", "password":"xxx", "database":"mydb"}
- Redis: {"host":"redis.server.com", "port":"6379", "password":"xxx"}
""")
public String addMonitorProtected(
@ToolParam(description = "The id for current conversation (required)", required = true) Long conversationId,
@ToolParam(description = "Monitor name (required)", required = true) String name,
@ToolParam(description = "Monitor type: website, mysql, postgresql, redis, linux, windows, etc.", required = true) String app,
@ToolParam(description = "Collection interval in seconds (default: 600)", required = false) Integer intervals,
@ToolParam(description = "Monitor-specific parameters as JSON string. "
+ "Use get_monitor_additional_params to see required fields. "
+ "Example: {\"host\":\"192.168.1.1\", \"port\":\"22\", \"username\":\"root\"}",
required = true) String params,
@ToolParam(description = "Monitor description (optional)", required = false) String description) {
try {
log.info("Adding monitor: name={}, app={}", name, app);
@@ -251,9 +307,22 @@ public class MonitorToolsImpl implements MonitorTools {
if (intervals == null || intervals < 10) {
intervals = 600;
}
// Parse params to extract host and port for instance
List<Param> paramList = parseParams(params);
// Query and add sensitive parameters
if (conversationId != null) {
Optional<ChatConversation> chatConversation = conversationDao.findById(conversationId);
if (chatConversation.isPresent() && StringUtils.isNotEmpty(chatConversation.get().getSecurityData())) {
List<Param> securityParams = JsonUtil.fromJson(
AesUtil.aesDecode(chatConversation.get().getSecurityData()),
new TypeReference<List<Param>>() {
});
if (CollectionUtils.isNotEmpty(securityParams)) {
paramList.addAll(securityParams);
}
}
}
String host = paramList.stream()
.filter(p -> "host".equals(p.getField()))
.map(Param::getParamValue)
@@ -280,7 +349,9 @@ public class MonitorToolsImpl implements MonitorTools {
// Validate that all required parameters for this monitor type are provided
try {
MonitorDto monitorDto = MonitorDto.builder().monitor(monitor).params(paramList).build();
MonitorDto monitorDto = new MonitorDto();
monitorDto.setMonitor(monitor);
monitorDto.setParams(paramList);
monitorService.validate(monitorDto, false);
} catch (IllegalArgumentException argumentException) {
if (argumentException.getMessage().contains("required")) {
@@ -456,7 +527,7 @@ public class MonitorToolsImpl implements MonitorTools {
}
// Get parameter definitions from app service
List<ParamDefine> paramDefines = appService.getAppParamDefines(app.toLowerCase().trim());
List<ParamDefineInfo> paramDefines = appService.getAppParamDefines(app.toLowerCase().trim());
if (paramDefines == null || paramDefines.isEmpty()) {
return String.format("No parameter definitions found for monitor type '%s'. "
@@ -468,7 +539,7 @@ public class MonitorToolsImpl implements MonitorTools {
response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n",
app, paramDefines.size()));
for (ParamDefine paramDefine : paramDefines) {
for (ParamDefineInfo paramDefine : paramDefines) {
response.append("• Field: ").append(paramDefine.getField()).append("\n");
// Add display name if available
@@ -0,0 +1,140 @@
# IMPORTANT
The following information requires special attention. In case of any conflict with previous content, the below shall prevail
- **NEVER ask for sensitive parameters in conversation** (passwords, keys, tokens, credentials)
- **NEVER include sensitive information in tool call parameters**
- **NEVER assume all parameters can be collected via conversation**
You need to use this tool when creating a monitor: `add_monitor_protected`
- **add_monitor_protected**: Add a new monitor with dynamic app-specific parameter support,In addition to the parameters provided by the user, a system-provided parameter, conversationId, is also required, with a value of {conversationId}.
## Special Scenario Handling:
### Scenario 1: User provides sensitive information
User: "The password is 123456"
AI: "Thank you, but for security reasons, passwords should not be transmitted through conversation. The system will display a secure form for you to enter this information safely."
### Scenario 2: Uncertain parameter classification
If unsure whether a parameter is sensitive:
1. Assume it's sensitive
2. Guide to secure form
3. Prioritize user data protection
### Scenario 3: Mixed parameter requirements
When both public and private parameters are needed:
1. Collect all public parameters first
2. Then guide to secure form for private parameters
3. Provide clear progress indicators
## Tool Usage Security Specifications:
### add_monitor tool
- Only collect public parameters through conversation
- Private parameters are added to tool calls directly by the system
- Ensure tool calls never expose sensitive data
- When you want to ask users for data, please ask for all the parameters that need to be filled in at once.
### Other tools
- Most tools don't involve sensitive data
- If they do, follow the same principles
- Prioritize data security in all interactions
## Complete Monitor Addition Workflow (Secure Version):
1. **Identify requirement**: What does the user want to monitor?
2. **Determine type**: Use `list_monitor_types` to find matching monitor type
3. **Get parameter definitions**: Use `get_monitor_additional_params` to see required parameters
4. **Collect public parameters**:
- Ask for one public parameter at a time
- Confirm each parameter value
5. **Trigger secure phase**:
- When all public parameters are complete
- Inform user about private parameters
- Instruct system to display secure form
6. **Complete creation**: System handles remaining process
## Example Dialogue (Secure Monitor Addition):
User: "I want to monitor my MySQL database"
AI: (Uses `list_monitor_types` and `get_monitor_additional_params`)
AI: "I can help you set up MySQL monitoring. First, I need some information:
1. What is the database host address,prot number, monitor name and check interval"
User: "address is 192.168.1.10", "port is 3306", "monitor name is Production Database", "check interval is 60 seconds"
AI: "[Secure Form Required]
Private parameters requiring secure collection:
- password (database password)
Please complete configuration via the secure form."
---
## Security and Privacy Principles
1. **Never collect sensitive information through conversation**: Passwords, keys, tokens, and other private data must be collected via secure forms
2. **Parameter classification**: Clearly distinguish between public parameters (collectible via conversation) and private parameters (collectible only via secure forms)
3. **Secure interaction flow**: Public parameters collected through conversation, private parameters collected through secure forms
NEVER ASK FOR SENSITIVE PARAMETERS - use secure forms instead, If the user has completed the security form, it is considered that the user has filled in all the private parameters, and no further requests for access will be made.
## Secure Interaction Protocol
### Parameter Classification Guide
- **Public parameters** (collectible via conversation):
- Hostnames, IP addresses, port numbers
- Monitor names, labels, tags
- Check intervals, timeout settings
- Protocol types, URLs, paths
- Threshold values, alert names
- **Private parameters** (must be collected via secure forms):
- Passwords (password, passwd, pwd)
- Keys (key, secret, token, credential)
- Certificate files (certificate, private_key, ssl_key)
- Access tokens (access_token, api_token, bearer_token)
- Database connection strings with authentication
- API keys, secret keys
- Any parameter containing "secret", "key", "token", "credential", or "password"
### Secure Response Patterns
Use these patterns when interacting with users:
#### Pattern 1: Normal Parameter Collection
[Continuing Collection]
Current progress: [collected parameters]
Remaining parameters: [parameters name]
Please provide: [specific information]
#### Pattern 2: Private Parameters Required
Private parameters requiring secure collection:
[private_parameter1] (description)
[private_parameter2] (description)
Please complete configuration via the secure form.
(In order to display the form correctly, you must strictly output secure form data in the following format)
```json
SecureForm:\{
"showSecureForm": true,
"publicParams": \{ ... \},
"privateParams": [
\{
"id": ...,
"app":...,
"name": \{
"zh-CN": "...",
"en-US": "...",
"ja-JP": "...",
"pt-BR": "...",
"zh-TW": "..."
\},
"field": "...",
"type": "password",
"required": ...,
"defaultValue": ...,
"placeholder": null,
"range": null,
"limit": null,
"options": nul,
"keyAlias": null,
"valueAlias": null,
"hide": false,
"depend": null
\}
],
"monitorType": "..."
\}
```
@@ -95,7 +95,7 @@ Common conversions:
- "List all Redis monitors with their connection status"
### Alert Configuration:
- ALERT RULE means when to alert a user
- ALERT RULE means when to alert a user
- "Create an alert for Kafka JVM when VmName equals 'vm-w2'"
- "Alert when OpenAI credit grants exceed 1000"
- "Set up HBase Master alert when heap memory usage is over 80%"
@@ -126,7 +126,7 @@ Common conversions:
- Collect all required parameters from the list_monitor_types tool and ask user to give them all, before calling add_monitor
- Example: "To monitor MySQL, I need host, port, username, password, and database name"
2. **Creating Alert Rules or Alerts**:
### 2. Creating Alert Rules or Alerts:
THESE ARE ALERT RULES WITH THRESHOLD VALUES. USERS CAN SPECIFY THE THRESHOLD VALUES FOR EXAMPLE,
IF THE USER SAYS "ALERT ME WHEN MY COST EXCEEDS 700, THE EXPRESSION SHOULD BE 'cost > 700' NOT 'cost < 700'.
APPLY THE SAME LOGIC FOR LESS THAN OPERATOR.
@@ -162,16 +162,16 @@ CRITICAL WORKFLOW Do all of this iteratively with user interaction at each step:
- Priority levels: 0=critical, 1=warning, 2=info
3. **Analyzing Performance**:
- Use get_realtime_metrics for current status
- Use get_historical_metrics for trends
- Use get_high_usage_monitors to find problems
### 3. Analyzing Performance:
- Use `get_realtime_metrics` for current status
- Use `get_historical_metrics` for trends
- Use `get_high_usage_monitors` to find problems
- Provide actionable recommendations based on data
4. **Troubleshooting Alerts**:
- Use query_alerts to find current issues
- Use get_monitor_alerts for specific monitor problems
- Use get_frequent_alerts to identify recurring issues
### 4. Troubleshooting Alerts:
- Use `query_alerts` to find current issues
- Use `get_monitor_alerts` for specific monitor problems
- Use `get_frequent_alerts` to identify recurring issues
- Suggest root cause analysis steps
## Parameter Guidelines:
@@ -196,7 +196,7 @@ CRITICAL WORKFLOW Do all of this iteratively with user interaction at each step:
- Provide clear explanations of monitoring data and actionable insights
## Avoid these common errors:
- Using Label name instead of the value from the heirarchy JSON while creating alert rules.
- Using Label name instead of the value from the hierarchy JSON while creating alert rules.
- Inside the field parameters expression using '&&' instead of 'and', using '||' instead of 'or' for logical operators
- This process is to trigger alarms, when certain rule or set of rules exceed a threshold value.
So when a user says that the threshold should be less than 1000. the operator used should be '>' not '<',
@@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.service.impl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hertzbeat.ai.dao.ChatConversationDao;
import org.apache.hertzbeat.ai.dao.ChatMessageDao;
import org.apache.hertzbeat.ai.pojo.dto.ChatRequestContext;
import org.apache.hertzbeat.ai.pojo.dto.ChatResponseChunk;
import org.apache.hertzbeat.ai.service.ChatClientProviderService;
import org.apache.hertzbeat.common.entity.ai.ChatConversation;
import org.apache.hertzbeat.common.entity.ai.ChatMessage;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.codec.ServerSentEvent;
import reactor.core.publisher.Flux;
/**
* Tests multi-turn conversation context handling in {@link ConversationServiceImpl}.
*/
@ExtendWith(MockitoExtension.class)
class ConversationServiceImplTest {
private static final long CONVERSATION_ID = 1L;
@Mock
private ChatConversationDao conversationDao;
@Mock
private ChatMessageDao messageDao;
@Mock
private ChatClientProviderService chatClientProviderService;
@InjectMocks
private ConversationServiceImpl conversationService;
@Test
void streamChatShouldKeepCompleteConversationHistory() {
ChatConversation conversation = ChatConversation.builder()
.id(CONVERSATION_ID)
.title("已命名会话")
.build();
List<ChatMessage> history = List.of(
ChatMessage.builder()
.id(11L)
.conversationId(CONVERSATION_ID)
.role("user")
.content("上一轮问题")
.build(),
ChatMessage.builder()
.id(12L)
.conversationId(CONVERSATION_ID)
.role("assistant")
.content("上一轮回答")
.build());
AtomicLong messageId = new AtomicLong(20L);
when(chatClientProviderService.isConfigured()).thenReturn(true);
when(conversationDao.findById(CONVERSATION_ID)).thenReturn(Optional.of(conversation));
when(messageDao.findByConversationIdOrderByGmtCreateAsc(CONVERSATION_ID)).thenReturn(history);
when(messageDao.save(any(ChatMessage.class))).thenAnswer(invocation -> {
ChatMessage savedMessage = invocation.getArgument(0);
savedMessage.setId(messageId.getAndIncrement());
return savedMessage;
});
when(chatClientProviderService.streamChat(any(ChatRequestContext.class)))
.thenReturn(Flux.just("本轮回答"));
List<ServerSentEvent<ChatResponseChunk>> events = conversationService
.streamChat("本轮问题", CONVERSATION_ID)
.collectList()
.block();
assertNotNull(events);
assertEquals(2, events.size());
ArgumentCaptor<ChatRequestContext> contextCaptor = ArgumentCaptor.forClass(ChatRequestContext.class);
verify(chatClientProviderService).streamChat(contextCaptor.capture());
assertEquals(history, contextCaptor.getValue().getConversationHistory());
}
}
+5
View File
@@ -40,6 +40,11 @@
<artifactId>hertzbeat-common-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common-spring</artifactId>
<scope>provided</scope>
</dependency>
<!-- plugin -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
@@ -74,6 +74,11 @@ public class AlerterProperties {
*/
private String gotifyWebhookUrl = "https://push.example.de/message?token=";
/**
* Ntfy default server url
*/
private String ntfyDefaultServerUrl = "https://ntfy.sh";
/**
* Data entry configuration properties
*/
@@ -18,12 +18,20 @@
package org.apache.hertzbeat.alert;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
@@ -31,16 +39,25 @@ import org.springframework.stereotype.Component;
*/
@Component
@Slf4j
public class AlerterWorkerPool {
public class AlerterWorkerPool implements DisposableBean {
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor notifyExecutor;
private ThreadPoolExecutor logWorkerExecutor;
private ManagedExecutor notifyExecutor;
private ManagedExecutor logWorkerExecutor;
private Map<Byte, Semaphore> notifyChannelPermits;
private int notifyMaxConcurrentPerChannel;
public AlerterWorkerPool() {
this(VirtualThreadProperties.defaults());
}
@Autowired
public AlerterWorkerPool(VirtualThreadProperties virtualThreadProperties) {
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
initWorkExecutor();
initNotifyExecutor();
initLogWorkerExecutor();
initNotifyExecutor(properties);
initLogWorkerExecutor(properties);
}
private void initWorkExecutor() {
@@ -61,16 +78,32 @@ public class AlerterWorkerPool {
new ThreadPoolExecutor.AbortPolicy());
}
private void initNotifyExecutor() {
private void initNotifyExecutor(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("Alerter notifyExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
if (properties.enabled()) {
VirtualThreadProperties.AlerterProperties alerterProperties = properties.alerter();
VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.notifyPool();
notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.notifyMaxConcurrentPerChannel());
notifyChannelPermits = new ConcurrentHashMap<>(8);
notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-",
notifyProperties.mode(), notifyProperties.maxConcurrentJobs(), handler);
return;
}
notifyMaxConcurrentPerChannel = 0;
notifyChannelPermits = null;
notifyExecutor = ManagedExecutors.wrap("notify-worker", createLegacyNotifyExecutor(handler));
}
private ThreadPoolExecutor createLegacyNotifyExecutor(Thread.UncaughtExceptionHandler handler) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter notifyExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setUncaughtExceptionHandler(handler)
.setDaemon(true)
.setNameFormat("notify-worker-%d")
.build();
notifyExecutor = new ThreadPoolExecutor(6,
return new ThreadPoolExecutor(6,
6,
10,
TimeUnit.SECONDS,
@@ -79,16 +112,27 @@ public class AlerterWorkerPool {
new ThreadPoolExecutor.AbortPolicy());
}
private void initLogWorkerExecutor() {
private void initLogWorkerExecutor(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("Alerter logWorkerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
if (properties.enabled()) {
VirtualThreadProperties.QueueProperties logWorkerProperties = properties.alerter().logWorker();
logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-",
logWorkerProperties.maxConcurrentJobs(), logWorkerProperties.queueCapacity(), handler);
return;
}
logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler));
}
private ThreadPoolExecutor createLegacyLogWorkerExecutor(Thread.UncaughtExceptionHandler handler) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter logWorkerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setUncaughtExceptionHandler(handler)
.setDaemon(true)
.setNameFormat("log-worker-%d")
.build();
logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS,
return new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(1000),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
@@ -113,6 +157,41 @@ public class AlerterWorkerPool {
notifyExecutor.execute(runnable);
}
/**
* Executes the given runnable task using the notify executor with per-channel concurrency control.
*
* @param channelType notification channel type
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeNotify(byte channelType, Runnable runnable) throws RejectedExecutionException {
if (notifyChannelPermits == null) {
notifyExecutor.execute(runnable);
return;
}
Semaphore semaphore = notifyChannelPermits.computeIfAbsent(channelType,
key -> new Semaphore(notifyMaxConcurrentPerChannel));
if (!semaphore.tryAcquire()) {
throw new RejectedExecutionException(
"notify-worker rejected task because channel concurrency limit was reached for type " + channelType);
}
boolean submitted = false;
try {
notifyExecutor.execute(() -> {
try {
runnable.run();
} finally {
semaphore.release();
}
});
submitted = true;
} finally {
if (!submitted) {
semaphore.release();
}
}
}
/**
* Executes the given runnable task using the logWorkerExecutor.
*
@@ -122,4 +201,11 @@ public class AlerterWorkerPool {
public void executeLogJob(Runnable runnable) throws RejectedExecutionException {
logWorkerExecutor.execute(runnable);
}
@Override
public void destroy() {
workerExecutor.shutdownNow();
notifyExecutor.close();
logWorkerExecutor.close();
}
}
@@ -25,12 +25,19 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
@@ -40,18 +47,33 @@ import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
*/
@Slf4j
@Component
public class PeriodicAlertRuleScheduler implements CommandLineRunner {
public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean {
private final MetricsPeriodicAlertCalculator metricsCalculator;
private final LogPeriodicAlertCalculator logCalculator;
private final AlertDefineDao alertDefineDao;
private final ScheduledExecutorService scheduledExecutor;
private final Map<Long, ScheduledFuture<?>> scheduledFutures;
private final ExecutorService periodicExecutor;
private final Semaphore periodicPermits;
private final boolean virtualThreadsEnabled;
private final Map<Long, ScheduledTaskState> scheduledTasks;
public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, LogPeriodicAlertCalculator logCalculator, AlertDefineDao alertDefineDao) {
this(metricsCalculator, logCalculator, alertDefineDao, VirtualThreadProperties.defaults());
}
@Autowired
public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator,
LogPeriodicAlertCalculator logCalculator,
AlertDefineDao alertDefineDao,
VirtualThreadProperties virtualThreadProperties) {
this.metricsCalculator = metricsCalculator;
this.logCalculator = logCalculator;
this.alertDefineDao = alertDefineDao;
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("Scheduled periodic alert threshold has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Scheduled periodic alert threshold has uncaughtException.");
@@ -61,17 +83,27 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner {
.setNameFormat("periodic-alert-threshold-worker-%d")
.build();
this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory);
this.scheduledFutures = new ConcurrentHashMap<>();
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.virtualThreadsEnabled = properties.enabled();
int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs());
this.periodicExecutor = virtualThreadsEnabled
? Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("periodic-alert-task-", 0)
.uncaughtExceptionHandler(handler)
.factory())
: null;
this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null;
this.scheduledTasks = new ConcurrentHashMap<>();
}
public void cancelSchedule(Long ruleId) {
if (ruleId == null) {
return;
}
ScheduledFuture<?> future = scheduledFutures.get(ruleId);
if (future != null) {
future.cancel(true);
scheduledFutures.remove(ruleId);
ScheduledTaskState state = scheduledTasks.remove(ruleId);
if (state != null) {
state.cancel();
}
}
@@ -83,14 +115,12 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner {
cancelSchedule(rule.getId());
if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)
|| rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) {
ScheduledFuture<?> future = scheduledExecutor.scheduleAtFixedRate(() -> {
if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) {
metricsCalculator.calculate(rule);
} else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) {
logCalculator.calculate(rule);
}
}, 0, rule.getPeriod(), java.util.concurrent.TimeUnit.SECONDS);
scheduledFutures.put(rule.getId(), future);
ScheduledTaskState state = new ScheduledTaskState(rule);
ScheduledFuture<?> future = scheduledExecutor.scheduleAtFixedRate(
virtualThreadsEnabled ? state::trigger : () -> executeRule(rule),
0, rule.getPeriod(), TimeUnit.SECONDS);
state.setScheduledFuture(future);
scheduledTasks.put(rule.getId(), state);
}
}
@@ -106,4 +136,107 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner {
updateSchedule(rule);
}
}
@Override
public void destroy() {
scheduledTasks.values().forEach(ScheduledTaskState::cancel);
scheduledTasks.clear();
scheduledExecutor.shutdownNow();
if (periodicExecutor != null) {
periodicExecutor.shutdownNow();
}
}
private void executeRule(AlertDefine rule) {
if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) {
metricsCalculator.calculate(rule);
} else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) {
logCalculator.calculate(rule);
}
}
private final class ScheduledTaskState {
private final AlertDefine rule;
private ScheduledFuture<?> scheduledFuture;
private Future<?> runningFuture;
private boolean running;
private boolean pending;
private boolean cancelled;
private ScheduledTaskState(AlertDefine rule) {
this.rule = rule;
}
private synchronized void setScheduledFuture(ScheduledFuture<?> scheduledFuture) {
this.scheduledFuture = scheduledFuture;
}
private synchronized void trigger() {
if (cancelled) {
return;
}
if (running) {
pending = true;
return;
}
running = true;
submitLocked();
}
private synchronized void cancel() {
cancelled = true;
pending = false;
ScheduledFuture<?> periodicFuture = scheduledFuture;
Future<?> currentFuture = runningFuture;
if (periodicFuture != null) {
periodicFuture.cancel(true);
}
if (currentFuture != null) {
currentFuture.cancel(true);
}
}
private void submitLocked() {
try {
runningFuture = periodicExecutor.submit(() -> {
boolean permitAcquired = false;
try {
periodicPermits.acquire();
permitAcquired = true;
if (!Thread.currentThread().isInterrupted()) {
executeRule(rule);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e);
} finally {
if (permitAcquired) {
periodicPermits.release();
}
onComplete();
}
});
} catch (RuntimeException e) {
running = false;
throw e;
}
}
private synchronized void onComplete() {
runningFuture = null;
if (cancelled) {
running = false;
pending = false;
return;
}
if (!pending) {
running = false;
return;
}
pending = false;
submitLocked();
}
}
}
@@ -20,6 +20,11 @@ package org.apache.hertzbeat.alert.calculate.realtime.window;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.hertzbeat.alert.util.AlertTemplateUtil;
@@ -32,10 +37,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Alarm Evaluator - Final alarm logic trigger
@@ -47,36 +48,48 @@ import java.util.concurrent.TimeUnit;
*/
@Component
@Slf4j
public class AlarmEvaluator {
public class AlarmEvaluator implements DisposableBean {
private static final String WINDOW_START_TIME = "window_start_time";
private static final String WINDOW_END_TIME = "window_end_time";
private static final String MATCHING_LOGS_COUNT = "matching_logs_count";
private final AlarmCommonReduce alarmCommonReduce;
private ThreadPoolExecutor workerExecutor;
private final ManagedExecutor workerExecutor;
public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce) {
this.alarmCommonReduce = alarmCommonReduce;
initAlarmEvaluator();
this(alarmCommonReduce, VirtualThreadProperties.defaults());
}
public void initAlarmEvaluator() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("alerter-reduce-worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("alerter-reduce-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(2,
@Autowired
public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce, VirtualThreadProperties virtualThreadProperties) {
this.alarmCommonReduce = alarmCommonReduce;
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.workerExecutor = initAlarmEvaluator(properties);
}
public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("alerter-reduce-worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
if (properties.enabled()) {
VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().windowEvaluator();
return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-",
queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler);
}
return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2,
10,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
java.util.concurrent.TimeUnit.SECONDS,
new java.util.concurrent.LinkedBlockingQueue<>(),
new ThreadFactoryBuilder()
.setUncaughtExceptionHandler(handler)
.setDaemon(true)
.setNameFormat("alerter-reduce-worker-%d")
.build(),
new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()));
}
public void sendAndProcessWindowData(WindowAggregator.WindowData windowData) {
@@ -314,4 +327,9 @@ public class AlarmEvaluator {
}
}
}
}
@Override
public void destroy() {
workerExecutor.close();
}
}
@@ -21,6 +21,7 @@ import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.RejectedExecutionException;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.AlerterWorkerPool;
import org.apache.hertzbeat.alert.config.AlertSseManager;
@@ -121,14 +122,27 @@ public class AlertNoticeDispatch {
}
private void sendNotify(GroupAlert alert) {
matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> workerPool.executeNotify(() -> rule.getReceiverId()
.forEach(receiverId -> {
try {
sendNoticeMsg(getOneReceiverById(receiverId),
getOneTemplateById(rule.getTemplateId()), alert);
} catch (AlertNoticeException e) {
log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage());
}
}))));
matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> {
NoticeTemplate noticeTemplate = getOneTemplateById(rule.getTemplateId());
rule.getReceiverId().forEach(receiverId -> {
NoticeReceiver receiver = getOneReceiverById(receiverId);
if (receiver == null || receiver.getType() == null) {
log.warn("DispatchTask skip invalid receiver, receiverId: {}, alertId: {}", receiverId, alert.getId());
return;
}
try {
workerPool.executeNotify(receiver.getType(), () -> {
try {
sendNoticeMsg(receiver, noticeTemplate, alert);
} catch (AlertNoticeException e) {
log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage());
}
});
} catch (RejectedExecutionException e) {
log.warn("DispatchTask rejected notify task, receiverId: {}, type: {}, message: {}",
receiverId, receiver.getType(), e.getMessage());
}
});
}));
}
}
@@ -0,0 +1,182 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.notice.impl;
import java.util.Map;
import java.util.StringJoiner;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.notice.AlertNoticeException;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
/**
* Send alert notification through ntfy push notification service.
* Supports priority mapping, tags/emoji, click action URL,
* and optional Bearer token authentication for self-hosted ntfy servers.
*
* @see <a href="https://docs.ntfy.sh/publish/">ntfy publish API</a>
*/
@Component
@RequiredArgsConstructor
@Slf4j
public class NtfyAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl {
private static final String STATUS_FIRING = "firing";
private static final String SEVERITY_CRITICAL = "critical";
private static final String SEVERITY_WARNING = "warning";
@Override
public void send(NoticeReceiver receiver, NoticeTemplate noticeTemplate, GroupAlert alert) throws AlertNoticeException {
try {
String content = renderContent(noticeTemplate, alert);
String url = buildNtfyUrl(receiver);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
headers.set("Title", bundle.getString("alerter.notify.title"));
headers.set("Markdown", "yes");
headers.set("Priority", String.valueOf(mapPriority(alert)));
headers.set("Tags", buildTags(alert));
// Set click action to console URL if available
if (alerterProperties != null && alerterProperties.getConsoleUrl() != null
&& !alerterProperties.getConsoleUrl().isEmpty()) {
headers.set("Click", alerterProperties.getConsoleUrl());
}
// Set Bearer token authentication for self-hosted ntfy servers
String token = receiver.getNtfyToken();
if (token != null && !token.isEmpty()) {
headers.set("Authorization", "Bearer " + token);
}
HttpEntity<String> httpEntity = new HttpEntity<>(content, headers);
ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, httpEntity, String.class);
if (responseEntity.getStatusCode() == HttpStatus.OK) {
log.debug("Send ntfy notification to {} Success", url);
} else {
log.warn("Send ntfy notification to {} Failed: {}", url, responseEntity.getBody());
throw new AlertNoticeException("Http StatusCode " + responseEntity.getStatusCode());
}
} catch (Exception e) {
throw new AlertNoticeException("[Ntfy Notify Error] " + e.getMessage());
}
}
@Override
public byte type() {
return 15;
}
/**
* Build the ntfy publish URL from receiver configuration.
*/
private String buildNtfyUrl(NoticeReceiver receiver) {
String serverUrl = receiver.getNtfyServerUrl();
if (serverUrl == null || serverUrl.isEmpty()) {
serverUrl = alerterProperties.getNtfyDefaultServerUrl();
}
if (serverUrl.endsWith("/")) {
serverUrl = serverUrl.substring(0, serverUrl.length() - 1);
}
return serverUrl + "/" + receiver.getNtfyTopic();
}
/**
* Map alert severity to ntfy priority level (1-5).
* <ul>
* <li>5 (max) - critical severity while firing</li>
* <li>4 (high) - warning severity while firing</li>
* <li>3 (default) - info or unknown severity while firing</li>
* <li>2 (low) - resolved alerts</li>
* </ul>
*
* @see <a href="https://docs.ntfy.sh/publish/#message-priority">ntfy message priority</a>
*/
protected int mapPriority(GroupAlert alert) {
String status = alert.getStatus();
if (!STATUS_FIRING.equalsIgnoreCase(status)) {
return 2;
}
String severity = extractSeverity(alert);
if (SEVERITY_CRITICAL.equalsIgnoreCase(severity)) {
return 5;
} else if (SEVERITY_WARNING.equalsIgnoreCase(severity)) {
return 4;
}
return 3;
}
/**
* Build ntfy tags string with emoji based on alert status and severity.
* Tags appear as emoji icons in the notification.
*
* @see <a href="https://docs.ntfy.sh/publish/#tags-emojis">ntfy tags &amp; emojis</a>
*/
protected String buildTags(GroupAlert alert) {
StringJoiner joiner = new StringJoiner(",");
String status = alert.getStatus();
if (STATUS_FIRING.equalsIgnoreCase(status)) {
String severity = extractSeverity(alert);
if (SEVERITY_CRITICAL.equalsIgnoreCase(severity)) {
joiner.add("rotating_light");
joiner.add("skull");
} else if (SEVERITY_WARNING.equalsIgnoreCase(severity)) {
joiner.add("warning");
} else {
joiner.add("information_source");
}
} else {
joiner.add("white_check_mark");
}
// Append alert name as a plain-text tag if available
Map<String, String> commonLabels = alert.getCommonLabels();
if (commonLabels != null && commonLabels.containsKey("alertname")) {
joiner.add(commonLabels.get("alertname"));
}
return joiner.toString();
}
/**
* Extract severity from alert's common labels.
* Checks "severity" key first, then falls back to "priority".
*/
private String extractSeverity(GroupAlert alert) {
Map<String, String> commonLabels = alert.getCommonLabels();
if (commonLabels == null) {
return null;
}
String severity = commonLabels.get("severity");
if (severity == null) {
severity = commonLabels.get("priority");
}
return severity;
}
}
@@ -18,16 +18,16 @@
package org.apache.hertzbeat.alert.reduce;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
@@ -35,33 +35,45 @@ import org.springframework.stereotype.Service;
*/
@Service
@Slf4j
public class AlarmCommonReduce {
public class AlarmCommonReduce implements DisposableBean {
private final AlarmGroupReduce alarmGroupReduce;
private ThreadPoolExecutor workerExecutor;
private final ManagedExecutor workerExecutor;
public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) {
initWorkExecutor();
this.alarmGroupReduce = alarmGroupReduce;
this(alarmGroupReduce, VirtualThreadProperties.defaults());
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("alerter-reduce-worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("alerter-reduce-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(2,
@Autowired
public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, VirtualThreadProperties virtualThreadProperties) {
this.alarmGroupReduce = alarmGroupReduce;
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.workerExecutor = initWorkExecutor(properties);
}
private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("alerter-reduce-worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
if (properties.enabled()) {
VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().reduce();
return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-",
queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler);
}
return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2,
2,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
java.util.concurrent.TimeUnit.SECONDS,
new java.util.concurrent.LinkedBlockingQueue<>(),
new ThreadFactoryBuilder()
.setUncaughtExceptionHandler(handler)
.setDaemon(true)
.setNameFormat("alerter-reduce-worker-%d")
.build(),
new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()));
}
@@ -103,10 +115,6 @@ public class AlarmCommonReduce {
* Fingerprint is based on labels excluding timestamp related fields
*/
private String generateAlertFingerprint(Map<String, String> labels) {
// Remove timestamp related fields
labels.remove("timestamp");
labels.remove("start_at");
labels.remove("active_at");
return labels.entrySet().stream()
.filter(e -> !"timestamp".equals(e.getKey())
&& !"starts_at".equals(e.getKey()) && !"actives_at".equals(e.getKey())
@@ -116,4 +124,9 @@ public class AlarmCommonReduce {
.map(e -> e.getKey() + ":" + e.getValue())
.collect(Collectors.joining(","));
}
@Override
public void destroy() {
workerExecutor.close();
}
}
@@ -26,17 +26,22 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
@@ -45,7 +50,7 @@ import org.springframework.stereotype.Component;
*/
@Component
@Slf4j
public class AlarmGroupReduce {
public class AlarmGroupReduce implements DisposableBean {
/**
* Default initial group wait time 30s
@@ -88,16 +93,60 @@ public class AlarmGroupReduce {
*/
private final Map<String, GroupAlertCache> groupCacheMap;
private final ScheduledExecutorService scheduledExecutor;
private final ExecutorService workerExecutor;
private final ScheduledDispatchTask checkTask;
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) {
this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true);
}
@Autowired
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
VirtualThreadProperties virtualThreadProperties) {
this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true);
}
AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
VirtualThreadProperties virtualThreadProperties, boolean autoStart) {
this.alarmInhibitReduce = alarmInhibitReduce;
this.groupDefines = new ConcurrentHashMap<>(8);
this.groupCacheMap = new ConcurrentHashMap<>(8);
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.scheduledExecutor = createScheduler();
this.workerExecutor = createVirtualExecutor(properties);
this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups);
List<AlertGroupConverge> groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue();
refreshGroupDefines(groupConverges);
startCheckAndSendGroups();
if (autoStart) {
startCheckAndSendGroups();
}
}
private void startCheckAndSendGroups() {
scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
}
void dispatchCheckAndSendGroups() {
checkTask.dispatch();
}
void beforeCheckAndSendGroupsRun() {
}
@Override
public void destroy() {
scheduledExecutor.shutdownNow();
if (workerExecutor != null) {
workerExecutor.shutdownNow();
}
}
private ScheduledExecutorService createScheduler() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Check alarm groups calculate has uncaughtException.");
@@ -106,21 +155,36 @@ public class AlarmGroupReduce {
.setDaemon(true)
.setNameFormat("alarm-group-calculate-%d")
.build();
ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory);
scheduledExecutor.scheduleAtFixedRate(() -> {
try {
long now = System.currentTimeMillis();
groupCacheMap.forEach((groupKey, cache) -> {
if (shouldSendGroup(cache, now)) {
sendGroupAlert(cache);
cache.setLastSendTime(now);
cache.getAlertFingerprints().clear();
}
});
} catch (Exception e) {
log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e);
}
}, 10000, CHECK_INTERVAL, java.util.concurrent.TimeUnit.MILLISECONDS);
return Executors.newSingleThreadScheduledExecutor(threadFactory);
}
private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) {
if (!properties.enabled()) {
return null;
}
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("alarm-group-calculate-vt-", 0)
.uncaughtExceptionHandler((thread, throwable) -> {
log.error("Check alarm groups calculate worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.factory());
}
private void runCheckAndSendGroups() {
beforeCheckAndSendGroupsRun();
try {
long now = System.currentTimeMillis();
groupCacheMap.forEach((groupKey, cache) -> {
if (shouldSendGroup(cache, now)) {
sendGroupAlert(cache);
cache.setLastSendTime(now);
cache.getAlertFingerprints().clear();
}
});
} catch (Exception e) {
log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e);
}
}
/**
@@ -339,4 +403,63 @@ public class AlarmGroupReduce {
private long lastSendTime;
private long lastRepeatTime;
}
private static final class ScheduledDispatchTask {
private final ExecutorService executor;
private final Runnable task;
private boolean running;
private int pendingRuns;
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
this.executor = executor;
this.task = task;
}
private void dispatch() {
boolean shouldSchedule;
synchronized (this) {
pendingRuns++;
shouldSchedule = !running;
if (shouldSchedule) {
running = true;
}
}
if (shouldSchedule) {
scheduleRun();
}
}
private void scheduleRun() {
if (executor != null) {
executor.execute(this::runOnce);
} else {
runOnce();
}
}
private void runOnce() {
try {
task.run();
} finally {
scheduleNextIfNeeded();
}
}
private void scheduleNextIfNeeded() {
boolean shouldSchedule;
synchronized (this) {
pendingRuns = Math.max(0, pendingRuns - 1);
shouldSchedule = pendingRuns > 0;
if (!shouldSchedule) {
running = false;
return;
}
}
scheduleRun();
}
}
}
@@ -18,25 +18,28 @@
package org.apache.hertzbeat.alert.reduce;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.dao.AlertInhibitDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertInhibit;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Collections;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.AllArgsConstructor;
@@ -46,7 +49,7 @@ import lombok.AllArgsConstructor;
*/
@Component
@Slf4j
public class AlarmInhibitReduce {
public class AlarmInhibitReduce implements DisposableBean {
/**
* Interval for checking and cleaning up expired source alerts
@@ -73,22 +76,69 @@ public class AlarmInhibitReduce {
/**
* Default TTL for source alerts (4 hours)
*/
private static long SOURCE_ALERT_TTL = 4 * 60 * 60 * 1000L;
private final long sourceAlertTtl;
private final ScheduledExecutorService cleanupScheduler;
private final ExecutorService cleanupExecutor;
private final ScheduledDispatchTask cleanupTask;
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao
, AlerterProperties alerterProperties) {
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true);
}
@Autowired
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) {
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true);
}
AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties,
boolean autoStart) {
this.alarmSilenceReduce = alarmSilenceReduce;
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) {
SOURCE_ALERT_TTL = alerterProperties.getInhibit().getTtl();
this.sourceAlertTtl = alerterProperties.getInhibit().getTtl();
} else {
this.sourceAlertTtl = 4 * 60 * 60 * 1000L;
}
inhibitRules = new ConcurrentHashMap<>(8);
sourceAlertCache = new ConcurrentHashMap<>(8);
this.cleanupScheduler = createCleanupScheduler();
this.cleanupExecutor = createCleanupExecutor(properties);
this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache);
List<AlertInhibit> inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue();
refreshInhibitRules(inhibits);
startScheduledCleanupCache();
if (autoStart) {
startScheduledCleanupCache();
}
}
private void startScheduledCleanupCache() {
cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
}
void dispatchCleanupCache() {
cleanupTask.dispatch();
}
void beforeCleanupCacheRun() {
}
@Override
public void destroy() {
cleanupScheduler.shutdownNow();
if (cleanupExecutor != null) {
cleanupExecutor.shutdownNow();
}
}
private ScheduledExecutorService createCleanupScheduler() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Scheduled clean up inhibit cache has uncaughtException.");
@@ -97,17 +147,30 @@ public class AlarmInhibitReduce {
.setDaemon(true)
.setNameFormat("inhibit-clean-up-%d")
.build();
ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory);
// Scheduled cleanup of all expired source alerts
scheduledExecutor.scheduleAtFixedRate(() -> {
try {
sourceAlertCache.values().forEach(this::cleanupExpiredEntries);
// Remove empty rule caches
sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty());
} catch (Exception e) {
log.error("Error during scheduled cleanup", e);
}
}, CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS);
return Executors.newSingleThreadScheduledExecutor(threadFactory);
}
private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) {
if (!properties.enabled()) {
return null;
}
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("inhibit-clean-up-vt-", 0)
.uncaughtExceptionHandler((thread, throwable) -> {
log.error("Scheduled clean up inhibit cache worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.factory());
}
private void runCleanupCache() {
beforeCleanupCacheRun();
try {
sourceAlertCache.values().forEach(this::cleanupExpiredEntries);
sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty());
} catch (Exception e) {
log.error("Error during scheduled cleanup", e);
}
}
/**
@@ -264,7 +327,7 @@ public class AlarmInhibitReduce {
SourceAlertEntry entry = new SourceAlertEntry(
alert,
System.currentTimeMillis(),
System.currentTimeMillis() + SOURCE_ALERT_TTL
System.currentTimeMillis() + sourceAlertTtl
);
ruleCache.put(alert.getFingerprint(), entry);
cleanupExpiredEntries(ruleCache);
@@ -315,4 +378,63 @@ public class AlarmInhibitReduce {
private final long createTime;
private final long expiryTime;
}
private static final class ScheduledDispatchTask {
private final ExecutorService executor;
private final Runnable task;
private boolean running;
private int pendingRuns;
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
this.executor = executor;
this.task = task;
}
private void dispatch() {
boolean shouldSchedule;
synchronized (this) {
pendingRuns++;
shouldSchedule = !running;
if (shouldSchedule) {
running = true;
}
}
if (shouldSchedule) {
scheduleRun();
}
}
private void scheduleRun() {
if (executor != null) {
executor.execute(this::runOnce);
} else {
runOnce();
}
}
private void runOnce() {
try {
task.run();
} finally {
scheduleNextIfNeeded();
}
}
private void scheduleNextIfNeeded() {
boolean shouldSchedule;
synchronized (this) {
pendingRuns = Math.max(0, pendingRuns - 1);
shouldSchedule = pendingRuns > 0;
if (!shouldSchedule) {
running = false;
return;
}
}
scheduleRun();
}
}
}
@@ -18,9 +18,12 @@
package org.apache.hertzbeat.alert.reduce;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.hertzbeat.alert.dao.AlertSilenceDao;
import org.apache.hertzbeat.alert.notice.AlertNoticeDispatch;
@@ -43,6 +46,7 @@ public class AlarmSilenceReduce {
/**
* Process alert with silence rules
* If alert matches any active silence rule, it will be silenced
*
* @param groupAlert The alert to be processed
*/
public void silenceAlarm(GroupAlert groupAlert) {
@@ -51,7 +55,7 @@ public class AlarmSilenceReduce {
alertSilenceList = alertSilenceDao.findAlertSilencesByEnableTrue();
CacheFactory.setAlertSilenceCache(alertSilenceList);
}
// Check each silence rule
for (AlertSilence alertSilence : alertSilenceList) {
// Check if alert matches silence rule
@@ -59,10 +63,10 @@ public class AlarmSilenceReduce {
if (!match && groupAlert.getGroupLabels() != null) {
Map<String, String> labels = alertSilence.getLabels();
Map<String, String> alertLabels = groupAlert.getGroupLabels();
match = labels.entrySet().stream().anyMatch(item ->
match = labels.entrySet().stream().anyMatch(item ->
alertLabels.containsKey(item.getKey()) && item.getValue().equals(alertLabels.get(item.getKey())));
}
if (match) {
LocalDateTime now = LocalDateTime.now();
if (alertSilence.getType() == 0) {
@@ -75,30 +79,59 @@ public class AlarmSilenceReduce {
} else if (alertSilence.getType() == 1) {
// Cyclic silence rule
int currentDayOfWeek = now.getDayOfWeek().getValue();
if (alertSilence.getDays() != null && alertSilence.getDays().contains((byte) currentDayOfWeek)
&& !checkAndSave(now, alertSilence)) {
if (alertSilence.getDays() != null && alertSilence.getDays().contains((byte) currentDayOfWeek)
&& !checkAndSave(now, alertSilence)) {
// Alert is silenced
return;
}
}
}
}
// No matching silence rule, forward the alert
dispatcherAlarm.dispatchAlarm(groupAlert);
}
/**
* Check if alert time is within silence period and update silence rule counter
* @param now Current time
*
* @param now Current time
* @param alertSilence Silence rule to check
* @return true if alert should not be silenced, false if alert should be silenced
*/
private boolean checkAndSave(LocalDateTime now, AlertSilence alertSilence) {
boolean startMatch = alertSilence.getPeriodStart() == null
|| now.isAfter(alertSilence.getPeriodStart().toLocalDateTime());
boolean endMatch = alertSilence.getPeriodEnd() == null
|| now.isBefore(alertSilence.getPeriodEnd().toLocalDateTime());
boolean startMatch;
boolean endMatch;
if (alertSilence.getType() == 1) {
LocalTime nowTime = now.toLocalTime();
// compare wall-clock times in the server time zone, the stored offset may differ from it
LocalTime startTime = alertSilence.getPeriodStart() == null
? null : alertSilence.getPeriodStart().withZoneSameInstant(ZoneId.systemDefault()).toLocalTime();
LocalTime endTime = alertSilence.getPeriodEnd() == null
? null : alertSilence.getPeriodEnd().withZoneSameInstant(ZoneId.systemDefault()).toLocalTime();
if (startTime == null && endTime == null) {
startMatch = true;
endMatch = true;
} else if (startTime == null) {
startMatch = true;
endMatch = !nowTime.isAfter(endTime);
} else if (endTime == null) {
startMatch = !nowTime.isBefore(startTime);
endMatch = true;
} else if (!startTime.isAfter(endTime)) {
startMatch = !nowTime.isBefore(startTime);
endMatch = !nowTime.isAfter(endTime);
} else {
// Cross-midnight window, e.g. 23:00-02:00.
startMatch = !nowTime.isBefore(startTime) || !nowTime.isAfter(endTime);
endMatch = true;
}
} else {
startMatch = alertSilence.getPeriodStart() == null
|| now.isAfter(alertSilence.getPeriodStart().withZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime());
endMatch = alertSilence.getPeriodEnd() == null
|| now.isBefore(alertSilence.getPeriodEnd().withZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime());
}
if (startMatch && endMatch) {
int time = Optional.ofNullable(alertSilence.getTimes()).orElse(0);
@@ -52,6 +52,7 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -69,18 +70,18 @@ import java.util.stream.Collectors;
@Transactional(rollbackFor = Exception.class)
@Slf4j
public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLineRunner {
private static final Map<Byte, NoticeTemplate> PRESET_TEMPLATE = new HashMap<>(16);
@Autowired
private NoticeReceiverDao noticeReceiverDao;
@Autowired
private NoticeRuleDao noticeRuleDao;
@Autowired
private NoticeTemplateDao noticeTemplateDao;
@Autowired
@Lazy
private AlertNoticeDispatch dispatcherAlarm;
@@ -91,7 +92,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
Predicate predicate = criteriaBuilder.conjunction();
if (StringUtils.isNotBlank(name)) {
Predicate predicateName = criteriaBuilder.like(
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
);
predicate = criteriaBuilder.and(predicateName);
}
@@ -113,9 +114,9 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
// Filter by name (case-insensitive)
List<NoticeTemplate> filteredDefaultTemplates = defaultTemplates.stream()
.filter(template -> StringUtils.isBlank(name)
|| template.getName().toLowerCase().contains(name.toLowerCase()))
.collect(Collectors.toList());
.filter(template -> StringUtils.isBlank(name)
|| template.getName().toLowerCase().contains(name.toLowerCase()))
.collect(Collectors.toList());
// Pagination logic
int totalItems = filteredDefaultTemplates.size();
@@ -134,7 +135,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
Predicate predicate = criteriaBuilder.conjunction();
if (StringUtils.isNotBlank(name)) {
Predicate predicateName = criteriaBuilder.like(
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
);
predicate = criteriaBuilder.and(predicateName);
}
@@ -146,7 +147,6 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
}
@Override
public List<NoticeTemplate> getAllNoticeTemplates() {
List<NoticeTemplate> defaultTemplates = new LinkedList<>(PRESET_TEMPLATE.values());
@@ -160,7 +160,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
Predicate predicate = criteriaBuilder.conjunction();
if (StringUtils.isNotBlank(name)) {
Predicate predicateName = criteriaBuilder.like(
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
criteriaBuilder.lower(root.get("name")), "%" + name.toLowerCase() + "%"
);
predicate = criteriaBuilder.and(predicateName);
}
@@ -212,44 +212,61 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
}
// The temporary rule is to forward all, and then implement more matching rules: alarm status selection, monitoring type selection, etc.
// TODO: This matches an already-grouped alert against notice rules (group-then-route). It cannot fully
// separate alerts that were grouped together but should reach different receivers, so a rule matched by
// one alert still notifies the whole group. The ideal design is route-then-group (like Alertmanager):
// route each single alert by its labels first, then group per receiver. Tracked as a follow-up to #3852.
return rules.stream()
.filter(rule -> {
if (!rule.isFilterAll()) {
// filter labels
if (rule.getLabels() != null && !rule.getLabels().isEmpty()) {
boolean labelMatch = rule.getLabels().entrySet().stream().allMatch(labelItem -> {
if (!alert.getCommonLabels().containsKey(labelItem.getKey())) {
return false;
}
String alertLabelValue = alert.getCommonLabels().get(labelItem.getKey());
return Objects.equals(labelItem.getValue(), alertLabelValue);
});
if (!labelMatch) {
.filter(rule -> {
if (!rule.isFilterAll()) {
// filter labels: a rule matches when ANY single alert in the group carries
if (rule.getLabels() != null && !rule.getLabels().isEmpty()) {
List<SingleAlert> singleAlerts = alert.getAlerts();
boolean labelMatch = singleAlerts != null && singleAlerts.stream().anyMatch(singleAlert -> {
Map<String, String> alertLabels = singleAlert.getLabels();
if (alertLabels == null) {
return false;
}
}
}
LocalDateTime nowDate = LocalDateTime.now();
// filter day
int currentDayOfWeek = nowDate.toLocalDate().getDayOfWeek().getValue();
if (rule.getDays() != null && !rule.getDays().isEmpty()) {
boolean dayMatch = rule.getDays().stream().anyMatch(item -> item == currentDayOfWeek);
if (!dayMatch) {
return rule.getLabels().entrySet().stream().allMatch(labelItem ->
Objects.equals(labelItem.getValue(), alertLabels.get(labelItem.getKey())));
});
if (!labelMatch) {
return false;
}
}
// filter time
LocalTime nowTime = nowDate.toLocalTime();
boolean startMatch = rule.getPeriodStart() == null
|| nowTime.isAfter(rule.getPeriodStart().toLocalTime())
|| (rule.getPeriodEnd() != null && rule.getPeriodStart().isAfter(rule.getPeriodEnd())
&& nowTime.isBefore(rule.getPeriodStart().toLocalTime()));
boolean endMatch = rule.getPeriodEnd() == null
|| nowTime.isBefore(rule.getPeriodEnd().toLocalTime());
return startMatch && endMatch;
})
.collect(Collectors.toList());
}
LocalDateTime nowDate = LocalDateTime.now();
// filter day
int currentDayOfWeek = nowDate.toLocalDate().getDayOfWeek().getValue();
if (rule.getDays() != null && !rule.getDays().isEmpty()) {
boolean dayMatch = rule.getDays().stream().anyMatch(item -> item == currentDayOfWeek);
if (!dayMatch) {
return false;
}
}
// filter time, compare wall-clock times in the server time zone,
// the stored date part is meaningless (it is the day the user picked the time on the ui)
LocalTime nowTime = nowDate.toLocalTime();
LocalTime startTime = rule.getPeriodStart() == null
? null : rule.getPeriodStart().withZoneSameInstant(ZoneId.systemDefault()).toLocalTime();
LocalTime endTime = rule.getPeriodEnd() == null
? null : rule.getPeriodEnd().withZoneSameInstant(ZoneId.systemDefault()).toLocalTime();
if (startTime == null && endTime == null) {
return true;
}
if (startTime == null) {
return !nowTime.isAfter(endTime);
}
if (endTime == null) {
return !nowTime.isBefore(startTime);
}
if (!startTime.isAfter(endTime)) {
return !nowTime.isBefore(startTime) && !nowTime.isAfter(endTime);
}
// cross-midnight window, e.g. 22:00-06:00
return !nowTime.isBefore(startTime) || !nowTime.isAfter(endTime);
}).collect(Collectors.toList());
}
@Override
@@ -308,31 +325,31 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
Map<String, String> annotations = new HashMap<>(8);
annotations.put("suggest", "Please check the CPU usage of the server");
SingleAlert singleAlert1 = SingleAlert.builder()
.labels(labels)
.content("test send msg! \\n This is the test data. It is proved that it can be received successfully")
.startAt(System.currentTimeMillis())
.activeAt(System.currentTimeMillis())
.endAt(System.currentTimeMillis())
.triggerTimes(2)
.annotations(annotations)
.status("firing")
.build();
.labels(labels)
.content("test send msg! \\n This is the test data. It is proved that it can be received successfully")
.startAt(System.currentTimeMillis())
.activeAt(System.currentTimeMillis())
.endAt(System.currentTimeMillis())
.triggerTimes(2)
.annotations(annotations)
.status("firing")
.build();
SingleAlert singleAlert2 = SingleAlert.builder()
.labels(labels)
.content("test send msg! \\n This is the test data. It is proved that it can be received successfully")
.startAt(System.currentTimeMillis())
.activeAt(System.currentTimeMillis())
.endAt(System.currentTimeMillis())
.triggerTimes(4)
.annotations(annotations)
.status("firing")
.build();
.labels(labels)
.content("test send msg! \\n This is the test data. It is proved that it can be received successfully")
.startAt(System.currentTimeMillis())
.activeAt(System.currentTimeMillis())
.endAt(System.currentTimeMillis())
.triggerTimes(4)
.annotations(annotations)
.status("firing")
.build();
GroupAlert groupAlert = GroupAlert.builder()
.commonLabels(Map.of(CommonConstants.LABEL_ALERT_NAME, "CPU Usage Alert"))
.commonAnnotations(annotations)
.alerts(List.of(singleAlert1, singleAlert2))
.status("firing")
.build();
.commonLabels(Map.of(CommonConstants.LABEL_ALERT_NAME, "CPU Usage Alert"))
.commonAnnotations(annotations)
.alerts(List.of(singleAlert1, singleAlert2))
.status("firing")
.build();
return dispatcherAlarm.sendNoticeMsg(noticeReceiver, null, groupAlert);
}
@@ -18,9 +18,18 @@
package org.apache.hertzbeat.alert;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.BeforeEach;
import org.apache.hertzbeat.common.concurrent.AdmissionMode;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
/**
@@ -28,60 +37,178 @@ import org.junit.jupiter.api.Test;
*/
class AlerterWorkerPoolTest {
private static final int NUMBER_OF_THREADS = 10;
private AlerterWorkerPool pool;
private AtomicInteger counter;
private CountDownLatch latch;
private static final int NUMBER_OF_TASKS = 10;
@BeforeEach
void setUp() {
pool = new AlerterWorkerPool();
counter = new AtomicInteger();
latch = new CountDownLatch(NUMBER_OF_THREADS);
private AlerterWorkerPool pool;
@AfterEach
void tearDown() {
if (pool != null) {
pool.destroy();
}
}
@Test
void executeJob() throws InterruptedException {
for (int i = 0; i < NUMBER_OF_THREADS; i++) {
pool = new AlerterWorkerPool();
AtomicInteger counter = new AtomicInteger();
CountDownLatch latch = new CountDownLatch(NUMBER_OF_TASKS);
for (int i = 0; i < NUMBER_OF_TASKS; i++) {
pool.executeJob(() -> {
counter.incrementAndGet();
latch.countDown();
});
}
latch.await();
assertEquals(NUMBER_OF_THREADS, counter.get());
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertEquals(NUMBER_OF_TASKS, counter.get());
}
@Test
void executeNotify() throws InterruptedException {
counter = new AtomicInteger();
latch = new CountDownLatch(NUMBER_OF_THREADS);
for (int i = 0; i < NUMBER_OF_THREADS; i++) {
pool.executeNotify(() -> {
counter.incrementAndGet();
latch.countDown();
});
}
latch.await();
void executeNotifyRunsOnVirtualThread() throws Exception {
pool = new AlerterWorkerPool();
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
assertEquals(NUMBER_OF_THREADS, counter.get());
pool.executeNotify((byte) 1, () -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
});
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
VirtualThreadProperties.QueueProperties.reduceDefaults(),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
8),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
pool = new AlerterWorkerPool(properties);
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
pool.executeNotify((byte) 1, () -> {
started.countDown();
try {
release.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
assertTrue(started.await(5, TimeUnit.SECONDS));
try {
assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 2, () -> {
}));
} finally {
release.countDown();
}
}
@Test
void executeNotifyRejectsWhenChannelLimitReached() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 8),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
VirtualThreadProperties.QueueProperties.reduceDefaults(),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
1),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
pool = new AlerterWorkerPool(properties);
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
pool.executeNotify((byte) 1, () -> {
started.countDown();
try {
release.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
assertTrue(started.await(5, TimeUnit.SECONDS));
try {
assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 1, () -> {
}));
} finally {
release.countDown();
}
}
@Test
void executeLogJob() throws InterruptedException {
counter = new AtomicInteger();
latch = new CountDownLatch(NUMBER_OF_THREADS);
for (int i = 0; i < NUMBER_OF_THREADS; i++) {
pool.executeLogJob(() -> {
counter.incrementAndGet();
latch.countDown();
});
}
latch.await();
pool = new AlerterWorkerPool();
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
assertEquals(NUMBER_OF_THREADS, counter.get());
pool.executeLogJob(() -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
});
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
10,
new VirtualThreadProperties.QueueProperties(1, 1),
VirtualThreadProperties.QueueProperties.reduceDefaults(),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
pool = new AlerterWorkerPool(properties);
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
pool.executeLogJob(() -> {
firstStarted.countDown();
try {
releaseFirst.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
pool.executeLogJob(secondStarted::countDown);
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
try {
assertThrows(RejectedExecutionException.class, () -> pool.executeLogJob(() -> {
}));
} finally {
releaseFirst.countDown();
}
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
}
}
@@ -0,0 +1,213 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.calculate.periodic;
import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Tests for {@link PeriodicAlertRuleScheduler}.
*/
@ExtendWith(MockitoExtension.class)
class PeriodicAlertRuleSchedulerTest {
@Mock
private MetricsPeriodicAlertCalculator metricsCalculator;
@Mock
private LogPeriodicAlertCalculator logCalculator;
@Mock
private AlertDefineDao alertDefineDao;
private PeriodicAlertRuleScheduler scheduler;
@BeforeEach
void setUp() {
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao,
VirtualThreadProperties.defaults());
}
@AfterEach
void tearDown() {
if (scheduler != null) {
scheduler.destroy();
}
}
@Test
void updateScheduleRunsPeriodicCalculationOnVirtualThread() throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
doAnswer(invocation -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
AlertDefine rule = metricRule(1L);
scheduler.updateSchedule(rule);
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void updateScheduleKeepsSingleInFlightExecutionPerRule() throws InterruptedException {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger concurrent = new AtomicInteger();
AtomicInteger maxConcurrent = new AtomicInteger();
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
int active = concurrent.incrementAndGet();
maxConcurrent.updateAndGet(current -> Math.max(current, active));
int count = invocations.incrementAndGet();
try {
if (count == 1) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (count == 2) {
secondStarted.countDown();
}
} finally {
concurrent.decrementAndGet();
}
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
scheduler.updateSchedule(metricRule(2L));
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
Thread.sleep(1200L);
assertEquals(1, maxConcurrent.get());
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
@Test
void cancelScheduleInterruptsRunningVirtualTask() throws InterruptedException {
CountDownLatch started = new CountDownLatch(1);
CountDownLatch interrupted = new CountDownLatch(1);
doAnswer(invocation -> {
started.countDown();
try {
Thread.sleep(5000L);
} catch (InterruptedException e) {
interrupted.countDown();
Thread.currentThread().interrupt();
}
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
AlertDefine rule = metricRule(3L);
scheduler.updateSchedule(rule);
assertTrue(started.await(5, TimeUnit.SECONDS));
scheduler.cancelSchedule(rule.getId());
assertTrue(interrupted.await(5, TimeUnit.SECONDS));
}
@Test
void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException {
scheduler.destroy();
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao,
periodicProperties(1));
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger concurrent = new AtomicInteger();
AtomicInteger maxConcurrent = new AtomicInteger();
doAnswer(invocation -> {
int active = concurrent.incrementAndGet();
maxConcurrent.updateAndGet(current -> Math.max(current, active));
AlertDefine rule = invocation.getArgument(0);
try {
if (rule.getId().equals(4L)) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (rule.getId().equals(5L)) {
secondStarted.countDown();
}
} finally {
concurrent.decrementAndGet();
}
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
scheduler.updateSchedule(metricRule(4L));
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
scheduler.updateSchedule(metricRule(5L));
Thread.sleep(200L);
assertEquals(1, maxConcurrent.get());
assertEquals(1L, secondStarted.getCount());
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
private AlertDefine metricRule(Long id) {
return AlertDefine.builder()
.id(id)
.name("periodic-rule-" + id)
.type(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)
.period(1)
.enable(true)
.build();
}
private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) {
return new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
maxConcurrentJobs,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
VirtualThreadProperties.QueueProperties.reduceDefaults(),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
}
}
@@ -19,9 +19,11 @@ package org.apache.hertzbeat.alert.calculate.realtime.window;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -32,14 +34,20 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -98,6 +106,13 @@ class AlarmEvaluatorTest {
windowData.addMatchingLog(matchingEvent);
}
@AfterEach
void tearDown() {
if (alarmEvaluator != null) {
alarmEvaluator.destroy();
}
}
@Test
void testProcessWindowDataWithIndividualMode() throws InterruptedException {
// Given - alert define with individual mode
@@ -343,4 +358,104 @@ class AlarmEvaluatorTest {
assertEquals(2, alerts.size());
assertEquals(2, alerts.get(0).getTriggerTimes()); // Each alert should have trigger times = total count
}
@Test
void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
doAnswer(invocation -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
return null;
}).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class));
alertDefine.setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL));
alarmEvaluator.sendAndProcessWindowData(windowData);
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
VirtualThreadProperties.QueueProperties.reduceDefaults(),
new VirtualThreadProperties.QueueProperties(1, 0),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
alarmEvaluator.destroy();
alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties);
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger invocationOrder = new AtomicInteger();
doAnswer(invocation -> {
int order = invocationOrder.incrementAndGet();
if (order == 1) {
firstStarted.countDown();
try {
releaseFirst.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
} else if (order == 2) {
secondStarted.countDown();
}
return null;
}).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class));
WindowAggregator.WindowData firstWindow = cloneWindowDataWithBody("first");
WindowAggregator.WindowData secondWindow = cloneWindowDataWithBody("second");
firstWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL));
secondWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL));
alarmEvaluator.sendAndProcessWindowData(firstWindow);
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmEvaluator.sendAndProcessWindowData(secondWindow);
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
}
private WindowAggregator.WindowData cloneWindowDataWithBody(String body) {
LogEntry logEntry = LogEntry.builder()
.timeUnixNano(System.currentTimeMillis() * 1_000_000L)
.severityText("ERROR")
.body(body)
.build();
AlertDefine define = AlertDefine.builder()
.id(alertDefine.getId())
.name(alertDefine.getName())
.type(alertDefine.getType())
.expr(alertDefine.getExpr())
.times(alertDefine.getTimes())
.template(alertDefine.getTemplate())
.labels(alertDefine.getLabels())
.annotations(alertDefine.getAnnotations())
.enable(alertDefine.isEnable())
.build();
MatchingLogEvent event = MatchingLogEvent.builder()
.logEntry(logEntry)
.alertDefine(define)
.eventTimestamp(System.currentTimeMillis())
.workerTimestamp(System.currentTimeMillis())
.build();
WindowAggregator.WindowData clonedWindowData = new WindowAggregator.WindowData(
new WindowAggregator.WindowKey(define.getId(), System.currentTimeMillis() - 60000, System.currentTimeMillis()),
define);
clonedWindowData.addMatchingLog(event);
return clonedWindowData;
}
}
@@ -19,7 +19,10 @@ package org.apache.hertzbeat.alert.notice;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyByte;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -149,4 +152,32 @@ class AlertNoticeDispatchTest {
assertFalse(alertNoticeDispatch.sendNoticeMsg(receiver, null, alert));
}
@Test
void testDispatchAlarmUsesTypedNotifyExecution() {
NoticeTemplate template = new NoticeTemplate();
template.setId(1L);
template.setName("default-template");
when(alertStoreHandler.store(alert)).thenReturn(alert);
when(noticeConfigService.getReceiverFilterRule(alert)).thenReturn(Collections.singletonList(
org.apache.hertzbeat.common.entity.alerter.NoticeRule.builder()
.receiverId(Collections.singletonList(1L))
.templateId(1L)
.build()));
when(noticeConfigService.getReceiverById(1L)).thenReturn(receiver);
when(noticeConfigService.getOneTemplateById(1L)).thenReturn(template);
doNothing().when(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert));
doAnswer(invocation -> {
Runnable task = invocation.getArgument(1);
task.run();
return null;
}).when(workerPool).executeNotify(anyByte(), any(Runnable.class));
alertNoticeDispatch.dispatchAlarm(alert);
verify(workerPool).executeNotify(eq((byte) 1), any(Runnable.class));
verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert));
verify(emitterManager).broadcast(any(String.class));
}
}
@@ -0,0 +1,288 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.notice.impl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.notice.AlertNoticeException;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
/**
* Test case for {@link NtfyAlertNotifyHandlerImpl}
*/
@ExtendWith(MockitoExtension.class)
class NtfyAlertNotifyHandlerImplTest {
@Mock
private RestTemplate restTemplate;
@Mock
private AlerterProperties alerterProperties;
@Mock
private ResourceBundle bundle;
@InjectMocks
private NtfyAlertNotifyHandlerImpl ntfyHandler;
private NoticeReceiver receiver;
private NoticeTemplate template;
@BeforeEach
public void setUp() {
receiver = new NoticeReceiver();
receiver.setId(1L);
receiver.setName("ntfy-test");
receiver.setNtfyServerUrl("https://ntfy.example.com");
receiver.setNtfyTopic("hertzbeat-alerts");
template = new NoticeTemplate();
template.setId(1L);
template.setName("test-template");
template.setContent("test alert content");
lenient().when(alerterProperties.getNtfyDefaultServerUrl()).thenReturn("https://ntfy.sh");
lenient().when(alerterProperties.getConsoleUrl()).thenReturn("https://console.hertzbeat.com");
lenient().when(bundle.getString("alerter.notify.title")).thenReturn("HertzBeat Alert");
}
@Test
void testType() {
assertEquals(15, ntfyHandler.type());
}
@Test
void testSendSuccess() {
GroupAlert alert = buildGroupAlert("firing", "critical");
ResponseEntity<String> responseEntity = new ResponseEntity<>("{\"id\":\"abc123\"}", HttpStatus.OK);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
ntfyHandler.send(receiver, template, alert);
ArgumentCaptor<String> urlCaptor = ArgumentCaptor.forClass(String.class);
verify(restTemplate).postForEntity(urlCaptor.capture(), any(HttpEntity.class), eq(String.class));
assertEquals("https://ntfy.example.com/hertzbeat-alerts", urlCaptor.getValue());
}
@Test
void testSendWithAuthToken() {
receiver.setNtfyToken("tk_testtoken123");
GroupAlert alert = buildGroupAlert("firing", "warning");
ResponseEntity<String> responseEntity = new ResponseEntity<>("{}", HttpStatus.OK);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
ntfyHandler.send(receiver, template, alert);
@SuppressWarnings("unchecked")
ArgumentCaptor<HttpEntity<String>> entityCaptor = ArgumentCaptor.forClass(HttpEntity.class);
verify(restTemplate).postForEntity(anyString(), entityCaptor.capture(), eq(String.class));
assertEquals("Bearer tk_testtoken123", entityCaptor.getValue().getHeaders().getFirst("Authorization"));
}
@Test
void testSendUsesDefaultServerWhenEmpty() {
receiver.setNtfyServerUrl("");
GroupAlert alert = buildGroupAlert("firing", "info");
ResponseEntity<String> responseEntity = new ResponseEntity<>("{}", HttpStatus.OK);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
ntfyHandler.send(receiver, template, alert);
ArgumentCaptor<String> urlCaptor = ArgumentCaptor.forClass(String.class);
verify(restTemplate).postForEntity(urlCaptor.capture(), any(HttpEntity.class), eq(String.class));
assertEquals("https://ntfy.sh/hertzbeat-alerts", urlCaptor.getValue());
}
@Test
void testSendFailure() {
GroupAlert alert = buildGroupAlert("firing", "critical");
ResponseEntity<String> responseEntity = new ResponseEntity<>("error", HttpStatus.INTERNAL_SERVER_ERROR);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
assertThrows(AlertNoticeException.class, () -> ntfyHandler.send(receiver, template, alert));
}
@Test
void testSendNetworkError() {
GroupAlert alert = buildGroupAlert("firing", "warning");
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenThrow(new org.springframework.web.client.ResourceAccessException("Connection refused"));
assertThrows(AlertNoticeException.class, () -> ntfyHandler.send(receiver, template, alert));
}
@Test
void testMapPriorityCriticalFiring() {
GroupAlert alert = buildGroupAlert("firing", "critical");
assertEquals(5, ntfyHandler.mapPriority(alert));
}
@Test
void testMapPriorityWarningFiring() {
GroupAlert alert = buildGroupAlert("firing", "warning");
assertEquals(4, ntfyHandler.mapPriority(alert));
}
@Test
void testMapPriorityInfoFiring() {
GroupAlert alert = buildGroupAlert("firing", "info");
assertEquals(3, ntfyHandler.mapPriority(alert));
}
@Test
void testMapPriorityResolved() {
GroupAlert alert = buildGroupAlert("resolved", "critical");
assertEquals(2, ntfyHandler.mapPriority(alert));
}
@Test
void testMapPriorityUnknownSeverity() {
GroupAlert alert = buildGroupAlert("firing", null);
assertEquals(3, ntfyHandler.mapPriority(alert));
}
@Test
void testBuildTagsCriticalFiring() {
GroupAlert alert = buildGroupAlert("firing", "critical");
String tags = ntfyHandler.buildTags(alert);
assertTrue(tags.contains("rotating_light"));
assertTrue(tags.contains("skull"));
}
@Test
void testBuildTagsWarningFiring() {
GroupAlert alert = buildGroupAlert("firing", "warning");
String tags = ntfyHandler.buildTags(alert);
assertTrue(tags.contains("warning"));
}
@Test
void testBuildTagsResolved() {
GroupAlert alert = buildGroupAlert("resolved", "critical");
String tags = ntfyHandler.buildTags(alert);
assertTrue(tags.contains("white_check_mark"));
}
@Test
void testBuildTagsIncludesAlertName() {
GroupAlert alert = buildGroupAlert("firing", "warning");
alert.getCommonLabels().put("alertname", "HighCPU");
String tags = ntfyHandler.buildTags(alert);
assertTrue(tags.contains("HighCPU"));
}
@Test
void testSendHeadersContainPriorityAndTags() {
GroupAlert alert = buildGroupAlert("firing", "critical");
ResponseEntity<String> responseEntity = new ResponseEntity<>("{}", HttpStatus.OK);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
ntfyHandler.send(receiver, template, alert);
@SuppressWarnings("unchecked")
ArgumentCaptor<HttpEntity<String>> entityCaptor = ArgumentCaptor.forClass(HttpEntity.class);
verify(restTemplate).postForEntity(anyString(), entityCaptor.capture(), eq(String.class));
HttpEntity<String> captured = entityCaptor.getValue();
assertEquals("5", captured.getHeaders().getFirst("Priority"));
assertEquals("yes", captured.getHeaders().getFirst("Markdown"));
assertEquals("https://console.hertzbeat.com", captured.getHeaders().getFirst("Click"));
assertTrue(captured.getHeaders().getFirst("Tags").contains("rotating_light"));
}
@Test
void testServerUrlTrailingSlashRemoved() {
receiver.setNtfyServerUrl("https://ntfy.example.com/");
GroupAlert alert = buildGroupAlert("firing", "info");
ResponseEntity<String> responseEntity = new ResponseEntity<>("{}", HttpStatus.OK);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
ntfyHandler.send(receiver, template, alert);
ArgumentCaptor<String> urlCaptor = ArgumentCaptor.forClass(String.class);
verify(restTemplate).postForEntity(urlCaptor.capture(), any(HttpEntity.class), eq(String.class));
assertEquals("https://ntfy.example.com/hertzbeat-alerts", urlCaptor.getValue());
}
private GroupAlert buildGroupAlert(String status, String severity) {
GroupAlert groupAlert = new GroupAlert();
groupAlert.setStatus(status);
Map<String, String> commonLabels = new HashMap<>();
if (severity != null) {
commonLabels.put("severity", severity);
}
groupAlert.setCommonLabels(commonLabels);
groupAlert.setCommonAnnotations(new HashMap<>());
groupAlert.setGroupLabels(new HashMap<>());
SingleAlert singleAlert = new SingleAlert();
singleAlert.setLabels(new HashMap<>());
singleAlert.setAnnotations(new HashMap<>());
if (severity != null) {
singleAlert.getLabels().put("severity", severity);
}
List<SingleAlert> alerts = new ArrayList<>();
alerts.add(singleAlert);
groupAlert.setAlerts(alerts);
return groupAlert;
}
}
@@ -17,7 +17,20 @@
package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -40,14 +53,86 @@ class AlarmCommonReduceTest {
@BeforeEach
void setUp() {
testAlert = SingleAlert.builder().build();
testAlert = SingleAlert.builder().labels(new HashMap<>(Map.of("alertname", "test"))).build();
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce);
}
@AfterEach
void tearDown() {
if (alarmCommonReduce != null) {
alarmCommonReduce.destroy();
}
}
@Test
void testReduceAndSendAlarm() {
alarmCommonReduce.reduceAndSendAlarm(testAlert);
}
@Test
void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
doAnswer(invocation -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
return null;
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
alarmCommonReduce.reduceAndSendAlarm(testAlert);
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
new VirtualThreadProperties.QueueProperties(1, 0),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
alarmCommonReduce.destroy();
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties);
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger invocationOrder = new AtomicInteger();
doAnswer(invocation -> {
int order = invocationOrder.incrementAndGet();
if (order == 1) {
firstStarted.countDown();
try {
releaseFirst.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
} else if (order == 2) {
secondStarted.countDown();
}
return null;
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder()
.labels(new HashMap<>(Map.of("name", "first"))).build());
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder()
.labels(new HashMap<>(Map.of("name", "second"))).build());
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
}
}
@@ -38,6 +38,9 @@
package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.never;
@@ -47,9 +50,15 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
@@ -73,7 +82,15 @@ class AlarmGroupReduceTest {
MockitoAnnotations.openMocks(this);
when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue())
.thenReturn(Collections.emptyList());
alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao);
alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), false);
}
@AfterEach
void tearDown() {
if (alarmGroupReduce != null) {
alarmGroupReduce.destroy();
}
}
@Test
@@ -112,6 +129,42 @@ class AlarmGroupReduceTest {
verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait
}
@Test
void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
alarmGroupReduce.destroy();
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
alarmGroupReduce.dispatchCheckAndSendGroups();
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger maxConcurrent = new AtomicInteger();
alarmGroupReduce.destroy();
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
maxConcurrent, new AtomicInteger());
alarmGroupReduce.dispatchCheckAndSendGroups();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmGroupReduce.dispatchCheckAndSendGroups();
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
private Map<String, String> createLabels(String... keyValues) {
Map<String, String> labels = new HashMap<>();
for (int i = 0; i < keyValues.length; i += 2) {
@@ -119,4 +172,67 @@ class AlarmGroupReduceTest {
}
return labels;
}
private static final class TestAlarmGroupReduce extends AlarmGroupReduce {
private final CountDownLatch virtualThreadLatch;
private final AtomicBoolean virtualThread;
private final CountDownLatch firstStarted;
private final CountDownLatch releaseFirst;
private final CountDownLatch secondStarted;
private final AtomicInteger maxConcurrent;
private final AtomicInteger concurrent;
private final AtomicInteger invocations;
private TestAlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
VirtualThreadProperties properties, CountDownLatch virtualThreadLatch,
AtomicBoolean virtualThread, CountDownLatch firstStarted,
CountDownLatch releaseFirst, CountDownLatch secondStarted,
AtomicInteger maxConcurrent, AtomicInteger invocations) {
super(alarmInhibitReduce, alertGroupConvergeDao, properties, false);
this.virtualThreadLatch = virtualThreadLatch;
this.virtualThread = virtualThread;
this.firstStarted = firstStarted;
this.releaseFirst = releaseFirst;
this.secondStarted = secondStarted;
this.maxConcurrent = maxConcurrent;
this.invocations = invocations;
this.concurrent = maxConcurrent == null ? null : new AtomicInteger();
}
@Override
void beforeCheckAndSendGroupsRun() {
if (virtualThread != null) {
virtualThread.set(Thread.currentThread().isVirtual());
}
if (virtualThreadLatch != null) {
virtualThreadLatch.countDown();
}
if (maxConcurrent == null || invocations == null) {
return;
}
int running = concurrent.incrementAndGet();
maxConcurrent.accumulateAndGet(running, Math::max);
int currentInvocation = invocations.incrementAndGet();
try {
if (currentInvocation == 1 && firstStarted != null && releaseFirst != null) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (currentInvocation == 2 && secondStarted != null) {
secondStarted.countDown();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} finally {
concurrent.decrementAndGet();
}
}
}
}
@@ -39,6 +39,7 @@
package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -49,14 +50,19 @@ import java.util.HashMap;
import java.util.Map;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.dao.AlertInhibitDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.AlertInhibit;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
@@ -89,7 +95,15 @@ class AlarmInhibitReduceTest {
inhibitProperties.setTtl(60000);
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties);
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), false);
}
@AfterEach
void tearDown() {
if (alarmInhibitReduce != null) {
alarmInhibitReduce.destroy();
}
}
@Test
@@ -290,7 +304,9 @@ class AlarmInhibitReduceTest {
AlerterProperties.InhibitProperties inhibitProperties = new AlerterProperties.InhibitProperties();
inhibitProperties.setTtl(100);
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties);
alarmInhibitReduce.destroy();
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), false);
AlertInhibit rule = AlertInhibit.builder()
.id(1L)
@@ -324,6 +340,42 @@ class AlarmInhibitReduceTest {
verify(alarmSilenceReduce).silenceAlarm(targetGroupAlert);
}
@Test
void dispatchCleanupCacheRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
alarmInhibitReduce.destroy();
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
alarmInhibitReduce.dispatchCleanupCache();
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger maxConcurrent = new AtomicInteger();
alarmInhibitReduce.destroy();
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
maxConcurrent, new AtomicInteger());
alarmInhibitReduce.dispatchCleanupCache();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmInhibitReduce.dispatchCleanupCache();
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
private GroupAlert createGroupAlert(String status, Map<String, String> labels, List<SingleAlert> alerts) {
return GroupAlert.builder()
.status(status)
@@ -347,4 +399,68 @@ class AlarmInhibitReduceTest {
.labels(labels)
.build();
}
}
private static final class TestAlarmInhibitReduce extends AlarmInhibitReduce {
private final CountDownLatch virtualThreadLatch;
private final AtomicBoolean virtualThread;
private final CountDownLatch firstStarted;
private final CountDownLatch releaseFirst;
private final CountDownLatch secondStarted;
private final AtomicInteger maxConcurrent;
private final AtomicInteger concurrent;
private final AtomicInteger invocations;
private TestAlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
AlerterProperties alerterProperties, VirtualThreadProperties properties,
CountDownLatch virtualThreadLatch, AtomicBoolean virtualThread,
CountDownLatch firstStarted, CountDownLatch releaseFirst,
CountDownLatch secondStarted, AtomicInteger maxConcurrent,
AtomicInteger invocations) {
super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties, false);
this.virtualThreadLatch = virtualThreadLatch;
this.virtualThread = virtualThread;
this.firstStarted = firstStarted;
this.releaseFirst = releaseFirst;
this.secondStarted = secondStarted;
this.maxConcurrent = maxConcurrent;
this.invocations = invocations;
this.concurrent = maxConcurrent == null ? null : new AtomicInteger();
}
@Override
void beforeCleanupCacheRun() {
if (virtualThread != null) {
virtualThread.set(Thread.currentThread().isVirtual());
}
if (virtualThreadLatch != null) {
virtualThreadLatch.countDown();
}
if (maxConcurrent == null || invocations == null) {
return;
}
int running = concurrent.incrementAndGet();
maxConcurrent.accumulateAndGet(running, Math::max);
int currentInvocation = invocations.incrementAndGet();
try {
if (currentInvocation == 1 && firstStarted != null && releaseFirst != null) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (currentInvocation == 2 && secondStarted != null) {
secondStarted.countDown();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} finally {
concurrent.decrementAndGet();
}
}
}
}
@@ -119,6 +119,88 @@ class AlarmSilenceReduceTest {
verify(alertSilenceDao).save(silenceRule);
}
@Test
void whenCyclicSilenceRuleTimeMatchesButDateDifferent_shouldNotForwardAlert() {
LocalDateTime now = LocalDateTime.now();
LocalDateTime yesterday = now.minusDays(1);
LocalDateTime start = yesterday.withHour(now.getHour())
.withMinute(Math.max(0, now.getMinute() - 10))
.withSecond(0)
.withNano(0);
LocalDateTime end = yesterday.withHour(now.getHour())
.withMinute(Math.min(59, now.getMinute() + 10))
.withSecond(59)
.withNano(0);
AlertSilence silenceRule = AlertSilence.builder()
.enable(true)
.matchAll(false)
.type((byte) 1)
.labels(createLabels("service", "web"))
.periodStart(start.atZone(ZoneId.systemDefault()))
.periodEnd(end.atZone(ZoneId.systemDefault()))
.days(Collections.singletonList((byte) now.getDayOfWeek().getValue()))
.times(0)
.build();
when(alertSilenceDao.findAlertSilencesByEnableTrue()).thenReturn(Collections.singletonList(silenceRule));
when(alertSilenceDao.save(any(AlertSilence.class))).thenReturn(silenceRule);
GroupAlert alert = createGroupAlert("firing", createLabels("service", "web"));
alarmSilenceReduce.silenceAlarm(alert);
verify(alertNoticeDispatch, never()).dispatchAlarm(alert);
verify(alertSilenceDao).save(silenceRule);
}
@Test
void whenMatchingCrossMidnightCyclicSilenceRule_shouldNotForwardAlert() {
LocalDateTime now = LocalDateTime.now();
AlertSilence silenceRule = AlertSilence.builder()
.enable(true)
.matchAll(false)
.type((byte) 1)
.labels(createLabels("service", "web"))
.periodStart(now.minusHours(1).atZone(ZoneId.systemDefault()))
.periodEnd(now.minusHours(2).atZone(ZoneId.systemDefault()))
.days(Collections.singletonList((byte) now.getDayOfWeek().getValue()))
.times(0)
.build();
when(alertSilenceDao.findAlertSilencesByEnableTrue()).thenReturn(Collections.singletonList(silenceRule));
when(alertSilenceDao.save(any(AlertSilence.class))).thenReturn(silenceRule);
GroupAlert alert = createGroupAlert("firing", createLabels("service", "web"));
alarmSilenceReduce.silenceAlarm(alert);
verify(alertNoticeDispatch, never()).dispatchAlarm(alert);
verify(alertSilenceDao).save(silenceRule);
}
@Test
void whenCrossMidnightCyclicSilenceRuleDoesNotIncludeCurrentDay_shouldForwardAlert() {
LocalDateTime now = LocalDateTime.now();
byte previousDay = (byte) now.minusDays(1).getDayOfWeek().getValue();
AlertSilence silenceRule = AlertSilence.builder()
.enable(true)
.matchAll(false)
.type((byte) 1)
.labels(createLabels("service", "web"))
.periodStart(now.minusHours(1).atZone(ZoneId.systemDefault()))
.periodEnd(now.minusHours(2).atZone(ZoneId.systemDefault()))
.days(Collections.singletonList(previousDay))
.times(0)
.build();
when(alertSilenceDao.findAlertSilencesByEnableTrue()).thenReturn(Collections.singletonList(silenceRule));
GroupAlert alert = createGroupAlert("firing", createLabels("service", "web"));
alarmSilenceReduce.silenceAlarm(alert);
verify(alertNoticeDispatch).dispatchAlarm(alert);
verify(alertSilenceDao, never()).save(any());
}
@Test
void whenSilenceRuleExpired_shouldForwardAlert() {
AlertSilence silenceRule = AlertSilence.builder()
@@ -22,10 +22,13 @@ import org.apache.hertzbeat.alert.dao.NoticeRuleDao;
import org.apache.hertzbeat.alert.dao.NoticeTemplateDao;
import org.apache.hertzbeat.alert.notice.AlertNoticeDispatch;
import org.apache.hertzbeat.alert.service.impl.NoticeConfigServiceImpl;
import org.apache.hertzbeat.common.cache.CacheFactory;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeRule;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -38,10 +41,16 @@ import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
@@ -284,4 +293,135 @@ class NoticeConfigServiceTest {
noticeConfigService.sendTestMsg(noticeReceiver);
verify(dispatcherAlarm, times(1)).sendNoticeMsg(eq(noticeReceiver), eq(noticeTemplate), any(GroupAlert.class));
}
@AfterEach
void tearDown() {
CacheFactory.clearNoticeCache();
}
private GroupAlert buildGroupAlertWithPartialLabel() {
SingleAlert alertA = SingleAlert.builder()
.labels(new HashMap<>(Map.of("alertname", "cpu", "instance", "s1", "department", "algorithm")))
.build();
SingleAlert alertB = SingleAlert.builder()
.labels(new HashMap<>(Map.of("alertname", "cpu", "instance", "s2")))
.build();
return GroupAlert.builder()
.commonLabels(new HashMap<>(Map.of("alertname", "cpu")))
.alerts(Arrays.asList(alertA, alertB))
.build();
}
/**
* A rule scoped to a label carried by only part of the group's alerts must still match,
* even though that label is absent from commonLabels. Regression test for issue #3852.
*/
@Test
void getReceiverFilterRuleMatchesLabelOnPartialAlert() {
NoticeRule rule = new NoticeRule();
rule.setId(1L);
rule.setName("algorithm-team");
rule.setFilterAll(false);
rule.setLabels(new HashMap<>(Map.of("department", "algorithm")));
CacheFactory.setNoticeCache(List.of(rule));
List<NoticeRule> matched = noticeConfigService.getReceiverFilterRule(buildGroupAlertWithPartialLabel());
assertEquals(1, matched.size());
assertEquals(1L, matched.get(0).getId());
}
/**
* A rule whose label value is not present on any alert in the group must not match.
*/
@Test
void getReceiverFilterRuleSkipsRuleWithUnmatchedLabel() {
NoticeRule rule = new NoticeRule();
rule.setId(2L);
rule.setName("infra-team");
rule.setFilterAll(false);
rule.setLabels(new HashMap<>(Map.of("department", "infra")));
CacheFactory.setNoticeCache(List.of(rule));
List<NoticeRule> matched = noticeConfigService.getReceiverFilterRule(buildGroupAlertWithPartialLabel());
assertTrue(matched.isEmpty());
}
/**
* A rule requiring several labels matches only when a single alert carries all of them,
* preventing false matches assembled across different alerts in the group.
*/
@Test
void getReceiverFilterRuleRequiresAllLabelsOnSameAlert() {
NoticeRule rule = new NoticeRule();
rule.setId(3L);
rule.setName("algorithm-on-s2");
rule.setFilterAll(false);
rule.setLabels(new HashMap<>(Map.of("department", "algorithm", "instance", "s2")));
CacheFactory.setNoticeCache(List.of(rule));
List<NoticeRule> matched = noticeConfigService.getReceiverFilterRule(buildGroupAlertWithPartialLabel());
assertTrue(matched.isEmpty());
}
/**
* A forward-all rule keeps matching regardless of labels.
*/
@Test
void getReceiverFilterRuleForwardAllAlwaysMatches() {
NoticeRule rule = new NoticeRule();
rule.setId(4L);
rule.setName("forward-all");
rule.setFilterAll(true);
CacheFactory.setNoticeCache(List.of(rule));
List<NoticeRule> matched = noticeConfigService.getReceiverFilterRule(buildGroupAlertWithPartialLabel());
assertEquals(1, matched.size());
assertEquals(4L, matched.get(0).getId());
}
@Test
void getReceiverFilterRuleMatchesPeriodContainingNow() {
ZonedDateTime now = ZonedDateTime.now();
List<NoticeRule> matched = filterWithPeriod(now.minusHours(6), now.plusHours(6));
assertEquals(1, matched.size());
}
@Test
void getReceiverFilterRuleFiltersPeriodExcludingNow() {
ZonedDateTime now = ZonedDateTime.now();
List<NoticeRule> matched = filterWithPeriod(now.plusHours(1), now.plusHours(2));
assertEquals(0, matched.size());
}
@Test
void getReceiverFilterRuleMatchesCrossMidnightPeriod() {
ZonedDateTime now = ZonedDateTime.now();
List<NoticeRule> matched = filterWithPeriod(now.minusHours(1), now.minusHours(2).plusDays(1));
assertEquals(1, matched.size());
}
@Test
void getReceiverFilterRuleNormalizesStoredOffsetToServerZone() {
ZonedDateTime now = ZonedDateTime.now();
List<NoticeRule> matched = filterWithPeriod(
now.minusHours(6).withZoneSameInstant(ZoneOffset.ofHours(-7)),
now.plusHours(6).withZoneSameInstant(ZoneOffset.ofHours(9)));
assertEquals(1, matched.size());
}
private List<NoticeRule> filterWithPeriod(ZonedDateTime periodStart, ZonedDateTime periodEnd) {
NoticeRule rule = new NoticeRule();
rule.setId(10L);
rule.setName("PeriodRule");
rule.setFilterAll(true);
rule.setPeriodStart(periodStart);
rule.setPeriodEnd(periodEnd);
CacheFactory.clearNoticeCache();
when(noticeRuleDao.findNoticeRulesByEnableTrue()).thenReturn(Collections.singletonList(rule));
return noticeConfigService.getReceiverFilterRule(new GroupAlert());
}
}
@@ -30,8 +30,8 @@
<name>${project.artifactId}</name>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<mqtt.version>1.2.5</mqtt.version>
</properties>
@@ -67,12 +67,6 @@
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</dependency>
<!-- mysql -->
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>provided</scope>
</dependency>
<!-- clickhouse -->
<dependency>
<groupId>com.clickhouse</groupId>
@@ -189,4 +183,4 @@
<version>${zookeeper.version}</version>
</dependency>
</dependencies>
</project>
</project>
@@ -20,7 +20,6 @@ package org.apache.hertzbeat.collector.collect.database;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
@@ -34,6 +33,9 @@ import org.apache.hertzbeat.collector.collect.common.cache.AbstractConnection;
import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier;
import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache;
import org.apache.hertzbeat.collector.collect.common.cache.JdbcConnect;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet;
import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -61,6 +63,8 @@ public class JdbcCommonCollect extends AbstractCollect {
private static final String QUERY_TYPE_MULTI_ROW = "multiRow";
private static final String QUERY_TYPE_COLUMNS = "columns";
private static final String RUN_SCRIPT = "runScript";
private static final int CONNECTION_LOCK_STRIPES = 64;
private static final Object[] CONNECTION_LOCKS = createConnectionLocks();
private static final String[] VULNERABLE_KEYWORDS = {"allowLoadLocalInfile", "allowLoadLocalInfileInPath", "useLocalInfile"};
@@ -205,31 +209,26 @@ public class JdbcCommonCollect extends AbstractCollect {
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
long startTime = System.currentTimeMillis();
JdbcProtocol jdbcProtocol = metrics.getJdbc();
SshTunnel sshTunnel = jdbcProtocol.getSshTunnel();
int timeout = CollectUtil.getTimeout(jdbcProtocol.getTimeout());
boolean reuseConnection = Boolean.parseBoolean(jdbcProtocol.getReuseConnection());
Statement statement = null;
String databaseUrl;
try {
if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) {
int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort());
databaseUrl = constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort));
} else {
databaseUrl = constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort());
}
statement = getConnection(jdbcProtocol.getUsername(),
jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection);
switch (jdbcProtocol.getQueryType()) {
case QUERY_TYPE_ONE_ROW -> queryOneRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime);
case QUERY_TYPE_MULTI_ROW -> queryMultiRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime);
case QUERY_TYPE_COLUMNS -> queryOneRowByMatchTwoColumns(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime);
case RUN_SCRIPT -> {
Connection connection = statement.getConnection();
FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql());
ScriptUtils.executeSqlScript(connection, rc);
case QUERY_TYPE_ONE_ROW -> {
try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1)) {
queryOneRow(rowSet, metrics.getAliasFields(), builder, startTime);
}
}
case QUERY_TYPE_MULTI_ROW -> {
try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) {
queryMultiRow(rowSet, metrics.getAliasFields(), builder, startTime);
}
}
case QUERY_TYPE_COLUMNS -> {
try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) {
queryOneRowByMatchTwoColumns(rowSet, metrics.getAliasFields(), builder, startTime);
}
}
case RUN_SCRIPT -> runScript(metrics, timeout, reuseConnection);
default -> {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Not support database query type: " + jdbcProtocol.getQueryType());
@@ -261,23 +260,6 @@ public class JdbcCommonCollect extends AbstractCollect {
log.error("Jdbc error: {}.", errorMessage, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Query Error: " + errorMessage);
} finally {
if (statement != null) {
Connection connection = null;
try {
connection = statement.getConnection();
statement.close();
} catch (Exception e) {
log.error("Jdbc close statement error: {}", e.getMessage());
}
try {
if (!reuseConnection && connection != null) {
connection.close();
}
} catch (Exception e) {
log.error("Jdbc close connection error: {}", e.getMessage());
}
}
}
}
@@ -286,11 +268,68 @@ public class JdbcCommonCollect extends AbstractCollect {
return DispatchConstants.PROTOCOL_JDBC;
}
private JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, boolean reuseConnection, int maxRows) throws Exception {
Optional<JdbcQueryExecutor> executor = JdbcQueryExecutorRegistry.resolve(metrics);
if (executor.isPresent()) {
return executor.get().executeQuery(metrics, timeout, maxRows);
}
return executeJdbcQuery(metrics.getJdbc(), timeout, reuseConnection, maxRows);
}
private JdbcQueryRowSet executeJdbcQuery(JdbcProtocol jdbcProtocol, int timeout, boolean reuseConnection,
int maxRows) throws Exception {
Statement statement = null;
try {
String databaseUrl = resolveDatabaseUrl(jdbcProtocol);
statement = getConnection(jdbcProtocol.getUsername(),
jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection);
statement.setMaxRows(maxRows);
return new ResultSetJdbcQueryRowSet(statement, statement.executeQuery(jdbcProtocol.getSql()), reuseConnection);
} catch (Exception exception) {
closeStatementAndConnection(statement, reuseConnection);
throw exception;
}
}
private void runScript(Metrics metrics, int timeout, boolean reuseConnection) throws Exception {
JdbcProtocol jdbcProtocol = metrics.getJdbc();
Statement statement = null;
try {
String databaseUrl = resolveDatabaseUrl(jdbcProtocol);
statement = getConnection(jdbcProtocol.getUsername(),
jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection);
Connection connection = statement.getConnection();
FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql());
ScriptUtils.executeSqlScript(connection, rc);
} finally {
closeStatementAndConnection(statement, reuseConnection);
}
}
private String resolveDatabaseUrl(JdbcProtocol jdbcProtocol) throws Exception {
SshTunnel sshTunnel = jdbcProtocol.getSshTunnel();
if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) {
int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort());
return constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort));
}
return constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort());
}
private Statement getConnection(String username, String password, String url, Integer timeout, boolean reuseConnection) throws Exception {
CacheIdentifier identifier = CacheIdentifier.builder()
.ip(url)
.username(username).password(password).build();
if (!reuseConnection) {
return getConnection(username, password, url, timeout, false, identifier);
}
Object lock = CONNECTION_LOCKS[Math.floorMod(identifier.hashCode(), CONNECTION_LOCKS.length)];
synchronized (lock) {
return getConnection(username, password, url, timeout, true, identifier);
}
}
private Statement getConnection(String username, String password, String url, Integer timeout,
boolean reuseConnection, CacheIdentifier identifier) throws Exception {
Statement statement = null;
if (reuseConnection) {
Optional<AbstractConnection<?>> cacheOption = connectionCommonCache.getCache(identifier, true);
@@ -322,50 +361,77 @@ public class JdbcCommonCollect extends AbstractCollect {
return statement;
}
}
// renew connection when failed
Connection connection = DriverManager.getConnection(url, username, password);
connection.setReadOnly(true);
statement = connection.createStatement();
int timeoutSecond = timeout / 1000;
timeoutSecond = timeoutSecond <= 0 ? 1 : timeoutSecond;
statement.setQueryTimeout(timeoutSecond);
statement.setMaxRows(1000);
if (reuseConnection) {
JdbcConnect jdbcConnect = new JdbcConnect(connection);
connectionCommonCache.addCache(identifier, jdbcConnect);
Connection connection = null;
try {
// renew connection when failed
connection = openConnection(url, username, password);
connection.setReadOnly(true);
statement = connection.createStatement();
int timeoutSecond = timeout / 1000;
timeoutSecond = timeoutSecond <= 0 ? 1 : timeoutSecond;
statement.setQueryTimeout(timeoutSecond);
statement.setMaxRows(1000);
if (reuseConnection) {
JdbcConnect jdbcConnect = new JdbcConnect(connection);
connectionCommonCache.addCache(identifier, jdbcConnect);
}
} catch (Exception exception) {
try {
if (statement != null) {
statement.close();
}
} catch (Exception closeException) {
log.error("Jdbc close statement error: {}", closeException.getMessage());
}
try {
if (connection != null) {
connection.close();
}
} catch (Exception closeException) {
log.error("Jdbc close connection error: {}", closeException.getMessage());
}
throw exception;
}
return statement;
}
private static Object[] createConnectionLocks() {
Object[] locks = new Object[CONNECTION_LOCK_STRIPES];
for (int i = 0; i < locks.length; i++) {
locks[i] = new Object();
}
return locks;
}
Connection openConnection(String url, String username, String password) throws SQLException {
return DriverManager.getConnection(url, username, password);
}
/**
* query one row record, response metrics header and one value row
* eg:
* query metricsone tow three four
* query sqlselect one, tow, three, four from book limit 1;
*
* @param statement statement
* @param sql sql
* @param rowSet row set
* @param columns query metrics field list
* @throws Exception when error happen
*/
private void queryOneRow(Statement statement, String sql, List<String> columns,
private void queryOneRow(JdbcQueryRowSet rowSet, List<String> columns,
CollectRep.MetricsData.Builder builder, long startTime) throws Exception {
statement.setMaxRows(1);
try (ResultSet resultSet = statement.executeQuery(sql)) {
if (resultSet.next()) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = resultSet.getString(column);
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
if (rowSet.next()) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = rowSet.getString(column);
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
builder.addValueRow(valueRowBuilder.build());
}
builder.addValueRow(valueRowBuilder.build());
}
}
@@ -380,33 +446,30 @@ public class JdbcCommonCollect extends AbstractCollect {
* three - value3
* four - value4
*
* @param statement statement
* @param sql sql
* @param rowSet row set
* @param columns query metrics field list
* @throws Exception when error happen
*/
private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List<String> columns,
private void queryOneRowByMatchTwoColumns(JdbcQueryRowSet rowSet, List<String> columns,
CollectRep.MetricsData.Builder builder, long startTime) throws Exception {
try (ResultSet resultSet = statement.executeQuery(sql)) {
HashMap<String, String> values = new HashMap<>(columns.size());
while (resultSet.next()) {
if (resultSet.getString(1) != null) {
values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2));
}
HashMap<String, String> values = new HashMap<>(columns.size());
while (rowSet.next()) {
if (rowSet.getString(1) != null) {
values.put(rowSet.getString(1).toLowerCase().trim(), rowSet.getString(2));
}
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = values.get(column.toLowerCase());
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
}
builder.addValueRow(valueRowBuilder.build());
}
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = values.get(column.toLowerCase());
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
}
builder.addValueRow(valueRowBuilder.build());
}
/**
@@ -416,28 +479,45 @@ public class JdbcCommonCollect extends AbstractCollect {
* query sqlselect one, tow, three, four from book;
* and return multi row record mapping with the metrics
*
* @param statement statement
* @param sql sql
* @param rowSet row set
* @param columns query metrics field list
* @throws Exception when error happen
*/
private void queryMultiRow(Statement statement, String sql, List<String> columns,
private void queryMultiRow(JdbcQueryRowSet rowSet, List<String> columns,
CollectRep.MetricsData.Builder builder, long startTime) throws Exception {
try (ResultSet resultSet = statement.executeQuery(sql)) {
while (resultSet.next()) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = resultSet.getString(column);
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
while (rowSet.next()) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String column : columns) {
if (CollectorConstants.RESPONSE_TIME.equals(column)) {
long time = System.currentTimeMillis() - startTime;
valueRowBuilder.addColumn(String.valueOf(time));
} else {
String value = rowSet.getString(column);
value = value == null ? CommonConstants.NULL_VALUE : value;
valueRowBuilder.addColumn(value);
}
builder.addValueRow(valueRowBuilder.build());
}
builder.addValueRow(valueRowBuilder.build());
}
}
private void closeStatementAndConnection(Statement statement, boolean reuseConnection) {
if (statement == null) {
return;
}
Connection connection = null;
try {
connection = statement.getConnection();
statement.close();
} catch (Exception exception) {
log.error("Jdbc close statement error: {}", exception.getMessage());
}
try {
if (!reuseConnection && connection != null) {
connection.close();
}
} catch (Exception exception) {
log.error("Jdbc close connection error: {}", exception.getMessage());
}
}
@@ -548,4 +628,53 @@ public class JdbcCommonCollect extends AbstractCollect {
default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform());
};
}
private static final class ResultSetJdbcQueryRowSet implements JdbcQueryRowSet {
private final Statement statement;
private final java.sql.ResultSet resultSet;
private final boolean reuseConnection;
private ResultSetJdbcQueryRowSet(Statement statement, java.sql.ResultSet resultSet, boolean reuseConnection) {
this.statement = statement;
this.resultSet = resultSet;
this.reuseConnection = reuseConnection;
}
@Override
public boolean next() throws Exception {
return resultSet.next();
}
@Override
public String getString(String column) throws Exception {
return resultSet.getString(column);
}
@Override
public String getString(int index) throws Exception {
return resultSet.getString(index);
}
@Override
public void close() throws Exception {
Connection connection = null;
try {
connection = statement.getConnection();
} catch (Exception ignored) {
// ignore
}
try {
resultSet.close();
} finally {
try {
statement.close();
} finally {
if (!reuseConnection && connection != null) {
connection.close();
}
}
}
}
}
}
@@ -15,26 +15,16 @@
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.common.ssh;
package org.apache.hertzbeat.collector.collect.database.query;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hertzbeat.common.entity.job.Metrics;
/**
* Test case for {@link CommonSshClient}
* Adapter point for replacing only the SQL query execution part of JdbcCommonCollect.
*/
class CommonSshClientTest {
public interface JdbcQueryExecutor {
@BeforeEach
void setUp() {
}
boolean supports(Metrics metrics);
@AfterEach
void tearDown() {
}
@Test
void getSshClient() {
}
JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) throws Exception;
}
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.query;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.hertzbeat.common.entity.job.Metrics;
/**
* Static registry used by JdbcCommonCollect to discover optional query executors.
*/
public final class JdbcQueryExecutorRegistry {
private static final List<JdbcQueryExecutor> EXECUTORS = new CopyOnWriteArrayList<>();
private JdbcQueryExecutorRegistry() {
}
public static void register(JdbcQueryExecutor executor) {
if (executor == null || EXECUTORS.contains(executor)) {
return;
}
EXECUTORS.add(executor);
}
public static void unregister(JdbcQueryExecutor executor) {
if (executor == null) {
return;
}
EXECUTORS.remove(executor);
}
public static Optional<JdbcQueryExecutor> resolve(Metrics metrics) {
return EXECUTORS.stream()
.filter(executor -> executor.supports(metrics))
.findFirst();
}
}
@@ -15,16 +15,19 @@
* limitations under the License.
*/
package org.apache.hertzbeat.templatehub.service;
package org.apache.hertzbeat.collector.collect.database.query;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
/**
* A minimal row cursor abstraction shared by JDBC and R2DBC-backed database queries.
*/
public interface JdbcQueryRowSet extends AutoCloseable {
public interface FileStorageService {
boolean next() throws Exception;
void uploadFile(MultipartFile file, String path, String fileName);
String getString(String column) throws Exception;
void deleteFile(String path, String fileName);
String getString(int index) throws Exception;
Resource downloadFile(String path, String fileName);
@Override
void close() throws Exception;
}
@@ -30,6 +30,7 @@ import java.io.InterruptedIOException;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.LinkedList;
@@ -82,6 +83,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
@@ -114,6 +116,10 @@ public class HttpCollectImpl extends AbstractCollect {
*/
private static final List<Pattern> DANGEROUS_XPATH_PATTERNS;
private static final Pattern XML_ENCODING_PATTERN = Pattern.compile(
"<\\?xml\\s+[^>]*encoding\\s*=\\s*[\"']([^\"']+)[\"']",
Pattern.CASE_INSENSITIVE);
static {
List<Pattern> patterns = new ArrayList<>();
for (String pattern : CollectorConstants.DANGEROUS_XPATH_PATTERNS) {
@@ -182,7 +188,7 @@ public class HttpCollectImpl extends AbstractCollect {
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8);
String resp = entity == null ? "" : readEntityAsString(entity, isXmlParseType(parseType));
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
@@ -240,6 +246,37 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
private boolean isXmlParseType(String parseType) {
return DispatchConstants.PARSE_XML_PATH.equals(parseType) || DispatchConstants.PARSE_SITE_MAP.equals(parseType);
}
private String readEntityAsString(HttpEntity entity, boolean xmlAware) throws IOException {
if (!xmlAware) {
return EntityUtils.toString(entity, StandardCharsets.UTF_8);
}
byte[] content = EntityUtils.toByteArray(entity);
ContentType contentType = ContentType.get(entity);
Charset charset = contentType == null ? null : contentType.getCharset();
if (charset == null) {
charset = detectXmlCharset(content);
}
return new String(content, charset);
}
private Charset detectXmlCharset(byte[] content) {
int previewLength = Math.min(content.length, 256);
String preview = new String(content, 0, previewLength, StandardCharsets.US_ASCII);
Matcher matcher = XML_ENCODING_PATTERN.matcher(preview);
if (matcher.find()) {
try {
return Charset.forName(matcher.group(1));
} catch (IllegalArgumentException ignored) {
return StandardCharsets.UTF_8;
}
}
return StandardCharsets.UTF_8;
}
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, CloseableHttpResponse response) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliases) {
@@ -18,9 +18,11 @@
package org.apache.hertzbeat.collector.collect.jmx;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -106,31 +108,42 @@ public class JmxCollectImpl extends AbstractCollect {
}
/**
* Validate JMX URL
*
* Validate JMX URL
*
* @param url JMX URL to validate
* @throws IllegalArgumentException if URL is potentially malicious
*/
private void validateJmxUrl(String url) throws IllegalArgumentException {
// Only allow service:jmx:rmi protocol
Assert.isTrue(url.startsWith("service:jmx:rmi:"), "Only service:jmx:rmi protocol is supported");
JMXServiceURL serviceUrl;
try {
serviceUrl = new JMXServiceURL(url);
} catch (MalformedURLException e) {
throw new IllegalArgumentException("Invalid JMX URL", e);
}
Assert.isTrue("rmi".equalsIgnoreCase(serviceUrl.getProtocol()), "Only service:jmx:rmi protocol is supported");
String[] disallowedPatterns = { "ldap:", "rmi:", "iiop:", "nis:", "dns:", "corbaname:", "http:", "https:" };
String lowerUrl = url.toLowerCase(Locale.ROOT);
String[] disallowedPatterns = {"ldap:", "iiop:", "nis:", "dns:", "corbaname:", "http:", "https:"};
for (String pattern : disallowedPatterns) {
if (url.contains(pattern) && !pattern.equals("rmi:///jndi/rmi:")) {
if (lowerUrl.contains(pattern)) {
throw new IllegalArgumentException("Potentially unsafe JNDI protocol detected in URL: " + pattern);
}
}
String lowerPath = serviceUrl.getURLPath().toLowerCase(Locale.ROOT);
if (lowerPath.startsWith("/jndi/") && !lowerPath.startsWith("/jndi/rmi://")) {
throw new IllegalArgumentException("Only rmi JNDI protocol is supported");
}
// Check for suspicious patterns
if (url.contains("${") || url.contains("$[") || url.contains(":#") || url.contains(":/")) {
if (url.contains("${") || url.contains("$[") || url.contains(":#")) {
throw new IllegalArgumentException("Potentially malicious pattern detected in JMX URL");
}
}
/**
* Validate hostname format
*
* Validate hostname format
*
* @param hostname Hostname to validate
* @return true if hostname is valid
*/
@@ -32,4 +32,9 @@ import lombok.Data;
public class ConnectConfig {
private String host;
private int port;
private String username;
private String password;
private String namespace;
private String serviceName;
private String groupName;
}
@@ -22,10 +22,14 @@ package org.apache.hertzbeat.collector.collect.registry.discovery.impl;
import com.alibaba.nacos.api.exception.NacosException;
import com.alibaba.nacos.api.naming.NamingFactory;
import com.alibaba.nacos.api.naming.NamingService;
import com.alibaba.nacos.api.naming.pojo.Instance;
import com.google.common.collect.Lists;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.registry.constant.DiscoveryClientHealthStatus;
import org.apache.hertzbeat.collector.collect.registry.discovery.DiscoveryClient;
@@ -33,6 +37,7 @@ import org.apache.hertzbeat.collector.collect.registry.discovery.entity.ConnectC
import org.apache.hertzbeat.collector.collect.registry.discovery.entity.ServerInfo;
import org.apache.hertzbeat.collector.collect.registry.discovery.entity.ServiceInstance;
import org.apache.hertzbeat.common.entity.job.protocol.RegistryProtocol;
import org.springframework.util.StringUtils;
/**
* DiscoveryClient impl of Nacos
@@ -45,9 +50,14 @@ public class NacosDiscoveryClient implements DiscoveryClient {
@Override
public ConnectConfig buildConnectConfig(RegistryProtocol registryProtocol) {
return ConnectConfig.builder()
.host(registryProtocol.getHost())
.port(Integer.parseInt(registryProtocol.getPort()))
.build();
.host(registryProtocol.getHost())
.port(Integer.parseInt(registryProtocol.getPort()))
.username(registryProtocol.getUsername())
.password(registryProtocol.getPassword())
.namespace(registryProtocol.getNamespace())
.serviceName(registryProtocol.getServiceName())
.groupName(registryProtocol.getGroupName())
.build();
}
@Override
@@ -55,7 +65,20 @@ public class NacosDiscoveryClient implements DiscoveryClient {
try {
localConnectConfig = connectConfig;
namingService = NamingFactory.createNamingService(connectConfig.getHost() + ":" + connectConfig.getPort());
Properties properties = new Properties();
properties.put("serverAddr", connectConfig.getHost() + ":" + connectConfig.getPort());
if (StringUtils.hasText(connectConfig.getUsername())) {
properties.put("username", connectConfig.getUsername());
}
if (StringUtils.hasText(connectConfig.getPassword())) {
properties.put("password", connectConfig.getPassword());
}
if (StringUtils.hasText(connectConfig.getNamespace())) {
properties.put("namespace", connectConfig.getNamespace());
}
namingService = NamingFactory.createNamingService(properties);
// Perform a synchronous probe to verify connectivity eagerly,
// because NamingFactory.createNamingService() establishes the TCP
@@ -75,9 +98,9 @@ public class NacosDiscoveryClient implements DiscoveryClient {
ServerInfo serverInfo;
if (healthCheck()) {
serverInfo = ServerInfo.builder()
.address(localConnectConfig.getHost())
.port(String.valueOf(localConnectConfig.getPort()))
.build();
.address(localConnectConfig.getHost())
.port(String.valueOf(localConnectConfig.getPort()))
.build();
} else {
throw new RuntimeException("NamingService is not healthy");
}
@@ -97,19 +120,36 @@ public class NacosDiscoveryClient implements DiscoveryClient {
}
List<ServiceInstance> serviceInstanceList = Lists.newArrayList();
try {
for (String serviceName : namingService.getServicesOfServer(0, 9999).getData()) {
namingService.getAllInstances(serviceName).forEach(instance ->
serviceInstanceList.add(ServiceInstance.builder()
.serviceId(instance.getInstanceId())
.serviceName(instance.getServiceName())
.address(instance.getIp())
.weight(instance.getWeight())
.metadata(instance.getMetadata())
.port(instance.getPort())
.healthStatus(instance.isHealthy()
? DiscoveryClientHealthStatus.UP
: DiscoveryClientHealthStatus.DOWN)
.build()));
List<String> services;
if (StringUtils.hasText(localConnectConfig.getServiceName())) {
services = List.of(localConnectConfig.getServiceName());
} else if (StringUtils.hasText(localConnectConfig.getGroupName())) {
services = namingService.getServicesOfServer(0, 9999, localConnectConfig.getGroupName()).getData();
} else {
services = namingService.getServicesOfServer(0, 9999).getData();
}
for (String serviceName : services) {
List<Instance> instances;
if (StringUtils.hasText(localConnectConfig.getGroupName())) {
instances = namingService.getAllInstances(serviceName, localConnectConfig.getGroupName());
} else {
instances = namingService.getAllInstances(serviceName);
}
instances.forEach(instance ->
serviceInstanceList.add(ServiceInstance.builder()
.serviceId(instance.getInstanceId())
.serviceName(instance.getServiceName())
.address(instance.getIp())
.weight(instance.getWeight())
.metadata(instance.getMetadata())
.port(instance.getPort())
.healthStatus(instance.isHealthy()
? DiscoveryClientHealthStatus.UP
: DiscoveryClientHealthStatus.DOWN)
.build()));
}
} catch (NacosException e) {
throw new RuntimeException("Failed to fetch instance info");
@@ -62,6 +62,11 @@ public class NacosSdCollectImpl extends AbstractCollect {
RegistryProtocol registryProtocol = RegistryProtocol.builder()
.host(metrics.getNacos_sd().getHost())
.port(metrics.getNacos_sd().getPort())
.username(metrics.getNacos_sd().getUsername())
.password(metrics.getNacos_sd().getPassword())
.namespace(metrics.getNacos_sd().getNamespace())
.groupName(metrics.getNacos_sd().getGroupName())
.serviceName(metrics.getNacos_sd().getServiceName())
.discoveryClientTypeName(DiscoveryClientInstance.NACOS.name())
.build();
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.collector.collect.ssh;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.security.GeneralSecurityException;
@@ -32,8 +33,6 @@ import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier;
import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache;
import org.apache.hertzbeat.collector.collect.common.ssh.CommonSshBlacklist;
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -49,6 +48,7 @@ import org.apache.sshd.client.channel.ClientChannelEvent;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.common.SshException;
import org.apache.sshd.common.channel.exception.SshChannelOpenException;
import org.apache.sshd.common.future.CloseFuture;
import org.apache.sshd.common.util.io.output.NoCloseOutputStream;
import org.springframework.util.StringUtils;
@@ -64,7 +64,6 @@ public class SshCollectImpl extends AbstractCollect {
private static final String PARSE_TYPE_LOG = "log";
private static final int DEFAULT_TIMEOUT = 10_000;
private final GlobalConnectionCache connectionCommonCache = GlobalConnectionCache.getInstance();
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
@@ -79,7 +78,8 @@ public class SshCollectImpl extends AbstractCollect {
long startTime = System.currentTimeMillis();
SshProtocol sshProtocol = metrics.getSsh();
boolean reuseConnection = Boolean.parseBoolean(sshProtocol.getReuseConnection());
boolean useProxy = Boolean.parseBoolean(sshProtocol.getUseProxy());
boolean useProxy = Boolean.parseBoolean(sshProtocol.getUseProxy())
&& StringUtils.hasText(sshProtocol.getProxyHost());
int timeout = CollectUtil.getTimeout(sshProtocol.getTimeout(), DEFAULT_TIMEOUT);
ClientChannel channel = null;
ClientSession clientSession = null;
@@ -147,17 +147,14 @@ public class SshCollectImpl extends AbstractCollect {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (channel != null && channel.isOpen()) {
try {
// Close the SSH channel with the 'false' parameter to ensure the session is not kept alive.
long st = System.currentTimeMillis();
channel.close(false).addListener(future ->
log.debug("channel is closed in {} ms", System.currentTimeMillis() - st));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
boolean channelClosed = true;
try {
channelClosed = closeChannel(channel, timeout);
} catch (Exception e) {
channelClosed = false;
log.error("Failed to close SSH channel", e);
}
if (clientSession != null && !reuseConnection && !useProxy) {
if (clientSession != null && (!channelClosed || (!reuseConnection && !useProxy))) {
try {
clientSession.close();
} catch (Exception e) {
@@ -167,6 +164,35 @@ public class SshCollectImpl extends AbstractCollect {
}
}
static boolean closeChannel(ClientChannel channel, int timeout) throws IOException {
if (channel == null || channel.isClosed()) {
return true;
}
long startTime = System.currentTimeMillis();
try {
CloseFuture closeFuture = channel.close(false);
if (!closeFuture.await(timeout)) {
log.warn("SSH channel graceful close timed out after {} ms, forcing local cleanup", timeout);
closeFuture = channel.close(true);
if (!closeFuture.await(timeout)) {
log.warn("SSH channel immediate close timed out after {} ms", timeout);
return false;
}
}
} catch (InterruptedIOException e) {
try {
channel.close(true);
} catch (RuntimeException closeException) {
e.addSuppressed(closeException);
} finally {
Thread.currentThread().interrupt();
}
throw e;
}
log.debug("SSH channel closed in {} ms", System.currentTimeMillis() - startTime);
return true;
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_SSH;
@@ -273,14 +299,6 @@ public class SshCollectImpl extends AbstractCollect {
}
}
private void removeConnectSessionCache(SshProtocol sshProtocol) {
CacheIdentifier identifier = CacheIdentifier.builder()
.ip(sshProtocol.getHost()).port(sshProtocol.getPort())
.username(sshProtocol.getUsername()).password(sshProtocol.getPassword())
.build();
connectionCommonCache.removeCache(identifier);
}
private ClientSession getConnectSession(SshProtocol sshProtocol, int timeout, boolean reuseConnection, boolean useProxy)
throws IOException, GeneralSecurityException {
return SshHelper.getConnectSession(sshProtocol, timeout, reuseConnection, useProxy);
@@ -17,20 +17,41 @@
package org.apache.hertzbeat.collector.collect.database;
import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier;
import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test case for {@link JdbcCommonCollect}
@@ -93,37 +114,6 @@ class JdbcCommonCollectTest {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
jdbcCommonCollect.collect(builder, metrics);
});
String[] platforms = new String[]{
"mysql", "mariadb",
"postgresql",
"clickhouse",
"sqlserver",
"oracle",
"dm"
};
for (String platform : platforms) {
JdbcProtocol jdbc = new JdbcProtocol();
jdbc.setPlatform(platform);
Metrics metrics = new Metrics();
metrics.setJdbc(jdbc);
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
jdbcCommonCollect.collect(builder, metrics);
assertNotEquals(builder.getMsg(), "Query Error: Not support database platform: " + platform);
}
// invalid platform
JdbcProtocol jdbc = new JdbcProtocol();
jdbc.setPlatform("invalid");
Metrics metrics = new Metrics();
metrics.setJdbc(jdbc);
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
jdbcCommonCollect.collect(builder, metrics);
assertEquals(builder.getCode(), CollectRep.Code.FAIL);
assertEquals(builder.getMsg(), "Query Error: Not support database platform: invalid");
}
@Test
@@ -151,13 +141,10 @@ class JdbcCommonCollectTest {
.database("test")
.url(originalUrl)
.build();
// Use reflection to call constructDatabaseUrl method
Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class);
constructMethod.setAccessible(true);
String processedUrl = (String) constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306");
String processedUrl = constructDatabaseUrl(jdbcCollect, jdbcProtocol, "localhost", "3306");
// Verify that the processed URL is the same as the original URL
assertEquals(originalUrl, processedUrl,
assertEquals(originalUrl, processedUrl,
"URL should be passed through without modification: " + originalUrl);
} catch (Exception e) {
System.out.println("URL rejected by security validation: " + originalUrl + ", reason: " + e.getMessage());
@@ -165,6 +152,98 @@ class JdbcCommonCollectTest {
}
}
@Test
void testConstructDatabaseUrlByPlatform() throws Exception {
Map<String, String> expectedUrls = new LinkedHashMap<>();
expectedUrls.put("mysql", "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false");
expectedUrls.put("mariadb", "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8&useSSL=false");
expectedUrls.put("postgresql", "jdbc:postgresql://localhost:3306/test");
expectedUrls.put("clickhouse", "jdbc:clickhouse://localhost:3306/test");
expectedUrls.put("sqlserver", "jdbc:sqlserver://localhost:3306;DatabaseName=test;trustServerCertificate=true;");
expectedUrls.put("oracle", "jdbc:oracle:thin:@localhost:3306/test");
expectedUrls.put("dm", "jdbc:dm://localhost:3306");
for (Map.Entry<String, String> entry : expectedUrls.entrySet()) {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.platform(entry.getKey())
.database("test")
.build();
assertEquals(entry.getValue(), constructDatabaseUrl(jdbcCommonCollect, jdbcProtocol, "localhost", "3306"));
}
}
@Test
void testConstructDatabaseUrlRejectsUnsupportedPlatform() {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.platform("invalid")
.database("test")
.build();
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> constructDatabaseUrl(jdbcCommonCollect, jdbcProtocol, "localhost", "3306"));
assertEquals("Not support database platform: invalid", exception.getMessage());
}
@Test
void testCloseConnectionWhenCreateStatementFails() throws Exception {
String url = "jdbc:postgresql://localhost:5432/hertzbeat";
String username = "root";
String password = "root";
Connection connection = mock(Connection.class);
when(connection.createStatement()).thenThrow(new SQLException("create statement failed"));
try (MockedStatic<DriverManager> driverManager = mockStatic(DriverManager.class)) {
driverManager.when(() -> DriverManager.getConnection(url, username, password)).thenReturn(connection);
SQLException exception = assertThrows(SQLException.class,
() -> getConnection(jdbcCommonCollect, username, password, url, 1000, false));
assertEquals("create statement failed", exception.getMessage());
verify(connection).close();
}
}
@Test
void testReuseConnectionSerializesCreationForSameIdentifier() throws Exception {
String url = "jdbc:hertzbeat-test:concurrent";
String username = "root";
String password = "root";
Connection firstConnection = mock(Connection.class);
Connection secondConnection = mock(Connection.class);
Statement firstStatement = mock(Statement.class);
Statement reusedStatement = mock(Statement.class);
when(firstConnection.createStatement()).thenReturn(firstStatement, reusedStatement);
when(secondConnection.createStatement()).thenReturn(mock(Statement.class));
BlockingJdbcCommonCollect collect = new BlockingJdbcCommonCollect(firstConnection, secondConnection);
CacheIdentifier identifier = CacheIdentifier.builder()
.ip(url).username(username).password(password).build();
GlobalConnectionCache.getInstance().removeCache(identifier);
try (ExecutorService executor = Executors.newFixedThreadPool(2)) {
Future<Statement> first = executor.submit(() -> getConnection(collect, username, password, url, 1000, true));
assertTrue(collect.firstOpenStarted.await(5, TimeUnit.SECONDS));
CountDownLatch secondTaskStarted = new CountDownLatch(1);
Future<Statement> second = executor.submit(() -> {
secondTaskStarted.countDown();
return getConnection(collect, username, password, url, 1000, true);
});
assertTrue(secondTaskStarted.await(5, TimeUnit.SECONDS));
boolean secondPhysicalConnectionOpened = collect.secondOpenStarted.await(1, TimeUnit.SECONDS);
collect.allowFirstOpen.countDown();
assertSame(firstStatement, first.get(5, TimeUnit.SECONDS));
assertSame(reusedStatement, second.get(5, TimeUnit.SECONDS));
assertFalse(secondPhysicalConnectionOpened);
assertEquals(1, collect.openCount.get());
} finally {
collect.allowFirstOpen.countDown();
GlobalConnectionCache.getInstance().removeCache(identifier);
}
}
@Test
void testConstructDatabaseUrlSecurityInterception() {
JdbcCommonCollect jdbcCollect = new JdbcCommonCollect();
@@ -197,15 +276,82 @@ class JdbcCommonCollectTest {
.build();
assertThrows(Exception.class, () -> {
try {
Method constructMethod = JdbcCommonCollect.class.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class);
constructMethod.setAccessible(true);
constructMethod.invoke(jdbcCollect, jdbcProtocol, "localhost", "3306");
} catch (InvocationTargetException e) {
throw e.getCause();
}
constructDatabaseUrl(jdbcCollect, jdbcProtocol, "localhost", "3306");
}, "Malicious URL should be blocked: " + maliciousUrl);
}
}
private String constructDatabaseUrl(JdbcCommonCollect jdbcCollect, JdbcProtocol jdbcProtocol,
String host, String port) throws Exception {
try {
Method constructMethod = JdbcCommonCollect.class
.getDeclaredMethod("constructDatabaseUrl", JdbcProtocol.class, String.class, String.class);
constructMethod.setAccessible(true);
return (String) constructMethod.invoke(jdbcCollect, jdbcProtocol, host, port);
} catch (InvocationTargetException e) {
Throwable cause = e.getCause();
if (cause instanceof Exception exception) {
throw exception;
}
if (cause instanceof Error error) {
throw error;
}
throw new RuntimeException(cause);
}
}
private Statement getConnection(JdbcCommonCollect jdbcCollect, String username, String password, String url,
Integer timeout, boolean reuseConnection) throws Exception {
try {
Method getConnectionMethod = JdbcCommonCollect.class
.getDeclaredMethod("getConnection", String.class, String.class, String.class,
Integer.class, boolean.class);
getConnectionMethod.setAccessible(true);
return (Statement) getConnectionMethod.invoke(jdbcCollect, username, password, url, timeout,
reuseConnection);
} catch (InvocationTargetException e) {
Throwable cause = e.getCause();
if (cause instanceof Exception exception) {
throw exception;
}
if (cause instanceof Error error) {
throw error;
}
throw new RuntimeException(cause);
}
}
private static final class BlockingJdbcCommonCollect extends JdbcCommonCollect {
private final Connection firstConnection;
private final Connection secondConnection;
private final CountDownLatch firstOpenStarted = new CountDownLatch(1);
private final CountDownLatch secondOpenStarted = new CountDownLatch(1);
private final CountDownLatch allowFirstOpen = new CountDownLatch(1);
private final AtomicInteger openCount = new AtomicInteger();
private BlockingJdbcCommonCollect(Connection firstConnection, Connection secondConnection) {
this.firstConnection = firstConnection;
this.secondConnection = secondConnection;
}
@Override
Connection openConnection(String url, String username, String password) throws SQLException {
if (openCount.incrementAndGet() == 1) {
firstOpenStarted.countDown();
try {
if (!allowFirstOpen.await(5, TimeUnit.SECONDS)) {
throw new SQLException("Timed out waiting to open the first connection");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new SQLException("Interrupted while opening the first connection", e);
}
return firstConnection;
}
secondOpenStarted.countDown();
return secondConnection;
}
}
}
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.collector.collect.http;
import com.google.common.collect.Lists;
import com.sun.net.httpserver.HttpServer;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
@@ -28,6 +29,7 @@ import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
@@ -190,6 +192,94 @@ class HttpCollectImplTest {
assertEquals("0", secondRow.getColumns(3), "Second server memory should be 0");
}
@Test
void parseResponseByXmlPathHonorsXmlEncodingDeclaration() throws Exception {
String xmlResponse = """
<?xml version="1.0" encoding="ISO-8859-1"?>
<root>
<item>
<name>café</name>
</item>
</root>
""";
byte[] responseBytes = xmlResponse.getBytes(StandardCharsets.ISO_8859_1);
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/metrics", exchange -> {
exchange.getResponseHeaders().add("Content-Type", "application/xml");
exchange.sendResponseHeaders(200, responseBytes.length);
exchange.getResponseBody().write(responseBytes);
exchange.close();
});
server.start();
try {
HttpProtocol http = HttpProtocol.builder()
.method("GET")
.host("127.0.0.1")
.port(String.valueOf(server.getAddress().getPort()))
.url("/metrics")
.parseType(DispatchConstants.PARSE_XML_PATH)
.parseScript("//item")
.build();
Metrics metrics = Metrics.builder()
.http(http)
.aliasFields(Lists.newArrayList("name"))
.build();
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
httpCollectImpl.collect(builder, metrics);
assertEquals(1, builder.getValuesCount());
assertEquals("café", builder.getValues(0).getColumns(0));
} finally {
server.stop(0);
}
}
@Test
void parseResponseByXmlPathFallsBackToUtf8ForUnsupportedXmlEncoding() throws Exception {
String xmlResponse = """
<?xml version="1.0" encoding="UNKNOWN-CHARSET"?>
<root>
<item>
<name>service</name>
</item>
</root>
""";
byte[] responseBytes = xmlResponse.getBytes(StandardCharsets.UTF_8);
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/metrics", exchange -> {
exchange.getResponseHeaders().add("Content-Type", "application/xml");
exchange.sendResponseHeaders(200, responseBytes.length);
exchange.getResponseBody().write(responseBytes);
exchange.close();
});
server.start();
try {
HttpProtocol http = HttpProtocol.builder()
.method("GET")
.host("127.0.0.1")
.port(String.valueOf(server.getAddress().getPort()))
.url("/metrics")
.parseType(DispatchConstants.PARSE_XML_PATH)
.parseScript("//item")
.build();
Metrics metrics = Metrics.builder()
.http(http)
.aliasFields(Lists.newArrayList("name"))
.build();
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
httpCollectImpl.collect(builder, metrics);
assertEquals(1, builder.getValuesCount());
assertEquals("service", builder.getValues(0).getColumns(0));
} finally {
server.stop(0);
}
}
@Test
void parseResponseByJsonPath() throws Exception {
String jsonResponse = "{"
@@ -50,7 +50,7 @@ class JmxCollectImplTest {
JmxProtocol jmx = JmxProtocol.builder().build();
jmx.setUrl("/stub/");
Metrics metrics = Metrics.builder().jmx(jmx).build();
jmxCollect.preCheck(metrics);
});
}
@@ -68,4 +68,25 @@ class JmxCollectImplTest {
void supportProtocol() {
assert DispatchConstants.PROTOCOL_JMX.equals(jmxCollect.supportProtocol());
}
@Test
void preCheckShouldAcceptSupportedJmxRmiUrl() {
JmxProtocol jmx = JmxProtocol.builder()
.url("service:jmx:rmi:///jndi/rmi://127.0.0.1:9999/jmxrmi")
.build();
Metrics metrics = Metrics.builder().jmx(jmx).build();
assertDoesNotThrow(() -> jmxCollect.preCheck(metrics));
}
@Test
void preCheckShouldRejectUnsafeJndiProtocolInJmxRmiUrl() {
JmxProtocol jmx = JmxProtocol.builder()
.url("service:jmx:rmi:///jndi/ldap://127.0.0.1:1389/jmxrmi")
.build();
Metrics metrics = Metrics.builder().jmx(jmx).build();
assertThrows(IllegalArgumentException.class, () -> jmxCollect.preCheck(metrics));
}
}
@@ -24,9 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -39,6 +41,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.hertzbeat.collector.collect.registry.constant.DiscoveryClientHealthStatus;
import org.apache.hertzbeat.collector.collect.registry.discovery.entity.ConnectConfig;
import org.apache.hertzbeat.collector.collect.registry.discovery.entity.ServerInfo;
@@ -77,6 +80,11 @@ class NacosDiscoveryClientTest {
RegistryProtocol protocol = RegistryProtocol.builder()
.host(HOST)
.port(String.valueOf(PORT))
.username("test-user")
.password("test-pass")
.namespace("test-namespace")
.serviceName("test-service")
.groupName("test-group")
.build();
ConnectConfig config = nacosDiscoveryClient.buildConnectConfig(protocol);
@@ -84,6 +92,11 @@ class NacosDiscoveryClientTest {
assertNotNull(config);
assertEquals(HOST, config.getHost());
assertEquals(PORT, config.getPort());
assertEquals("test-user", config.getUsername());
assertEquals("test-pass", config.getPassword());
assertEquals("test-namespace", config.getNamespace());
assertEquals("test-service", config.getServiceName());
assertEquals("test-group", config.getGroupName());
}
@Test
@@ -91,7 +104,7 @@ class NacosDiscoveryClientTest {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).build();
try (MockedStatic<NamingFactory> mockedFactory = Mockito.mockStatic(NamingFactory.class)) {
mockedFactory.when(() -> NamingFactory.createNamingService(HOST + ":" + PORT))
mockedFactory.when(() -> NamingFactory.createNamingService(any(Properties.class)))
.thenReturn(namingService);
ListView<String> emptyView = new ListView<>();
emptyView.setData(Collections.emptyList());
@@ -99,17 +112,17 @@ class NacosDiscoveryClientTest {
nacosDiscoveryClient.initClient(config);
mockedFactory.verify(() -> NamingFactory.createNamingService(HOST + ":" + PORT));
mockedFactory.verify(() -> NamingFactory.createNamingService(any(Properties.class)));
verify(namingService).getServicesOfServer(0, 1);
}
}
@Test
void testInitClientFailedOnCreate() throws NacosException {
void testInitClientFailedOnCreate() {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).build();
try (MockedStatic<NamingFactory> mockedFactory = Mockito.mockStatic(NamingFactory.class)) {
mockedFactory.when(() -> NamingFactory.createNamingService(anyString()))
mockedFactory.when(() -> NamingFactory.createNamingService(any(Properties.class)))
.thenThrow(new NacosException(500, "connection refused"));
assertThrows(RuntimeException.class, () -> nacosDiscoveryClient.initClient(config));
@@ -121,7 +134,7 @@ class NacosDiscoveryClientTest {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).build();
try (MockedStatic<NamingFactory> mockedFactory = Mockito.mockStatic(NamingFactory.class)) {
mockedFactory.when(() -> NamingFactory.createNamingService(anyString()))
mockedFactory.when(() -> NamingFactory.createNamingService(any(Properties.class)))
.thenReturn(namingService);
when(namingService.getServicesOfServer(0, 1))
.thenThrow(new NacosException(500, "host unreachable"));
@@ -244,6 +257,94 @@ class NacosDiscoveryClientTest {
assertThrows(RuntimeException.class, () -> nacosDiscoveryClient.getServices());
}
@Test
void testGetServicesWithGroupNameOnly() throws NacosException {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).groupName("test-group").build();
injectNamingServiceAndConfig(config);
when(namingService.getServerStatus()).thenReturn(DiscoveryClientHealthStatus.UP);
ListView<String> serviceNames = new ListView<>();
serviceNames.setData(Collections.singletonList("test-service"));
when(namingService.getServicesOfServer(0, 9999, "test-group")).thenReturn(serviceNames);
Instance instance = new Instance();
instance.setInstanceId("inst-2");
instance.setServiceName("test-service");
instance.setIp("192.168.1.2");
instance.setPort(9090);
instance.setWeight(1.0);
instance.setHealthy(true);
when(namingService.getAllInstances("test-service", "test-group")).thenReturn(Collections.singletonList(instance));
List<ServiceInstance> result = nacosDiscoveryClient.getServices();
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("inst-2", result.get(0).getServiceId());
verify(namingService).getServicesOfServer(0, 9999, "test-group");
verify(namingService).getAllInstances("test-service", "test-group");
}
@Test
void testGetServicesWithServiceNameOnly() throws NacosException {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).serviceName("test-service-only").build();
injectNamingServiceAndConfig(config);
when(namingService.getServerStatus()).thenReturn(DiscoveryClientHealthStatus.UP);
Instance instance = new Instance();
instance.setInstanceId("inst-3");
instance.setServiceName("test-service-only");
instance.setIp("192.168.1.3");
instance.setPort(9090);
instance.setWeight(1.0);
instance.setHealthy(true);
when(namingService.getAllInstances("test-service-only")).thenReturn(Collections.singletonList(instance));
List<ServiceInstance> result = nacosDiscoveryClient.getServices();
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("inst-3", result.get(0).getServiceId());
// Assert getServicesOfServer is NOT called
verify(namingService, never()).getServicesOfServer(anyInt(), anyInt());
verify(namingService, never()).getServicesOfServer(anyInt(), anyInt(), anyString());
verify(namingService).getAllInstances("test-service-only");
}
@Test
void testGetServicesWithGroupAndServiceName() throws NacosException {
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).groupName("test-group").serviceName("target-service").build();
injectNamingServiceAndConfig(config);
when(namingService.getServerStatus()).thenReturn(DiscoveryClientHealthStatus.UP);
Instance instance = new Instance();
instance.setInstanceId("inst-4");
instance.setServiceName("target-service");
instance.setIp("192.168.1.4");
instance.setPort(9090);
instance.setWeight(1.0);
instance.setHealthy(true);
when(namingService.getAllInstances("target-service", "test-group")).thenReturn(Collections.singletonList(instance));
List<ServiceInstance> result = nacosDiscoveryClient.getServices();
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("inst-4", result.get(0).getServiceId());
// Assert getServicesOfServer is NOT called
verify(namingService, never()).getServicesOfServer(anyInt(), anyInt());
verify(namingService, never()).getServicesOfServer(anyInt(), anyInt(), anyString());
verify(namingService).getAllInstances("target-service", "test-group");
}
@Test
void testHealthCheckReturnsTrue() {
injectNamingServiceAndConfig();
@@ -285,8 +386,11 @@ class NacosDiscoveryClientTest {
}
private void injectNamingServiceAndConfig() {
injectNamingServiceAndConfig(ConnectConfig.builder().host(HOST).port(PORT).build());
}
private void injectNamingServiceAndConfig(ConnectConfig config) {
ReflectionTestUtils.setField(nacosDiscoveryClient, "namingService", namingService);
ConnectConfig config = ConnectConfig.builder().host(HOST).port(PORT).build();
ReflectionTestUtils.setField(nacosDiscoveryClient, "localConnectConfig", config);
}
}
@@ -19,14 +19,32 @@ package org.apache.hertzbeat.collector.collect.ssh;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.List;
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.SshProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.sshd.client.channel.ChannelExec;
import org.apache.sshd.client.channel.ClientChannel;
import org.apache.sshd.client.future.OpenFuture;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.common.SshException;
import org.apache.sshd.common.future.CloseFuture;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
/**
* Test case for {@link SshCollectImpl}
@@ -74,4 +92,153 @@ class SshCollectImplTest {
void supportProtocol() {
assertEquals(DispatchConstants.PROTOCOL_SSH, sshCollect.supportProtocol());
}
@Test
void closeChannelWaitsForGracefulClose() throws IOException {
ClientChannel channel = mock(ClientChannel.class);
CloseFuture closeFuture = mock(CloseFuture.class);
when(channel.close(false)).thenReturn(closeFuture);
when(closeFuture.await(1_000)).thenReturn(true);
assertTrue(SshCollectImpl.closeChannel(channel, 1_000));
verify(channel).close(false);
verify(channel, never()).close(true);
}
@Test
void closeChannelForcesCleanupAfterGracefulTimeout() throws IOException {
ClientChannel channel = mock(ClientChannel.class);
CloseFuture gracefulClose = mock(CloseFuture.class);
CloseFuture immediateClose = mock(CloseFuture.class);
when(channel.close(false)).thenReturn(gracefulClose);
when(gracefulClose.await(1_000)).thenReturn(false);
when(channel.close(true)).thenReturn(immediateClose);
when(immediateClose.await(1_000)).thenReturn(true);
assertTrue(SshCollectImpl.closeChannel(channel, 1_000));
verify(channel).close(false);
verify(channel).close(true);
}
@Test
void closeChannelForcesCleanupAndPreservesInterrupt() throws IOException {
ClientChannel channel = mock(ClientChannel.class);
CloseFuture gracefulClose = mock(CloseFuture.class);
when(channel.close(false)).thenReturn(gracefulClose);
when(gracefulClose.await(1_000)).thenThrow(new InterruptedIOException("interrupted"));
try {
assertThrows(InterruptedIOException.class,
() -> SshCollectImpl.closeChannel(channel, 1_000));
assertTrue(Thread.currentThread().isInterrupted());
verify(channel).close(true);
} finally {
Thread.interrupted();
}
}
@Test
void closeChannelSkipsAlreadyClosedChannel() throws IOException {
ClientChannel channel = mock(ClientChannel.class);
when(channel.isClosed()).thenReturn(true);
assertTrue(SshCollectImpl.closeChannel(channel, 1_000));
verify(channel, never()).close(false);
verify(channel, never()).close(true);
}
@Test
void closeChannelSuppressesForcedCloseFailureAndPreservesInterrupt() throws IOException {
ClientChannel channel = mock(ClientChannel.class);
CloseFuture gracefulClose = mock(CloseFuture.class);
InterruptedIOException interrupted = new InterruptedIOException("interrupted");
IllegalStateException closeFailure = new IllegalStateException("force close failed");
when(channel.close(false)).thenReturn(gracefulClose);
when(gracefulClose.await(1_000)).thenThrow(interrupted);
when(channel.close(true)).thenThrow(closeFailure);
try {
InterruptedIOException thrown = assertThrows(InterruptedIOException.class,
() -> SshCollectImpl.closeChannel(channel, 1_000));
assertSame(interrupted, thrown);
assertEquals(List.of(closeFailure), List.of(thrown.getSuppressed()));
assertTrue(Thread.currentThread().isInterrupted());
} finally {
Thread.interrupted();
}
}
@Test
void collectFinallyAwaitsGracefulChannelClose() throws Exception {
int timeout = 1_000;
SshProtocol protocol = protocol(timeout);
Metrics metrics = Metrics.builder().ssh(protocol).build();
ClientSession clientSession = mock(ClientSession.class);
ChannelExec channel = mock(ChannelExec.class);
OpenFuture openFuture = mock(OpenFuture.class);
CloseFuture closeFuture = mock(CloseFuture.class);
when(clientSession.createExecChannel("echo ok")).thenReturn(channel);
when(channel.open()).thenReturn(openFuture);
when(openFuture.verify(timeout)).thenThrow(new SshException("channel open failed"));
when(channel.close(false)).thenReturn(closeFuture);
when(closeFuture.await(timeout)).thenReturn(true);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(protocol, timeout, true, false))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.UN_CONNECTABLE, builder.getCode());
verify(channel).close(false);
verify(closeFuture).await(timeout);
verify(channel, never()).close(true);
verify(clientSession, never()).close();
}
@Test
void collectClosesExactSessionWhenChannelCannotClose() throws Exception {
int timeout = 1_000;
SshProtocol protocol = protocol(timeout);
Metrics metrics = Metrics.builder().ssh(protocol).build();
ClientSession clientSession = mock(ClientSession.class);
ChannelExec channel = mock(ChannelExec.class);
OpenFuture openFuture = mock(OpenFuture.class);
CloseFuture gracefulClose = mock(CloseFuture.class);
CloseFuture immediateClose = mock(CloseFuture.class);
when(clientSession.createExecChannel("echo ok")).thenReturn(channel);
when(channel.open()).thenReturn(openFuture);
when(openFuture.verify(timeout)).thenThrow(new SshException("channel open failed"));
when(channel.close(false)).thenReturn(gracefulClose);
when(gracefulClose.await(timeout)).thenReturn(false);
when(channel.close(true)).thenReturn(immediateClose);
when(immediateClose.await(timeout)).thenReturn(false);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(protocol, timeout, true, false))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
verify(channel).close(false);
verify(gracefulClose).await(timeout);
verify(channel).close(true);
verify(immediateClose).await(timeout);
verify(clientSession).close();
}
private SshProtocol protocol(int timeout) {
return SshProtocol.builder()
.host("target.example.com")
.port("22")
.username("root")
.password("password")
.timeout(String.valueOf(timeout))
.reuseConnection("true")
.useProxy("false")
.script("echo ok")
.build();
}
}
@@ -17,12 +17,15 @@
package org.apache.hertzbeat.collector.dispatch;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import org.junit.jupiter.api.BeforeEach;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hertzbeat.common.concurrent.AdmissionMode;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
/**
@@ -32,33 +35,57 @@ class WorkerPoolTest {
private WorkerPool workerPool;
private Runnable mockTask;
@BeforeEach
void setUp() {
@AfterEach
void tearDown() throws Exception {
if (workerPool != null) {
workerPool.destroy();
}
}
@Test
void testExecuteJobRunsOnVirtualThread() throws Exception {
workerPool = new WorkerPool();
mockTask = mock(Runnable.class);
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
workerPool.executeJob(() -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
});
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void testExecuteJob() {
void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
VirtualThreadProperties.AlerterProperties.defaults(),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
workerPool = new WorkerPool(properties);
assertDoesNotThrow(() -> workerPool.executeJob(mockTask));
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
workerPool.executeJob(() -> {
started.countDown();
try {
release.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
assertTrue(started.await(5, TimeUnit.SECONDS));
try {
assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(() -> {
}));
} finally {
release.countDown();
}
}
@Test
void testExecuteJobThrowsException() {
workerPool = mock(WorkerPool.class);
doThrow(new RejectedExecutionException()).when(workerPool).executeJob(mockTask);
assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(mockTask));
}
@Test
void testDestroy() {
assertDoesNotThrow(() -> workerPool.destroy());
}
}
@@ -18,19 +18,27 @@
package org.apache.hertzbeat.collector.dispatch.entrance;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import io.netty.channel.Channel;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -62,7 +70,7 @@ class CollectServerTest {
private DispatchProperties.EntranceProperties.NettyProperties nettyProperties;
@Mock
private CommonThreadPool threadPool;
private BackgroundTaskExecutor threadPool;
@Mock
private CollectorInfoProperties infoProperties;
@@ -142,4 +150,69 @@ class CollectServerTest {
assertNotNull(scheduledExecutor);
}
@Test
void testDispatchHeartbeatRunsOnVirtualThread() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties();
collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties);
RemotingClient remotingClient = mock(RemotingClient.class);
ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient);
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
org.mockito.Mockito.doAnswer(invocation -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
return null;
}).when(remotingClient).sendMsg(any(ClusterMsg.Message.class));
collectServer.dispatchHeartbeat("collector1");
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void testDispatchHeartbeatDoesNotRunConcurrently() throws Exception {
VirtualThreadProperties properties = new VirtualThreadProperties();
collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties);
RemotingClient remotingClient = mock(RemotingClient.class);
ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient);
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger concurrent = new AtomicInteger();
AtomicInteger maxConcurrent = new AtomicInteger();
AtomicInteger invocations = new AtomicInteger();
org.mockito.Mockito.doAnswer(invocation -> {
int running = concurrent.incrementAndGet();
maxConcurrent.accumulateAndGet(running, Math::max);
int currentInvocation = invocations.incrementAndGet();
if (currentInvocation == 1) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (currentInvocation == 2) {
secondStarted.countDown();
}
concurrent.decrementAndGet();
return null;
}).when(remotingClient).sendMsg(any(ClusterMsg.Message.class));
collectServer.dispatchHeartbeat("collector1");
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
collectServer.dispatchHeartbeat("collector1");
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
private DispatchProperties properties() {
return properties;
}
}
@@ -29,12 +29,18 @@
<name>${project.artifactId}</name>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common-spring</artifactId>
<version>${hertzbeat.version}</version>
</dependency>
<!-- collector-basic -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
@@ -42,6 +48,12 @@
<version>${hertzbeat.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-collector-mysql-r2dbc</artifactId>
<version>${hertzbeat.version}</version>
</dependency>
<!-- collector-kafka -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
@@ -93,10 +105,59 @@
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>apache-hertzbeat-collector-${hzb.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<configuration>
<classesDirectory>target/classes/</classesDirectory>
<archive>
<!--Generated JAR does not include Maven descriptor-related files-->
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<!--Project startup class-->
<mainClass>org.apache.hertzbeat.collector.Collector</mainClass>
<useUniqueVersions>false</useUniqueVersions>
<!--Third-party JARs are added to the classpath using maven-dependency-plugin-->
<addClasspath>true</addClasspath>
<!--Location of external dependency JARs-->
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Class-Path>. config</Class-Path>
</manifestEntries>
</archive>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
@@ -142,32 +203,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>17</source>
<target>17</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<configuration>
<classesDirectory>target/classes/</classesDirectory>
<archive>
<!--Generated JAR does not include Maven descriptor-related files-->
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<!--Project startup class-->
<mainClass>org.apache.hertzbeat.collector.Collector</mainClass>
<useUniqueVersions>false</useUniqueVersions>
<!--Third-party JARs are added to the classpath using maven-dependency-plugin-->
<addClasspath>true</addClasspath>
<!--Location of external dependency JARs-->
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Class-Path>. config</Class-Path>
</manifestEntries>
</archive>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
@@ -216,32 +253,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>17</source>
<target>17</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<configuration>
<classesDirectory>target/classes/</classesDirectory>
<archive>
<!--Generated JAR does not include Maven descriptor-related files-->
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<!--Project startup class-->
<mainClass>org.apache.hertzbeat.collector.Collector</mainClass>
<useUniqueVersions>false</useUniqueVersions>
<!--Third-party JARs are added to the classpath using maven-dependency-plugin-->
<addClasspath>true</addClasspath>
<!--Location of external dependency JARs-->
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Class-Path>. config</Class-Path>
</manifestEntries>
</archive>
<source>${java.version}</source>
<target>${java.version}</target>
</configuration>
</plugin>
<plugin>
@@ -344,5 +357,193 @@
</plugins>
</build>
</profile>
<profile>
<id>native</id>
<properties>
<native.image.name>hertzbeat-collector-collector</native.image.name>
<native.binary.extension></native.binary.extension>
<native.launcher.dir>../../script/assembly/collector/bin-native</native.launcher.dir>
<native.package.format>tar.gz</native.package.format>
<native.target.platform>unsupported</native.target.platform>
<native.package.id>${native.target.platform}-bin</native.package.id>
<native.package.baseDirectory>apache-hertzbeat-collector-native-${hzb.version}-${native.target.platform}-bin</native.package.baseDirectory>
<native.binary.source>target/${native.image.name}${native.binary.extension}</native.binary.source>
<native.executable.packageName>${project.build.finalName}${native.binary.extension}</native.executable.packageName>
</properties>
<build>
<finalName>apache-hertzbeat-collector-native-${hzb.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<proc>full</proc>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>3.5.0</version>
<executions>
<execution>
<id>require-supported-native-platform</id>
<phase>validate</phase>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireProperty>
<property>native.target.platform</property>
<regex>^(linux-amd64|linux-arm64|macos-amd64|macos-arm64|windows-amd64)$</regex>
<message>Unsupported native collector platform. Build the native collector on a supported GitHub runner or matching local host: linux-amd64, linux-arm64, macos-amd64, macos-arm64, windows-amd64.</message>
</requireProperty>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.graalvm.buildtools</groupId>
<artifactId>native-maven-plugin</artifactId>
<version>0.11.4</version>
<executions>
<execution>
<id>build-native-image</id>
<phase>package</phase>
<goals>
<goal>compile-no-fork</goal>
</goals>
</execution>
</executions>
<configuration>
<imageName>${native.image.name}</imageName>
<buildArgs>
<buildArg>-H:+UnlockExperimentalVMOptions</buildArg>
<buildArg>-H:-AddAllFileSystemProviders</buildArg>
<buildArg>-H:ServiceLoaderFeatureExcludeServiceProviders=org.apache.sshd.common.file.root.RootedFileSystemProvider,org.apache.sshd.sftp.client.fs.SftpFileSystemProvider</buildArg>
<buildArg>--initialize-at-build-time=net.i2p.crypto.eddsa.EdDSASecurityProvider,org.apache.arrow.memory.util.MemoryUtil</buildArg>
<buildArg>-J--add-opens=java.base/java.nio=ALL-UNNAMED</buildArg>
<buildArg>-J-Dorg.apache.sshd.security.registrars=org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar</buildArg>
</buildArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
<executions>
<execution>
<id>native-bin</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>../../script/assembly/collector/assembly-native.xml</descriptor>
</descriptors>
<outputDirectory>../../dist</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>native-platform-linux-amd64</id>
<activation>
<os>
<name>Linux</name>
<arch>amd64</arch>
</os>
</activation>
<properties>
<native.target.platform>linux-amd64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-linux-x86_64</id>
<activation>
<os>
<name>Linux</name>
<arch>x86_64</arch>
</os>
</activation>
<properties>
<native.target.platform>linux-amd64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-linux-arm64</id>
<activation>
<os>
<name>Linux</name>
<arch>aarch64</arch>
</os>
</activation>
<properties>
<native.target.platform>linux-arm64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-macos-amd64</id>
<activation>
<os>
<family>mac</family>
<arch>x86_64</arch>
</os>
</activation>
<properties>
<native.target.platform>macos-amd64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-macos-arm64</id>
<activation>
<os>
<family>mac</family>
<arch>aarch64</arch>
</os>
</activation>
<properties>
<native.target.platform>macos-arm64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-windows-amd64</id>
<activation>
<os>
<family>windows</family>
<arch>amd64</arch>
</os>
</activation>
<properties>
<native.binary.extension>.exe</native.binary.extension>
<native.launcher.dir>../../script/assembly/collector/bin-native-win</native.launcher.dir>
<native.package.format>zip</native.package.format>
<native.target.platform>windows-amd64</native.target.platform>
</properties>
</profile>
<profile>
<id>native-platform-windows-x86_64</id>
<activation>
<os>
<family>windows</family>
<arch>x86_64</arch>
</os>
</activation>
<properties>
<native.binary.extension>.exe</native.binary.extension>
<native.launcher.dir>../../script/assembly/collector/bin-native-win</native.launcher.dir>
<native.package.format>zip</native.package.format>
<native.target.platform>windows-amd64</native.target.platform>
</properties>
</profile>
</profiles>
</project>
@@ -18,10 +18,13 @@
package org.apache.hertzbeat.collector;
import jakarta.annotation.PostConstruct;
import org.apache.hertzbeat.collector.nativex.CollectorRuntimeHintsRegistrar;
import org.apache.hertzbeat.collector.nativex.NativeCollectorDefaults;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.ImportRuntimeHints;
/**
* collector startup
@@ -29,13 +32,20 @@ import org.springframework.context.annotation.ComponentScan;
@ComponentScan(basePackages = {"org.apache.hertzbeat"})
@ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"})
@SpringBootApplication
@ImportRuntimeHints(CollectorRuntimeHintsRegistrar.class)
public class Collector {
public static void main(String[] args) {
SpringApplication.run(Collector.class, args);
SpringApplication application = new SpringApplication(Collector.class);
NativeCollectorDefaults.applyTo(application);
application.run(args);
}
@PostConstruct
public void init() {
System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory");
if (System.getProperty("arrow.allocation.manager.type") == null
&& System.getenv("ARROW_ALLOCATION_MANAGER_TYPE") == null) {
System.setProperty("arrow.allocation.manager.type", "Netty");
}
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Collector-side MySQL query engine routing.
*/
@Getter
@Setter
@ConfigurationProperties(prefix = "hertzbeat.collector.mysql")
public class MysqlCollectorProperties {
private QueryEngine queryEngine = QueryEngine.AUTO;
public QueryEngine resolveQueryEngine(boolean mysqlJdbcDriverAvailable) {
if (queryEngine == QueryEngine.AUTO) {
return mysqlJdbcDriverAvailable ? QueryEngine.JDBC : QueryEngine.R2DBC;
}
return queryEngine;
}
public boolean useR2dbc(boolean mysqlJdbcDriverAvailable) {
return resolveQueryEngine(mysqlJdbcDriverAvailable) == QueryEngine.R2DBC;
}
/**
* Supported collector-side query engines.
*/
public enum QueryEngine {
AUTO,
JDBC,
R2DBC
}
}
@@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import java.net.URL;
import java.security.CodeSource;
import java.util.Locale;
import org.springframework.stereotype.Component;
import org.springframework.util.ClassUtils;
import org.springframework.util.StringUtils;
/**
* Detects whether a MySQL JDBC driver is available from the external ext-lib path.
*/
@Component
public class MysqlJdbcDriverAvailability {
private static final String[] MYSQL_DRIVER_CLASSES = {
"com.mysql.cj.jdbc.Driver",
"com.mysql.jdbc.Driver"
};
public boolean hasMysqlJdbcDriver() {
ClassLoader classLoader = ClassUtils.getDefaultClassLoader();
for (String driverClass : MYSQL_DRIVER_CLASSES) {
if (!ClassUtils.isPresent(driverClass, classLoader)) {
continue;
}
try {
if (isExternalExtLibDriver(ClassUtils.forName(driverClass, classLoader))) {
return true;
}
} catch (ClassNotFoundException ignored) {
// Race-free enough for runtime detection: keep probing other known driver class names.
}
}
return false;
}
boolean isExternalExtLibDriver(Class<?> driverClass) {
String location = resolveLocation(driverClass);
return isExtLibLocation(location);
}
static boolean isExtLibLocation(String location) {
if (!StringUtils.hasText(location)) {
return false;
}
String normalized = location
.replace('\\', '/')
.toLowerCase(Locale.ROOT);
return normalized.contains("/ext-lib/");
}
private String resolveLocation(Class<?> driverClass) {
CodeSource codeSource = driverClass.getProtectionDomain().getCodeSource();
if (codeSource != null && codeSource.getLocation() != null) {
return codeSource.getLocation().toExternalForm();
}
String resourceName = ClassUtils.convertClassNameToResourcePath(driverClass.getName()) + ".class";
ClassLoader classLoader = driverClass.getClassLoader();
URL resource = classLoader != null ? classLoader.getResource(resourceName) : ClassLoader.getSystemResource(resourceName);
return resource != null ? resource.toExternalForm() : null;
}
}
@@ -0,0 +1,214 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import java.net.URI;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions;
import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.SshTunnel;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
/**
* MySQL-compatible query-only adapter that lets JdbcCommonCollect execute read-only queries through the built-in
* R2DBC path when no MySQL JDBC driver is present.
*/
@Component
public class MysqlR2dbcJdbcQueryExecutor implements JdbcQueryExecutor, InitializingBean, DisposableBean {
private static final String QUERY_TYPE_ONE_ROW = "oneRow";
private static final String QUERY_TYPE_MULTI_ROW = "multiRow";
private static final String QUERY_TYPE_COLUMNS = "columns";
private final MysqlCollectorProperties properties;
private final MysqlQueryExecutor mysqlQueryExecutor;
private final MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability;
public MysqlR2dbcJdbcQueryExecutor(MysqlCollectorProperties properties,
MysqlQueryExecutor mysqlQueryExecutor,
MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability) {
this.properties = properties;
this.mysqlQueryExecutor = mysqlQueryExecutor;
this.mysqlJdbcDriverAvailability = mysqlJdbcDriverAvailability;
}
@Override
public boolean supports(Metrics metrics) {
if (metrics == null || metrics.getJdbc() == null) {
return false;
}
JdbcProtocol jdbcProtocol = metrics.getJdbc();
if (!isMysqlCompatiblePlatform(jdbcProtocol.getPlatform())) {
return false;
}
String queryType = jdbcProtocol.getQueryType();
return properties.useR2dbc(mysqlJdbcDriverAvailability.hasMysqlJdbcDriver())
&& (QUERY_TYPE_ONE_ROW.equals(queryType)
|| QUERY_TYPE_MULTI_ROW.equals(queryType)
|| QUERY_TYPE_COLUMNS.equals(queryType));
}
private boolean isMysqlCompatiblePlatform(String platform) {
return "mysql".equalsIgnoreCase(platform) || "mariadb".equalsIgnoreCase(platform);
}
@Override
public JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) {
JdbcProtocol jdbcProtocol = metrics.getJdbc();
QueryOptions options = buildQueryOptions(jdbcProtocol, timeout, maxRows);
QueryResult queryResult = mysqlQueryExecutor.execute(jdbcProtocol.getSql(), options);
if (queryResult.hasError()) {
throw new IllegalStateException("R2DBC MySQL query failed: " + queryResult.getError());
}
return new QueryResultRowSet(queryResult);
}
@Override
public void afterPropertiesSet() {
JdbcQueryExecutorRegistry.register(this);
}
@Override
public void destroy() {
JdbcQueryExecutorRegistry.unregister(this);
}
private QueryOptions buildQueryOptions(JdbcProtocol jdbcProtocol, int timeout, int maxRows) {
MysqlTarget target = resolveTarget(jdbcProtocol);
SshTunnel sshTunnel = jdbcProtocol.getSshTunnel();
String host = target.host();
int port = target.port();
if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) {
try {
int localPort = SshTunnelHelper.localPortForward(sshTunnel, host, String.valueOf(port));
host = "127.0.0.1";
port = localPort;
} catch (Exception exception) {
throw new IllegalStateException("R2DBC MySQL query adapter failed to establish SSH tunnel", exception);
}
}
return QueryOptions.builder()
.host(host)
.port(port)
.username(jdbcProtocol.getUsername())
.password(jdbcProtocol.getPassword())
.database(target.database())
.schema(target.database())
.timeout(Duration.ofMillis(timeout))
.maxRows(maxRows)
.fetchSize(256)
.readOnly(true)
.build();
}
private MysqlTarget resolveTarget(JdbcProtocol jdbcProtocol) {
if (StringUtils.hasText(jdbcProtocol.getUrl())) {
return parseJdbcUrl(jdbcProtocol.getUrl(), jdbcProtocol.getDatabase());
}
if (!StringUtils.hasText(jdbcProtocol.getHost()) || !StringUtils.hasText(jdbcProtocol.getPort())) {
throw new IllegalArgumentException("R2DBC MySQL query adapter requires host/port or a jdbc:mysql URL");
}
return new MysqlTarget(jdbcProtocol.getHost(), Integer.parseInt(jdbcProtocol.getPort()), jdbcProtocol.getDatabase());
}
private MysqlTarget parseJdbcUrl(String url, String fallbackDatabase) {
String trimmed = url.trim();
if (!(trimmed.startsWith("jdbc:mysql://") || trimmed.startsWith("jdbc:mariadb://"))) {
throw new IllegalArgumentException("R2DBC MySQL query adapter only supports jdbc:mysql:// or jdbc:mariadb:// URLs");
}
URI uri = URI.create(trimmed.substring("jdbc:".length()));
String host = uri.getHost();
int port = uri.getPort() > 0 ? uri.getPort() : 3306;
if (!StringUtils.hasText(host)) {
throw new IllegalArgumentException("R2DBC MySQL query adapter URL must include a host");
}
String path = uri.getPath();
String database = StringUtils.hasText(path) && path.length() > 1 ? path.substring(1) : fallbackDatabase;
return new MysqlTarget(host, port, database);
}
private record MysqlTarget(String host, int port, String database) {
}
private static final class QueryResultRowSet implements JdbcQueryRowSet {
private final List<List<String>> rows;
private final Map<String, Integer> columnIndexMap;
private int currentIndex = -1;
private QueryResultRowSet(QueryResult queryResult) {
this.rows = queryResult.getRows();
this.columnIndexMap = buildColumnIndexMap(queryResult.getColumns());
}
@Override
public boolean next() {
currentIndex++;
return currentIndex < rows.size();
}
@Override
public String getString(String column) {
Integer index = columnIndexMap.get(column.toLowerCase(Locale.ROOT));
if (index == null) {
throw new IllegalArgumentException("Column not found in R2DBC MySQL result: " + column);
}
return getString(index + 1);
}
@Override
public String getString(int index) {
if (currentIndex < 0 || currentIndex >= rows.size()) {
throw new IllegalStateException("R2DBC MySQL result cursor is not positioned on a row");
}
int zeroBased = index - 1;
List<String> row = rows.get(currentIndex);
if (zeroBased < 0 || zeroBased >= row.size()) {
throw new IllegalArgumentException("Column index out of bounds in R2DBC MySQL result: " + index);
}
return row.get(zeroBased);
}
@Override
public void close() {
// QueryResult is fully materialized, so there is nothing left to close here.
}
private static Map<String, Integer> buildColumnIndexMap(List<String> columns) {
Map<String, Integer> indexMap = new HashMap<>(columns.size());
for (int index = 0; index < columns.size(); index++) {
indexMap.put(columns.get(index).toLowerCase(Locale.ROOT), index);
}
return indexMap;
}
}
}
@@ -20,6 +20,7 @@ package org.apache.hertzbeat.collector.collect.strategy;
import java.util.ServiceLoader;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Configuration;
@@ -29,6 +30,7 @@ import org.springframework.core.annotation.Order;
/**
* Specific metrics collection factory
*/
@Slf4j
@Configuration
@Order(value = Ordered.HIGHEST_PRECEDENCE + 1)
public class CollectStrategyFactory implements CommandLineRunner {
@@ -49,10 +51,18 @@ public class CollectStrategyFactory implements CommandLineRunner {
@Override
public void run(String... args) throws Exception {
COLLECT_STRATEGY.clear();
// spi load and registry protocol and collect instance
ServiceLoader<AbstractCollect> loader = ServiceLoader.load(AbstractCollect.class, AbstractCollect.class.getClassLoader());
for (AbstractCollect collect : loader) {
COLLECT_STRATEGY.put(collect.supportProtocol(), collect);
}
if (COLLECT_STRATEGY.isEmpty()) {
throw new IllegalStateException(
"No collect strategies were registered. "
+ "Verify META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect "
+ "is present on the runtime classpath.");
}
log.info("Registered {} collect strategies: {}", COLLECT_STRATEGY.size(), COLLECT_STRATEGY.keySet());
}
}
@@ -114,7 +114,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
public void start() {
try {
// Pull the collection task from the task queue and put it into the thread pool for execution
workerPool.executeJob(() -> {
workerPool.executeLongRunning(() -> {
Thread.currentThread().setName("metrics-task-dispatcher");
while (!Thread.currentThread().isInterrupted()) {
MetricsCollect metricsCollect = null;
@@ -175,6 +175,9 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
.setId(job.getMonitorId())
.setTenantId(job.getTenantId())
.setApp(job.getApp())
.setLabels(job.getLabels())
.setAnnotations(job.getAnnotations())
.addMetadataAll(job.getMetadata())
.setMetrics(metricsTime.getMetrics().getName())
.setPriority(metricsTime.getMetrics().getPriority())
.setTime(System.currentTimeMillis())
@@ -379,4 +382,4 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
private Metrics metrics;
private Timeout timeout;
}
}
}
@@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.nativex;
import java.util.LinkedHashSet;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.memory.netty.NettyAllocationManager;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.IntervalUnit;
import org.apache.arrow.vector.types.MetadataVersion;
import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
import org.springframework.aot.hint.BindingReflectionHintsRegistrar;
import org.springframework.aot.hint.MemberCategory;
import org.springframework.aot.hint.RuntimeHints;
import org.springframework.aot.hint.RuntimeHintsRegistrar;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
import org.springframework.core.type.filter.TypeFilter;
import org.springframework.lang.NonNull;
import org.springframework.util.ClassUtils;
/**
* Registers native binding hints for collector-side message payloads.
*/
@Slf4j
public class CollectorRuntimeHintsRegistrar implements RuntimeHintsRegistrar {
private static final String JOB_PACKAGE = "org.apache.hertzbeat.common.entity.job";
private static final String JOB_PROTOCOL_PACKAGE = "org.apache.hertzbeat.common.entity.job.protocol";
@Override
public void registerHints(@NonNull RuntimeHints hints, ClassLoader classLoader) {
BindingReflectionHintsRegistrar bindingRegistrar = new BindingReflectionHintsRegistrar();
registerType(bindingRegistrar, hints, ServerInfo.class);
scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PACKAGE);
scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PROTOCOL_PACKAGE);
hints.reflection().registerType(NettyAllocationManager.class, MemberCategory.DECLARED_FIELDS);
registerType(bindingRegistrar, hints, Schema.class);
registerType(bindingRegistrar, hints, Field.class);
registerType(bindingRegistrar, hints, FieldType.class);
registerType(bindingRegistrar, hints, DictionaryEncoding.class);
registerType(bindingRegistrar, hints, DateUnit.class);
registerType(bindingRegistrar, hints, FloatingPointPrecision.class);
registerType(bindingRegistrar, hints, IntervalUnit.class);
registerType(bindingRegistrar, hints, MetadataVersion.class);
registerType(bindingRegistrar, hints, TimeUnit.class);
registerType(bindingRegistrar, hints, UnionMode.class);
for (Class<?> nestedClass : ArrowType.class.getDeclaredClasses()) {
if (!nestedClass.isAnnotation() && !nestedClass.isInterface()) {
registerType(bindingRegistrar, hints, nestedClass);
}
}
}
private void scanBindingPackage(ClassLoader classLoader, BindingReflectionHintsRegistrar bindingRegistrar,
RuntimeHints hints, String basePackage) {
for (Class<?> clazz : findBindingTypes(basePackage, classLoader)) {
registerType(bindingRegistrar, hints, clazz);
}
}
private void registerType(BindingReflectionHintsRegistrar bindingRegistrar, RuntimeHints hints, Class<?> clazz) {
bindingRegistrar.registerReflectionHints(hints.reflection(), clazz);
}
private Set<Class<?>> findBindingTypes(String basePackage, ClassLoader classLoader) {
Set<Class<?>> bindingTypes = new LinkedHashSet<>();
ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false);
TypeFilter includeAll = (metadataReader, metadataReaderFactory) -> true;
scanner.addIncludeFilter(includeAll);
for (BeanDefinition candidate : scanner.findCandidateComponents(basePackage)) {
String className = candidate.getBeanClassName();
if (className == null) {
continue;
}
try {
Class<?> clazz = ClassUtils.forName(className, classLoader);
if (!clazz.isAnnotation() && !clazz.isInterface()) {
bindingTypes.add(clazz);
}
} catch (Throwable ex) {
log.debug("Skip native binding hint registration for {}", className, ex);
}
}
return bindingTypes;
}
}
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.nativex;
import java.util.Map;
import org.springframework.boot.SpringApplication;
import org.springframework.core.NativeDetector;
/**
* Applies collector defaults without forking {@code application.yml}.
*/
public final class NativeCollectorDefaults {
static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude";
static final String JVM_AUTOCONFIGURE_EXCLUDES = String.join(",",
"org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration",
"org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration",
"org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration",
"org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration");
static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",",
JVM_AUTOCONFIGURE_EXCLUDES,
"org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration",
"org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration",
"org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration",
"org.springframework.boot.jdbc.autoconfigure.health.DataSourceHealthContributorAutoConfiguration",
"org.springframework.boot.jdbc.autoconfigure.metrics.DataSourcePoolMetricsAutoConfiguration",
"org.springframework.boot.tomcat.autoconfigure.metrics.TomcatMetricsAutoConfiguration");
private NativeCollectorDefaults() {
}
public static void applyTo(SpringApplication application) {
application.setDefaultProperties(defaultProperties(NativeDetector.inNativeImage()));
}
static Map<String, Object> defaultProperties(boolean nativeImage) {
return Map.of(
AUTOCONFIGURE_EXCLUDE_PROPERTY,
nativeImage ? NATIVE_AUTOCONFIGURE_EXCLUDES : JVM_AUTOCONFIGURE_EXCLUDES);
}
}
@@ -35,3 +35,4 @@ org.apache.hertzbeat.collector.collect.sd.EurekaSdCollectImpl
org.apache.hertzbeat.collector.collect.sd.ConsulSdCollectImpl
org.apache.hertzbeat.collector.collect.modbus.ModbusCollectImpl
org.apache.hertzbeat.collector.collect.s7.S7CollectImpl
org.apache.hertzbeat.collector.collect.sd.ZookeeperSdCollectImpl
@@ -1,4 +1,3 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -6,17 +5,13 @@
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
### share
GET http://localhost:8080/api/share/getShareURL/1
### downloadShare
GET http://localhost:8080/api/share/download/6laZF
# Intentionally left blank.
# Native collector defaults are applied from Collector.main.
@@ -27,9 +27,6 @@ spring:
timeout-per-shutdown-phase: 10s
jackson:
default-property-inclusion: ALWAYS
# need to disable spring boot mongodb auto config, or default mongodb connection tried and failed...
autoconfigure:
exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
management:
endpoints:
web:
@@ -75,3 +72,19 @@ push:
common:
queue:
type: netty
hertzbeat:
collector:
mysql:
# MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics.
# auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine
# jdbc : always use JDBC
# r2dbc : always use the built-in query engine
query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto}
# Optional virtual-thread overrides. Remove this whole block to use built-in defaults.
vthreads:
enabled: true
common:
mode: UNBOUNDED_VT
collector:
mode: LIMIT_AND_REJECT
@@ -0,0 +1,54 @@
[
{
"name": "org.apache.hertzbeat.collector.Collector__ApplicationContextInitializer",
"allDeclaredConstructors": true,
"allDeclaredMethods": true
},
{
"name": "org.apache.hertzbeat.collector.Collector__BeanFactoryRegistrations",
"allDeclaredConstructors": true,
"allDeclaredMethods": true
},
{
"name": "org.apache.hertzbeat.common.entity.dto.ServerInfo",
"allDeclaredConstructors": true,
"allDeclaredFields": true,
"allDeclaredMethods": true
},
{
"name": "io.netty.channel.kqueue.KQueueDatagramChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.kqueue.KQueueSocketChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.kqueue.KQueueEventLoopGroup",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.epoll.EpollDatagramChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.epoll.EpollSocketChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.epoll.EpollEventLoopGroup",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.socket.nio.NioDatagramChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.socket.nio.NioSocketChannel",
"allPublicConstructors": true
},
{
"name": "io.netty.channel.nio.NioEventLoopGroup",
"allPublicConstructors": true
}
]
@@ -0,0 +1,38 @@
org.apache.hertzbeat.collector.collect.http.HttpCollectImpl
org.apache.hertzbeat.collector.collect.http.SslCertificateCollectImpl
org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect
org.apache.hertzbeat.collector.collect.icmp.IcmpCollectImpl
org.apache.hertzbeat.collector.collect.jmx.JmxCollectImpl
org.apache.hertzbeat.collector.collect.redis.RedisCommonCollectImpl
org.apache.hertzbeat.collector.collect.mongodb.MongodbSingleCollectImpl
org.apache.hertzbeat.collector.collect.rocketmq.RocketmqSingleCollectImpl
org.apache.hertzbeat.collector.collect.snmp.SnmpCollectImpl
org.apache.hertzbeat.collector.collect.ssh.SshCollectImpl
org.apache.hertzbeat.collector.collect.telnet.TelnetCollectImpl
org.apache.hertzbeat.collector.collect.smtp.SmtpCollectImpl
org.apache.hertzbeat.collector.collect.ntp.NtpCollectImpl
org.apache.hertzbeat.collector.collect.websocket.WebsocketCollectImpl
org.apache.hertzbeat.collector.collect.ftp.FtpCollectImpl
org.apache.hertzbeat.collector.collect.udp.UdpCollectImpl
org.apache.hertzbeat.collector.collect.push.PushCollectImpl
org.apache.hertzbeat.collector.collect.dns.DnsCollectImpl
org.apache.hertzbeat.collector.collect.nginx.NginxCollectImpl
org.apache.hertzbeat.collector.collect.memcached.MemcachedCollectImpl
org.apache.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl
org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl
org.apache.hertzbeat.collector.collect.registry.RegistryImpl
org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl
org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl
org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl
org.apache.hertzbeat.collector.collect.script.ScriptCollectImpl
org.apache.hertzbeat.collector.collect.mqtt.MqttCollectImpl
org.apache.hertzbeat.collector.collect.ipmi2.IpmiCollectImpl
org.apache.hertzbeat.collector.collect.kafka.KafkaCollectImpl
org.apache.hertzbeat.collector.collect.sd.HttpSdCollectImpl
org.apache.hertzbeat.collector.collect.sd.NacosSdCollectImpl
org.apache.hertzbeat.collector.collect.sd.DnsSdCollectImpl
org.apache.hertzbeat.collector.collect.sd.EurekaSdCollectImpl
org.apache.hertzbeat.collector.collect.sd.ConsulSdCollectImpl
org.apache.hertzbeat.collector.collect.modbus.ModbusCollectImpl
org.apache.hertzbeat.collector.collect.s7.S7CollectImpl
org.apache.hertzbeat.collector.collect.sd.ZookeeperSdCollectImpl
@@ -0,0 +1,320 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class MariadbJdbcQueryAdapterTemplateIntegrationTest {
private static final String TEST_DATABASE = "hzb";
private static final String TEST_USERNAME = "test";
private static final String TEST_PASSWORD = "test123";
private static final String ROOT_PASSWORD = "root123";
private GenericContainer<?> container;
private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor;
private List<Metrics> mariadbTemplateMetrics;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
container = new GenericContainer<>(DockerImageName.parse("mariadb:11.4"))
.withExposedPorts(3306)
.withEnv("MARIADB_DATABASE", TEST_DATABASE)
.withEnv("MARIADB_USER", TEST_USERNAME)
.withEnv("MARIADB_PASSWORD", TEST_PASSWORD)
.withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD)
.waitingFor(Wait.forListeningPort());
container.start();
awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE);
initMonitoringData(container);
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
jdbcQueryExecutor.afterPropertiesSet();
mariadbTemplateMetrics = loadMariadbTemplate().getMetrics();
}
@AfterAll
void tearDown() throws Exception {
if (jdbcQueryExecutor != null) {
jdbcQueryExecutor.destroy();
}
if (container != null) {
container.stop();
}
}
@TestFactory
Stream<DynamicTest> shouldCollectOfficialMariadbTemplateThroughJdbcQueryAdapter() {
return mariadbTemplateMetrics.stream()
.map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(),
() -> verifyTemplateMetric(templateMetric)));
}
private void verifyTemplateMetric(Metrics templateMetric) throws Exception {
Metrics metric = materializeMetric(templateMetric);
if ("process_state".equals(metric.getName())) {
startBackgroundSleepQuery(container);
}
if ("slow_sql".equals(metric.getName())) {
generateSlowQuery(container);
}
CollectRep.MetricsData metricsData = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(),
() -> metric.getName() + " failed: " + metricsData.getMsg());
assertEquals(metric.getFields().size(), metricsData.getFieldsCount(),
() -> metric.getName() + " fields should still be produced by the original parser");
if ("columns".equals(metric.getJdbc().getQueryType())) {
assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape");
}
if ("basic".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0);
assertNotNull(metricsData.getValues().getFirst().getColumns(0));
assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)),
"basic.version should be collected through the adapted query path");
}
if ("process_state".equals(metric.getName()) || "slow_sql".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row");
}
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("mariadb")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data");
return collectDataDispatch.metricsData;
}
private Metrics materializeMetric(Metrics templateMetric) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306)));
jdbcProtocol.setUsername(TEST_USERNAME);
jdbcProtocol.setPassword(TEST_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setUrl(null);
jdbcProtocol.setSshTunnel(null);
if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) {
jdbcProtocol.setDatabase(TEST_DATABASE);
}
if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
return metric;
}
private Job loadMariadbTemplate() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mariadb.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
return yaml.loadAs(reader, Job.class);
}
}
private void initMonitoringData(GenericContainer<?> mariaDb) throws Exception {
execRoot(mariaDb,
"GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';"
+ " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';"
+ " SET GLOBAL log_output='TABLE';"
+ " SET GLOBAL slow_query_log='ON';"
+ " SET GLOBAL long_query_time=0;"
+ " FLUSH PRIVILEGES;");
generateSlowQuery(mariaDb);
}
private void generateSlowQuery(GenericContainer<?> mariaDb) throws Exception {
execUser(mariaDb, TEST_DATABASE, "SELECT SLEEP(0.2);");
Thread.sleep(300);
}
private void startBackgroundSleepQuery(GenericContainer<?> mariaDb) throws Exception {
String command = String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"nohup sh -lc",
"'$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + TEST_USERNAME,
"-p" + TEST_PASSWORD,
TEST_DATABASE,
"-e",
"\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'",
">/dev/null 2>&1 &");
mariaDb.execInContainer("sh", "-lc", command);
Thread.sleep(500);
}
private void awaitTcpLoginReady(GenericContainer<?> mariaDb, String username, String password, String database) throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1"));
if (result.getExitCode() == 0) {
return;
}
} catch (Exception ignored) {
// Wait for the MariaDB entrypoint to finish bootstrapping and switch to the final TCP listener.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for MariaDB TCP login to become ready");
}
private void execRoot(GenericContainer<?> mariaDb, String sql) throws Exception {
var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("root mysql command failed: " + result.getStderr());
}
}
private void execUser(GenericContainer<?> mariaDb, String database, String sql) throws Exception {
var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("user mysql command failed: " + result.getStderr());
}
}
private String mysqlCliCommand(String username, String password, String database, String sql) {
return String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + username,
"-p" + password,
database,
"-e",
"\"" + sql.replace("\"", "\\\"") + "\"");
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
class MysqlJdbcDriverAvailabilityTest {
@Test
void shouldTreatOnlyExtLibLocationsAsAutoJdbcSignal() {
assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("/opt/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar"));
assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("file:/C:/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar"));
assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation("/Users/dev/.m2/repository/com/mysql/mysql-connector-j/9.0.0/mysql-connector-j-9.0.0.jar"));
assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation(null));
}
@Test
void shouldIgnoreTestClasspathMysqlDriverWhenItIsNotFromExtLib() {
MysqlJdbcDriverAvailability availability = new MysqlJdbcDriverAvailability();
assertFalse(availability.hasMysqlJdbcDriver());
}
}
@@ -0,0 +1,312 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class MysqlJdbcQueryAdapterCompatibilityIntegrationTest {
private static final String TEST_DATABASE = "hzb";
private static final String TEST_USERNAME = "test";
private static final String TEST_PASSWORD = "test123";
private static final String ROOT_PASSWORD = "root123";
private static final Set<String> REPRESENTATIVE_TEMPLATE_METRICS = Set.of("basic", "process_state");
private List<Metrics> representativeTemplateMetrics;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
representativeTemplateMetrics = loadMysqlTemplate().getMetrics().stream()
.filter(metric -> REPRESENTATIVE_TEMPLATE_METRICS.contains(metric.getName()))
.collect(Collectors.toList());
}
@TestFactory
Stream<DynamicTest> shouldCollectRepresentativeTemplateMetricsAcrossCompatibilityMatrix() {
return Stream.of(
new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false),
new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false),
new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true))
.map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyRepresentativeMetrics(target)));
}
private void verifyRepresentativeMetrics(DatabaseTarget target) throws Exception {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
try (GenericContainer<?> container = createContainer(target)) {
jdbcQueryExecutor.afterPropertiesSet();
container.start();
awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE);
initMonitoringData(container);
for (Metrics templateMetric : representativeTemplateMetrics) {
Metrics metric = materializeMetric(templateMetric, container);
if ("process_state".equals(metric.getName())) {
startBackgroundSleepQuery(container);
}
CollectRep.MetricsData metricsData = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(),
() -> target.name() + " " + metric.getName() + " failed: " + metricsData.getMsg());
assertEquals(metric.getFields().size(), metricsData.getFieldsCount(),
() -> target.name() + " " + metric.getName() + " should keep the original parser output shape");
if ("basic".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0, () -> target.name() + " basic should return data");
assertNotNull(metricsData.getValues().getFirst().getColumns(0));
assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)),
() -> target.name() + " basic.version should be collected");
}
if ("process_state".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0,
() -> target.name() + " process_state should return at least one grouped state row");
}
}
} finally {
jdbcQueryExecutor.destroy();
}
}
private GenericContainer<?> createContainer(DatabaseTarget target) {
GenericContainer<?> container = new GenericContainer<>(target.image())
.withExposedPorts(3306)
.waitingFor(Wait.forListeningPort());
if (target.mariaDb()) {
return container.withEnv("MARIADB_DATABASE", TEST_DATABASE)
.withEnv("MARIADB_USER", TEST_USERNAME)
.withEnv("MARIADB_PASSWORD", TEST_PASSWORD)
.withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD);
}
return container.withEnv("MYSQL_DATABASE", TEST_DATABASE)
.withEnv("MYSQL_USER", TEST_USERNAME)
.withEnv("MYSQL_PASSWORD", TEST_PASSWORD)
.withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD);
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("mysql")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
return collectDataDispatch.metricsData;
}
private Metrics materializeMetric(Metrics templateMetric, GenericContainer<?> container) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306)));
jdbcProtocol.setUsername(TEST_USERNAME);
jdbcProtocol.setPassword(TEST_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setUrl(null);
jdbcProtocol.setSshTunnel(null);
if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) {
jdbcProtocol.setDatabase(TEST_DATABASE);
}
if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
return metric;
}
private Job loadMysqlTemplate() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
return yaml.loadAs(reader, Job.class);
}
}
private void initMonitoringData(GenericContainer<?> mysql) throws Exception {
execRoot(mysql,
"GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';"
+ " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';"
+ " SET GLOBAL log_output='TABLE';"
+ " SET GLOBAL slow_query_log='ON';"
+ " SET GLOBAL long_query_time=0;"
+ " FLUSH PRIVILEGES;");
}
private void startBackgroundSleepQuery(GenericContainer<?> mysql) throws Exception {
String command = String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"nohup sh -lc",
"'$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + TEST_USERNAME,
"-p" + TEST_PASSWORD,
TEST_DATABASE,
"-e",
"\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'",
">/dev/null 2>&1 &");
mysql.execInContainer("sh", "-lc", command);
Thread.sleep(500);
}
private void awaitTcpLoginReady(GenericContainer<?> mysql, String username, String password, String database) throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1"));
if (result.getExitCode() == 0) {
return;
}
} catch (Exception ignored) {
// Wait for the database entrypoint to finish bootstrapping and switch to the final TCP listener.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready");
}
private void execRoot(GenericContainer<?> mysql, String sql) throws Exception {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("root mysql command failed: " + result.getStderr());
}
}
private String mysqlCliCommand(String username, String password, String database, String sql) {
return String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + username,
"-p" + password,
database,
"-e",
"\"" + sql.replace("\"", "\\\"") + "\"");
}
private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) {
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}
@@ -0,0 +1,322 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class MysqlJdbcQueryAdapterTemplateIntegrationTest {
private static final String TEST_DATABASE = "hzb";
private static final String TEST_USERNAME = "test";
private static final String TEST_PASSWORD = "test123";
private static final String ROOT_PASSWORD = "root123";
private GenericContainer<?> container;
private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor;
private List<Metrics> mysqlTemplateMetrics;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36"))
.withExposedPorts(3306)
.withEnv("MYSQL_DATABASE", TEST_DATABASE)
.withEnv("MYSQL_USER", TEST_USERNAME)
.withEnv("MYSQL_PASSWORD", TEST_PASSWORD)
.withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD)
.waitingFor(Wait.forListeningPort());
container.start();
awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE);
initMonitoringData(container);
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
jdbcQueryExecutor.afterPropertiesSet();
mysqlTemplateMetrics = loadMysqlTemplate().getMetrics();
}
@AfterAll
void tearDown() throws Exception {
if (jdbcQueryExecutor != null) {
jdbcQueryExecutor.destroy();
}
if (container != null) {
container.stop();
}
}
@TestFactory
Stream<DynamicTest> shouldCollectOfficialMysqlTemplateThroughJdbcQueryAdapter() {
return mysqlTemplateMetrics.stream()
.map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(),
() -> verifyTemplateMetric(templateMetric)));
}
private void verifyTemplateMetric(Metrics templateMetric) throws Exception {
Metrics metric = materializeMetric(templateMetric);
if ("process_state".equals(metric.getName())) {
startBackgroundSleepQuery(container);
}
if ("slow_sql".equals(metric.getName())) {
generateSlowQuery(container);
}
CollectRep.MetricsData metricsData = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(),
() -> metric.getName() + " failed: " + metricsData.getMsg());
assertEquals(metric.getFields().size(), metricsData.getFieldsCount(),
() -> metric.getName() + " fields should still be produced by the original parser");
if ("columns".equals(metric.getJdbc().getQueryType())) {
assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape");
}
if ("basic".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0);
assertNotNull(metricsData.getValues().getFirst().getColumns(0));
assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)),
"basic.version should be collected through the adapted query path");
}
if ("process_state".equals(metric.getName())
|| "slow_sql".equals(metric.getName())
|| "account_expiry".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row");
}
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("mysql")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data");
return collectDataDispatch.metricsData;
}
private Metrics materializeMetric(Metrics templateMetric) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306)));
jdbcProtocol.setUsername(TEST_USERNAME);
jdbcProtocol.setPassword(TEST_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setUrl(null);
jdbcProtocol.setSshTunnel(null);
if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) {
jdbcProtocol.setDatabase(TEST_DATABASE);
}
if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
return metric;
}
private Job loadMysqlTemplate() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
return yaml.loadAs(reader, Job.class);
}
}
private void initMonitoringData(GenericContainer<?> mysql) throws Exception {
execRoot(mysql,
"GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';"
+ " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';"
+ " SET GLOBAL log_output='TABLE';"
+ " SET GLOBAL slow_query_log='ON';"
+ " SET GLOBAL long_query_time=0;"
+ " FLUSH PRIVILEGES;");
generateSlowQuery(mysql);
}
private void generateSlowQuery(GenericContainer<?> mysql) throws Exception {
execUser(mysql, TEST_DATABASE, "SELECT SLEEP(0.2);");
Thread.sleep(300);
}
private void startBackgroundSleepQuery(GenericContainer<?> mysql) throws Exception {
String command = String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"nohup sh -lc",
"'$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + TEST_USERNAME,
"-p" + TEST_PASSWORD,
TEST_DATABASE,
"-e",
"\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'",
">/dev/null 2>&1 &");
mysql.execInContainer("sh", "-lc", command);
Thread.sleep(500);
}
private void awaitTcpLoginReady(GenericContainer<?> mysql, String username, String password, String database) throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1"));
if (result.getExitCode() == 0) {
return;
}
} catch (Exception ignored) {
// Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready");
}
private void execRoot(GenericContainer<?> mysql, String sql) throws Exception {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("root mysql command failed: " + result.getStderr());
}
}
private void execUser(GenericContainer<?> mysql, String database, String sql) throws Exception {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("user mysql command failed: " + result.getStderr());
}
}
private String mysqlCliCommand(String username, String password, String database, String sql) {
return String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + username,
"-p" + password,
database,
"-e",
"\"" + sql.replace("\"", "\\\"") + "\"");
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}
@@ -0,0 +1,393 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class MysqlJdbcQueryParityIntegrationTest {
private static final String TEST_DATABASE = "hzb";
private static final String TEST_USERNAME = "test";
private static final String TEST_PASSWORD = "test123";
private static final String ROOT_PASSWORD = "root123";
private static final String PARITY_TABLE = "collector_parity_metrics";
private Metrics basicTemplateMetric;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
assertDoesNotThrow(() -> Class.forName("com.mysql.cj.jdbc.Driver"));
basicTemplateMetric = loadMysqlTemplate().getMetrics().stream()
.filter(metric -> "basic".equals(metric.getName()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-mysql.yml"));
}
@AfterEach
void clearRegisteredExecutors() throws Exception {
Field executorsField = JdbcQueryExecutorRegistry.class.getDeclaredField("EXECUTORS");
executorsField.setAccessible(true);
@SuppressWarnings("unchecked")
CopyOnWriteArrayList<Object> executors = (CopyOnWriteArrayList<Object>) executorsField.get(null);
executors.clear();
}
@TestFactory
Stream<DynamicTest> shouldMatchJdbcResultsForRepresentativeMysqlQueryShapes() {
return Stream.of(
new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44")),
new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36")))
.map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyParityAcrossTarget(target)));
}
private void verifyParityAcrossTarget(DatabaseTarget target) throws Exception {
try (GenericContainer<?> container = createContainer(target)) {
container.start();
awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE);
initParityData(container);
List<Metrics> parityMetrics = List.of(
materializeMetric(basicTemplateMetric, container),
buildColumnsParityMetric(container),
buildOneRowParityMetric(container),
buildMultiRowParityMetric(container));
for (Metrics parityMetric : parityMetrics) {
CollectRep.MetricsData jdbcResult = collectWithJdbc(parityMetric);
CollectRep.MetricsData r2dbcResult = collectWithR2dbc(parityMetric);
assertEquals(CollectRep.Code.SUCCESS, jdbcResult.getCode(),
() -> target.name() + " JDBC baseline failed for " + parityMetric.getName() + ": " + jdbcResult.getMsg());
assertEquals(CollectRep.Code.SUCCESS, r2dbcResult.getCode(),
() -> target.name() + " R2DBC path failed for " + parityMetric.getName() + ": " + r2dbcResult.getMsg());
assertEquals(jdbcResult.getFields(), r2dbcResult.getFields(),
() -> target.name() + " field set differs for " + parityMetric.getName());
assertEquals(normalizeRows(jdbcResult), normalizeRows(r2dbcResult),
() -> target.name() + " row payload differs for " + parityMetric.getName());
assertFalse(r2dbcResult.getValues().isEmpty(),
() -> target.name() + " " + parityMetric.getName() + " should return at least one row");
}
}
}
private CollectRep.MetricsData collectWithJdbc(Metrics metric) throws Exception {
clearRegisteredExecutors();
return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class));
}
private CollectRep.MetricsData collectWithR2dbc(Metrics metric) throws Exception {
clearRegisteredExecutors();
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
try {
jdbcQueryExecutor.afterPropertiesSet();
return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class));
} finally {
jdbcQueryExecutor.destroy();
clearRegisteredExecutors();
}
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("mysql")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
return collectDataDispatch.metricsData;
}
private Metrics buildColumnsParityMetric(GenericContainer<?> container) {
return buildMetric(
"columns-parity",
List.of(field("version"), field("max_connections"), field("character_set_server")),
List.of("version", "max_connections", "character_set_server"),
"columns",
"SHOW VARIABLES WHERE Variable_name IN ('version', 'max_connections', 'character_set_server')",
container);
}
private Metrics buildOneRowParityMetric(GenericContainer<?> container) {
return buildMetric(
"one-row-parity",
List.of(field("answer"), field("label"), field("nullable_value")),
List.of("answer", "label", "nullable_value"),
"oneRow",
"SELECT 42 AS answer, 'adapter-parity' AS label, NULL AS nullable_value",
container);
}
private Metrics buildMultiRowParityMetric(GenericContainer<?> container) {
return buildMetric(
"multi-row-parity",
List.of(field("metric_name"), field("metric_value"), field("metric_note")),
List.of("metric_name", "metric_value", "metric_note"),
"multiRow",
"SELECT metric_name, metric_value, metric_note FROM " + PARITY_TABLE + " ORDER BY metric_name",
container);
}
private Metrics buildMetric(String name, List<Metrics.Field> fields, List<String> aliasFields,
String queryType, String sql, GenericContainer<?> container) {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.host(container.getHost())
.port(String.valueOf(container.getMappedPort(3306)))
.platform("mysql")
.database(TEST_DATABASE)
.username(TEST_USERNAME)
.password(TEST_PASSWORD)
.timeout(String.valueOf(Duration.ofSeconds(8).toMillis()))
.queryType(queryType)
.reuseConnection("false")
.url(buildJdbcUrl(container))
.sql(sql)
.build();
return Metrics.builder()
.name(name)
.protocol("jdbc")
.priority((byte) 1)
.fields(fields)
.aliasFields(aliasFields)
.jdbc(jdbcProtocol)
.build();
}
private Metrics materializeMetric(Metrics templateMetric, GenericContainer<?> container) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306)));
jdbcProtocol.setUsername(TEST_USERNAME);
jdbcProtocol.setPassword(TEST_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setUrl(buildJdbcUrl(container));
jdbcProtocol.setSshTunnel(null);
if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) {
jdbcProtocol.setDatabase(TEST_DATABASE);
}
if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
return metric;
}
private Job loadMysqlTemplate() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
return yaml.loadAs(reader, Job.class);
}
}
private GenericContainer<?> createContainer(DatabaseTarget target) {
return new GenericContainer<>(target.image())
.withExposedPorts(3306)
.withEnv("MYSQL_DATABASE", TEST_DATABASE)
.withEnv("MYSQL_USER", TEST_USERNAME)
.withEnv("MYSQL_PASSWORD", TEST_PASSWORD)
.withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD)
.waitingFor(Wait.forListeningPort());
}
private void initParityData(GenericContainer<?> mysql) throws Exception {
execRoot(mysql,
"GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';"
+ " GRANT SELECT ON " + TEST_DATABASE + ".* TO '" + TEST_USERNAME + "'@'%';"
+ " DROP TABLE IF EXISTS " + TEST_DATABASE + "." + PARITY_TABLE + ";"
+ " CREATE TABLE " + TEST_DATABASE + "." + PARITY_TABLE + " ("
+ " metric_name VARCHAR(32) PRIMARY KEY,"
+ " metric_value VARCHAR(32) NOT NULL,"
+ " metric_note VARCHAR(32) NULL"
+ " );"
+ " INSERT INTO " + TEST_DATABASE + "." + PARITY_TABLE
+ " (metric_name, metric_value, metric_note) VALUES"
+ " ('alpha', '1', NULL),"
+ " ('beta', '2', 'steady');"
+ " FLUSH PRIVILEGES;");
}
private void awaitTcpLoginReady(GenericContainer<?> mysql, String username, String password, String database) throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1"));
if (result.getExitCode() == 0) {
return;
}
} catch (Exception ignored) {
// Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready");
}
private void execRoot(GenericContainer<?> mysql, String sql) throws Exception {
var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql));
if (result.getExitCode() != 0) {
throw new IllegalStateException("root mysql command failed: " + result.getStderr());
}
}
private String mysqlCliCommand(String username, String password, String database, String sql) {
return String.join(" ",
"CLIENT=$(command -v mysql || command -v mariadb)",
"&&",
"$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
"-u" + username,
"-p" + password,
database,
"-e",
"\"" + sql.replace("\"", "\\\"") + "\"");
}
private String buildJdbcUrl(GenericContainer<?> container) {
return "jdbc:mysql://%s:%d/%s?allowPublicKeyRetrieval=true&useSSL=false"
.formatted(container.getHost(), container.getMappedPort(3306), TEST_DATABASE);
}
private List<List<String>> normalizeRows(CollectRep.MetricsData metricsData) {
return metricsData.getValues().stream()
.map(valueRow -> new ArrayList<>(valueRow.getColumnsList()))
.sorted(Comparator.comparing(row -> String.join("\u0001", row)))
.collect(Collectors.toList());
}
private Metrics.Field field(String name) {
return Metrics.Field.builder().field(name).type((byte) 1).build();
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
private record DatabaseTarget(String name, DockerImageName image) {
}
}
@@ -0,0 +1,143 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.List;
import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.junit.jupiter.api.Test;
class MysqlR2dbcJdbcQueryExecutorTest {
@Test
void shouldAutoRouteToR2dbcOnlyWhenMysqlJdbcDriverIsAbsent() {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class);
when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(false);
MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor(
properties, mock(MysqlQueryExecutor.class), driverAvailability);
assertTrue(executor.supports(metrics("mysql", "columns")));
assertTrue(executor.supports(metrics("mariadb", "columns")));
assertTrue(executor.supports(metrics("mysql", "multiRow")));
assertFalse(executor.supports(metrics("mysql", "runScript")));
assertFalse(executor.supports(metrics("postgresql", "columns")));
}
@Test
void shouldPreferJdbcWhenMysqlJdbcDriverIsPresentInAutoMode() {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class);
when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true);
MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor(
properties, mock(MysqlQueryExecutor.class), driverAvailability);
assertFalse(executor.supports(metrics("mysql", "columns")));
}
@Test
void shouldHonorExplicitQueryEngineOverrides() {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class);
when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true);
MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor(
properties, mock(MysqlQueryExecutor.class), driverAvailability);
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
assertTrue(executor.supports(metrics("mysql", "columns")));
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.JDBC);
assertFalse(executor.supports(metrics("mysql", "columns")));
}
@Test
void shouldExposeQueryResultsAsJdbcStyleRowSet() throws Exception {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class);
when(mysqlQueryExecutor.execute(anyString(), any()))
.thenReturn(QueryResult.builder()
.columns(List.of("Variable_name", "Value"))
.rows(List.of(
List.of("Threads_connected", "5"),
List.of("Uptime", "10")))
.elapsedMs(11)
.rowCount(2)
.build());
MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor(
properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class));
try (JdbcQueryRowSet rowSet = executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)) {
assertTrue(rowSet.next());
assertEquals("Threads_connected", rowSet.getString(1));
assertEquals("5", rowSet.getString(2));
assertEquals("5", rowSet.getString("value"));
assertTrue(rowSet.next());
assertEquals("Uptime", rowSet.getString("variable_name"));
assertEquals("10", rowSet.getString(2));
assertFalse(rowSet.next());
}
}
@Test
void shouldFailFastWhenR2dbcQueryReturnsError() {
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class);
when(mysqlQueryExecutor.execute(anyString(), any()))
.thenReturn(QueryResult.builder().error("query timeout").build());
MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor(
properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class));
IllegalStateException exception = assertThrows(IllegalStateException.class,
() -> executor.executeQuery(metrics("mysql", "columns"), 6000, 1000));
assertTrue(exception.getMessage().contains("query timeout"));
}
private Metrics metrics(String platform, String queryType) {
JdbcProtocol jdbcProtocol = JdbcProtocol.builder()
.host("127.0.0.1")
.port("3306")
.platform(platform)
.database("hzb")
.username("test")
.password("test123")
.queryType(queryType)
.sql("SHOW GLOBAL STATUS")
.timeout("6000")
.build();
Metrics metrics = new Metrics();
metrics.setProtocol("jdbc");
metrics.setName("status");
metrics.setJdbc(jdbcProtocol);
return metrics;
}
}
@@ -0,0 +1,275 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class OceanbaseJdbcQueryAdapterIntegrationTest {
private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce:latest";
private static final String OCEANBASE_USERNAME = "root@sys";
private static final String OCEANBASE_PASSWORD = "";
private static final String OCEANBASE_DATABASE = "oceanbase";
private GenericContainer<?> container;
private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor;
private List<Metrics> oceanbaseTemplateMetrics;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
container = new GenericContainer<>(DockerImageName.parse(OCEANBASE_IMAGE))
.withExposedPorts(2881)
.withEnv("MODE", "MINI")
.withEnv("OB_MEMORY_LIMIT", "4096M")
.withEnv("OB_SYSTEM_MEMORY", "1024M")
.withEnv("OB_DATAFILE_SIZE", "2048M")
.withEnv("OB_LOG_DISK_SIZE", "2048M")
.withCommand("bash", "-lc", "/usr/sbin/sshd || true; /root/boot/start.sh || true; tail -f /dev/null")
.waitingFor(Wait.forListeningPort())
.withStartupTimeout(Duration.ofMinutes(5));
container.start();
awaitSysLoginReady();
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
jdbcQueryExecutor.afterPropertiesSet();
oceanbaseTemplateMetrics = loadOceanbaseTemplate().getMetrics();
}
@AfterAll
void tearDown() throws Exception {
if (jdbcQueryExecutor != null) {
jdbcQueryExecutor.destroy();
}
if (container != null) {
container.stop();
}
}
@TestFactory
Stream<DynamicTest> shouldCollectOfficialOceanbaseTemplateThroughJdbcQueryAdapter() {
return oceanbaseTemplateMetrics.stream()
.map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(),
() -> verifyTemplateMetric(templateMetric)));
}
private void verifyTemplateMetric(Metrics templateMetric) throws Exception {
Metrics metric = materializeMetric(templateMetric);
if ("process_state".equals(metric.getName())) {
startBackgroundSleepQuery();
}
CollectRep.MetricsData metricsData = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(),
() -> metric.getName() + " failed: " + metricsData.getMsg());
assertEquals(metric.getFields().size(), metricsData.getFieldsCount(),
() -> metric.getName() + " fields should still be produced by the original parser");
if ("basic".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0, "basic should return data");
assertNotNull(metricsData.getValues().getFirst().getColumns(0));
assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)),
"basic.version should be collected through the adapted query path");
}
if ("tenant".equals(metric.getName()) || "sql".equals(metric.getName()) || "process_state".equals(metric.getName())) {
assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row");
}
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("oceanbase")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data");
return collectDataDispatch.metricsData;
}
private Metrics materializeMetric(Metrics templateMetric) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(2881)));
jdbcProtocol.setUsername(OCEANBASE_USERNAME);
jdbcProtocol.setPassword(OCEANBASE_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(12).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setUrl(null);
jdbcProtocol.setSshTunnel(null);
jdbcProtocol.setDatabase(OCEANBASE_DATABASE);
if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) {
metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList()));
}
return metric;
}
private Job loadOceanbaseTemplate() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-oceanbase.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
return yaml.loadAs(reader, Job.class);
}
}
private void awaitSysLoginReady() throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofMinutes(3).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
var result = container.execInContainer("sh", "-lc", obclientCommand("select 1"));
if (result.getExitCode() == 0) {
return;
}
} catch (Exception ignored) {
// Wait for OceanBase observer bootstrap to finish and accept sys tenant logins.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for OceanBase sys tenant login to become ready");
}
private void startBackgroundSleepQuery() throws Exception {
String command = "nohup sh -lc '" + obclientCommand("select sleep(15);").replace("'", "'\"'\"'") + " >/tmp/oceanbase-process-state.log 2>&1' >/dev/null 2>&1 &";
container.execInContainer("sh", "-lc", command);
Thread.sleep(500);
}
private String obclientCommand(String sql) {
StringBuilder command = new StringBuilder("obclient -h127.0.0.1 -P2881 -u")
.append(OCEANBASE_USERNAME)
.append(" -D")
.append(OCEANBASE_DATABASE)
.append(" -A ");
if (!OCEANBASE_PASSWORD.isEmpty()) {
command.append("-p").append(OCEANBASE_PASSWORD).append(' ');
}
command.append("-e \"").append(sql.replace("\"", "\\\"")).append('"');
return command.toString();
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}
@@ -0,0 +1,240 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.mysql;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.Reader;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TidbJdbcQueryAdapterIntegrationTest {
private static final String TIDB_USERNAME = "root";
private static final String TIDB_PASSWORD = "";
private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient();
private GenericContainer<?> container;
private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor;
private Metrics tidbBasicMetric;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests");
new CollectStrategyFactory().run();
container = new GenericContainer<>(DockerImageName.parse("pingcap/tidb:v7.5.1"))
.withCommand("--store=unistore", "--path=")
.withExposedPorts(4000, 10080)
.waitingFor(Wait.forLogMessage(".*server is running MySQL protocol.*", 1));
container.start();
waitForStatusEndpoint();
MysqlCollectorProperties properties = new MysqlCollectorProperties();
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
properties,
new MysqlR2dbcQueryExecutor(
new MysqlR2dbcConnectionFactoryProvider(),
new ResultSetMapper(),
new SqlGuard()),
new MysqlJdbcDriverAvailability());
jdbcQueryExecutor.afterPropertiesSet();
tidbBasicMetric = loadTidbBasicMetric();
}
@AfterAll
void tearDown() throws Exception {
if (jdbcQueryExecutor != null) {
jdbcQueryExecutor.destroy();
}
if (container != null) {
container.stop();
}
}
@Test
void shouldCollectTidbBasicMetricThroughMysqlCompatibleQueryAdapter() {
Metrics metric = materializeMetric(tidbBasicMetric);
CollectRep.MetricsData metricsData = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), metricsData.getMsg());
assertEquals(metric.getFields().size(), metricsData.getFieldsCount());
assertTrue(metricsData.getValuesCount() > 0);
assertNotNull(metricsData.getValues().getFirst());
assertTrue(metricsData.getValues().getFirst().getColumnsList().stream()
.anyMatch(value -> !Objects.equals(CommonConstants.NULL_VALUE, value) && !value.isEmpty()),
"TiDB basic should still return at least one concrete field value through the adapted query path");
}
private Metrics materializeMetric(Metrics templateMetric) {
Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbcProtocol = metric.getJdbc();
jdbcProtocol.setHost(container.getHost());
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(4000)));
jdbcProtocol.setUsername(TIDB_USERNAME);
jdbcProtocol.setPassword(TIDB_PASSWORD);
jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis()));
jdbcProtocol.setReuseConnection("false");
jdbcProtocol.setDatabase(null);
jdbcProtocol.setUrl(null);
jdbcProtocol.setSshTunnel(null);
return metric;
}
private Metrics loadTidbBasicMetric() throws IOException {
Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-tidb.yml")
.toAbsolutePath()
.normalize();
Yaml yaml = new Yaml();
try (Reader reader = Files.newBufferedReader(template)) {
Job job = yaml.loadAs(reader, Job.class);
return job.getMetrics().stream()
.filter(metric -> "basic".equals(metric.getName()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-tidb.yml"));
}
}
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L)
.tenantId(1L)
.app("tidb")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch();
MetricsCollect metricsCollect = new MetricsCollect(
metric,
new StubTimeout(timerTask),
collectDataDispatch,
"collector-test",
List.of());
metricsCollect.run();
return collectDataDispatch.metricsData;
}
private void waitForStatusEndpoint() throws Exception {
long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis();
String statusUrl = "http://" + container.getHost() + ":" + container.getMappedPort(10080) + "/status";
while (System.currentTimeMillis() < deadline) {
try {
HttpRequest request = HttpRequest.newBuilder(URI.create(statusUrl))
.GET()
.timeout(Duration.ofSeconds(3))
.build();
HttpResponse<String> response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() == 200) {
return;
}
} catch (Exception ignored) {
// Wait for the TiDB status endpoint to become available.
}
Thread.sleep(1000);
}
throw new IllegalStateException("Timed out waiting for TiDB status endpoint");
}
private static final class CapturingCollectDataDispatch implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) {
this.metricsData = metricsData;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRep.MetricsData> metricsDataList) {
if (metricsDataList != null && !metricsDataList.isEmpty()) {
this.metricsData = metricsDataList.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}
@@ -0,0 +1,435 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.database.sqlserver;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.DynamicContainer;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.TestInstance;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.yaml.snakeyaml.Yaml;
/**
* Integration test for app-sqlserver.yml against all supported SQL Server
* versions. Waits for container readiness via JDBC (same mechanism the
* collector uses) so timing is independent of the machine.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class SqlServerJdbcTemplateIntegrationTest {
private static final String SA_PASSWORD = "HertzBeat!2026";
private static final String TEST_LOGIN = "test_login";
private static final String TEST_PASSWORD = "TestLogin!2026";
private static final List<String> SUPPORTED_VERSIONS = List.of(
"2017-latest",
"2019-latest",
"2022-latest",
"2025-latest"
);
private record VersionedContainer(
String versionTag, GenericContainer<?> container) {
}
private List<VersionedContainer> containers;
private List<Metrics> templateMetrics;
@BeforeAll
void setUp() throws Exception {
Assumptions.assumeTrue(
DockerClientFactory.instance().isDockerAvailable(),
"Docker is required for integration tests");
new CollectStrategyFactory().run();
templateMetrics = loadTemplate().getMetrics();
containers = new ArrayList<>();
for (String tag : SUPPORTED_VERSIONS) {
GenericContainer<?> c = startContainer(tag);
containers.add(new VersionedContainer(tag, c));
}
}
@AfterAll
void tearDown() {
if (containers != null) {
containers.forEach(vc -> vc.container().stop());
}
}
// ── Contract: account_expiry must exist ──
@Test
@DisplayName("account_expiry metric group must be defined")
void accountExpiryShouldBeDefined() {
assertTrue(templateMetrics.stream()
.anyMatch(m -> "account_expiry".equals(m.getName())),
"app-sqlserver.yml must define an account_expiry metric group");
}
@Test
@DisplayName("account_expiry field types must match schema")
void accountExpiryFieldTypesShouldMatch() {
Metrics expiry = templateMetrics.stream()
.filter(m -> "account_expiry".equals(m.getName()))
.findFirst().orElseThrow();
for (Metrics.Field f : expiry.getFields()) {
switch (f.getField()) {
case "login_name", "password_expired", "is_disabled" ->
assertEquals(1, f.getType(),
f.getField() + " must be type 1 (string)");
case "days_left" ->
assertEquals(0, f.getType(),
"days_left must be type 0 (number)");
default ->
throw new IllegalStateException(
"Unexpected field: " + f.getField());
}
}
}
// ── Every metric group must collect on every version ──
@TestFactory
Stream<DynamicContainer> shouldCollectOnAllSupportedVersions() {
return containers.stream()
.map(vc -> DynamicContainer.dynamicContainer(vc.versionTag(),
templateMetrics.stream()
.map(m -> DynamicTest.dynamicTest(m.getName(),
() -> verifyMetric(m, vc)))));
}
private void verifyMetric(Metrics tmpl, VersionedContainer vc)
throws Exception {
Metrics metric = materialize(tmpl, vc);
CollectRep.MetricsData data = collect(metric);
assertEquals(CollectRep.Code.SUCCESS, data.getCode(),
() -> vc.versionTag() + " " + metric.getName()
+ " failed: " + data.getMsg());
assertEquals(metric.getFields().size(), data.getFieldsCount(),
() -> vc.versionTag() + " " + metric.getName()
+ " field count mismatch");
if ("columns".equals(metric.getJdbc().getQueryType())) {
assertEquals(1, data.getValuesCount(),
() -> vc.versionTag() + " " + metric.getName()
+ " should be single-row");
}
if ("basic".equals(metric.getName())) {
assertTrue(data.getValuesCount() > 0);
assertNotNull(data.getValues().getFirst().getColumns(0),
vc.versionTag() + " basic.version should be non-null");
}
if ("account_expiry".equals(metric.getName())) {
assertTrue(data.getValuesCount() > 0,
vc.versionTag()
+ " account_expiry must return at least one row");
}
}
// ── Edge-case assertions on account_expiry data ──
@TestFactory
@DisplayName("account_expiry edge cases across all versions")
Stream<DynamicTest> accountExpiryEdgeCases() {
return containers.stream()
.flatMap(vc -> Stream.of(
DynamicTest.dynamicTest(
vc.versionTag() + " sa must appear",
() -> assertSaLoginAppears(vc)),
DynamicTest.dynamicTest(
vc.versionTag() + " days_left must be integer",
() -> assertDaysLeftIsInteger(vc)),
DynamicTest.dynamicTest(
vc.versionTag()
+ " password_expired must be 0 or 1",
() -> assertPasswordExpiredIsBinary(vc)),
DynamicTest.dynamicTest(
vc.versionTag()
+ " is_disabled must be 0 or 1",
() -> assertIsDisabledIsBinary(vc))));
}
private void assertSaLoginAppears(VersionedContainer vc) throws Exception {
CollectRep.MetricsData data = collect(expiryMetric(vc));
boolean found = false;
for (int i = 0; i < data.getValuesCount(); i++) {
if ("sa".equalsIgnoreCase(
data.getValues().get(i).getColumns(0))) {
found = true;
break;
}
}
assertTrue(found,
vc.versionTag() + " sa login must appear in account_expiry");
}
private void assertDaysLeftIsInteger(VersionedContainer vc)
throws Exception {
CollectRep.MetricsData data = collect(expiryMetric(vc));
for (int i = 0; i < data.getValuesCount(); i++) {
String v = data.getValues().get(i).getColumns(1);
assertNotNull(v,
vc.versionTag() + " days_left must not be null");
try {
Integer.parseInt(v);
} catch (NumberFormatException e) {
throw new AssertionError(
vc.versionTag() + " days_left not an integer: " + v);
}
}
}
private void assertPasswordExpiredIsBinary(VersionedContainer vc)
throws Exception {
CollectRep.MetricsData data = collect(expiryMetric(vc));
for (int i = 0; i < data.getValuesCount(); i++) {
String v = data.getValues().get(i).getColumns(2);
assertNotNull(v,
vc.versionTag() + " password_expired must not be null");
assertTrue("Y".equals(v) || "N".equals(v),
vc.versionTag() + " password_expired must be Y/N: " + v);
}
}
private void assertIsDisabledIsBinary(VersionedContainer vc)
throws Exception {
CollectRep.MetricsData data = collect(expiryMetric(vc));
for (int i = 0; i < data.getValuesCount(); i++) {
String v = data.getValues().get(i).getColumns(3);
assertNotNull(v,
vc.versionTag() + " is_disabled must not be null");
assertTrue("Y".equals(v) || "N".equals(v),
vc.versionTag() + " is_disabled must be Y/N: " + v);
}
}
private Metrics expiryMetric(VersionedContainer vc) {
return materialize(
templateMetrics.stream()
.filter(m -> "account_expiry".equals(m.getName()))
.findFirst().orElseThrow(),
vc);
}
// ── Container lifecycle ──
private static GenericContainer<?> startContainer(String versionTag)
throws Exception {
GenericContainer<?> container = new GenericContainer<>(
DockerImageName.parse(
"mcr.microsoft.com/mssql/server:" + versionTag))
.withExposedPorts(1433)
.withEnv("ACCEPT_EULA", "Y")
.withEnv("MSSQL_SA_PASSWORD", SA_PASSWORD)
.waitingFor(Wait.forLogMessage(
".*SQL Server is now ready for client connections.*",
1));
container.start();
awaitJdbc(container, versionTag);
// Create a test login with CHECK_EXPIRATION=ON
String jdbcUrl = "jdbc:sqlserver://"
+ container.getHost() + ":"
+ container.getMappedPort(1433)
+ ";encrypt=false;trustServerCertificate=true";
try (Connection c = DriverManager.getConnection(
jdbcUrl, "sa", SA_PASSWORD);
var stmt = c.createStatement()) {
stmt.execute("CREATE LOGIN [" + TEST_LOGIN
+ "] WITH PASSWORD = '" + TEST_PASSWORD
+ "', CHECK_POLICY = ON, CHECK_EXPIRATION = ON");
}
return container;
}
/**
* Poll the container via JDBC until a login succeeds.
* Uses a generous timeout because SQL Server startup time varies
* significantly across versions and hardware.
*/
private static void awaitJdbc(GenericContainer<?> container,
String versionTag) throws Exception {
String jdbcUrl = "jdbc:sqlserver://"
+ container.getHost() + ":"
+ container.getMappedPort(1433)
+ ";encrypt=false;trustServerCertificate=true;loginTimeout=5";
long deadline = System.currentTimeMillis()
+ Duration.ofMinutes(3).toMillis();
while (System.currentTimeMillis() < deadline) {
try {
try (Connection c = DriverManager.getConnection(
jdbcUrl, "sa", SA_PASSWORD)) {
if (c.isValid(3)) {
return;
}
}
} catch (Exception ignored) {
// SQL Server not accepting logins yet
}
Thread.sleep(5000);
}
throw new IllegalStateException(
"Timed out waiting for " + versionTag + " JDBC connectivity");
}
// ── Collection helpers (same pattern as Mysql test) ──
private CollectRep.MetricsData collect(Metrics metric) {
Job job = Job.builder()
.monitorId(1L).tenantId(1L).app("sqlserver")
.defaultInterval(600L)
.metadata(new HashMap<>(0))
.labels(new HashMap<>(0))
.annotations(new HashMap<>(0))
.configmap(new ArrayList<>(0))
.metrics(new ArrayList<>(List.of(metric)))
.build();
WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> {
});
var dispatch = new CapturingCollectDataDispatch();
var collector = new MetricsCollect(
metric, new StubTimeout(timerTask), dispatch,
"collector-test", List.of());
collector.run();
assertNotNull(dispatch.metricsData,
metric.getName() + " should dispatch metrics data");
return dispatch.metricsData;
}
private Metrics materialize(Metrics templateMetric,
VersionedContainer vc) {
Metrics m = JsonUtil.fromJson(
JsonUtil.toJson(templateMetric), Metrics.class);
JdbcProtocol jdbc = m.getJdbc();
jdbc.setHost(vc.container().getHost());
jdbc.setPort(
String.valueOf(vc.container().getMappedPort(1433)));
jdbc.setUsername("sa");
jdbc.setPassword(SA_PASSWORD);
jdbc.setTimeout(
String.valueOf(Duration.ofSeconds(15).toMillis()));
jdbc.setReuseConnection("false");
jdbc.setUrl(null);
jdbc.setSshTunnel(null);
if (jdbc.getDatabase() == null
|| jdbc.getDatabase().contains("^_^")) {
jdbc.setDatabase("master");
}
if (m.getAliasFields() == null
|| m.getAliasFields().isEmpty()) {
m.setAliasFields(
m.getFields().stream()
.map(Metrics.Field::getField)
.collect(Collectors.toList()));
}
return m;
}
private Job loadTemplate() throws Exception {
Path path = Path.of(
"..", "..", "hertzbeat-manager", "src", "main",
"resources", "define", "app-sqlserver.yml")
.toAbsolutePath().normalize();
try (Reader r = Files.newBufferedReader(path)) {
return new Yaml().loadAs(r, Job.class);
}
}
// ── inner types ──
private static final class CapturingCollectDataDispatch
implements CollectDataDispatch {
private CollectRep.MetricsData metricsData;
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics,
CollectRep.MetricsData data) {
this.metricsData = data;
}
@Override
public void dispatchCollectData(Timeout timeout, Metrics metrics,
List<CollectRep.MetricsData> list) {
if (list != null && !list.isEmpty()) {
this.metricsData = list.getFirst();
}
}
}
private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout {
@Override
public org.apache.hertzbeat.common.timer.Timer timer() {
return null;
}
@Override
public org.apache.hertzbeat.common.timer.TimerTask task() {
return wheelTimerTask;
}
@Override
public boolean isExpired() {
return false;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean cancel() {
return false;
}
}
}

Some files were not shown because too many files have changed in this diff Show More