From 42307a9928ebe967c31ad7840d57d55a398bd73e Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:10:00 -0700 Subject: [PATCH 01/18] [feat] Support etcd monitoring (#4306) Co-authored-by: Duansg --- .../basic/http/EtcdMonitorE2eTest.java | 158 +++++++++ .../src/test/resources/http/etcd/metrics.txt | 17 + .../src/main/resources/define/app-etcd.yml | 328 ++++++++++++++++++ home/docs/help/etcd.md | 65 ++++ .../current/help/etcd.md | 65 ++++ home/sidebars.json | 1 + 6 files changed, 634 insertions(+) create mode 100644 hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/EtcdMonitorE2eTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/etcd/metrics.txt create mode 100644 hertzbeat-manager/src/main/resources/define/app-etcd.yml create mode 100644 home/docs/help/etcd.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/etcd.md diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/EtcdMonitorE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/EtcdMonitorE2eTest.java new file mode 100644 index 0000000000..3233adf8f7 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/EtcdMonitorE2eTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.basic.http; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; +import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.dispatch.unit.impl.DataSizeConvert; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol; +import org.apache.hertzbeat.common.entity.job.protocol.Protocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.util.ResourceUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Integration test for etcd monitoring functionality. + * Fixture at src/test/resources/http/etcd/metrics.txt is a real capture from a live + * etcd v3.5.17 /metrics endpoint (see app-etcd.yml for the corresponding template). + */ +@Slf4j +@ExtendWith(MockitoExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class EtcdMonitorE2eTest extends AbstractCollectE2eTest { + + private static final int MOCK_SERVER_PORT = 52379; + private static final String LOCALHOST = "127.0.0.1"; + private static HttpServer mockServer; + + @AfterAll + public static void tearDown() { + if (mockServer != null) { + mockServer.stop(0); + } + } + + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + collect = new HttpCollectImpl(); + + // the shared harness wires MetricsCollect with an empty unit-convert list, + // but this template relies on B->MB conversion + Timeout convertTimeout = mock(Timeout.class); + WheelTimerTask convertTimerJob = mock(WheelTimerTask.class); + when(convertTimeout.task()).thenReturn(convertTimerJob); + when(convertTimerJob.getJob()).thenReturn(mock(Job.class)); + metricsCollect = new MetricsCollect(mock(Metrics.class), convertTimeout, + mock(CollectDataDispatch.class), null, List.of(new DataSizeConvert())); + + String metricsResponse = loadResponseFromFile("classpath:http/etcd/metrics.txt"); + + mockServer = HttpServer.create(new InetSocketAddress(MOCK_SERVER_PORT), 0); + mockServer.setExecutor(null); + mockServer.start(); + mockServer.createContext("/metrics", exchange -> sendTextResponse(exchange, metricsResponse)); + } + + private String loadResponseFromFile(String resourcePath) throws Exception { + return new String(Files.readAllBytes(ResourceUtils.getFile(resourcePath).toPath())); + } + + private void sendTextResponse(HttpExchange exchange, String response) throws IOException { + exchange.getResponseHeaders().set("Content-Type", "text/plain"); + final byte[] array = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, array.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(array); + } + } + + private static final Map EXPECTED_VALUES = Map.of( + "etcd_server_has_leader", "1", + "etcd_mvcc_db_total_size_in_bytes", "0.0195", + "etcd_server_leader_changes_seen_total", "1", + "process_cpu_seconds_total", "42.41", + "process_resident_memory_bytes", "29.9844"); + + @Test + public void testEtcdMonitor() { + Job etcdJob = appService.getAppDefine("etcd"); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : etcdJob.getMetrics()) { + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, + !configmapFromPreCollectData.isEmpty() ? configmapFromPreCollectData.get(0) : new HashMap<>()); + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName()); + Assertions.assertEquals(EXPECTED_VALUES.get(metricsDef.getName()), + metricsData.getValues().get(0).getColumns(0), + metricsDef.getName() + " collected value mismatch"); + configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData); + } + } + + @Override + protected Protocol buildProtocol(Metrics metricsDef) { + HttpProtocol protocol = new HttpProtocol(); + protocol.setHost(LOCALHOST); + protocol.setPort(String.valueOf(MOCK_SERVER_PORT)); + protocol.setMethod(metricsDef.getHttp().getMethod()); + protocol.setParseType(metricsDef.getHttp().getParseType()); + protocol.setParseScript(metricsDef.getHttp().getParseScript()); + protocol.setUrl(metricsDef.getHttp().getUrl()); + return protocol; + } + + @Override + protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) { + HttpProtocol protocol = (HttpProtocol) buildProtocol(metricsDef); + metrics.setHttp(protocol); + // prometheus parseType filters by builder.getMetrics(); production sets it in + // MetricsCollect.run() but this test harness does not, so set it here + CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder() + .setMetrics(metricsDef.getName()); + return collectMetricsData(metrics, metricsDef, metricsData); + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/etcd/metrics.txt b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/etcd/metrics.txt new file mode 100644 index 0000000000..45969a7b6b --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/etcd/metrics.txt @@ -0,0 +1,17 @@ +# Captured from a live etcd v3.5.17 /metrics endpoint, trimmed to the metric +# families used by app-etcd.yml. +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 20480 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 42.41 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 3.1440896e+07 diff --git a/hertzbeat-manager/src/main/resources/define/app-etcd.yml b/hertzbeat-manager/src/main/resources/define/app-etcd.yml new file mode 100644 index 0000000000..d177dd9dc8 --- /dev/null +++ b/hertzbeat-manager/src/main/resources/define/app-etcd.yml @@ -0,0 +1,328 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The monitoring type category：service-application service monitoring db-database monitoring custom-custom monitoring os-operating system monitoring mid-middleware +category: mid +# The monitoring type eg: linux windows tomcat mysql aws... +app: etcd +# The monitoring i18n name +name: + zh-CN: etcd + en-US: etcd + ja-JP: etcd +# The description and help of this monitoring type +help: + zh-CN: HertzBeat 通过调用 etcd Prometheus Metrics 接口（默认在客户端端口 2379 的 /metrics 路径）对 etcd 键值存储（3.4+）的领导者状态、数据库大小、进程资源等指标进行采集监控。
您可以点击“新建 etcd”并配置 HOST 端口等相关参数进行添加。
⚠️注意：请确保 HertzBeat 能访问 etcd 的 /metrics 接口。该接口默认由 client listener 提供；若 etcd 仅监听 localhost 或客户端启用了双向 TLS，请通过 --listen-metrics-urls 配置独立的 metrics 地址。 + en-US: HertzBeat monitors the etcd key-value store's (3.4+) leader status, database size and process resource usage by calling the etcd Prometheus Metrics endpoint (default at the /metrics path on the client port 2379).
You can click "New etcd" and configure the host, port and other related params to add it.
Note - make sure HertzBeat can reach etcd's /metrics endpoint. It is served on the client listener by default; if etcd only listens on localhost or client mutual TLS is enabled, configure a dedicated metrics address via --listen-metrics-urls. + zh-TW: HertzBeat 透過調用 etcd Prometheus Metrics 介面（預設在客戶端連接埠 2379 的 /metrics 路徑）對 etcd 鍵值儲存（3.4+）的領導者狀態、資料庫大小、程序資源等指標進行採集監控。
您可以點擊“新建 etcd”並配置 HOST 連接埠等相關參數進行添加。
⚠️注意：請確保 HertzBeat 能訪問 etcd 的 /metrics 介面。該介面預設由 client listener 提供；若 etcd 僅監聽 localhost 或客戶端啟用了雙向 TLS，請透過 --listen-metrics-urls 配置獨立的 metrics 地址。 + ja-JP: HertzBeat は etcd（3.4+）が公開する Prometheus metrics エンドポイント（デフォルトではクライアントポート 2379 の /metrics パス）から、リーダー状態・データベースサイズ・プロセスリソース等の指標を収集し、etcd キーバリューストアを監視します。
「新規 etcd」をクリックしてホストやポートなどのパラメータを設定して追加できます。
⚠️注意：HertzBeat が etcd の /metrics エンドポイントへ到達できることを確認してください。デフォルトでは client listener が提供しますが、etcd が localhost のみを監視している場合やクライアント相互 TLS が有効な場合は、--listen-metrics-urls で専用の metrics アドレスを設定してください。 +helpLink: + zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/etcd/ + en-US: https://hertzbeat.apache.org/docs/help/etcd/ +# Input params define for monitoring(render web ui by the definition) +params: + # field-param field key + - field: host + # name-param field display i18n name + name: + zh-CN: 目标Host + en-US: Target Host + ja-JP: 目標ホスト + # type-param field type(most mapping the html input type) + type: host + # required-true or false + required: true + - field: port + name: + zh-CN: 端口 + en-US: Port + ja-JP: ポート + # type-param field type(most mapping the html input type) + type: number + # when type is number, range is required + range: '[0,65535]' + # default value: etcd client port that serves /metrics + defaultValue: 2379 + required: true + - field: timeout + name: + zh-CN: 查询超时时间 + en-US: Query Timeout + ja-JP: クエリタイムアウト + type: number + required: false + # hide param-true or false + hide: true + defaultValue: 6000 + # field-param field key + - field: ssl + # name-param field display i18n name + name: + zh-CN: 启用HTTPS + en-US: HTTPS + ja-JP: HTTPS + # type-param field type(most mapping the html input type) + type: boolean + hide: true + # field-param field key + - field: headers + # name-param field display i18n name + name: + zh-CN: 请求Headers + en-US: Headers + ja-JP: ヘッダ + # type-param field type(most mapping the html input type) + type: key-value + # required-true or false + required: false + hide: true + # when type is key-value, use keyAlias to config key alias name + keyAlias: Header Name + # when type is key-value, use valueAlias to config value alias name + valueAlias: Header Value + # field-param field key + - field: authType + # name-param field display i18n name + name: + zh-CN: 认证方式 + en-US: Auth Type + ja-JP: 認証方法 + # type-param field type(most mapping the html input type) + type: radio + # required-true or false + required: false + # hide param-true or false + hide: true + # when type is radio checkbox, use option to show optional values {name1:value1,name2:value2} + options: + - label: Basic Auth + value: Basic Auth + - label: Digest Auth + value: Digest Auth + # field-param field key + - field: username + # name-param field display i18n name + name: + zh-CN: 用户名 + en-US: Username + ja-JP: ユーザー名 + # type-param field type(most mapping the html input type) + type: text + # when type is text, use limit to limit string length + limit: 50 + # required-true or false + required: false + # hide param-true or false + hide: true + # field-param field key + - field: password + # name-param field display i18n name + name: + zh-CN: 密码 + en-US: Password + ja-JP: パスワード + # type-param field type(most mapping the html input type) + type: password + # required-true or false + required: false + # hide param-true or false + hide: true +# collect metrics config list +# each metrics group name must exactly match a Prometheus metric family name exposed by etcd /metrics +metrics: + # metrics - etcd_server_has_leader (availability: whether this etcd member has a raft leader) + - name: etcd_server_has_leader + i18n: + zh-CN: 领导者状态 + en-US: Leader Status + ja-JP: リーダー状態 + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + # priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue + priority: 0 + fields: + - field: hasLeader + type: 0 + i18n: + zh-CN: 是否存在领导者(1有0无) + en-US: Has Leader(1=yes 0=no) + ja-JP: リーダー有無(1=有 0=無) + aliasFields: + - value + calculates: + - hasLeader=value + protocol: http + http: + host: ^_^host^_^ + port: ^_^port^_^ + url: /metrics + timeout: ^_^timeout^_^ + ssl: ^_^ssl^_^ + method: GET + headers: + ^_^headers^_^: ^_^headers^_^ + authorization: + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + parseType: prometheus + # metrics - etcd_mvcc_db_total_size_in_bytes + - name: etcd_mvcc_db_total_size_in_bytes + i18n: + zh-CN: 数据库大小 + en-US: Database Size + ja-JP: データベースサイズ + priority: 1 + fields: + - field: dbSize + type: 0 + unit: MB + i18n: + zh-CN: 物理分配的数据库大小 + en-US: Physically Allocated DB Size + ja-JP: 物理割当済みDBサイズ + aliasFields: + - value + calculates: + - dbSize=value + units: + - dbSize=B->MB + protocol: http + http: + host: ^_^host^_^ + port: ^_^port^_^ + url: /metrics + timeout: ^_^timeout^_^ + ssl: ^_^ssl^_^ + method: GET + headers: + ^_^headers^_^: ^_^headers^_^ + authorization: + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + parseType: prometheus + # metrics - etcd_server_leader_changes_seen_total + - name: etcd_server_leader_changes_seen_total + i18n: + zh-CN: 领导者变更次数 + en-US: Leader Changes + ja-JP: リーダー変更回数 + priority: 1 + fields: + - field: leaderChanges + type: 0 + i18n: + zh-CN: 已观测到的领导者变更总次数 + en-US: Total Leader Changes Seen + ja-JP: 観測されたリーダー変更総数 + aliasFields: + - value + calculates: + - leaderChanges=value + protocol: http + http: + host: ^_^host^_^ + port: ^_^port^_^ + url: /metrics + timeout: ^_^timeout^_^ + ssl: ^_^ssl^_^ + method: GET + headers: + ^_^headers^_^: ^_^headers^_^ + authorization: + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + parseType: prometheus + # metrics - process_cpu_seconds_total (standard Go/Prometheus client process collector, always present) + - name: process_cpu_seconds_total + i18n: + zh-CN: 进程CPU时间 + en-US: Process CPU Time + ja-JP: プロセスCPU時間 + priority: 1 + fields: + - field: cpuSeconds + type: 0 + unit: s + i18n: + zh-CN: 累计用户与系统CPU时间 + en-US: Total User+System CPU Time + ja-JP: 累計ユーザー+システムCPU時間 + aliasFields: + - value + calculates: + - cpuSeconds=value + protocol: http + http: + host: ^_^host^_^ + port: ^_^port^_^ + url: /metrics + timeout: ^_^timeout^_^ + ssl: ^_^ssl^_^ + method: GET + headers: + ^_^headers^_^: ^_^headers^_^ + authorization: + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + parseType: prometheus + # metrics - process_resident_memory_bytes + - name: process_resident_memory_bytes + i18n: + zh-CN: 进程内存占用 + en-US: Process Resident Memory + ja-JP: プロセス常駐メモリ + priority: 1 + fields: + - field: memory + type: 0 + unit: MB + i18n: + zh-CN: 常驻内存大小 + en-US: Resident Memory Size + ja-JP: 常駐メモリサイズ + aliasFields: + - value + calculates: + - memory=value + units: + - memory=B->MB + protocol: http + http: + host: ^_^host^_^ + port: ^_^port^_^ + url: /metrics + timeout: ^_^timeout^_^ + ssl: ^_^ssl^_^ + method: GET + headers: + ^_^headers^_^: ^_^headers^_^ + authorization: + type: ^_^authType^_^ + basicAuthUsername: ^_^username^_^ + basicAuthPassword: ^_^password^_^ + digestAuthUsername: ^_^username^_^ + digestAuthPassword: ^_^password^_^ + parseType: prometheus diff --git a/home/docs/help/etcd.md b/home/docs/help/etcd.md new file mode 100644 index 0000000000..e9395a85eb --- /dev/null +++ b/home/docs/help/etcd.md @@ -0,0 +1,65 @@ +--- +id: etcd +title: Monitoring：etcd monitoring +sidebar_label: etcd +keywords: [open source monitoring tool, open source middleware monitoring tool, monitoring etcd metrics] +--- + +> HertzBeat monitors the etcd key-value store by collecting metrics from the Prometheus metrics endpoint that etcd exposes. +> +> etcd 3.4+ is supported (the database size metric `etcd_mvcc_db_total_size_in_bytes` replaced the old `etcd_debugging_*` name in 3.4). + +## PreRequisites + +### Make sure HertzBeat can reach etcd's metrics endpoint + +etcd exposes Prometheus-format metrics on its client port (default `2379`) at the `/metrics` path. Make sure this address is reachable from HertzBeat: + +1. If etcd only listens on localhost, or client mutual TLS is enabled on the client port, configure a dedicated metrics listener via [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/). It serves the metrics and health-check endpoints; if exposed without TLS, restrict it to a trusted network. +2. Access `{metrics-host}:{metrics-port}/metrics` (the client port `2379` by default) from the HertzBeat host to confirm metrics data can be fetched. + +More information see [etcd monitoring documentation](https://etcd.io/docs/latest/op-guide/monitoring/). + +### Configuration parameter + +| Parameter name | Parameter help description | +|----------------------|---------------------------------------------------------------------------------------------| +| Target Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) | +| Port | Port of the etcd metrics endpoint, default 2379 when using the client listener | +| Query Timeout | HTTP request timeout in milliseconds, default 6000 | +| HTTPS | Whether to use HTTPS to request the metrics endpoint | +| Headers | Optional extra HTTP request headers | +| Auth Type | Optional Basic/Digest auth if the metrics endpoint sits behind an auth proxy | +| Username / Password | Credentials used when Auth Type is set | + +### Collection Metric + +#### Metric set：etcd_server_has_leader + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|---------------------------------------------------------------| +| hasLeader | none | Whether this etcd member has a raft leader (1=yes, 0=no) | + +#### Metric set：etcd_mvcc_db_total_size_in_bytes + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|--------------------------------------------------------| +| dbSize | MB | Total size of the underlying database physically allocated | + +#### Metric set：etcd_server_leader_changes_seen_total + +| Metric name | Metric unit | Metric help description | +|-----------------|-------------|-------------------------------------------| +| leaderChanges | none | Total number of leader changes observed | + +#### Metric set：process_cpu_seconds_total + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|---------------------------------------------------| +| cpuSeconds | second | Cumulative user and system CPU time consumed | + +#### Metric set：process_resident_memory_bytes + +| Metric name | Metric unit | Metric help description | +|-------------|-------------|-----------------------------------| +| memory | MB | Resident memory size of the process | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/etcd.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/etcd.md new file mode 100644 index 0000000000..5a672fc08c --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/etcd.md @@ -0,0 +1,65 @@ +--- +id: etcd +title: 监控：etcd 监控 +sidebar_label: etcd +keywords: [开源监控系统, 中间件监控, etcd监控] +--- + +> HertzBeat 通过采集 etcd 暴露的 Prometheus metrics 接口数据，对 etcd 键值存储进行监控。 +> +> 支持 etcd 3.4 及以上版本（数据库大小指标 `etcd_mvcc_db_total_size_in_bytes` 自 3.4 起替代旧的 `etcd_debugging_*` 命名）。 + +## 监控前操作 + +### 确认 HertzBeat 能访问 etcd 的 metrics 接口 + +etcd 会在客户端端口（默认 `2379`）的 `/metrics` 路径暴露 Prometheus 格式的指标。请确保 HertzBeat 能访问该地址： + +1. 若 etcd 仅监听 localhost，或客户端端口启用了双向 TLS，请通过 [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/) 配置独立的 metrics 监听地址。该地址提供 metrics 与健康检查端点；若不加 TLS 暴露，请仅限受信任网络访问。 +2. 从 HertzBeat 所在机器访问 `{metrics-host}:{metrics-port}/metrics`（默认为客户端端口 `2379`），确认能获取到 metrics 数据。 + +更多信息请参考 [etcd 监控文档](https://etcd.io/docs/latest/op-guide/monitoring/)。 + +### 配置参数 + +| 参数名称 | 参数帮助描述 | +|--------|-----------------------------------------------| +| 目标Host | 被监控的对端IPV4，IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 | +| 端口 | metrics 接口端口，使用客户端 listener 时默认为 2379 | +| 查询超时时间 | HTTP请求超时时间，单位毫秒，默认6000 | +| 启用HTTPS | 是否使用 HTTPS 请求 metrics 接口 | +| 请求Headers | 可选的额外 HTTP 请求头 | +| 认证方式 | 若 metrics 接口在认证代理后面，可选 Basic/Digest 认证 | +| 用户名/密码 | 配置认证方式后使用的凭据 | + +### 采集指标 + +#### 指标集合：etcd_server_has_leader + +| 指标名称 | 指标单位 | 指标帮助描述 | +|-----------|------|------------------------------| +| hasLeader | 无 | 该 etcd 成员是否存在 raft 领导者(1有0无) | + +#### 指标集合：etcd_mvcc_db_total_size_in_bytes + +| 指标名称 | 指标单位 | 指标帮助描述 | +|--------|------|-----------------| +| dbSize | MB | 物理分配的数据库总大小 | + +#### 指标集合：etcd_server_leader_changes_seen_total + +| 指标名称 | 指标单位 | 指标帮助描述 | +|----------------|------|--------------| +| leaderChanges | 无 | 已观测到的领导者变更总次数 | + +#### 指标集合：process_cpu_seconds_total + +| 指标名称 | 指标单位 | 指标帮助描述 | +|------------|------|------------------| +| cpuSeconds | 秒 | 累计用户与系统CPU使用时间 | + +#### 指标集合：process_resident_memory_bytes + +| 指标名称 | 指标单位 | 指标帮助描述 | +|--------|------|---------| +| memory | MB | 进程常驻内存大小 | diff --git a/home/sidebars.json b/home/sidebars.json index fd24405bbe..b42a8db679 100755 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -245,6 +245,7 @@ "help/kafka_client", "help/pulsar", "help/nacos", + "help/etcd", "help/rabbitmq", "help/rocketmq", "help/shenyu", From 3e7c2bc67f28d9415a4574bc9918442ed368d5db Mon Sep 17 00:00:00 2001 From: lynx009 <2030509072@qq.com> Date: Mon, 10 Aug 2026 00:41:18 +0800 Subject: [PATCH 02/18] feat(alerter): support Alibaba Cloud Monitor webhook (#4296) Co-authored-by: Duansg --- .../alert/dto/AlibabaCloudCmsExternAlert.java | 142 +++++++++++++ .../AlibabaCloudCmsExternAlertService.java | 195 ++++++++++++++++++ ...AlibabaCloudCmsExternAlertServiceTest.java | 180 ++++++++++++++++ .../alert-integration.component.ts | 5 + .../alibabacloud-cms.en-US.md | 75 +++++++ .../alibabacloud-cms.zh-CN.md | 75 +++++++ web-app/src/assets/i18n/en-US.json | 1 + web-app/src/assets/i18n/ja-JP.json | 1 + web-app/src/assets/i18n/ko-KR.json | 1 + web-app/src/assets/i18n/pt-BR.json | 1 + web-app/src/assets/i18n/zh-CN.json | 1 + web-app/src/assets/i18n/zh-TW.json | 1 + 12 files changed, 678 insertions(+) create mode 100644 hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlibabaCloudCmsExternAlert.java create mode 100644 hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlibabaCloudCmsExternAlertService.java create mode 100644 hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlibabaCloudCmsExternAlertServiceTest.java create mode 100644 web-app/src/assets/doc/alert-integration/alibabacloud-cms.en-US.md create mode 100644 web-app/src/assets/doc/alert-integration/alibabacloud-cms.zh-CN.md diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlibabaCloudCmsExternAlert.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlibabaCloudCmsExternAlert.java new file mode 100644 index 0000000000..2970dcd63a --- /dev/null +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlibabaCloudCmsExternAlert.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Alibaba Cloud Monitor 2.0 webhook alert entity. + * + * @see + * Alibaba Cloud Monitor webhook payload fields + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class AlibabaCloudCmsExternAlert { + + private String specversion; + + private String id; + + private String type; + + private String subtype; + + private String source; + + private String sourcetype; + + private String time; + + private Long timestamp; + + private String subject; + + private String datacontenttype; + + private String severity; + + private String status; + + private String userId; + + private String ruleId; + + private String workspace; + + private String traceId; + + private String alertMessage; + + private String alertEntityId; + + private Resource resource; + + private Map labels; + + private Map annotations; + + private AlertData data; + + private Map alertEntityFields; + + private String ruleUrl; + + private String entityUrl; + + private String alertRuleUrl; + + private String alertHistoryUrl; + + /** + * Alert resource. + */ + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class Resource { + + private Entity entity; + + private Map tags; + } + + /** + * Alert resource entity. + */ + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class Entity { + + private String domain; + + @JsonAlias("entity_type") + private String entityType; + + @JsonAlias("entity_id") + private String entityId; + + private Map prop; + } + + /** + * Threshold alert data. + */ + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class AlertData { + + private Object value; + + private Object threshold; + + private String comparisonOperator; + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlibabaCloudCmsExternAlertService.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlibabaCloudCmsExternAlertService.java new file mode 100644 index 0000000000..3b032e9fa0 --- /dev/null +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlibabaCloudCmsExternAlertService.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.service.impl; + +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.alert.dto.AlibabaCloudCmsExternAlert; +import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.alert.service.ExternAlertService; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.springframework.stereotype.Service; + +/** + * Alibaba Cloud Monitor 2.0 external alert service. + */ +@Slf4j +@Service +public class AlibabaCloudCmsExternAlertService implements ExternAlertService { + + private static final String SOURCE = "alibabacloud-cms"; + + private final AlarmCommonReduce alarmCommonReduce; + + public AlibabaCloudCmsExternAlertService(AlarmCommonReduce alarmCommonReduce) { + this.alarmCommonReduce = alarmCommonReduce; + } + + @Override + public void addExternAlert(String content) { + AlibabaCloudCmsExternAlert externAlert = JsonUtil.fromJson(content, AlibabaCloudCmsExternAlert.class); + if (externAlert == null || StringUtils.isBlank(externAlert.getStatus())) { + log.warn("Failed to parse Alibaba Cloud Monitor external alert content: {}", content); + return; + } + alarmCommonReduce.reduceAndSendAlarm(convert(externAlert)); + } + + @Override + public String supportSource() { + return SOURCE; + } + + private SingleAlert convert(AlibabaCloudCmsExternAlert externAlert) { + boolean resolved = isResolved(externAlert); + long eventTime = getEventTime(externAlert); + return SingleAlert.builder() + .content(getAlertContent(externAlert)) + .status(resolved ? CommonConstants.ALERT_STATUS_RESOLVED : CommonConstants.ALERT_STATUS_FIRING) + .startAt(eventTime) + .activeAt(resolved ? null : eventTime) + .endAt(resolved ? eventTime : null) + .labels(buildLabels(externAlert)) + .annotations(buildAnnotations(externAlert)) + .triggerTimes(1) + .build(); + } + + private boolean isResolved(AlibabaCloudCmsExternAlert externAlert) { + return "RESOLVED".equalsIgnoreCase(externAlert.getStatus()) + || "RECOVERED".equalsIgnoreCase(externAlert.getStatus()) + || "NORMAL_RESOLVE".equalsIgnoreCase(externAlert.getSubtype()); + } + + private long getEventTime(AlibabaCloudCmsExternAlert externAlert) { + if (externAlert.getTimestamp() != null && externAlert.getTimestamp() > 0) { + return externAlert.getTimestamp(); + } + if (StringUtils.isNotBlank(externAlert.getTime())) { + try { + return Instant.parse(externAlert.getTime()).toEpochMilli(); + } catch (DateTimeParseException e) { + log.warn("Failed to parse Alibaba Cloud Monitor event time: {}", externAlert.getTime()); + } + } + return Instant.now().toEpochMilli(); + } + + private Map buildLabels(AlibabaCloudCmsExternAlert externAlert) { + Map labels = new HashMap<>(16); + putValues(labels, externAlert.getLabels()); + AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource(); + if (resource != null) { + putValues(labels, resource.getTags()); + AlibabaCloudCmsExternAlert.Entity entity = resource.getEntity(); + if (entity != null) { + putIfNotBlank(labels, "resourceDomain", entity.getDomain()); + putIfNotBlank(labels, "resourceType", entity.getEntityType()); + putIfNotBlank(labels, "resourceId", entity.getEntityId()); + } + } + labels.put("__source__", SOURCE); + putIfNotBlank(labels, CommonConstants.LABEL_ALERT_NAME, externAlert.getSubject()); + putIfNotBlank(labels, CommonConstants.LABEL_ALERT_SEVERITY, convertSeverity(externAlert.getSeverity())); + putIfNotBlank(labels, "ruleId", externAlert.getRuleId()); + putIfNotBlank(labels, "workspace", externAlert.getWorkspace()); + putIfNotBlank(labels, "alertEntityId", externAlert.getAlertEntityId()); + putIfNotBlank(labels, "userId", externAlert.getUserId()); + return labels; + } + + private Map buildAnnotations(AlibabaCloudCmsExternAlert externAlert) { + Map annotations = new HashMap<>(16); + putValues(annotations, externAlert.getAnnotations()); + AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource(); + if (resource != null && resource.getEntity() != null) { + putValues(annotations, resource.getEntity().getProp()); + } + putValues(annotations, externAlert.getAlertEntityFields()); + AlibabaCloudCmsExternAlert.AlertData data = externAlert.getData(); + if (data != null) { + putValue(annotations, "value", data.getValue()); + putValue(annotations, "threshold", data.getThreshold()); + putIfNotBlank(annotations, "comparisonOperator", data.getComparisonOperator()); + } + putIfNotBlank(annotations, "alertMessage", externAlert.getAlertMessage()); + putIfNotBlank(annotations, "traceId", externAlert.getTraceId()); + putIfNotBlank(annotations, "ruleUrl", externAlert.getRuleUrl()); + putIfNotBlank(annotations, "entityUrl", externAlert.getEntityUrl()); + putIfNotBlank(annotations, "alertRuleUrl", externAlert.getAlertRuleUrl()); + putIfNotBlank(annotations, "alertHistoryUrl", externAlert.getAlertHistoryUrl()); + return annotations; + } + + private String getAlertContent(AlibabaCloudCmsExternAlert externAlert) { + if (StringUtils.isNotBlank(externAlert.getAlertMessage())) { + return externAlert.getAlertMessage(); + } + if (StringUtils.isNotBlank(externAlert.getSubject())) { + return externAlert.getSubject(); + } + return "Alibaba Cloud Monitor alert"; + } + + private String convertSeverity(String severity) { + if (StringUtils.isBlank(severity)) { + return null; + } + return switch (severity.toUpperCase(Locale.ROOT)) { + case "EMERGENCY" -> CommonConstants.ALERT_SEVERITY_EMERGENCY; + case "CRITICAL" -> CommonConstants.ALERT_SEVERITY_CRITICAL; + case "WARN", "WARNING" -> CommonConstants.ALERT_SEVERITY_WARNING; + case "INFO", "INFORMATIONAL" -> CommonConstants.ALERT_SEVERITY_INFO; + default -> severity.toLowerCase(Locale.ROOT); + }; + } + + private void putValues(Map target, Map values) { + if (values == null || values.isEmpty()) { + return; + } + values.forEach((key, value) -> putValue(target, key, value)); + } + + private void putValue(Map target, String key, Object value) { + if (StringUtils.isBlank(key) || value == null) { + return; + } + String stringValue; + if (value instanceof Map || value instanceof Collection) { + stringValue = JsonUtil.toJson(value); + } else { + stringValue = String.valueOf(value); + } + putIfNotBlank(target, key, stringValue); + } + + private void putIfNotBlank(Map target, String key, String value) { + if (StringUtils.isNotBlank(value)) { + target.put(key, value); + } + } +} diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlibabaCloudCmsExternAlertServiceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlibabaCloudCmsExternAlertServiceTest.java new file mode 100644 index 0000000000..53c89d113d --- /dev/null +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlibabaCloudCmsExternAlertServiceTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import java.time.Instant; +import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.alert.service.impl.AlibabaCloudCmsExternAlertService; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Unit test for {@link AlibabaCloudCmsExternAlertService}. + */ +@ExtendWith(MockitoExtension.class) +class AlibabaCloudCmsExternAlertServiceTest { + + private static final long EVENT_TIME = 1785300000123L; + + @Mock + private AlarmCommonReduce alarmCommonReduce; + + private AlibabaCloudCmsExternAlertService externAlertService; + + @BeforeEach + void setUp() { + externAlertService = new AlibabaCloudCmsExternAlertService(alarmCommonReduce); + } + + @Test + void shouldConvertTriggeredAlert() { + externAlertService.addExternAlert(""" + { + "specversion": "1.0", + "id": "alert-event-1", + "type": "ALERT", + "subtype": "NORMAL_TRIGGER", + "time": "2026-07-29T06:00:00Z", + "timestamp": 1785300000123, + "subject": "ECS CPU usage is high", + "severity": "WARNING", + "status": "OCCURRED", + "userId": "123456", + "ruleId": "rule-1", + "workspace": "default-cms-123456-cn-hangzhou", + "traceId": "trace-1", + "alertMessage": "CPU usage exceeded 80%", + "alertEntityId": "ecs:i-123", + "resource": { + "entity": { + "domain": "ecs", + "entity_type": "instance", + "entity_id": "i-123", + "prop": { + "instanceName": "api-server" + } + }, + "tags": { + "regionId": "cn-hangzhou", + "environment": "production" + } + }, + "labels": { + "_cms_region": "cn-hangzhou", + "customNumber": 7 + }, + "annotations": { + "current_value": "92.5" + }, + "data": { + "value": 92.5, + "threshold": 80, + "comparisonOperator": ">" + }, + "alertEntityFields": { + "privateIp": "10.0.0.1" + }, + "alertHistoryUrl": "https://cmsnext.console.aliyun.com/history", + "futureField": "ignored" + } + """); + + SingleAlert alert = captureAlert(); + assertEquals(CommonConstants.ALERT_STATUS_FIRING, alert.getStatus()); + assertEquals(EVENT_TIME, alert.getStartAt()); + assertEquals(EVENT_TIME, alert.getActiveAt()); + assertNull(alert.getEndAt()); + assertEquals("CPU usage exceeded 80%", alert.getContent()); + assertEquals("alibabacloud-cms", alert.getLabels().get("__source__")); + assertEquals("ECS CPU usage is high", alert.getLabels().get("alertname")); + assertEquals(CommonConstants.ALERT_SEVERITY_WARNING, alert.getLabels().get("severity")); + assertEquals("instance", alert.getLabels().get("resourceType")); + assertEquals("i-123", alert.getLabels().get("resourceId")); + assertEquals("7", alert.getLabels().get("customNumber")); + assertEquals("api-server", alert.getAnnotations().get("instanceName")); + assertEquals("92.5", alert.getAnnotations().get("value")); + assertEquals("80", alert.getAnnotations().get("threshold")); + assertEquals("10.0.0.1", alert.getAnnotations().get("privateIp")); + } + + @Test + void shouldConvertResolvedAlertAndIsoTime() { + externAlertService.addExternAlert(""" + { + "subtype": "NORMAL_RESOLVE", + "time": "2026-07-29T06:00:00Z", + "subject": "ECS CPU usage is high", + "severity": "CRITICAL", + "status": "RESOLVED", + "labels": { + "instanceId": "i-123" + } + } + """); + + SingleAlert alert = captureAlert(); + long expectedTime = Instant.parse("2026-07-29T06:00:00Z").toEpochMilli(); + assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus()); + assertEquals(expectedTime, alert.getStartAt()); + assertNull(alert.getActiveAt()); + assertEquals(expectedTime, alert.getEndAt()); + assertEquals("ECS CPU usage is high", alert.getContent()); + assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL, alert.getLabels().get("severity")); + } + + @Test + void shouldTreatRecoveredStatusAsResolved() { + externAlertService.addExternAlert(""" + { + "timestamp": 1785300000123, + "subject": "Recovered alert", + "status": "RECOVERED" + } + """); + + SingleAlert alert = captureAlert(); + assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus()); + assertEquals(EVENT_TIME, alert.getEndAt()); + } + + @Test + void shouldIgnoreInvalidPayload() { + externAlertService.addExternAlert("invalid json"); + externAlertService.addExternAlert("{\"subject\":\"missing status\"}"); + + verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class)); + assertEquals("alibabacloud-cms", externAlertService.supportSource()); + } + + private SingleAlert captureAlert() { + ArgumentCaptor captor = ArgumentCaptor.forClass(SingleAlert.class); + verify(alarmCommonReduce).reduceAndSendAlarm(captor.capture()); + return captor.getValue(); + } +} diff --git a/web-app/src/app/routes/alert/alert-integration/alert-integration.component.ts b/web-app/src/app/routes/alert/alert-integration/alert-integration.component.ts index 57eab71d8f..f4e539821d 100644 --- a/web-app/src/app/routes/alert/alert-integration/alert-integration.component.ts +++ b/web-app/src/app/routes/alert/alert-integration/alert-integration.component.ts @@ -80,6 +80,11 @@ export class AlertIntegrationComponent implements OnInit { name: this.i18nSvc.fanyi('alert.integration.source.tencent'), icon: 'assets/img/integration/tencent.svg' }, + { + id: 'alibabacloud-cms', + name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-cms'), + icon: 'assets/img/integration/alibabacloud.svg' + }, { id: 'alibabacloud-sls', name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-sls'), diff --git a/web-app/src/assets/doc/alert-integration/alibabacloud-cms.en-US.md b/web-app/src/assets/doc/alert-integration/alibabacloud-cms.en-US.md new file mode 100644 index 0000000000..e36af233f7 --- /dev/null +++ b/web-app/src/assets/doc/alert-integration/alibabacloud-cms.en-US.md @@ -0,0 +1,75 @@ +> Send Alibaba Cloud Monitor 2.0 alerts to the HertzBeat alert platform through a webhook. + +### Prepare a HertzBeat API token + +1. Click **Manage API Tokens** in the upper-right corner of this page. +2. Create a token and save it securely. The complete token is displayed only once. + +### Create an Alibaba Cloud Monitor webhook + +1. Log on to the [Alibaba Cloud Monitor 2.0 console](https://cmsnext.console.aliyun.com/). +2. Select or create the target workspace, then go to **Alert Center** > **Notification Management** > **Notification Objects**. +3. Open the **Custom Webhook** tab and click **Create Webhook**. +4. Configure the webhook: + - Name: `HertzBeat` + - Identifier: for example, `hertzbeat` + - URL: + + ```text + http://{hertzbeat_host}:1157/api/alerts/report/alibabacloud-cms + ``` + + - Headers: add `Authorization` with the value `Bearer {token}` + - Method: `POST` + - Data format: `JSON` + - Language: select as needed +5. Save the webhook. + +> Use the notification objects in an Alibaba Cloud Monitor 2.0 workspace. Notification objects in Prometheus Monitoring or ARMS Alert Management use a different webhook format and are not supported by this integration. + +> `{hertzbeat_host}` must be publicly reachable from Alibaba Cloud Monitor. Expose this endpoint through an HTTPS reverse proxy in production. + +### Bind an alert rule + +1. Go to **Alert Center** > **Alert Management** > **Alert Rules**. +2. Create or edit an alert rule. +3. Select the HertzBeat custom webhook in the alert notification settings. +4. Enable recovery notifications if alerts should be resolved automatically in HertzBeat. +5. Save and enable the alert rule. + +### Field mapping + +| Alibaba Cloud Monitor field | HertzBeat field | +| --- | --- | +| `status: OCCURRED/PERSISTENT` | `status: firing` | +| `status: RESOLVED/RECOVERED` | `status: resolved` | +| `subject` | `labels.alertname` | +| `severity` | `labels.severity` | +| `alertMessage` | Alert content | +| `labels`, `resource.tags` | Alert labels | +| `annotations`, threshold, and resource properties | Alert annotations | +| `timestamp` or `time` | Alert time | + +### Troubleshooting + +#### The webhook returns 401 or 403 + +- Make sure the HertzBeat API token is active. +- Make sure the header is named `Authorization` and its value starts with `Bearer `. + +#### HertzBeat does not receive an alert + +- Make sure the webhook URL and port are publicly reachable. +- Make sure the data format is `JSON` and the request method is `POST`. +- Check Alibaba Cloud Monitor alert history and the HertzBeat service logs. +- If source IP allowlisting is enabled, use the latest CIDR list in the Alibaba Cloud documentation. + +#### An alert is not resolved automatically + +- Make sure recovery notifications are enabled in the alert rule or notification policy. +- Make sure the recovery webhook contains a `status` of `RESOLVED` or `RECOVERED`. + +For more information: + +- [Alibaba Cloud Monitor notification objects and webhook fields](https://help.aliyun.com/en/cms/cloudmonitor-2-0/notification-object) +- [Alibaba Cloud Monitor alert rules](https://help.aliyun.com/en/cms/cloudmonitor-2-0/alert-rules-cms-2-0) diff --git a/web-app/src/assets/doc/alert-integration/alibabacloud-cms.zh-CN.md b/web-app/src/assets/doc/alert-integration/alibabacloud-cms.zh-CN.md new file mode 100644 index 0000000000..72ba07b4e3 --- /dev/null +++ b/web-app/src/assets/doc/alert-integration/alibabacloud-cms.zh-CN.md @@ -0,0 +1,75 @@ +> 将阿里云云监控 2.0 的告警通过 Webhook 发送到 HertzBeat 告警平台。 + +### 准备 HertzBeat API Token + +1. 单击页面右上角的 **管理 API Token**。 +2. 创建一个 Token 并立即妥善保存。Token 只会完整显示一次。 + +### 创建阿里云云监控 Webhook + +1. 登录 [阿里云云监控 2.0 控制台](https://cmsnext.console.aliyun.com/)。 +2. 选择或创建目标工作空间，然后进入 **告警中心** > **通知管理** > **通知对象**。 +3. 选择 **自定义 Webhook** 页签，单击 **新建 Webhook**。 +4. 填写 Webhook 配置： + - 名称：`HertzBeat` + - 标识符：例如 `hertzbeat` + - URL： + + ```text + http://{hertzbeat_host}:1157/api/alerts/report/alibabacloud-cms + ``` + + - Headers：添加 `Authorization`，值为 `Bearer {token}` + - Method：`POST` + - 数据格式：`JSON` + - 语言：按需选择 +5. 保存 Webhook。 + +> 请使用云监控 2.0 工作空间内的通知对象。Prometheus 监控或 ARMS 告警管理中的通知对象使用不同的 Webhook 格式，不适用于此集成。 + +> `{hertzbeat_host}` 必须是阿里云云监控可以访问的公网地址。生产环境建议通过 HTTPS 反向代理暴露此接口。 + +### 绑定告警规则 + +1. 进入 **告警中心** > **告警管理** > **告警规则**。 +2. 创建或编辑告警规则。 +3. 在告警通知中选择上一步创建的 HertzBeat 自定义 Webhook。 +4. 如需在 HertzBeat 中自动恢复告警，请同时启用恢复通知。 +5. 保存并启用告警规则。 + +### 字段映射 + +| 阿里云云监控字段 | HertzBeat 字段 | +| --- | --- | +| `status: OCCURRED/PERSISTENT` | `status: firing` | +| `status: RESOLVED/RECOVERED` | `status: resolved` | +| `subject` | `labels.alertname` | +| `severity` | `labels.severity` | +| `alertMessage` | 告警内容 | +| `labels`、`resource.tags` | 告警标签 | +| `annotations`、阈值和资源属性 | 告警注解 | +| `timestamp` 或 `time` | 告警时间 | + +### 常见问题 + +#### 返回 401 或 403 + +- 确认已创建有效的 HertzBeat API Token。 +- 确认 Webhook Header 名称为 `Authorization`，值以 `Bearer ` 开头。 + +#### HertzBeat 未收到告警 + +- 确认 Webhook URL 可从公网访问，并且端口已放行。 +- 确认数据格式选择为 `JSON`，请求方法选择为 `POST`。 +- 检查阿里云云监控的告警历史以及 HertzBeat 服务日志。 +- 若配置了来源 IP 白名单，请以阿里云官方文档中的最新地址段为准。 + +#### 告警没有自动恢复 + +- 确认告警规则或通知策略已启用恢复通知。 +- 确认恢复 Webhook 中的 `status` 为 `RESOLVED` 或 `RECOVERED`。 + +更多信息请参考： + +- [阿里云云监控通知对象与 Webhook 字段](https://help.aliyun.com/zh/cms/cloudmonitor-2-0/notification-object) +- [阿里云云监控告警规则](https://help.aliyun.com/zh/cms/cloudmonitor-2-0/alert-rules-cms-2-0) diff --git a/web-app/src/assets/i18n/en-US.json b/web-app/src/assets/i18n/en-US.json index f2eaf8c63f..167866115d 100644 --- a/web-app/src/assets/i18n/en-US.json +++ b/web-app/src/assets/i18n/en-US.json @@ -100,6 +100,7 @@ "alert.integration.source.skywalking": "SkyWalking", "alert.integration.source.uptime-kuma": "Uptime Kuma", "alert.integration.source.zabbix": "Zabbix", + "alert.integration.source.alibabacloud-cms": "Alibaba Cloud Monitor", "alert.integration.source.alibabacloud-sls": "AlibabaCloud-SLS", "alert.integration.source.huaweicloud-ces": "Huawei Cloud Eye", "alert.integration.source.volcengine": "Volcengine Monitoring", diff --git a/web-app/src/assets/i18n/ja-JP.json b/web-app/src/assets/i18n/ja-JP.json index 98e25ac181..69534995f9 100644 --- a/web-app/src/assets/i18n/ja-JP.json +++ b/web-app/src/assets/i18n/ja-JP.json @@ -100,6 +100,7 @@ "alert.integration.source.skywalking": "SkyWalking", "alert.integration.source.uptime-kuma": "Uptime Kuma", "alert.integration.source.zabbix": "Zabbix", + "alert.integration.source.alibabacloud-cms": "Alibaba Cloud Monitor", "alert.integration.source.alibabacloud-sls": "AlibabaCloud-SLS", "alert.integration.source.huaweicloud-ces": "Huawei Cloud Eye", "alert.integration.source.volcengine": "火山エンジン監視", diff --git a/web-app/src/assets/i18n/ko-KR.json b/web-app/src/assets/i18n/ko-KR.json index 0a3fa12580..df7b93bcb9 100644 --- a/web-app/src/assets/i18n/ko-KR.json +++ b/web-app/src/assets/i18n/ko-KR.json @@ -100,6 +100,7 @@ "alert.integration.source.skywalking": "SkyWalking", "alert.integration.source.uptime-kuma": "Uptime Kuma", "alert.integration.source.zabbix": "Zabbix", + "alert.integration.source.alibabacloud-cms": "Alibaba Cloud Monitor", "alert.integration.source.alibabacloud-sls": "AlibabaCloud-SLS", "alert.integration.source.huaweicloud-ces": "Huawei Cloud Eye", "alert.integration.source.volcengine": "Volcengine Monitoring", diff --git a/web-app/src/assets/i18n/pt-BR.json b/web-app/src/assets/i18n/pt-BR.json index e03d13f9e1..e68a10984a 100644 --- a/web-app/src/assets/i18n/pt-BR.json +++ b/web-app/src/assets/i18n/pt-BR.json @@ -251,6 +251,7 @@ "alert.integration.source.prometheus": "Prometheus", "alert.integration.source.tencent": "Monitoramento de nuvem Tencent", "alert.integration.source.webhook": "PadrãoWebhook", + "alert.integration.source.alibabacloud-cms": "Alibaba Cloud Monitor", "alert.integration.source.alibabacloud-sls": "AlibabaCloud-SLS", "alert.integration.source.huaweicloud-ces": "Huawei Cloud Eye", "alert.integration.source.volcengine": "Volcengine", diff --git a/web-app/src/assets/i18n/zh-CN.json b/web-app/src/assets/i18n/zh-CN.json index d6397daa0f..0eda95a3c0 100644 --- a/web-app/src/assets/i18n/zh-CN.json +++ b/web-app/src/assets/i18n/zh-CN.json @@ -100,6 +100,7 @@ "alert.integration.source.skywalking": "SkyWalking", "alert.integration.source.uptime-kuma": "Uptime Kuma", "alert.integration.source.zabbix": "Zabbix", + "alert.integration.source.alibabacloud-cms": "阿里云云监控", "alert.integration.source.alibabacloud-sls": "阿里云日志服务 SLS", "alert.integration.source.huaweicloud-ces": "华为云监控服务", "alert.integration.source.volcengine": "火山引擎云监控", diff --git a/web-app/src/assets/i18n/zh-TW.json b/web-app/src/assets/i18n/zh-TW.json index 70b7847ef0..547b4f290c 100644 --- a/web-app/src/assets/i18n/zh-TW.json +++ b/web-app/src/assets/i18n/zh-TW.json @@ -100,6 +100,7 @@ "alert.integration.source.skywalking": "SkyWalking", "alert.integration.source.uptime-kuma": "Uptime Kuma", "alert.integration.source.zabbix": "Zabbix", + "alert.integration.source.alibabacloud-cms": "阿里雲雲監控", "alert.integration.source.alibabacloud-sls": "阿里雲端日誌服務 SLS", "alert.integration.source.huaweicloud-ces": "華為雲監控服務", "alert.integration.source.volcengine": "火山引擎監控", From 4feff12e320edfd5e79628d9ae328373362b60fa Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 9 Aug 2026 10:34:47 -0700 Subject: [PATCH 03/18] fix(alert): keep datasource in alert rule export/import (#4264) Co-authored-by: Duansg --- .../hertzbeat/alert/dto/AlertDefineDTO.java | 2 ++ .../AlertDefineExcelImExportServiceImpl.java | 6 +++- .../AlertDefineExcelImExportServiceTest.java | 5 ++++ .../AlertDefineJsonImExportServiceTest.java | 30 ++++++++++++++++++- .../AlertDefineYamlImExportServiceTest.java | 3 ++ 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlertDefineDTO.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlertDefineDTO.java index 3ac01585db..674ee9deb3 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlertDefineDTO.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/dto/AlertDefineDTO.java @@ -51,4 +51,6 @@ public class AlertDefineDTO { private String template; @Excel(name = "Enable") private Boolean enable; + @Excel(name = "Datasource") + private String datasource; } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineExcelImExportServiceImpl.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineExcelImExportServiceImpl.java index 91feab35d4..ca6af4f41c 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineExcelImExportServiceImpl.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/AlertDefineExcelImExportServiceImpl.java @@ -155,6 +155,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx alertDefineDTO.setAnnotations(JsonUtil.fromJson(getCellValueAsString(row.getCell(6)), typeReference)); alertDefineDTO.setTemplate(getCellValueAsString(row.getCell(7))); alertDefineDTO.setEnable(getCellValueAsBoolean(row.getCell(8))); + alertDefineDTO.setDatasource(getCellValueAsString(row.getCell(9))); return alertDefineDTO; } @@ -186,7 +187,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx CellStyle cellStyle = workbook.createCellStyle(); cellStyle.setAlignment(HorizontalAlignment.CENTER); // set header - String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable"}; + String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable", "Datasource"}; Row headerRow = sheet.createRow(0); for (int i = 0; i < headers.length; i++) { Cell cell = headerRow.createCell(i); @@ -227,6 +228,9 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx Cell enableCell = row.createCell(8); enableCell.setCellValue(alertDefineDTO.getEnable()); enableCell.setCellStyle(cellStyle); + Cell datasourceCell = row.createCell(9); + datasourceCell.setCellValue(alertDefineDTO.getDatasource()); + datasourceCell.setCellStyle(cellStyle); } workbook.write(os); os.close(); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineExcelImExportServiceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineExcelImExportServiceTest.java index 885898f0a7..34875e3d27 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineExcelImExportServiceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineExcelImExportServiceTest.java @@ -69,6 +69,7 @@ public class AlertDefineExcelImExportServiceTest { row.createCell(6).setCellValue(JsonUtil.toJson(Map.of("key", "value"))); row.createCell(7).setCellValue("template1"); row.createCell(8).setCellValue(true); + row.createCell(9).setCellValue("promql"); ByteArrayInputStream inputStream = new ByteArrayInputStream(toByteArray(initialWorkbook)); @@ -93,6 +94,7 @@ public class AlertDefineExcelImExportServiceTest { assertEquals(Map.of("key", "value"), alertDefineDTO.getAnnotations()); assertEquals("template1", alertDefineDTO.getTemplate()); assertTrue(alertDefineDTO.getEnable()); + assertEquals("promql", alertDefineDTO.getDatasource()); } } @@ -111,6 +113,7 @@ public class AlertDefineExcelImExportServiceTest { alertDefineDTO.setAnnotations(Map.of("key", "value")); alertDefineDTO.setTemplate("template1"); alertDefineDTO.setEnable(true); + alertDefineDTO.setDatasource("promql"); exportAlertDefineDTO.setAlertDefine(alertDefineDTO); exportAlertDefineList.add(exportAlertDefineDTO); @@ -129,6 +132,7 @@ public class AlertDefineExcelImExportServiceTest { assertEquals("Annotations", headerRow.getCell(6).getStringCellValue()); assertEquals("Template", headerRow.getCell(7).getStringCellValue()); assertEquals("Enable", headerRow.getCell(8).getStringCellValue()); + assertEquals("Datasource", headerRow.getCell(9).getStringCellValue()); Row dataRow = resultSheet.getRow(1); assertEquals("app1", dataRow.getCell(0).getStringCellValue()); @@ -140,6 +144,7 @@ public class AlertDefineExcelImExportServiceTest { assertEquals(JsonUtil.toJson(Map.of("key", "value")), dataRow.getCell(6).getStringCellValue()); assertEquals("template1", dataRow.getCell(7).getStringCellValue()); assertTrue(dataRow.getCell(8).getBooleanCellValue()); + assertEquals("promql", dataRow.getCell(9).getStringCellValue()); } } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineJsonImExportServiceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineJsonImExportServiceTest.java index d0727c0dca..9674f6ec8b 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineJsonImExportServiceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineJsonImExportServiceTest.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -29,8 +31,10 @@ import java.util.List; import org.apache.hertzbeat.alert.dto.AlertDefineDTO; import org.apache.hertzbeat.alert.dto.ExportAlertDefineDTO; import org.apache.hertzbeat.alert.service.impl.AlertDefineJsonImExportServiceImpl; +import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; /** * test case for {@link AlertDefineJsonImExportServiceImpl} @@ -43,7 +47,7 @@ class AlertDefineJsonImExportServiceTest { @SuppressWarnings("checkstyle:OperatorWrap") private static final String JSON_DATA = "[{\"alertDefine\":{\"name\":\"App1\",\"type\":\"realtime\"," + "\"expr\":\"Expr1\",\"period\":3000,\"times\":3," + - "\"enable\":true,\"template\":\"Template1\"}}]"; + "\"enable\":true,\"template\":\"Template1\",\"datasource\":\"promql\"}}]"; private InputStream inputStream; private List alertDefineList; @@ -77,6 +81,7 @@ class AlertDefineJsonImExportServiceTest { assertEquals(1, result.size()); assertEquals("App1", result.get(0).getAlertDefine().getName()); assertEquals("realtime", result.get(0).getAlertDefine().getType()); + assertEquals("promql", result.get(0).getAlertDefine().getDatasource()); } @Test @@ -100,6 +105,29 @@ class AlertDefineJsonImExportServiceTest { assertTrue(result.contains("realtime")); } + @Test + void testExportKeepsDatasource() { + AlertDefineService alertDefineService = mock(AlertDefineService.class); + AlertDefine define = AlertDefine.builder() + .name("test") + .type("periodic_metric") + .expr("cpu_usage{instance=\"server1\"} > 80") + .datasource("promql") + .period(300) + .times(3) + .template("test") + .enable(true) + .build(); + when(alertDefineService.getAlertDefine(1L)).thenReturn(define); + ReflectionTestUtils.setField(service, "alertDefineService", alertDefineService); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + service.exportConfig(outputStream, List.of(1L)); + + String result = outputStream.toString(StandardCharsets.UTF_8); + assertTrue(result.contains("promql"), "exported config should keep datasource, but got: " + result); + } + @Test void testType() { assertEquals("JSON", service.type()); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineYamlImExportServiceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineYamlImExportServiceTest.java index 692b633b8d..8383d82377 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineYamlImExportServiceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/service/AlertDefineYamlImExportServiceTest.java @@ -64,6 +64,7 @@ class AlertDefineYamlImExportServiceTest { times: 3 enable: true template: Template1 + datasource: promql """; private InputStream inputStream; @@ -82,6 +83,7 @@ class AlertDefineYamlImExportServiceTest { alertDefine.setExpr("Expr1"); alertDefine.setEnable(true); alertDefine.setTemplate("Template1"); + alertDefine.setDatasource("promql"); ExportAlertDefineDTO exportAlertDefine = new ExportAlertDefineDTO(); exportAlertDefine.setAlertDefine(alertDefine); @@ -135,6 +137,7 @@ class AlertDefineYamlImExportServiceTest { assertTrue(yamlOutput.contains("name: App1")); assertTrue(yamlOutput.contains("type: realtime")); assertTrue(yamlOutput.contains("expr: Expr1")); + assertTrue(yamlOutput.contains("datasource: promql")); } @Test From cfb3dee490439aebe8ecaa12c4a8219731833c7e Mon Sep 17 00:00:00 2001 From: Duansg Date: Mon, 10 Aug 2026 01:45:40 +0800 Subject: [PATCH 04/18] [fix] fix align cors configuration with header based authentication (#4267) --- .../config/SecurityCorsConfiguration.java | 4 +- .../config/SecurityCorsConfigurationTest.java | 70 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/SecurityCorsConfigurationTest.java diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SecurityCorsConfiguration.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SecurityCorsConfiguration.java index 69065eee42..28fd6ba38e 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SecurityCorsConfiguration.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SecurityCorsConfiguration.java @@ -35,7 +35,9 @@ public class SecurityCorsConfiguration { public FilterRegistrationBean corsFilter() { UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); CorsConfiguration corsConfiguration = new CorsConfiguration(); - corsConfiguration.setAllowCredentials(true); + // Requests authenticate with a token in the Authorization header rather than a + // cookie, so no request relies on ambient credentials being sent cross origin. + corsConfiguration.setAllowCredentials(false); corsConfiguration.setAllowedOriginPatterns(Collections.singletonList(CorsConfiguration.ALL)); corsConfiguration.addAllowedHeader(CorsConfiguration.ALL); corsConfiguration.addAllowedMethod(CorsConfiguration.ALL); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/SecurityCorsConfigurationTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/SecurityCorsConfigurationTest.java new file mode 100644 index 0000000000..18b96a6a03 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/SecurityCorsConfigurationTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import jakarta.servlet.Filter; +import org.junit.jupiter.api.Test; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.mock.web.MockFilterChain; + +/** + * Test case for {@link SecurityCorsConfiguration}. + * + *

The filter answers every origin, which is intentional, and requests authenticate with + * a token in the Authorization header rather than a cookie, so credentials do not need to + * be allowed. Both halves are asserted: the credentials header is not sent, and a preflight + * still succeeds so the api stays reachable cross origin. + */ +class SecurityCorsConfigurationTest { + + private static final String OTHER_ORIGIN = "https://other.example"; + + @Test + void testCredentialsAreNotAllowedForCrossOriginRequests() throws Exception { + MockHttpServletResponse response = handlePreflight(); + + assertNotEquals("true", response.getHeader("Access-Control-Allow-Credentials")); + } + + @Test + void testCrossOriginRequestsAreStillAnswered() throws Exception { + MockHttpServletResponse response = handlePreflight(); + + assertNotNull(response.getHeader("Access-Control-Allow-Origin"), + "the api is meant to stay reachable cross origin"); + assertEquals(200, response.getStatus()); + } + + private MockHttpServletResponse handlePreflight() throws Exception { + FilterRegistrationBean registration = new SecurityCorsConfiguration().corsFilter(); + Filter filter = (Filter) registration.getFilter(); + + MockHttpServletRequest request = new MockHttpServletRequest("OPTIONS", "/api/monitors"); + request.addHeader("Origin", OTHER_ORIGIN); + request.addHeader("Access-Control-Request-Method", "GET"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + filter.doFilter(request, response, new MockFilterChain()); + return response; + } +} From 41c4fb7ff56fe25e0f3b4b430bebe08ca2318420 Mon Sep 17 00:00:00 2001 From: Duansg Date: Tue, 11 Aug 2026 00:50:11 +0800 Subject: [PATCH 05/18] [fix] add missing rbac rules for the remaining unruled routes (#4271) --- .../src/main/resources/sureness.yml | 17 +++ .../security/SurenessUnruledEndpointTest.java | 130 ++++++++++++++++++ .../hertzbeat-mysql-iotdb/conf/sureness.yml | 17 +++ .../conf/sureness.yml | 17 +++ .../conf/sureness.yml | 17 +++ .../conf/sureness.yml | 17 +++ .../conf/sureness.yml | 17 +++ script/sureness.yml | 17 +++ 8 files changed, 249 insertions(+) create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index ee29402442..f63bee0e8f 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java new file mode 100644 index 0000000000..49ef4a867e --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import com.usthe.sureness.matcher.util.TirePathTree; +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +/** + * Guards the routes that carried no rbac rule at all. + * + *

A route absent from {@code sureness.yml} leaves `supportRoles` null, and + * `BaseProcessor.authorized` returns early when no role is required, so every one of + * these was reachable by any authenticated account including {@code guest}: a raw promql + * passthrough to the time series database, log deletion, log and alert injection, label + * management and the internal queue metrics of the hertzbeat process. + */ +class SurenessUnruledEndpointTest { + + private static final String SEPARATOR = "==="; + + private static TirePathTree roleTree; + + @BeforeAll + @SuppressWarnings("unchecked") + static void loadSurenessConfig() throws IOException { + List resourceRole; + try (InputStream in = SurenessUnruledEndpointTest.class.getResourceAsStream("/sureness.yml")) { + assertNotNull(in, "sureness.yml must be on the classpath"); + Map document = new Yaml().load(in); + resourceRole = (List) document.get("resourceRole"); + } + assertNotNull(resourceRole, "resourceRole must be present"); + roleTree = new TirePathTree(); + roleTree.buildTree(new LinkedHashSet<>(resourceRole)); + } + + private static String rolesFor(String path, String method) { + return roleTree.searchPathFilterRoles(path + SEPARATOR + method); + } + + /** + * `PromqlQueryExecutor` forwards the submitted expression verbatim, so this route reads + * the whole metric store regardless of which monitors the caller may see. + */ + @Test + void queryingTheWarehouseDirectlyIsRestrictedToAdmin() { + assertEquals("[admin]", rolesFor("/api/warehouse/query", "post")); + } + + @Test + void probingStorageAvailabilityStaysOpenToEveryRole() { + assertEquals("[admin,user,guest]", rolesFor("/api/warehouse/storage/status", "get")); + } + + @Test + void deletingLogsIsRestrictedToAdmin() { + assertEquals("[admin]", rolesFor("/api/logs", "delete")); + } + + @Test + void readingLogsStaysOpenToEveryRole() { + assertEquals("[admin,user,guest]", rolesFor("/api/logs/list", "get")); + } + + /** + * Matches how the sibling ingestion routes `/api/otlp/**` and `/api/logs/ingest/**` + * are already scoped, so a low privileged account can no longer forge log records. + */ + @Test + void ingestingOtlpLogsRequiresAtLeastUser() { + assertEquals("[admin,user]", rolesFor("/api/logs/otlp/v1/logs", "post")); + } + + /** + * The prometheus alertmanager webhook injects alerts, which drive notifications. + * Scoped like the sibling `/api/alerts/report` route. + */ + @Test + void injectingPrometheusAlertsRequiresAtLeastUser() { + assertEquals("[admin,user]", rolesFor("/api/v2/alerts", "post")); + } + + @Test + void labelWritesFollowTheUsualScoping() { + assertEquals("[admin,user,guest]", rolesFor("/api/label", "get")); + assertEquals("[admin,user]", rolesFor("/api/label", "post")); + assertEquals("[admin,user]", rolesFor("/api/label", "put")); + assertEquals("[admin]", rolesFor("/api/label", "delete")); + } + + @Test + void processQueueMetricsAreRestrictedToAdmin() { + assertEquals("[admin]", rolesFor("/api/metrics", "get")); + } + + /** + * Favourites are stored per account and rendered on the monitor pages, so they stay + * reachable by every role even though they sit under the same path prefix. + */ + @Test + void metricFavouritesStayOpenToEveryRole() { + assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1", "get")); + assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "post")); + assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "delete")); + } +} diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml index fd283a60c6..069fba3069 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml index fd283a60c6..069fba3069 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml index fd283a60c6..069fba3069 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml index 2c056a8448..d15a6ac5d2 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml index fd283a60c6..069fba3069 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] diff --git a/script/sureness.yml b/script/sureness.yml index 2c056a8448..d15a6ac5d2 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -70,6 +70,23 @@ resourceRole: - /api/config/**===post===[admin] - /api/config/**===put===[admin] - /api/config/**===delete===[admin] + # queue depth of the hertzbeat process itself, operational data + - /api/metrics===get===[admin] + # per account metric favourites rendered on the monitor pages + - /api/metrics/**===get===[admin,user,guest] + - /api/metrics/**===post===[admin,user,guest] + - /api/metrics/**===delete===[admin,user,guest] + - /api/label/**===get===[admin,user,guest] + - /api/label/**===post===[admin,user] + - /api/label/**===put===[admin,user] + - /api/label/**===delete===[admin] + # the storage availability probe is read by every monitor page, while the query + # route forwards a raw promql expression straight to the time series database + - /api/warehouse/**===get===[admin,user,guest] + - /api/warehouse/query===post===[admin] + - /api/logs/otlp/**===post===[admin,user] + - /api/logs===delete===[admin] + - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] From f07215639dec96015537f81f39b48414dbf15540 Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Wed, 12 Aug 2026 07:41:23 -0700 Subject: [PATCH 06/18] [feat] paginate the monitoring metrics data table (#4309) --- .../monitor-data-table.component.html | 12 +++++++++--- .../monitor-data-table.component.ts | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html index ee25a56259..1dfc67af21 100644 --- a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html +++ b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html @@ -92,10 +92,16 @@ *ngIf="!monitor && isTable" nzSize="small" [nzNoResult]="'monitor.detail.chart.no-data' | i18n" - [nzFrontPagination]="false" - [nzShowPagination]="false" + [nzFrontPagination]="true" + [nzShowPagination]="valueRows.length > 10" + [nzPageSize]="pageSize" + [nzShowSizeChanger]="true" + [nzPageSizeOptions]="[10, 20, 50, 100]" + [nzPageIndex]="pageIndex" + (nzPageIndexChange)="pageIndex = $event" + (nzPageSizeChange)="pageSize = $event; pageIndex = 1" [nzData]="valueRows" - [nzScroll]="height ? { y: scrollY } : { x: '100%' }" + [nzScroll]="height ? { y: valueRows.length > 10 ? pagedScrollY : scrollY } : { x: '100%' }" #smallTable > diff --git a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.ts b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.ts index cfb4885bb4..8cc6931814 100644 --- a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.ts +++ b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.ts @@ -60,16 +60,20 @@ export class MonitorDataTableComponent implements OnInit { showModal!: boolean; time!: any; fields!: any[]; - valueRows!: any[]; + valueRows: any[] = []; rowValues!: any[]; isTable: boolean = true; scrollY: string = '100%'; + pagedScrollY: string = '100%'; loading: boolean = false; + pageSize: number = 10; + pageIndex: number = 1; constructor(private monitorSvc: MonitorService, private notifySvc: NzNotificationService) {} ngOnInit(): void { this.scrollY = `calc(${this.height} - 130px)`; + this.pagedScrollY = `calc(${this.height} - 170px)`; } loadData() { @@ -85,6 +89,7 @@ export class MonitorDataTableComponent implements OnInit { this.time = message.data.time; this.fields = message.data.fields; this.valueRows = message.data.valueRows; + this.pageIndex = 1; if (this.valueRows.length == 1) { this.isTable = false; this.rowValues = this.valueRows[0].values; From 8a62215e8b19d2f16908e517b5230eece694d2e4 Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:26:38 -0700 Subject: [PATCH 07/18] test(web): make web-app unit tests runnable (#4314) Co-authored-by: Duansg --- .github/workflows/frontend-build-test.yml | 4 + web-app/karma.conf.js | 3 +- web-app/package.json | 4 +- .../src/app/core/i18n/i18n.service.spec.ts | 2 +- .../setting-drawer-i18n.directive.spec.ts | 43 +++++------ .../alert-center.component.spec.ts | 5 +- .../alert-group-converge.component.spec.ts | 5 +- .../alert-inhibit.component.spec.ts | 5 +- .../alert-integration.component.spec.ts | 6 +- .../alert-notice-receiver.component.spec.ts | 5 +- .../alert-notice-rule.component.spec.ts | 5 +- .../alert-notice-template.component.spec.ts | 5 +- .../alert-notice.component.spec.ts | 5 +- .../alert-setting.component.spec.ts | 5 +- .../alert-silence.component.spec.ts | 5 +- .../bulletin/bulletin.component.spec.ts | 5 +- .../monitor-data-chart.component.spec.ts | 5 +- .../monitor-data-table.component.spec.ts | 5 +- .../monitor-detail.component.spec.ts | 5 +- .../monitor-edit.component.spec.ts | 5 +- .../monitor-form.component.spec.ts | 8 +- .../monitor-list.component.spec.ts | 5 +- .../monitor-new/monitor-new.component.spec.ts | 5 +- .../collector/collector.component.spec.ts | 5 +- .../setting/define/define.component.spec.ts | 5 +- .../setting/label/label.component.spec.ts | 5 +- .../setting/plugins/plugin.component.spec.ts | 5 +- .../message-server.component.spec.ts | 5 +- .../object-store.component.spec.ts | 5 +- .../settings/settings.component.spec.ts | 7 +- .../system-config.component.spec.ts | 5 +- .../setting/status/status.component.spec.ts | 5 +- .../status-public.component.spec.ts | 5 +- .../app/service/alert-define.service.spec.ts | 3 +- .../app/service/alert-group.service.spec.ts | 3 +- .../app/service/alert-inhibit.service.spec.ts | 3 +- .../app/service/alert-silence.service.spec.ts | 3 +- web-app/src/app/service/alert.service.spec.ts | 3 +- .../app/service/app-define.service.spec.ts | 3 +- web-app/src/app/service/auth.service.spec.ts | 3 +- .../src/app/service/collector.service.spec.ts | 3 +- web-app/src/app/service/label.service.spec.ts | 3 +- .../src/app/service/monitor.service.spec.ts | 3 +- .../service/notice-receiver.service.spec.ts | 3 +- .../app/service/notice-rule.service.spec.ts | 3 +- .../service/notice-template.service.spec.ts | 3 +- .../src/app/service/plugin.service.spec.ts | 3 +- .../status-page-public.service.spec.ts | 3 +- .../app/service/status-page.service.spec.ts | 3 +- .../configurable-field.component.spec.ts | 5 +- .../form-field/form-field.component.spec.ts | 6 +- .../help-message-show.component.spec.ts | 5 +- .../label-selector.component.spec.ts | 5 +- .../monitor-select-list.component.spec.ts | 5 +- .../monitor-select-menu.component.spec.ts | 5 +- .../multi-func-input.component.spec.ts | 6 +- .../toolbar/toolbar.component.spec.ts | 5 +- web-app/src/testing.ts | 76 +++++++++++++++++++ web-app/tsconfig.json | 3 + 59 files changed, 220 insertions(+), 151 deletions(-) create mode 100644 web-app/src/testing.ts diff --git a/.github/workflows/frontend-build-test.yml b/.github/workflows/frontend-build-test.yml index 9a403df8d4..2998dfad75 100644 --- a/.github/workflows/frontend-build-test.yml +++ b/.github/workflows/frontend-build-test.yml @@ -37,6 +37,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -53,3 +54,6 @@ jobs: - name: EsLint Test working-directory: web-app run: pnpm lint:ts + - name: Unit Test + working-directory: web-app + run: pnpm test diff --git a/web-app/karma.conf.js b/web-app/karma.conf.js index 78f65e57f0..ca61318899 100644 --- a/web-app/karma.conf.js +++ b/web-app/karma.conf.js @@ -18,8 +18,7 @@ module.exports = function (config) { // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html // for example, you can disable the random execution with `random: false` // or set a specific seed with `seed: 4321` - }, - clearContext: false // leave Jasmine Spec Runner output visible in browser + } }, jasmineHtmlReporter: { suppressAll: true // removes the duplicated traces diff --git a/web-app/package.json b/web-app/package.json index 4bf1e9bee5..ccb88ce1d1 100644 --- a/web-app/package.json +++ b/web-app/package.json @@ -23,12 +23,12 @@ "start": "ng serve --proxy-config proxy.conf.json", "build": "npm run ng-high-memory build", "watch": "ng build --watch --configuration development", - "test": "ng test", + "test": "ng test --watch=false --browsers=ChromeHeadless", "ng-high-memory": "node --max_old_space_size=8000 ./node_modules/@angular/cli/bin/ng", "hmr": "ng s -o --hmr", "analyze": "npm run ng-high-memory build -- --source-map", "analyze:view": "source-map-explorer dist/**/*.js", - "test-coverage": "ng test --code-coverage --watch=false", + "test-coverage": "ng test --code-coverage --watch=false --browsers=ChromeHeadless", "color-less": "ng-alain-plugin-theme -t=colorLess", "theme": "ng-alain-plugin-theme -t=themeCss", "icon": "ng g ng-alain:plugin icon", diff --git a/web-app/src/app/core/i18n/i18n.service.spec.ts b/web-app/src/app/core/i18n/i18n.service.spec.ts index 4bb33b95fe..87afd66bd7 100644 --- a/web-app/src/app/core/i18n/i18n.service.spec.ts +++ b/web-app/src/app/core/i18n/i18n.service.spec.ts @@ -71,7 +71,7 @@ describe('Service: I18n', () => { it('should be use default language when the browser language is not in the list', () => { spyOnProperty(navigator, 'languages').and.returnValue(['es-419']); genModule(); - expect(srv.defaultLang).toBe('zh-CN'); + expect(srv.defaultLang).toBe('en-US'); }); it('should be trigger notify when changed language', () => { diff --git a/web-app/src/app/layout/basic/directives/setting-drawer-i18n.directive.spec.ts b/web-app/src/app/layout/basic/directives/setting-drawer-i18n.directive.spec.ts index 84fc9c8fbd..d6dcc4a4cd 100644 --- a/web-app/src/app/layout/basic/directives/setting-drawer-i18n.directive.spec.ts +++ b/web-app/src/app/layout/basic/directives/setting-drawer-i18n.directive.spec.ts @@ -18,7 +18,7 @@ */ import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; -import { Component, NgZone } from '@angular/core'; +import { Component } from '@angular/core'; import { ComponentFixture, TestBed, fakeAsync, tick } from '@angular/core/testing'; import { By } from '@angular/platform-browser'; import { I18NService } from '@core'; @@ -45,6 +45,7 @@ describe('SettingDrawerI18nDirective', () => { let i18nService: jasmine.SpyObj; let httpMock: HttpTestingController; let mockTranslations: { [key: string]: string }; + const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR']; const mockI18nData = { 'zh-CN': { @@ -102,7 +103,7 @@ describe('SettingDrawerI18nDirective', () => { await TestBed.configureTestingModule({ imports: [HttpClientTestingModule], declarations: [TestComponent, SettingDrawerI18nDirective], - providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }, NgZone] + providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }] }).compileComponents(); fixture = TestBed.createComponent(TestComponent); @@ -120,36 +121,33 @@ describe('SettingDrawerI18nDirective', () => { httpMock.verify(); }); - it('should create', () => { + function loadMappings(): void { + fixture.detectChanges(); + languages.forEach(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`).flush(mockI18nData[lang as keyof typeof mockI18nData])); + } + + function destroyFixture(): void { + fixture.destroy(); + tick(2000); + } + + it('should create', fakeAsync(() => { + loadMappings(); expect(directive).toBeTruthy(); - }); + destroyFixture(); + })); it('should load mappings from i18n files', fakeAsync(() => { - fixture.detectChanges(); - - const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR']; - const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`)); - - languages.forEach((lang, index) => { - requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]); - }); - + loadMappings(); tick(100); fixture.detectChanges(); expect(i18nService.fanyi).toHaveBeenCalled(); + destroyFixture(); })); it('should replace Chinese text with translations', fakeAsync(() => { - fixture.detectChanges(); - - const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR']; - const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`)); - - languages.forEach((lang, index) => { - requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]); - }); - + loadMappings(); tick(2000); fixture.detectChanges(); tick(100); @@ -161,5 +159,6 @@ describe('SettingDrawerI18nDirective', () => { expect(themeColorDiv.textContent).toContain('Theme Color'); } } + destroyFixture(); })); }); diff --git a/web-app/src/app/routes/alert/alert-center/alert-center.component.spec.ts b/web-app/src/app/routes/alert/alert-center/alert-center.component.spec.ts index 08d092b8d0..64b70ba9f2 100644 --- a/web-app/src/app/routes/alert/alert-center/alert-center.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-center/alert-center.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertCenterComponent } from './alert-center.component'; @@ -26,9 +27,7 @@ describe('AlertCenterComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [AlertCenterComponent] - }).compileComponents(); + await configureShallowTest(AlertCenterComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/alert/alert-group/alert-group-converge.component.spec.ts b/web-app/src/app/routes/alert/alert-group/alert-group-converge.component.spec.ts index f1d775cf00..af6dd8934b 100644 --- a/web-app/src/app/routes/alert/alert-group/alert-group-converge.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-group/alert-group-converge.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertGroupConvergeComponent } from './alert-group-converge.component'; @@ -26,9 +27,7 @@ describe('AlertConvergeComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [AlertGroupConvergeComponent] - }).compileComponents(); + await configureShallowTest(AlertGroupConvergeComponent).compileComponents(); fixture = TestBed.createComponent(AlertGroupConvergeComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-inhibit/alert-inhibit.component.spec.ts b/web-app/src/app/routes/alert/alert-inhibit/alert-inhibit.component.spec.ts index 98fcb9c5d9..63e79b229e 100644 --- a/web-app/src/app/routes/alert/alert-inhibit/alert-inhibit.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-inhibit/alert-inhibit.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertInhibitComponent } from './alert-inhibit.component'; @@ -26,9 +27,7 @@ describe('AlertInhibitComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [AlertInhibitComponent] - }).compileComponents(); + await configureShallowTest(AlertInhibitComponent).compileComponents(); fixture = TestBed.createComponent(AlertInhibitComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-integration/alert-integration.component.spec.ts b/web-app/src/app/routes/alert/alert-integration/alert-integration.component.spec.ts index c5c205dc53..28d6084128 100644 --- a/web-app/src/app/routes/alert/alert-integration/alert-integration.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-integration/alert-integration.component.spec.ts @@ -18,6 +18,8 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureStandaloneTest } from '@testing'; +import { MarkdownModule } from 'ngx-markdown'; import { AlertIntegrationComponent } from './alert-integration.component'; @@ -26,9 +28,7 @@ describe('AlertIntegrationComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [AlertIntegrationComponent] - }).compileComponents(); + await configureStandaloneTest(AlertIntegrationComponent, [MarkdownModule.forRoot()]).compileComponents(); fixture = TestBed.createComponent(AlertIntegrationComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-notice/alert-notice-receiver/alert-notice-receiver.component.spec.ts b/web-app/src/app/routes/alert/alert-notice/alert-notice-receiver/alert-notice-receiver.component.spec.ts index 348b6c67ce..cdd7671620 100644 --- a/web-app/src/app/routes/alert/alert-notice/alert-notice-receiver/alert-notice-receiver.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-notice/alert-notice-receiver/alert-notice-receiver.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertNoticeReceiverComponent } from './alert-notice-receiver.component'; @@ -26,9 +27,7 @@ describe('AlertNoticeReceiverComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [AlertNoticeReceiverComponent] - }).compileComponents(); + await configureShallowTest(AlertNoticeReceiverComponent).compileComponents(); fixture = TestBed.createComponent(AlertNoticeReceiverComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-notice/alert-notice-rule/alert-notice-rule.component.spec.ts b/web-app/src/app/routes/alert/alert-notice/alert-notice-rule/alert-notice-rule.component.spec.ts index 5d53537033..0936badeb5 100644 --- a/web-app/src/app/routes/alert/alert-notice/alert-notice-rule/alert-notice-rule.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-notice/alert-notice-rule/alert-notice-rule.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertNoticeRuleComponent } from './alert-notice-rule.component'; @@ -26,9 +27,7 @@ describe('AlertNoticeRuleComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [AlertNoticeRuleComponent] - }).compileComponents(); + await configureShallowTest(AlertNoticeRuleComponent).compileComponents(); fixture = TestBed.createComponent(AlertNoticeRuleComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-notice/alert-notice-template/alert-notice-template.component.spec.ts b/web-app/src/app/routes/alert/alert-notice/alert-notice-template/alert-notice-template.component.spec.ts index cca99270e5..328f9aa438 100644 --- a/web-app/src/app/routes/alert/alert-notice/alert-notice-template/alert-notice-template.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-notice/alert-notice-template/alert-notice-template.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertNoticeTemplateComponent } from './alert-notice-template.component'; @@ -26,9 +27,7 @@ describe('AlertNoticeTemplateComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [AlertNoticeTemplateComponent] - }).compileComponents(); + await configureShallowTest(AlertNoticeTemplateComponent).compileComponents(); fixture = TestBed.createComponent(AlertNoticeTemplateComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/alert/alert-notice/alert-notice.component.spec.ts b/web-app/src/app/routes/alert/alert-notice/alert-notice.component.spec.ts index 61f97deb62..e635918d31 100644 --- a/web-app/src/app/routes/alert/alert-notice/alert-notice.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-notice/alert-notice.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertNoticeComponent } from './alert-notice.component'; @@ -26,9 +27,7 @@ describe('AlertNoticeComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [AlertNoticeComponent] - }).compileComponents(); + await configureShallowTest(AlertNoticeComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.spec.ts b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.spec.ts index b36b6deeb5..5875360bd6 100644 --- a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertSettingComponent } from './alert-setting.component'; @@ -26,9 +27,7 @@ describe('AlertSettingComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [AlertSettingComponent] - }).compileComponents(); + await configureShallowTest(AlertSettingComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/alert/alert-silence/alert-silence.component.spec.ts b/web-app/src/app/routes/alert/alert-silence/alert-silence.component.spec.ts index f6f240176b..3dc81863ca 100644 --- a/web-app/src/app/routes/alert/alert-silence/alert-silence.component.spec.ts +++ b/web-app/src/app/routes/alert/alert-silence/alert-silence.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { AlertSilenceComponent } from './alert-silence.component'; @@ -26,9 +27,7 @@ describe('AlertSilenceComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [AlertSilenceComponent] - }).compileComponents(); + await configureShallowTest(AlertSilenceComponent).compileComponents(); fixture = TestBed.createComponent(AlertSilenceComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/bulletin/bulletin.component.spec.ts b/web-app/src/app/routes/bulletin/bulletin.component.spec.ts index 5e1fc10402..f2fd28b826 100644 --- a/web-app/src/app/routes/bulletin/bulletin.component.spec.ts +++ b/web-app/src/app/routes/bulletin/bulletin.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { BulletinComponent } from './bulletin.component'; @@ -26,9 +27,7 @@ describe('BulletinComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [BulletinComponent] - }).compileComponents(); + await configureShallowTest(BulletinComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.spec.ts b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.spec.ts index 6ae8d454c1..a320fd17c1 100644 --- a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorDataChartComponent } from './monitor-data-chart.component'; @@ -26,9 +27,7 @@ describe('MonitorDataChartComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorDataChartComponent] - }).compileComponents(); + await configureShallowTest(MonitorDataChartComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.spec.ts b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.spec.ts index de753cfb96..8aee5f90b7 100644 --- a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorDataTableComponent } from './monitor-data-table.component'; @@ -26,9 +27,7 @@ describe('MonitorDataChartComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorDataTableComponent] - }).compileComponents(); + await configureShallowTest(MonitorDataTableComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.spec.ts b/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.spec.ts index 74b4b1442d..958c5f401e 100644 --- a/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorDetailComponent } from './monitor-detail.component'; @@ -26,9 +27,7 @@ describe('MonitorDetailComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorDetailComponent] - }).compileComponents(); + await configureShallowTest(MonitorDetailComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-edit/monitor-edit.component.spec.ts b/web-app/src/app/routes/monitor/monitor-edit/monitor-edit.component.spec.ts index bed99b721b..688ea06b56 100644 --- a/web-app/src/app/routes/monitor/monitor-edit/monitor-edit.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-edit/monitor-edit.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorEditComponent } from './monitor-edit.component'; @@ -26,9 +27,7 @@ describe('MonitorModifyComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorEditComponent] - }).compileComponents(); + await configureShallowTest(MonitorEditComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.spec.ts b/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.spec.ts index 9a3dbd6e28..0b43dad05e 100644 --- a/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.spec.ts @@ -18,7 +18,10 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { FormsModule } from '@angular/forms'; +import { configureShallowTest } from '@testing'; +import { Monitor } from '../../../pojo/Monitor'; import { MonitorFormComponent } from './monitor-form.component'; describe('MonitorFormComponent', () => { @@ -26,14 +29,13 @@ describe('MonitorFormComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorFormComponent] - }).compileComponents(); + await configureShallowTest(MonitorFormComponent, [FormsModule]).compileComponents(); }); beforeEach(() => { fixture = TestBed.createComponent(MonitorFormComponent); component = fixture.componentInstance; + component.monitor = new Monitor(); fixture.detectChanges(); }); diff --git a/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.spec.ts b/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.spec.ts index 349dbf6ca2..9538bc22f3 100644 --- a/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorListComponent } from './monitor-list.component'; @@ -26,9 +27,7 @@ describe('MonitorListComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorListComponent] - }).compileComponents(); + await configureShallowTest(MonitorListComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.spec.ts b/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.spec.ts index f41a293f2c..5cf96a8b50 100644 --- a/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.spec.ts +++ b/web-app/src/app/routes/monitor/monitor-new/monitor-new.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorNewComponent } from './monitor-new.component'; @@ -26,9 +27,7 @@ describe('MonitorAddComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorNewComponent] - }).compileComponents(); + await configureShallowTest(MonitorNewComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/routes/setting/collector/collector.component.spec.ts b/web-app/src/app/routes/setting/collector/collector.component.spec.ts index 81c9ca07ad..3c41095d60 100644 --- a/web-app/src/app/routes/setting/collector/collector.component.spec.ts +++ b/web-app/src/app/routes/setting/collector/collector.component.spec.ts @@ -18,6 +18,7 @@ */ import { waitForAsync, ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { CollectorComponent } from './collector.component'; @@ -26,9 +27,7 @@ describe('CollectorComponent', () => { let fixture: ComponentFixture; beforeEach(waitForAsync(() => { - TestBed.configureTestingModule({ - declarations: [CollectorComponent] - }).compileComponents(); + configureShallowTest(CollectorComponent).compileComponents(); })); beforeEach(() => { diff --git a/web-app/src/app/routes/setting/define/define.component.spec.ts b/web-app/src/app/routes/setting/define/define.component.spec.ts index 371e0003e8..71bffbabc5 100644 --- a/web-app/src/app/routes/setting/define/define.component.spec.ts +++ b/web-app/src/app/routes/setting/define/define.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { DefineComponent } from './define.component'; @@ -26,9 +27,7 @@ describe('DefineComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [DefineComponent] - }).compileComponents(); + await configureShallowTest(DefineComponent).compileComponents(); fixture = TestBed.createComponent(DefineComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/setting/label/label.component.spec.ts b/web-app/src/app/routes/setting/label/label.component.spec.ts index 86cd42b462..e6077f5de2 100644 --- a/web-app/src/app/routes/setting/label/label.component.spec.ts +++ b/web-app/src/app/routes/setting/label/label.component.spec.ts @@ -18,6 +18,7 @@ */ import { waitForAsync, ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { SettingLabelComponent } from './label.component'; @@ -26,9 +27,7 @@ describe('SettingLabelComponent', () => { let fixture: ComponentFixture; beforeEach(waitForAsync(() => { - TestBed.configureTestingModule({ - declarations: [SettingLabelComponent] - }).compileComponents(); + configureShallowTest(SettingLabelComponent).compileComponents(); })); beforeEach(() => { diff --git a/web-app/src/app/routes/setting/plugins/plugin.component.spec.ts b/web-app/src/app/routes/setting/plugins/plugin.component.spec.ts index ffc816aa58..4733c00180 100644 --- a/web-app/src/app/routes/setting/plugins/plugin.component.spec.ts +++ b/web-app/src/app/routes/setting/plugins/plugin.component.spec.ts @@ -18,6 +18,7 @@ */ import { waitForAsync, ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { SettingPluginsComponent } from './plugin.component'; @@ -26,9 +27,7 @@ describe('SettingPluginsComponent', () => { let fixture: ComponentFixture; beforeEach(waitForAsync(() => { - TestBed.configureTestingModule({ - declarations: [SettingPluginsComponent] - }).compileComponents(); + configureShallowTest(SettingPluginsComponent).compileComponents(); })); beforeEach(() => { diff --git a/web-app/src/app/routes/setting/settings/message-server/message-server.component.spec.ts b/web-app/src/app/routes/setting/settings/message-server/message-server.component.spec.ts index 1e293537d4..26adb31da8 100644 --- a/web-app/src/app/routes/setting/settings/message-server/message-server.component.spec.ts +++ b/web-app/src/app/routes/setting/settings/message-server/message-server.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MessageServerComponent } from './message-server.component'; @@ -26,9 +27,7 @@ describe('MessageServerComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MessageServerComponent] - }).compileComponents(); + await configureShallowTest(MessageServerComponent).compileComponents(); fixture = TestBed.createComponent(MessageServerComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/setting/settings/object-store/object-store.component.spec.ts b/web-app/src/app/routes/setting/settings/object-store/object-store.component.spec.ts index 2b4215d1e0..ca857add72 100644 --- a/web-app/src/app/routes/setting/settings/object-store/object-store.component.spec.ts +++ b/web-app/src/app/routes/setting/settings/object-store/object-store.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { ObjectStoreComponent } from './object-store.component'; @@ -26,9 +27,7 @@ describe('ObjectStoreComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [ObjectStoreComponent] - }).compileComponents(); + await configureShallowTest(ObjectStoreComponent).compileComponents(); fixture = TestBed.createComponent(ObjectStoreComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/setting/settings/settings.component.spec.ts b/web-app/src/app/routes/setting/settings/settings.component.spec.ts index 34f682b177..d8de701aa6 100644 --- a/web-app/src/app/routes/setting/settings/settings.component.spec.ts +++ b/web-app/src/app/routes/setting/settings/settings.component.spec.ts @@ -18,6 +18,8 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { Router } from '@angular/router'; +import { configureShallowTest } from '@testing'; import { SettingsComponent } from './settings.component'; @@ -26,10 +28,9 @@ describe('SettingsComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [SettingsComponent] - }).compileComponents(); + await configureShallowTest(SettingsComponent).compileComponents(); + spyOnProperty(TestBed.inject(Router), 'url').and.returnValue('/setting/settings/config'); fixture = TestBed.createComponent(SettingsComponent); component = fixture.componentInstance; fixture.detectChanges(); diff --git a/web-app/src/app/routes/setting/settings/system-config/system-config.component.spec.ts b/web-app/src/app/routes/setting/settings/system-config/system-config.component.spec.ts index a0da5b178d..2aa8004823 100644 --- a/web-app/src/app/routes/setting/settings/system-config/system-config.component.spec.ts +++ b/web-app/src/app/routes/setting/settings/system-config/system-config.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { SystemConfigComponent } from './system-config.component'; @@ -26,9 +27,7 @@ describe('SystemConfigComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [SystemConfigComponent] - }).compileComponents(); + await configureShallowTest(SystemConfigComponent).compileComponents(); fixture = TestBed.createComponent(SystemConfigComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/setting/status/status.component.spec.ts b/web-app/src/app/routes/setting/status/status.component.spec.ts index 04e0e93373..00e9ab4ecd 100644 --- a/web-app/src/app/routes/setting/status/status.component.spec.ts +++ b/web-app/src/app/routes/setting/status/status.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { StatusComponent } from './status.component'; @@ -26,9 +27,7 @@ describe('StatusComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [StatusComponent] - }).compileComponents(); + await configureShallowTest(StatusComponent).compileComponents(); fixture = TestBed.createComponent(StatusComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/routes/status-public/status-public.component.spec.ts b/web-app/src/app/routes/status-public/status-public.component.spec.ts index 220804c9b7..878261d119 100644 --- a/web-app/src/app/routes/status-public/status-public.component.spec.ts +++ b/web-app/src/app/routes/status-public/status-public.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { StatusPublicComponent } from './status-public.component'; @@ -26,9 +27,7 @@ describe('StatusPublicComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [StatusPublicComponent] - }).compileComponents(); + await configureShallowTest(StatusPublicComponent).compileComponents(); fixture = TestBed.createComponent(StatusPublicComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/service/alert-define.service.spec.ts b/web-app/src/app/service/alert-define.service.spec.ts index d8e963fb6b..aa844c209f 100644 --- a/web-app/src/app/service/alert-define.service.spec.ts +++ b/web-app/src/app/service/alert-define.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AlertDefineService } from './alert-define.service'; @@ -25,7 +26,7 @@ describe('AlertDefineService', () => { let service: AlertDefineService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AlertDefineService); }); diff --git a/web-app/src/app/service/alert-group.service.spec.ts b/web-app/src/app/service/alert-group.service.spec.ts index 8964c07956..2e53b9fe22 100644 --- a/web-app/src/app/service/alert-group.service.spec.ts +++ b/web-app/src/app/service/alert-group.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AlertGroupService } from './alert-group.service'; @@ -25,7 +26,7 @@ describe('AlertConvergeService', () => { let service: AlertGroupService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AlertGroupService); }); diff --git a/web-app/src/app/service/alert-inhibit.service.spec.ts b/web-app/src/app/service/alert-inhibit.service.spec.ts index fb23e4ffa3..9e6f0648b5 100644 --- a/web-app/src/app/service/alert-inhibit.service.spec.ts +++ b/web-app/src/app/service/alert-inhibit.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AlertInhibitService } from './alert-inhibit.service'; @@ -25,7 +26,7 @@ describe('AlertInhibitService', () => { let service: AlertInhibitService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AlertInhibitService); }); diff --git a/web-app/src/app/service/alert-silence.service.spec.ts b/web-app/src/app/service/alert-silence.service.spec.ts index 5619c9a96b..7232756a95 100644 --- a/web-app/src/app/service/alert-silence.service.spec.ts +++ b/web-app/src/app/service/alert-silence.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AlertSilenceService } from './alert-silence.service'; @@ -25,7 +26,7 @@ describe('AlertSilenceService', () => { let service: AlertSilenceService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AlertSilenceService); }); diff --git a/web-app/src/app/service/alert.service.spec.ts b/web-app/src/app/service/alert.service.spec.ts index 782a0418ea..893d62b166 100644 --- a/web-app/src/app/service/alert.service.spec.ts +++ b/web-app/src/app/service/alert.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AlertService } from './alert.service'; @@ -25,7 +26,7 @@ describe('AlertService', () => { let service: AlertService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AlertService); }); diff --git a/web-app/src/app/service/app-define.service.spec.ts b/web-app/src/app/service/app-define.service.spec.ts index 540dfab3f7..01be720764 100644 --- a/web-app/src/app/service/app-define.service.spec.ts +++ b/web-app/src/app/service/app-define.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AppDefineService } from './app-define.service'; @@ -25,7 +26,7 @@ describe('AppDefineService', () => { let service: AppDefineService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AppDefineService); }); diff --git a/web-app/src/app/service/auth.service.spec.ts b/web-app/src/app/service/auth.service.spec.ts index bd5f2bbce6..650102d387 100644 --- a/web-app/src/app/service/auth.service.spec.ts +++ b/web-app/src/app/service/auth.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { AuthService } from './auth.service'; @@ -25,7 +26,7 @@ describe('AuthService', () => { let service: AuthService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(AuthService); }); diff --git a/web-app/src/app/service/collector.service.spec.ts b/web-app/src/app/service/collector.service.spec.ts index 235e5a7542..b533e2773b 100644 --- a/web-app/src/app/service/collector.service.spec.ts +++ b/web-app/src/app/service/collector.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { CollectorService } from './collector.service'; @@ -25,7 +26,7 @@ describe('CollectorService', () => { let service: CollectorService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(CollectorService); }); diff --git a/web-app/src/app/service/label.service.spec.ts b/web-app/src/app/service/label.service.spec.ts index 5d0950c5e5..b77da248d0 100644 --- a/web-app/src/app/service/label.service.spec.ts +++ b/web-app/src/app/service/label.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { LabelService } from './label.service'; @@ -25,7 +26,7 @@ describe('LabelService', () => { let service: LabelService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(LabelService); }); diff --git a/web-app/src/app/service/monitor.service.spec.ts b/web-app/src/app/service/monitor.service.spec.ts index 08cc30e450..c431e2b85c 100644 --- a/web-app/src/app/service/monitor.service.spec.ts +++ b/web-app/src/app/service/monitor.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { MonitorService } from './monitor.service'; @@ -25,7 +26,7 @@ describe('MonitorService', () => { let service: MonitorService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(MonitorService); }); diff --git a/web-app/src/app/service/notice-receiver.service.spec.ts b/web-app/src/app/service/notice-receiver.service.spec.ts index e7ef04d912..dad090d9b7 100644 --- a/web-app/src/app/service/notice-receiver.service.spec.ts +++ b/web-app/src/app/service/notice-receiver.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { NoticeReceiverService } from './notice-receiver.service'; @@ -25,7 +26,7 @@ describe('NoticeReceiverService', () => { let service: NoticeReceiverService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(NoticeReceiverService); }); diff --git a/web-app/src/app/service/notice-rule.service.spec.ts b/web-app/src/app/service/notice-rule.service.spec.ts index 136c6bcbbf..f0d90348aa 100644 --- a/web-app/src/app/service/notice-rule.service.spec.ts +++ b/web-app/src/app/service/notice-rule.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { NoticeRuleService } from './notice-rule.service'; @@ -25,7 +26,7 @@ describe('NoticeRuleService', () => { let service: NoticeRuleService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(NoticeRuleService); }); diff --git a/web-app/src/app/service/notice-template.service.spec.ts b/web-app/src/app/service/notice-template.service.spec.ts index b329d7b084..31c1a02fd9 100644 --- a/web-app/src/app/service/notice-template.service.spec.ts +++ b/web-app/src/app/service/notice-template.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { NoticeTemplateService } from './notice-template.service'; @@ -25,7 +26,7 @@ describe('NoticeTemplateService', () => { let service: NoticeTemplateService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(NoticeTemplateService); }); diff --git a/web-app/src/app/service/plugin.service.spec.ts b/web-app/src/app/service/plugin.service.spec.ts index 8855934fea..f7d6fcbe16 100644 --- a/web-app/src/app/service/plugin.service.spec.ts +++ b/web-app/src/app/service/plugin.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { PluginService } from './plugin.service'; @@ -25,7 +26,7 @@ describe('PluginService', () => { let service: PluginService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(PluginService); }); diff --git a/web-app/src/app/service/status-page-public.service.spec.ts b/web-app/src/app/service/status-page-public.service.spec.ts index 3f625076a2..fc13e47db8 100644 --- a/web-app/src/app/service/status-page-public.service.spec.ts +++ b/web-app/src/app/service/status-page-public.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { StatusPagePublicService } from './status-page-public.service'; @@ -25,7 +26,7 @@ describe('StatusPagePublicService', () => { let service: StatusPagePublicService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(StatusPagePublicService); }); diff --git a/web-app/src/app/service/status-page.service.spec.ts b/web-app/src/app/service/status-page.service.spec.ts index 318c00bb91..6d7c4961a8 100644 --- a/web-app/src/app/service/status-page.service.spec.ts +++ b/web-app/src/app/service/status-page.service.spec.ts @@ -18,6 +18,7 @@ */ import { TestBed } from '@angular/core/testing'; +import { configureHttpServiceTest } from '@testing'; import { StatusPageService } from './status-page.service'; @@ -25,7 +26,7 @@ describe('StatusPageService', () => { let service: StatusPageService; beforeEach(() => { - TestBed.configureTestingModule({}); + configureHttpServiceTest(); service = TestBed.inject(StatusPageService); }); diff --git a/web-app/src/app/shared/components/configurable-field/configurable-field.component.spec.ts b/web-app/src/app/shared/components/configurable-field/configurable-field.component.spec.ts index 91fad76ccc..b797930f0a 100644 --- a/web-app/src/app/shared/components/configurable-field/configurable-field.component.spec.ts +++ b/web-app/src/app/shared/components/configurable-field/configurable-field.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { ConfigurableFieldComponent } from './configurable-field.component'; @@ -26,9 +27,7 @@ describe('ConfigurableFieldComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [ConfigurableFieldComponent] - }).compileComponents(); + await configureShallowTest(ConfigurableFieldComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/shared/components/form-field/form-field.component.spec.ts b/web-app/src/app/shared/components/form-field/form-field.component.spec.ts index 971578a9d5..df92cae1bc 100644 --- a/web-app/src/app/shared/components/form-field/form-field.component.spec.ts +++ b/web-app/src/app/shared/components/form-field/form-field.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { FormFieldComponent } from './form-field.component'; @@ -26,14 +27,13 @@ describe('FormFieldComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [FormFieldComponent] - }).compileComponents(); + await configureShallowTest(FormFieldComponent).compileComponents(); }); beforeEach(() => { fixture = TestBed.createComponent(FormFieldComponent); component = fixture.componentInstance; + component.item = { type: 'text' }; fixture.detectChanges(); }); diff --git a/web-app/src/app/shared/components/help-message-show/help-message-show.component.spec.ts b/web-app/src/app/shared/components/help-message-show/help-message-show.component.spec.ts index f9089a95ae..787eb3123d 100644 --- a/web-app/src/app/shared/components/help-message-show/help-message-show.component.spec.ts +++ b/web-app/src/app/shared/components/help-message-show/help-message-show.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { HelpMessageShowComponent } from './help-message-show.component'; @@ -26,9 +27,7 @@ describe('HelpMessageShowComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [HelpMessageShowComponent] - }).compileComponents(); + await configureShallowTest(HelpMessageShowComponent).compileComponents(); fixture = TestBed.createComponent(HelpMessageShowComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/shared/components/label-selector/label-selector.component.spec.ts b/web-app/src/app/shared/components/label-selector/label-selector.component.spec.ts index 3a6958faa9..f1b828658f 100644 --- a/web-app/src/app/shared/components/label-selector/label-selector.component.spec.ts +++ b/web-app/src/app/shared/components/label-selector/label-selector.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { LabelSelectorComponent } from './label-selector.component'; @@ -26,9 +27,7 @@ describe('LabelSelectorComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [LabelSelectorComponent] - }).compileComponents(); + await configureShallowTest(LabelSelectorComponent).compileComponents(); fixture = TestBed.createComponent(LabelSelectorComponent); component = fixture.componentInstance; diff --git a/web-app/src/app/shared/components/monitor-select-list/monitor-select-list.component.spec.ts b/web-app/src/app/shared/components/monitor-select-list/monitor-select-list.component.spec.ts index fd903d3ed3..278e1c6124 100755 --- a/web-app/src/app/shared/components/monitor-select-list/monitor-select-list.component.spec.ts +++ b/web-app/src/app/shared/components/monitor-select-list/monitor-select-list.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorSelectListComponent } from './monitor-select-list.component'; @@ -26,9 +27,7 @@ describe('MonitorSelectListComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorSelectListComponent] - }).compileComponents(); + await configureShallowTest(MonitorSelectListComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/shared/components/monitor-select-menu/monitor-select-menu.component.spec.ts b/web-app/src/app/shared/components/monitor-select-menu/monitor-select-menu.component.spec.ts index 4d7af44eba..0364619396 100755 --- a/web-app/src/app/shared/components/monitor-select-menu/monitor-select-menu.component.spec.ts +++ b/web-app/src/app/shared/components/monitor-select-menu/monitor-select-menu.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { MonitorSelectMenuComponent } from './monitor-select-menu.component'; @@ -26,9 +27,7 @@ describe('MonitorSelectMenuComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MonitorSelectMenuComponent] - }).compileComponents(); + await configureShallowTest(MonitorSelectMenuComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/shared/components/multi-func-input/multi-func-input.component.spec.ts b/web-app/src/app/shared/components/multi-func-input/multi-func-input.component.spec.ts index 6720e9917d..345e77c0a6 100755 --- a/web-app/src/app/shared/components/multi-func-input/multi-func-input.component.spec.ts +++ b/web-app/src/app/shared/components/multi-func-input/multi-func-input.component.spec.ts @@ -18,6 +18,8 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { FormsModule } from '@angular/forms'; +import { configureShallowTest } from '@testing'; import { MultiFuncInputComponent } from './multi-func-input.component'; @@ -26,9 +28,7 @@ describe('MultiFuncInputComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [MultiFuncInputComponent] - }).compileComponents(); + await configureShallowTest(MultiFuncInputComponent, [FormsModule]).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/app/shared/components/toolbar/toolbar.component.spec.ts b/web-app/src/app/shared/components/toolbar/toolbar.component.spec.ts index 7f243894f6..1a7e214e52 100755 --- a/web-app/src/app/shared/components/toolbar/toolbar.component.spec.ts +++ b/web-app/src/app/shared/components/toolbar/toolbar.component.spec.ts @@ -18,6 +18,7 @@ */ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { configureShallowTest } from '@testing'; import { ToolbarComponent } from './toolbar.component'; @@ -26,9 +27,7 @@ describe('ToolbarComponent', () => { let fixture: ComponentFixture; beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [ToolbarComponent] - }).compileComponents(); + await configureShallowTest(ToolbarComponent).compileComponents(); }); beforeEach(() => { diff --git a/web-app/src/testing.ts b/web-app/src/testing.ts new file mode 100644 index 0000000000..b226dadb6b --- /dev/null +++ b/web-app/src/testing.ts @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { CUSTOM_ELEMENTS_SCHEMA, Type } from '@angular/core'; +import { TestBed, TestModuleMetadata } from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { provideRouter, RouterModule } from '@angular/router'; +import { ALAIN_I18N_TOKEN, AlainThemeModule } from '@delon/theme'; +import { NzGridModule } from 'ng-zorro-antd/grid'; +import { NzMenuModule } from 'ng-zorro-antd/menu'; +import { NzModalModule } from 'ng-zorro-antd/modal'; +import { NzTableModule } from 'ng-zorro-antd/table'; +import { NzToolTipModule } from 'ng-zorro-antd/tooltip'; +import { of } from 'rxjs'; + +const imports = [ + HttpClientTestingModule, + NoopAnimationsModule, + RouterModule, + NzGridModule, + NzMenuModule, + NzModalModule, + NzTableModule, + NzToolTipModule, + AlainThemeModule.forRoot() +]; + +const providers = [ + provideRouter([]), + { + provide: ALAIN_I18N_TOKEN, + useValue: { + change: of(null), + currentLang: 'en-US', + defaultLang: 'en-US', + fanyi: (key: string) => key + } + } +]; + +export function configureShallowTest(component: Type, extraImports: NonNullable = []): TestBed { + return TestBed.configureTestingModule({ + imports: [...imports, ...extraImports], + declarations: [component], + providers, + schemas: [CUSTOM_ELEMENTS_SCHEMA] + }); +} + +export function configureStandaloneTest(component: Type, extraImports: NonNullable = []): TestBed { + return TestBed.configureTestingModule({ + imports: [HttpClientTestingModule, component, ...extraImports], + providers + }); +} + +export function configureHttpServiceTest(): TestBed { + return TestBed.configureTestingModule({ imports: [HttpClientTestingModule] }); +} diff --git a/web-app/tsconfig.json b/web-app/tsconfig.json index 5428ce4511..511e32fed8 100644 --- a/web-app/tsconfig.json +++ b/web-app/tsconfig.json @@ -26,6 +26,9 @@ "@core": [ "src/app/core/index" ], + "@testing": [ + "src/testing" + ], "@env/*": [ "src/environments/*" ] From 2cea398910e5971ad24f46e0f27ae2b31af618bd Mon Sep 17 00:00:00 2001 From: shown Date: Thu, 13 Aug 2026 00:03:24 +0800 Subject: [PATCH 08/18] fix(ai): create conversation when conversation ID is missing (#4315) Co-authored-by: Duansg Signed-off-by: yuluo-yx --- .../service/impl/ConversationServiceImpl.java | 40 +++++++++++------- .../impl/ConversationServiceImplTest.java | 42 +++++++++++++++++++ 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImpl.java index c674617dea..b66b839e08 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImpl.java @@ -78,24 +78,32 @@ public class ConversationServiceImpl implements ConversationService { .build()); } - log.info("Starting streaming conversation: {}", conversationId); - ChatConversation conversation = conversationDao.findById(conversationId) - .orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId)); + ChatConversation conversation; + if (conversationId == null) { + // The API contract makes conversationId optional, so create a conversation for the first message. + conversation = new ChatConversation(); + conversation.setTitle(buildConversationTitle(message)); + conversation = conversationDao.save(conversation); + } else { + conversation = conversationDao.findById(conversationId) + .orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId)); + } + Long currentConversationId = conversation.getId(); + log.info("Starting streaming conversation: {}", currentConversationId); // Manually load messages for conversation history - List messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId); + List messages = messageDao.findByConversationIdOrderByGmtCreateAsc(currentConversationId); conversation.setMessages(messages); if (conversation.getTitle().startsWith("conversation")) { // Auto-generate title from first user message - String title = message.length() > 30 ? message.substring(0, 27) + "..." : message; - conversation.setTitle(title); + conversation.setTitle(buildConversationTitle(message)); conversationDao.save(conversation); } // Add user message to conversation ChatMessage chatMessage = ChatMessage.builder() - .conversationId(conversationId) + .conversationId(currentConversationId) .content(message) .role("user") .build(); @@ -103,7 +111,7 @@ public class ConversationServiceImpl implements ConversationService { ChatRequestContext context = ChatRequestContext.builder() .message(message) - .conversationId(conversationId) + .conversationId(currentConversationId) .conversationHistory(messages) .build(); @@ -116,7 +124,7 @@ public class ConversationServiceImpl implements ConversationService { .map(chunk -> { fullResponse.append(chunk); ChatResponseChunk responseChunk = ChatResponseChunk.builder() - .conversationId(conversationId) + .conversationId(currentConversationId) .userMessageId(finalChatMessage.getId()) .response(chunk) .build(); @@ -128,13 +136,13 @@ public class ConversationServiceImpl implements ConversationService { .concatWith(Flux.defer(() -> { // Add the complete AI response to conversation ChatMessage assistantMessage = ChatMessage.builder() - .conversationId(conversationId) + .conversationId(currentConversationId) .content(fullResponse.toString()) .role("assistant") .build(); assistantMessage = messageDao.save(assistantMessage); ChatResponseChunk finalResponse = ChatResponseChunk.builder() - .conversationId(conversationId) + .conversationId(currentConversationId) .response("") .assistantMessageId(assistantMessage.getId()) .build(); @@ -143,12 +151,12 @@ public class ConversationServiceImpl implements ConversationService { .event("complete") .build()); })) - .doOnComplete(() -> log.info("Streaming completed for conversation: {}", conversationId)) - .doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", conversationId, + .doOnComplete(() -> log.info("Streaming completed for conversation: {}", currentConversationId)) + .doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", currentConversationId, error.getMessage(), error)) .onErrorResume(error -> { ChatResponseChunk errorResponse = ChatResponseChunk.builder() - .conversationId(conversationId) + .conversationId(currentConversationId) .response("An error occurred: " + error.getMessage()) .userMessageId(finalChatMessage.getId()) .build(); @@ -165,6 +173,10 @@ public class ConversationServiceImpl implements ConversationService { return conversationDao.save(conversation); } + private String buildConversationTitle(String message) { + return message.length() > 30 ? message.substring(0, 27) + "..." : message; + } + @Override public ChatConversation getConversation(Long conversationId) { if (conversationId == null) { diff --git a/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImplTest.java b/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImplTest.java index b192073e45..43cdf88d8f 100644 --- a/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImplTest.java +++ b/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/service/impl/ConversationServiceImplTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import com.usthe.sureness.subject.SubjectSum; @@ -123,6 +124,47 @@ class ConversationServiceImplTest { assertEquals(subject, contextCaptor.getValue().getSubject()); } + /** + * The service should create a conversation and return its ID when the client omits the optional conversation ID. + */ + @Test + void streamChatShouldCreateConversationWhenConversationIdIsMissing() { + AtomicLong messageId = new AtomicLong(20L); + when(chatClientProviderService.isConfigured()).thenReturn(true); + when(conversationDao.save(any(ChatConversation.class))).thenAnswer(invocation -> { + ChatConversation savedConversation = invocation.getArgument(0); + savedConversation.setId(CONVERSATION_ID); + return savedConversation; + }); + when(messageDao.findByConversationIdOrderByGmtCreateAsc(CONVERSATION_ID)).thenReturn(List.of()); + when(messageDao.save(any(ChatMessage.class))).thenAnswer(invocation -> { + ChatMessage savedMessage = invocation.getArgument(0); + savedMessage.setId(messageId.getAndIncrement()); + return savedMessage; + }); + when(chatClientProviderService.streamChat(any(ChatRequestContext.class))) + .thenReturn(Flux.just("本轮回答")); + + List> events = conversationService + .streamChat("本轮问题", null) + .collectList() + .block(); + + assertNotNull(events); + assertEquals(2, events.size()); + assertEquals(CONVERSATION_ID, events.get(0).data().getConversationId()); + assertEquals(CONVERSATION_ID, events.get(1).data().getConversationId()); + + ArgumentCaptor contextCaptor = ArgumentCaptor.forClass(ChatRequestContext.class); + verify(chatClientProviderService).streamChat(contextCaptor.capture()); + assertEquals(CONVERSATION_ID, contextCaptor.getValue().getConversationId()); + assertEquals(List.of(), contextCaptor.getValue().getConversationHistory()); + ArgumentCaptor conversationCaptor = ArgumentCaptor.forClass(ChatConversation.class); + verify(conversationDao).save(conversationCaptor.capture()); + assertEquals("本轮问题", conversationCaptor.getValue().getTitle()); + verifyNoMoreInteractions(conversationDao); + } + /** * Deleting a conversation must remove its schedules before they can push more messages. */ From 001e2292363917adb454d5f42af5ed0a1c6a563a Mon Sep 17 00:00:00 2001 From: shown Date: Thu, 13 Aug 2026 22:38:55 +0800 Subject: [PATCH 09/18] fix(ai): persist parameters for scheduled skills (#4317) --- .../ai/tools/impl/ScheduleToolsImpl.java | 75 +++++++++- .../ai/tools/impl/ScheduleToolsImplTest.java | 129 ++++++++++++++++++ 2 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImplTest.java diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImpl.java index f82260c4ee..28b11818c7 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImpl.java @@ -19,17 +19,23 @@ package org.apache.hertzbeat.ai.tools.impl; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.ai.service.SopScheduleService; +import org.apache.hertzbeat.ai.sop.model.SopDefinition; +import org.apache.hertzbeat.ai.sop.model.SopParameter; import org.apache.hertzbeat.ai.sop.registry.SkillRegistry; import org.apache.hertzbeat.ai.utils.SopMessageUtil; import org.apache.hertzbeat.ai.tools.ScheduleTools; import org.apache.hertzbeat.common.entity.ai.SopSchedule; +import org.apache.hertzbeat.common.util.JsonUtil; import org.springframework.ai.tool.annotation.Tool; import org.springframework.ai.tool.annotation.ToolParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import tools.jackson.core.type.TypeReference; /** * Implementation of ScheduleTools for AI-driven schedule management. @@ -78,18 +84,22 @@ public class ScheduleToolsImpl implements ScheduleTools { @Tool(name = "createScheduleWithConversation", description = "Create a scheduled task for a specific conversation. " + "Use the conversationId from the system context. " + + "Pass skill parameters as a JSON object when the skill requires inputs. " + "The cron expression should be in 6-digit Spring format.") public String createScheduleWithConversation( @ToolParam(description = "Conversation ID from the system context", required = true) Long conversationId, @ToolParam(description = "Name of the skill to schedule (e.g., 'daily_inspection')", required = true) String skillName, @ToolParam(description = "Cron expression in Spring format (e.g., '0 0 9 * * ?')", required = true) String cronExpression, - @ToolParam(description = "Description of the schedule", required = false) String description) { + @ToolParam(description = "Description of the schedule", required = false) String description, + @ToolParam(description = "Skill parameters as a JSON object (e.g., '{\"monitorId\":123}')", + required = false) String paramsJson) { log.info("AI creating schedule: conversationId={}, skill={}, cron={}, desc={}", conversationId, skillName, cronExpression, description); // Validate skill exists - if (skillRegistry.getSkill(skillName) == null) { + SopDefinition skill = skillRegistry.getSkill(skillName); + if (skill == null) { String available = String.join(", ", skillRegistry.getAllSkills().stream() .map(s -> s.getName()) @@ -103,11 +113,16 @@ public class ScheduleToolsImpl implements ScheduleTools { } try { - // Check for duplicate schedule (same skill + cron expression) + Map params = parseSkillParams(paramsJson); + validateRequiredParameters(skill, params); + String serializedParams = params.isEmpty() ? null : JsonUtil.toJson(params); + + // Parameters are part of a schedule's identity so the same skill and cron can target different inputs. List existing = scheduleService.getSchedulesByConversation(conversationId); boolean duplicate = existing.stream() - .anyMatch(s -> s.getSopName().equals(skillName) - && s.getCronExpression().equals(cronExpression)); + .anyMatch(schedule -> Objects.equals(schedule.getSopName(), skillName) + && Objects.equals(schedule.getCronExpression(), cronExpression) + && hasSameParams(schedule.getSopParams(), params)); if (duplicate) { return SopMessageUtil.getMessage("schedule.create.duplicate", new Object[]{skillName, cronExpression}, null) @@ -118,6 +133,7 @@ public class ScheduleToolsImpl implements ScheduleTools { schedule.setConversationId(conversationId); schedule.setSopName(skillName); schedule.setCronExpression(cronExpression); + schedule.setSopParams(serializedParams); schedule.setEnabled(true); SopSchedule created = scheduleService.createSchedule(schedule); @@ -150,6 +166,55 @@ public class ScheduleToolsImpl implements ScheduleTools { } } + private Map parseSkillParams(String paramsJson) { + if (paramsJson == null || paramsJson.isBlank()) { + return Map.of(); + } + Map params; + try { + params = JsonUtil.fromJson(paramsJson, new TypeReference<>() {}); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Skill parameters must be a valid JSON object", e); + } + if (params == null) { + throw new IllegalArgumentException("Skill parameters must be a valid JSON object"); + } + return params; + } + + private void validateRequiredParameters(SopDefinition skill, Map params) { + if (skill.getParameters() == null) { + return; + } + for (SopParameter parameter : skill.getParameters()) { + Object value = params.get(parameter.getName()); + if (isMissing(value)) { + value = parameter.getDefaultValue(); + } + if (parameter.isRequired() && isMissing(value)) { + throw new IllegalArgumentException( + "Required skill parameter is missing: " + parameter.getName()); + } + } + } + + private boolean isMissing(Object value) { + return value == null || value instanceof String text && text.isBlank(); + } + + private boolean hasSameParams(String existingJson, Map params) { + if (existingJson == null || existingJson.isBlank()) { + return params.isEmpty(); + } + try { + Map existingParams = JsonUtil.fromJson(existingJson, new TypeReference<>() {}); + return Objects.equals(existingParams, params); + } catch (RuntimeException e) { + log.warn("Failed to parse parameters of an existing schedule", e); + return false; + } + } + @Override @Tool(name = "listSchedulesForConversation", description = "List all scheduled tasks for a specific conversation. " diff --git a/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImplTest.java b/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImplTest.java new file mode 100644 index 0000000000..c8e8535f59 --- /dev/null +++ b/hertzbeat-ai/src/test/java/org/apache/hertzbeat/ai/tools/impl/ScheduleToolsImplTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.ai.tools.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.hertzbeat.ai.service.SopScheduleService; +import org.apache.hertzbeat.ai.sop.model.SopDefinition; +import org.apache.hertzbeat.ai.sop.model.SopParameter; +import org.apache.hertzbeat.ai.sop.registry.SkillRegistry; +import org.apache.hertzbeat.common.entity.ai.SopSchedule; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Verifies that AI-created SOP schedules validate and persist skill parameters. + */ +@ExtendWith(MockitoExtension.class) +class ScheduleToolsImplTest { + + private static final String CRON = "0 0 9 * * ?"; + + @Mock + private SopScheduleService scheduleService; + + @Mock + private SkillRegistry skillRegistry; + + private ScheduleToolsImpl scheduleTools; + + @BeforeEach + void setUp() { + scheduleTools = new ScheduleToolsImpl(scheduleService, skillRegistry); + } + + @Test + void createScheduleShouldPersistSkillParameters() { + when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill()); + when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of()); + when(scheduleService.createSchedule(any())).thenAnswer(invocation -> { + SopSchedule schedule = invocation.getArgument(0); + schedule.setId(9L); + return schedule; + }); + + String result = scheduleTools.createScheduleWithConversation( + 7L, "diagnosis", CRON, "daily diagnosis", "{\"monitorId\":42}"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SopSchedule.class); + verify(scheduleService).createSchedule(captor.capture()); + assertEquals("{\"monitorId\":42}", captor.getValue().getSopParams()); + assertTrue(result.contains("9")); + } + + @Test + void createScheduleShouldAllowDifferentParametersAtTheSameTime() { + SopSchedule existing = SopSchedule.builder() + .sopName("diagnosis") + .cronExpression(CRON) + .sopParams("{\"monitorId\":41}") + .build(); + when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill()); + when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of(existing)); + when(scheduleService.createSchedule(any())).thenAnswer(invocation -> invocation.getArgument(0)); + + scheduleTools.createScheduleWithConversation( + 7L, "diagnosis", CRON, null, "{\"monitorId\":42}"); + + verify(scheduleService).createSchedule(any()); + } + + @Test + void createScheduleShouldRejectMissingRequiredParameter() { + when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill()); + + String result = scheduleTools.createScheduleWithConversation( + 7L, "diagnosis", CRON, null, "{}"); + + assertTrue(result.contains("monitorId")); + verify(scheduleService, never()).createSchedule(any()); + } + + @Test + void createScheduleShouldRejectInvalidParameterJson() { + when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill()); + + String result = scheduleTools.createScheduleWithConversation( + 7L, "diagnosis", CRON, null, "not-json"); + + assertTrue(result.contains("valid JSON object")); + verify(scheduleService, never()).createSchedule(any()); + } + + private SopDefinition parameterizedSkill() { + SopParameter monitorId = SopParameter.builder() + .name("monitorId") + .required(true) + .build(); + return SopDefinition.builder() + .name("diagnosis") + .parameters(List.of(monitorId)) + .build(); + } +} From cbb62a335806c416e9f8883a14efa68f91b30874 Mon Sep 17 00:00:00 2001 From: Duansg Date: Fri, 14 Aug 2026 10:41:02 +0800 Subject: [PATCH 10/18] [fix] bound what one anonymous push request can consume (#4273) --- .../prometheus/parser/OnlineParser.java | 35 +- .../prometheus/parser/OnlineParserTest.java | 25 ++ .../service/impl/PushGatewayServiceImpl.java | 315 +++++++++++++-- .../impl/PushGatewayServiceImplTest.java | 367 ++++++++++++++++++ .../src/main/resources/application.yml | 5 + script/application.yml | 5 + .../conf/application.yml | 13 +- .../conf/application.yml | 13 +- .../conf/application.yml | 13 +- .../conf/application.yml | 11 +- .../conf/application.yml | 11 +- 11 files changed, 767 insertions(+), 46 deletions(-) create mode 100644 hertzbeat-push/src/test/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImplTest.java diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParser.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParser.java index de48f59656..d5f9fae07f 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParser.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParser.java @@ -60,16 +60,37 @@ public class OnlineParser { } public static Map parseMetrics(InputStream inputStream) throws IOException { - Map metricFamilyMap = new ConcurrentHashMap<>(10); + return parseMetrics(inputStream, Integer.MAX_VALUE); + } + + /** + * Parses at most {@code maxSamples} samples from the supplied stream. + * + * @param inputStream The Prometheus text stream + * @param maxSamples The maximum number of samples to materialize + * @return The parsed metric families, or {@code null} when the text format is invalid + * @throws IOException When the stream cannot be read + * @throws SampleLimitExceededException When another sample follows the configured limit + */ + public static Map parseMetrics(InputStream inputStream, int maxSamples) throws IOException { + if (maxSamples < 0) { + throw new IllegalArgumentException("maxSamples must not be negative"); + } + final Map metricFamilyMap = new ConcurrentHashMap<>(10); + int sampleCount = 0; try { int i = getChar(inputStream); while (i != -1) { if (i == '#' || i == '\n') { skipToLineEnd(inputStream).maybeEol().maybeEof().noElse(); } else { - StringBuilder stringBuilder = new StringBuilder(); + if (sampleCount >= maxSamples) { + throw new SampleLimitExceededException(maxSamples); + } + final StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append((char) i); parseMetric(inputStream, metricFamilyMap, stringBuilder); + sampleCount++; } i = getChar(inputStream); // To address the `\n\r` scenario, it is necessary to skip @@ -84,6 +105,16 @@ public class OnlineParser { return metricFamilyMap; } + /** + * Signals that parsing stopped before materializing a sample beyond the configured limit. + */ + public static final class SampleLimitExceededException extends IOException { + + public SampleLimitExceededException(int limit) { + super("prometheus payload exceeds the " + limit + " sample limit"); + } + } + /** * Parses Prometheus metrics from the given {@link InputStream}, but only for the specified metric name. *

diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParserTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParserTest.java index 4f515b8696..370ac9a3bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParserTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/prometheus/parser/OnlineParserTest.java @@ -30,6 +30,8 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; class OnlineParserTest { @@ -459,4 +461,27 @@ class OnlineParserTest { assertEquals("run_as", metricFamily.getMetricList().get(0).getLabels().get(3).getName()); assertEquals("NT AUTHORITY\nLocalService", metricFamily.getMetricList().get(0).getLabels().get(3).getValue()); } + + @Test + void testParseMetricsStopsBeforeSampleBeyondLimit() { + final String metrics = "metric_a 1\nmetric_b 2\nmetric_c 3\nmetric_d 4\n"; + final ByteArrayInputStream inputStream = + new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8)); + + assertThrows(OnlineParser.SampleLimitExceededException.class, + () -> OnlineParser.parseMetrics(inputStream, 2)); + + assertTrue(inputStream.available() > 0, "samples after the limit should remain unread"); + } + + @Test + void testParseMetricsAllowsExactlyTheSampleLimit() throws Exception { + final String metrics = "metric_a 1\nmetric_b 2\n"; + final InputStream inputStream = new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8)); + + final Map metricFamilyMap = OnlineParser.parseMetrics(inputStream, 2); + + assertNotNull(metricFamilyMap); + assertEquals(2, metricFamilyMap.size()); + } } diff --git a/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java b/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java index 1be1a9203f..e0b706bff9 100644 --- a/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java +++ b/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java @@ -19,13 +19,19 @@ package org.apache.hertzbeat.push.service.impl; +import java.io.IOException; import java.io.InputStream; import java.time.Instant; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; + +import jakarta.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily; @@ -37,6 +43,7 @@ import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator; import org.apache.hertzbeat.push.dao.PushMonitorDao; import org.apache.hertzbeat.push.service.PushGatewayService; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; /** @@ -46,26 +53,81 @@ import org.springframework.stereotype.Service; @Slf4j @Service public class PushGatewayServiceImpl implements PushGatewayService { - + + private static final byte PUSH_MONITOR_TYPE = (byte) 1; + private final CommonDataQueue commonDataQueue; private final PushMonitorDao pushMonitorDao; - private final Map jobInstanceMap; - - public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao) { + private final Map jobInstanceMap; + + /** + * Cap on push monitors created automatically from unknown job/instance pairs. + * + *

The route is unauthenticated by design, and every new pair used to persist a + * monitor row and add a `jobInstanceMap` entry that is never removed, so a caller + * iterating over made up names could grow the database and the heap without bound. + * Above the cap an unknown pair is refused while the pairs already known keep working, + * which is why eviction is not used here: evicting a live entry would make the next + * push for that pair create a second monitor for the same job and instance. + */ + private final int maxAutoCreatedMonitors; + + /** + * Cap on how many bytes a single push body may carry. + * + *

The parser materializes its result in memory, and the servlet container does not bound + * a non form request body, so an independent byte limit is still needed alongside the sample + * limit to keep long names and label values from exhausting the heap. + */ + private final long maxBodyBytes; + + /** + * Cap on how many samples a single push body may carry. The parser stops before allocating + * a sample beyond this limit, so a compact body cannot create an unbounded object graph. + */ + private final int maxSamples; + + /** + * One entry per pair whose monitor is being created, so that concurrent pushes naming the + * same unknown pair wait for one creation instead of each starting their own. Persistence + * stays outside any shared lock: a slow database would otherwise hold every request for an + * unknown pair, and each of those requests is already holding its parsed samples. + */ + private final Map> monitorCreationMap; + + /** + * Successful and in flight creations together, so that the cap is claimed before the + * database write rather than counted after it. + */ + private final AtomicInteger trackedMonitorCount; + + public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao, + @Value("${hertzbeat.push.max-auto-created-monitors:10000}") int maxAutoCreatedMonitors, + @Value("${hertzbeat.push.max-body-bytes:5242880}") long maxBodyBytes, + @Value("${hertzbeat.push.max-samples:10000}") int maxSamples) { + if (maxAutoCreatedMonitors < 0 || maxBodyBytes < 0 || maxSamples < 0) { + throw new IllegalArgumentException("push gateway limits must not be negative"); + } this.commonDataQueue = commonDataQueue; this.pushMonitorDao = pushMonitorDao; + this.maxAutoCreatedMonitors = maxAutoCreatedMonitors; + this.maxBodyBytes = maxBodyBytes; + this.maxSamples = maxSamples; jobInstanceMap = new ConcurrentHashMap<>(); - pushMonitorDao.findMonitorsByType((byte) 1).forEach(monitor -> - jobInstanceMap.put(monitor.getApp() + "_" + monitor.getName(), monitor.getId())); + pushMonitorDao.findMonitorsByType(PUSH_MONITOR_TYPE).forEach(monitor -> + jobInstanceMap.put(new JobInstance(monitor.getApp(), monitor.getName()), monitor.getId())); + monitorCreationMap = new ConcurrentHashMap<>(); + trackedMonitorCount = new AtomicInteger(jobInstanceMap.size()); } @Override public boolean pushPrometheusMetrics(InputStream inputStream, String job, String instance) { try { - long curTime = Instant.now().toEpochMilli(); - Map metricFamilyMap = OnlineParser.parseMetrics(inputStream); + final long curTime = Instant.now().toEpochMilli(); + final Map metricFamilyMap = OnlineParser.parseMetrics( + new BoundedInputStream(inputStream, maxBodyBytes), maxSamples); if (metricFamilyMap == null) { log.error("parse prometheus metrics is null, job: {}, instance: {}", job, instance); return false; @@ -74,20 +136,11 @@ public class PushGatewayServiceImpl implements PushGatewayService { if (job != null && instance != null) { // auto create monitor when job and instance not null // job is app, instance is the name - id = jobInstanceMap.computeIfAbsent(job + "_" + instance, key -> { - log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance); - long monitorId = SnowFlakeIdGenerator.generateId(); - Monitor monitor = Monitor.builder() - .id(monitorId) - .app(job) - .name(instance) - .instance(instance) - .type((byte) 1) - .status(CommonConstants.MONITOR_UP_CODE) - .build(); - this.pushMonitorDao.save(monitor); - return monitorId; - }); + final Long monitorId = resolveMonitorId(new JobInstance(job, instance)); + if (monitorId == null) { + return false; + } + id = monitorId; } for (Map.Entry entry : metricFamilyMap.entrySet()) { CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); @@ -110,9 +163,18 @@ public class PushGatewayServiceImpl implements PushGatewayService { builder.addField(CollectRep.Field.newBuilder().setName("value") .setType(CommonConstants.TYPE_NUMBER).setLabel(false).build()); } - Map labelMap = metric.getLabels() - .stream() - .collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue)); + // A repeated label name is refused rather than resolved: the exposition + // format requires the names of a label set to be unique, and keeping one + // of the values would emit a schema carrying that name twice. Built by + // hand so the refusal is a rejection this method can answer with a + // warning, not the error trace a collector's exception would produce. + Map labelMap = new HashMap<>(metric.getLabels().size()); + for (MetricFamily.Label label : metric.getLabels()) { + if (labelMap.containsKey(label.getName())) { + throw new DuplicateLabelException(label.getName()); + } + labelMap.put(label.getName(), label.getValue()); + } CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); for (String field : metricsFields) { String fieldValue = labelMap.get(field); @@ -125,9 +187,210 @@ public class PushGatewayServiceImpl implements PushGatewayService { } } return true; + } catch (BodyTooLargeException e) { + // A rejection, not a failure: caught apart from the generic handler below so that a + // caller repeating oversized bodies costs one warning line each, not a stack trace + log.warn("reject prometheus push over the {} byte body limit, job: {}, instance: {}", + maxBodyBytes, job, instance); + return false; + } catch (OnlineParser.SampleLimitExceededException e) { + log.warn("reject prometheus push over the {} sample limit, job: {}, instance: {}", + maxSamples, job, instance); + return false; + } catch (DuplicateLabelException e) { + log.warn("reject prometheus push repeating a label name, job: {}, instance: {}: {}", + job, instance, e.getMessage()); + return false; } catch (Exception e) { log.error("push prometheus metrics error", e); return false; } } + + /** + * Returns the monitor id a job/instance pair resolves to, creating the monitor on first + * sight, or null once {@link #maxAutoCreatedMonitors} is reached. + * + *

Concurrent pushes naming the same unknown pair share one creation through a future, and + * the cap is claimed by an atomic count before the database write. Nothing here holds a lock + * across that write: unrelated pairs persist in parallel, and a slow database delays only the + * requests naming the pair being created, which matters because every waiting request is + * holding the samples it already parsed. + * + * @param pair Job and instance the push named + * @return The monitor id, or null when the cap leaves no room for a new one + */ + @Nullable + private Long resolveMonitorId(JobInstance pair) { + final Long known = jobInstanceMap.get(pair); + if (known != null) { + return known; + } + + final CompletableFuture proposedCreation = new CompletableFuture<>(); + final CompletableFuture ongoingCreation = monitorCreationMap.putIfAbsent(pair, proposedCreation); + if (ongoingCreation != null) { + try { + return ongoingCreation.join(); + } catch (CompletionException e) { + // The request owning the creation reports the failure once; joining its exception + // here would multiply a single database error by every request that waited + return null; + } + } + + try { + // Looked up again now that the creation is claimed: another request may have finished + // this pair between the lookup above and this claim, and going on to create it would + // leave two monitors for one pair and spend a second slot of the cap + final Long createdMeanwhile = jobInstanceMap.get(pair); + if (createdMeanwhile != null) { + proposedCreation.complete(createdMeanwhile); + return createdMeanwhile; + } + if (!reserveMonitorSlot()) { + proposedCreation.complete(null); + log.warn("reject prometheus push for unknown job: {}, instance: {}, " + + "already tracking {} push monitors, limit is {}", + pair.job(), pair.instance(), trackedMonitorCount.get(), maxAutoCreatedMonitors); + return null; + } + boolean created = false; + try { + final long monitorId = createMonitor(pair); + jobInstanceMap.put(pair, monitorId); + created = true; + proposedCreation.complete(monitorId); + return monitorId; + } finally { + if (!created) { + trackedMonitorCount.decrementAndGet(); + } + } + } catch (RuntimeException | Error e) { + proposedCreation.completeExceptionally(e); + throw e; + } finally { + monitorCreationMap.remove(pair, proposedCreation); + } + } + + /** + * Claims one slot of the cap, or reports that none is left. Claiming before the database + * write is what keeps concurrent creations from exceeding the cap together. + */ + private boolean reserveMonitorSlot() { + int tracked = trackedMonitorCount.get(); + while (tracked < maxAutoCreatedMonitors) { + if (trackedMonitorCount.compareAndSet(tracked, tracked + 1)) { + return true; + } + tracked = trackedMonitorCount.get(); + } + return false; + } + + /** + * Persists a push monitor after its slot of the cap has been claimed. + */ + private long createMonitor(JobInstance pair) { + final String job = pair.job(); + final String instance = pair.instance(); + log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance); + final long monitorId = SnowFlakeIdGenerator.generateId(); + final Monitor monitor = Monitor.builder() + .id(monitorId) + .app(job) + .name(instance) + .instance(instance) + .type(PUSH_MONITOR_TYPE) + .status(CommonConstants.MONITOR_UP_CODE) + .build(); + this.pushMonitorDao.save(monitor); + return monitorId; + } + + /** + * Identifies the monitor a push belongs to. + * + *

The two names are kept apart instead of being joined into one string: a separator + * carries no meaning in either name, so `job + "_" + instance` maps ("a", "b_c") and + * ("a_b", "c") onto the same key. Colliding pairs would push their samples into whichever + * monitor was created first, and at startup they would collapse into a single map entry, + * making the cap count fewer monitors than the database actually holds. + */ + private record JobInstance(String job, String instance) { + } + + /** + * Raised when a sample repeats a label name, which the exposition format does not allow. + * Kept apart from the generic handler so a malformed body costs one warning line rather + * than an error trace on a route that takes its input from anyone. + */ + static final class DuplicateLabelException extends IOException { + + DuplicateLabelException(String name) { + super("sample repeats the label name " + name); + } + } + + /** + * Raised when a body goes past {@link #maxBodyBytes}. It is kept apart from the other read + * failures so the caller can answer a body that is merely too large without an error trace. + */ + static final class BodyTooLargeException extends IOException { + + BodyTooLargeException(long limit) { + super("push body exceeds the " + limit + " byte limit"); + } + } + + /** + * Fails the read once the body has delivered more than {@code limit} bytes, instead of + * letting the parser accumulate an unbounded body in memory. Reading stops at the + * failure, so the bytes beyond the limit are never buffered. + */ + static final class BoundedInputStream extends InputStream { + + private final InputStream delegate; + + private final long limit; + + private long bytesRead; + + BoundedInputStream(InputStream delegate, long limit) { + this.delegate = delegate; + this.limit = limit; + } + + @Override + public int read() throws IOException { + final int value = delegate.read(); + if (value != -1) { + recordBytesRead(1); + } + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + final int bytesReadNow = delegate.read(buffer, offset, length); + if (bytesReadNow > 0) { + recordBytesRead(bytesReadNow); + } + return bytesReadNow; + } + + private void recordBytesRead(int increment) throws IOException { + bytesRead += increment; + if (bytesRead > limit) { + throw new BodyTooLargeException(limit); + } + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } } diff --git a/hertzbeat-push/src/test/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImplTest.java b/hertzbeat-push/src/test/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImplTest.java new file mode 100644 index 0000000000..d1991824e4 --- /dev/null +++ b/hertzbeat-push/src/test/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImplTest.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.push.service.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.entity.manager.Monitor; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.queue.CommonDataQueue; +import org.apache.hertzbeat.push.dao.PushMonitorDao; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Test case for {@link PushGatewayServiceImpl}. + * + *

`/api/push/prometheus/**` is unauthenticated by design, so the resource a single + * anonymous request may consume has to be bounded: the body it may carry, the samples it + * may enqueue, and the number of push monitors it may bring into existence. + */ +@ExtendWith(MockitoExtension.class) +class PushGatewayServiceImplTest { + + private static final String BODY = "sample_metric{label=\"a\"} 1\n"; + + @Mock + private CommonDataQueue commonDataQueue; + + @Mock + private PushMonitorDao pushMonitorDao; + + @BeforeEach + void setUp() { + // The stream test below builds no service, so this default must not be strict + lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of()); + } + + private PushGatewayServiceImpl createService(int maxMonitors, long maxBodyBytes, int maxSamples) { + return new PushGatewayServiceImpl(commonDataQueue, pushMonitorDao, maxMonitors, maxBodyBytes, maxSamples); + } + + private static ByteArrayInputStream createBody(String content) { + return new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + } + + /** + * The exposition format requires the names of a label set to be unique, so a sample that + * repeats one is refused rather than resolved. It stays a rejection though: answering it + * with an error trace would let a malformed body fill the log on an anonymous route. + */ + @Test + void testSampleRepeatingTheLabelNameIsRejected() { + final PushGatewayServiceImpl service = createService(10, 1024, 100); + + assertFalse(service.pushPrometheusMetrics( + createBody("sample_metric{label=\"a\",label=\"b\"} 1\n"), "job1", "instance1")); + + verify(commonDataQueue, never()).sendMetricsData(any()); + } + + @Test + void testPushIsAcceptedWithinTheLimits() { + final PushGatewayServiceImpl service = createService(10, 1024, 100); + + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + + verify(pushMonitorDao).save(any(Monitor.class)); + } + + @Test + void testBodyBeyondTheByteLimitIsRejected() { + final PushGatewayServiceImpl service = createService(10, 16, 100); + + assertFalse(service.pushPrometheusMetrics(createBody(BODY.repeat(100)), "job1", "instance1")); + + verify(pushMonitorDao, never()).save(any(Monitor.class)); + } + + @Test + void testBodyBeyondTheSampleLimitIsRejected() { + final PushGatewayServiceImpl service = createService(10, 1024 * 1024, 2); + final StringBuilder many = new StringBuilder(); + for (int index = 0; index < 10; index++) { + many.append("sample_metric{label=\"value").append(index).append("\"} 1\n"); + } + final ByteArrayInputStream inputStream = createBody(many.toString()); + + assertFalse(service.pushPrometheusMetrics(inputStream, "job1", "instance1")); + assertTrue(inputStream.available() > 0, "the parser should stop before consuming the remaining samples"); + + verify(pushMonitorDao, never()).save(any(Monitor.class)); + } + + @Test + void testBodyAtTheSampleLimitIsAccepted() { + final PushGatewayServiceImpl service = createService(10, 1024, 2); + final String twoSamples = "sample_metric{label=\"a\"} 1\n" + + "sample_metric{label=\"b\"} 2\n"; + + assertTrue(service.pushPrometheusMetrics(createBody(twoSamples), "job1", "instance1")); + } + + /** + * An unknown job/instance pair persists a monitor row and adds a map entry that is + * never removed, so without a cap an anonymous caller iterating over made up names + * grows the database and the heap without bound. + */ + @Test + void testAutoCreationStopsAtTheMonitorLimit() { + final PushGatewayServiceImpl service = createService(2, 1024, 100); + + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2")); + assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3")); + + verify(pushMonitorDao, times(2)).save(any(Monitor.class)); + } + + /** + * The cap must not turn into eviction: a pair already known has to keep resolving to + * the monitor it created, otherwise a later push would create a second monitor for the + * same job and instance. + */ + @Test + void testKnownPairsKeepWorkingAtTheLimit() { + final PushGatewayServiceImpl service = createService(1, 1024, 100); + + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + assertFalse(service.pushPrometheusMetrics(createBody(BODY), "other", "instance")); + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + + verify(pushMonitorDao, times(1)).save(any(Monitor.class)); + } + + /** + * The route is anonymous, so nothing stops a caller from sending its unknown pairs all at + * once. Testing the cap and claiming the entry in two steps lets every request that already + * passed the test create a monitor of its own, which is the cap being exceeded by as many + * requests as the container serves in parallel. + */ + @Test + void testConcurrentPushesForUnknownPairsStopAtTheMonitorLimit() throws Exception { + final int callers = 16; + final PushGatewayServiceImpl service = createService(1, 1024, 100); + final CyclicBarrier startTogether = new CyclicBarrier(callers); + final ExecutorService pool = Executors.newFixedThreadPool(callers); + final AtomicInteger accepted = new AtomicInteger(); + try { + final List> pushes = new ArrayList<>(); + for (int index = 0; index < callers; index++) { + final String instance = "instance" + index; + pushes.add(pool.submit(() -> { + startTogether.await(); + if (service.pushPrometheusMetrics(createBody(BODY), "job", instance)) { + accepted.incrementAndGet(); + } + return null; + })); + } + for (final Future push : pushes) { + push.get(30, TimeUnit.SECONDS); + } + } finally { + pool.shutdownNow(); + } + + assertEquals(1, accepted.get()); + verify(pushMonitorDao, times(1)).save(any(Monitor.class)); + } + + @Test + void testFailedSaveDoesNotConsumeTheMonitorLimit() { + when(pushMonitorDao.save(any(Monitor.class))) + .thenThrow(new IllegalStateException("database unavailable")) + .thenAnswer(invocation -> invocation.getArgument(0)); + final PushGatewayServiceImpl service = createService(1, 1024, 100); + + assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2")); + + verify(pushMonitorDao, times(2)).save(any(Monitor.class)); + } + + @Test + void testNegativeLimitsAreRejectedAtConstruction() { + assertThrows(IllegalArgumentException.class, () -> createService(-1, 1024, 100)); + assertThrows(IllegalArgumentException.class, () -> createService(1, -1, 100)); + assertThrows(IllegalArgumentException.class, () -> createService(1, 1024, -1)); + } + + @Test + void testConcurrentPushesForTheSamePairCreateOneMonitor() throws Exception { + final int callers = 8; + final CyclicBarrier startTogether = new CyclicBarrier(callers); + final PushGatewayServiceImpl service = createService(1, 1024, 100); + final ExecutorService pool = Executors.newFixedThreadPool(callers); + try { + final List> pushes = new ArrayList<>(); + for (int index = 0; index < callers; index++) { + pushes.add(pool.submit(() -> { + startTogether.await(); + return service.pushPrometheusMetrics(createBody(BODY), "job", "instance"); + })); + } + for (final Future push : pushes) { + assertTrue(push.get(30, TimeUnit.SECONDS)); + } + } finally { + pool.shutdownNow(); + } + + verify(pushMonitorDao).save(any(Monitor.class)); + } + + /** + * A request may read no entry for a pair and only then claim the creation, by which time + * another request may have created that pair and cleared its claim. Without a second lookup + * once the claim is won, this request goes on to create the same pair again, leaving two + * monitors for one pair and a slot of the cap spent for good. + * + *

The interleaving is forced rather than raced: the map hands back a miss, and while it + * does, a competing push runs its whole creation. + */ + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + void testStaleMissDoesNotCreateTwoMonitorsForOnePair() throws Exception { + final PushGatewayServiceImpl service = createService(2, 1024, 100); + final Field trackedPairs = PushGatewayServiceImpl.class.getDeclaredField("jobInstanceMap"); + trackedPairs.setAccessible(true); + final AtomicBoolean competed = new AtomicBoolean(); + final Map probing = new ConcurrentHashMap() { + @Override + public Object get(Object key) { + final Object value = super.get(key); + if (value == null && competed.compareAndSet(false, true)) { + service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"); + } + return value; + } + }; + trackedPairs.set(service, probing); + + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + + verify(pushMonitorDao, times(1)).save(any(Monitor.class)); + assertEquals(1, probing.size()); + // The slot the duplicate would have taken is still there for another pair + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2")); + } + + @Test + void testMonitorsLoadedAtStartupCountTowardsTheLimit() { + lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of( + Monitor.builder().id(1L).app("job1").name("instance1").build())); + final PushGatewayServiceImpl service = createService(1, 1024, 100); + + assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2")); + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1")); + + verify(pushMonitorDao, never()).save(any(Monitor.class)); + } + + /** + * A separator carries no meaning inside a job or an instance name, so the two names must not + * be joined into a single key: ("job", "a_b") and ("job_a", "b") are different monitors, and + * the second pair must not push its samples into the monitor the first one created. + */ + @Test + void testPairsSharingTheSeparatorAreDistinctMonitors() { + final PushGatewayServiceImpl service = createService(10, 1024, 100); + + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job", "a_b")); + assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job_a", "b")); + + verify(pushMonitorDao, times(2)).save(any(Monitor.class)); + final ArgumentCaptor pushed = + ArgumentCaptor.forClass(CollectRep.MetricsData.class); + verify(commonDataQueue, times(2)).sendMetricsData(pushed.capture()); + assertNotEquals(pushed.getAllValues().get(0).getId(), pushed.getAllValues().get(1).getId()); + } + + /** + * Colliding pairs must not collapse into one entry while the monitors are loaded either, + * which would let the cap count fewer monitors than the database actually holds. + */ + @Test + void testPairsSharingTheSeparatorCountSeparatelyAtStartup() { + lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of( + Monitor.builder().id(1L).app("job").name("a_b").build(), + Monitor.builder().id(2L).app("job_a").name("b").build())); + final PushGatewayServiceImpl service = createService(2, 1024, 100); + + assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3")); + + verify(pushMonitorDao, never()).save(any(Monitor.class)); + } + + @Test + void testBoundedStreamStopsAtTheLimit() throws Exception { + final PushGatewayServiceImpl.BoundedInputStream stream = + new PushGatewayServiceImpl.BoundedInputStream(createBody("abcdef"), 3); + + assertEquals('a', stream.read()); + assertEquals('b', stream.read()); + assertEquals('c', stream.read()); + assertThrows(PushGatewayServiceImpl.BodyTooLargeException.class, stream::read); + } + + /** + * A body over the limit is a rejection rather than a failure, so it must be distinguishable + * from a read that genuinely broke: the route is anonymous, and answering every oversized + * body with an error trace lets a caller fill the log at will. + */ + @Test + void testBodyOverTheByteLimitIsRejectedNotFailed() { + final PushGatewayServiceImpl.BoundedInputStream stream = + new PushGatewayServiceImpl.BoundedInputStream(createBody(BODY.repeat(100)), 16); + + final IOException raised = assertThrows(IOException.class, stream::readAllBytes); + + assertInstanceOf(PushGatewayServiceImpl.BodyTooLargeException.class, raised); + } +} diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index 90c0ea4bb8..fa6cda6e10 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -373,3 +373,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/application.yml b/script/application.yml index 90c0ea4bb8..fa6cda6e10 100644 --- a/script/application.yml +++ b/script/application.yml @@ -373,3 +373,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index 33b0537468..7695271d03 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -109,7 +109,7 @@ spring: hibernate: format_sql: true dialect: org.hibernate.dialect.MySQLDialect - + flyway: enabled: true clean-disabled: true @@ -117,7 +117,7 @@ spring: baseline-version: 1 locations: - classpath:db/migration/mysql - + # Not Require, Please config if you need email notify mail: # Attention: this is mail server address. @@ -138,7 +138,7 @@ common: queue: # memory or kafka type: memory - + warehouse: store: # store history metrics data, enable only one below @@ -222,7 +222,7 @@ alerter: region: AWS_REGION_FOR_END_USER_MESSAGING twilio: account-sid: YOUR_ACCOUNT_SID - auth-token: YOUR_AUTH_TOKEN + auth-token: YOUR_AUTH_TOKEN twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER scheduler: server: @@ -273,3 +273,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 8d795ed643..90220d0669 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -109,7 +109,7 @@ spring: hibernate: format_sql: true dialect: org.hibernate.dialect.MySQLDialect - + flyway: enabled: true clean-disabled: true @@ -117,7 +117,7 @@ spring: baseline-version: 1 locations: - classpath:db/migration/mysql - + # Not Require, Please config if you need email notify mail: # Attention: this is mail server address. @@ -138,7 +138,7 @@ common: queue: # memory or kafka type: memory - + warehouse: store: # store history metrics data, enable only one below @@ -219,7 +219,7 @@ alerter: region: AWS_REGION_FOR_END_USER_MESSAGING twilio: account-sid: YOUR_ACCOUNT_SID - auth-token: YOUR_AUTH_TOKEN + auth-token: YOUR_AUTH_TOKEN twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER scheduler: server: @@ -270,3 +270,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index f92abb9f98..0074a9dd3f 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -109,7 +109,7 @@ spring: hibernate: format_sql: true dialect: org.hibernate.dialect.MySQLDialect - + flyway: enabled: true clean-disabled: true @@ -117,7 +117,7 @@ spring: baseline-version: 1 locations: - classpath:db/migration/mysql - + # Not Require, Please config if you need email notify mail: # Attention: this is mail server address. @@ -142,7 +142,7 @@ warehouse: expire-time: 1h victoria-metrics: enabled: true - url: http://victoria-metrics:8428 + url: http://victoria-metrics:8428 username: root password: root insert: @@ -222,7 +222,7 @@ alerter: region: AWS_REGION_FOR_END_USER_MESSAGING twilio: account-sid: YOUR_ACCOUNT_SID - auth-token: YOUR_AUTH_TOKEN + auth-token: YOUR_AUTH_TOKEN twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER scheduler: server: @@ -273,3 +273,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index 6cc1490030..c25b533a48 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -108,7 +108,7 @@ spring: hibernate: format_sql: true dialect: org.hibernate.dialect.PostgreSQLDialect - + flyway: enabled: true clean-disabled: true @@ -116,7 +116,7 @@ spring: baseline-version: 1 locations: - classpath:db/migration/postgresql - + # Not Require, Please config if you need email notify mail: # Attention: this is mail server address. @@ -219,7 +219,7 @@ alerter: region: AWS_REGION_FOR_END_USER_MESSAGING twilio: account-sid: YOUR_ACCOUNT_SID - auth-token: YOUR_AUTH_TOKEN + auth-token: YOUR_AUTH_TOKEN twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER scheduler: server: @@ -270,3 +270,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index 3108f5b994..71d7fb2f58 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -108,7 +108,7 @@ spring: hibernate: format_sql: true dialect: org.hibernate.dialect.PostgreSQLDialect - + flyway: enabled: true clean-disabled: true @@ -116,7 +116,7 @@ spring: baseline-version: 1 locations: - classpath:db/migration/postgresql - + # Not Require, Please config if you need email notify mail: # Attention: this is mail server address. @@ -221,7 +221,7 @@ alerter: region: AWS_REGION_FOR_END_USER_MESSAGING twilio: account-sid: YOUR_ACCOUNT_SID - auth-token: YOUR_AUTH_TOKEN + auth-token: YOUR_AUTH_TOKEN twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER scheduler: server: @@ -272,3 +272,8 @@ hertzbeat: concurrency-limit: 256 reject-when-limit-reached: true task-termination-timeout: 5000 + # Bounds on what a single request to the push gateway may consume. + push: + max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000} + max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880} + max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000} From cf737db2eebb3028f4445bd757280d74a716f376 Mon Sep 17 00:00:00 2001 From: Duansg Date: Fri, 14 Aug 2026 10:44:48 +0800 Subject: [PATCH 11/18] [fix] stop serving the openapi document to anonymous callers (#4276) Co-authored-by: Claude Opus 5 (1M context) --- .../src/test/resources/sureness.yml | 15 +- .../AuthorizedSwaggerIndexTransformer.java | 126 +++++++++++ .../manager/config/SwaggerConfig.java | 28 +++ ...AuthorizedSwaggerIndexTransformerTest.java | 105 +++++++++ .../src/test/resources/sureness.yml | 15 +- .../src/main/resources/application.yml | 10 + .../src/main/resources/sureness.yml | 15 +- .../OpenApiDocumentDisabledByDefaultTest.java | 142 ++++++++++++ .../security/SurenessOpenApiDocRuleTest.java | 207 ++++++++++++++++++ home/docs/start/account-modify.md | 37 +++- .../current/start/account-modify.md | 37 +++- script/application.yml | 10 + .../conf/application.yml | 10 + .../hertzbeat-mysql-iotdb/conf/sureness.yml | 15 +- .../conf/application.yml | 10 + .../conf/sureness.yml | 15 +- .../conf/application.yml | 10 + .../conf/sureness.yml | 15 +- .../conf/application.yml | 10 + .../conf/sureness.yml | 15 +- .../conf/application.yml | 10 + .../conf/sureness.yml | 15 +- script/sureness.yml | 15 +- 23 files changed, 825 insertions(+), 62 deletions(-) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformer.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformerTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/OpenApiDocumentDisabledByDefaultTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessOpenApiDocRuleTest.java diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml b/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml index 1763868f1e..dff4e65966 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml +++ b/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -86,9 +86,16 @@ resourceRole: - /api/ingestion/otlp/**===get===[admin,user,guest] - /api/logs/**===get===[admin,user,guest] - /api/traces/**===get===[admin,user,guest] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -121,10 +128,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformer.java new file mode 100644 index 0000000000..9b71128609 --- /dev/null +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformer.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import jakarta.servlet.http.HttpServletRequest; +import org.springdoc.core.properties.SwaggerUiConfigProperties; +import org.springdoc.core.properties.SwaggerUiOAuthProperties; +import org.springdoc.core.providers.ObjectMapperProvider; +import org.springdoc.webmvc.ui.SwaggerIndexPageTransformer; +import org.springdoc.webmvc.ui.SwaggerWelcomeCommon; +import org.springframework.core.io.Resource; +import org.springframework.web.servlet.resource.ResourceTransformerChain; +import org.springframework.web.servlet.resource.TransformedResource; + +/** + * Adds the HertzBeat access token to same-origin requests made by Swagger UI. + */ +final class AuthorizedSwaggerIndexTransformer extends SwaggerIndexPageTransformer { + + private static final String SWAGGER_INITIALIZER = "swagger-initializer.js"; + + private static final String PRESETS_MARKER = "presets: ["; + + private static final String INTERCEPTOR_MARKER = "requestInterceptor: (request) => {"; + + private static final String INTERCEPTOR_RETURN = "return request;"; + + private static final String AUTHORIZATION_LOGIC = """ + const currentUrl = new URL(document.URL); + const requestUrl = new URL(request.url, document.location.origin); + const sameOrigin = currentUrl.protocol === requestUrl.protocol && currentUrl.host === requestUrl.host; + const token = window.localStorage.getItem('Authorization'); + if (sameOrigin && token) { + request.headers['Authorization'] = `Bearer ${token}`; + } + """; + + private static final String AUTHORIZATION_INTERCEPTOR = """ + requestInterceptor: (request) => { + %s + return request; + }, + """.formatted(AUTHORIZATION_LOGIC.stripTrailing()); + + AuthorizedSwaggerIndexTransformer(SwaggerUiConfigProperties swaggerUiConfig, + SwaggerUiOAuthProperties swaggerUiOauthProperties, + SwaggerWelcomeCommon swaggerWelcomeCommon, + ObjectMapperProvider objectMapperProvider) { + super(swaggerUiConfig, swaggerUiOauthProperties, swaggerWelcomeCommon, objectMapperProvider); + } + + @Override + public Resource transform(HttpServletRequest request, Resource resource, + ResourceTransformerChain transformerChain) throws IOException { + final Resource transformed = super.transform(request, resource, transformerChain); + if (!SWAGGER_INITIALIZER.equals(resource.getFilename())) { + return transformed; + } + final String initializer; + try (final var input = transformed.getInputStream()) { + initializer = new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + return new TransformedResource(transformed, + addAuthorizationInterceptor(initializer).getBytes(StandardCharsets.UTF_8)); + } + + /** + * Springdoc writes an interceptor of its own when csrf support is turned on. Two keys + * of the same name would silently drop one of them, so the token is appended to the + * body already there. It goes in at the end because the last write to a header wins, + * and springdoc can be configured to write to {@code Authorization} as well. The + * declarations above are named apart from the ones springdoc emits on purpose: the + * two bodies share a scope, so a collision would be a syntax error. + * + * @param initializer the swagger initializer script + * @return the script with the token attached to its outgoing requests + */ + static String addAuthorizationInterceptor(String initializer) { + final int interceptor = initializer.indexOf(INTERCEPTOR_MARKER); + if (interceptor >= 0) { + final int returnStatement = initializer.indexOf(INTERCEPTOR_RETURN, + interceptor + INTERCEPTOR_MARKER.length()); + if (returnStatement < 0) { + throw new IllegalStateException("the swagger initializer interceptor no longer returns the request"); + } + return insertLineBefore(initializer, returnStatement, AUTHORIZATION_LOGIC); + } + final int presets = initializer.indexOf(PRESETS_MARKER); + if (presets < 0) { + throw new IllegalStateException("the swagger initializer no longer contains the presets marker"); + } + return insertLineBefore(initializer, presets, AUTHORIZATION_INTERCEPTOR); + } + + /** + * @param initializer the swagger initializer script + * @param index an index into the line to insert in front of + * @param insertion the lines to insert, newline terminated + * @return the script with the insertion on its own lines, leaving the indentation of + * the line at {@code index} alone + */ + private static String insertLineBefore(String initializer, int index, String insertion) { + final int lineStart = initializer.lastIndexOf('\n', index) + 1; + return initializer.substring(0, lineStart) + insertion + initializer.substring(lineStart); + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SwaggerConfig.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SwaggerConfig.java index 295cbc91de..2ea060bf59 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SwaggerConfig.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/SwaggerConfig.java @@ -25,6 +25,13 @@ import io.swagger.v3.oas.models.info.Info; import io.swagger.v3.oas.models.info.License; import io.swagger.v3.oas.models.security.SecurityRequirement; import io.swagger.v3.oas.models.security.SecurityScheme; +import org.springdoc.core.properties.SwaggerUiConfigProperties; +import org.springdoc.core.properties.SwaggerUiOAuthProperties; +import org.springdoc.core.providers.ObjectMapperProvider; +import org.springdoc.webmvc.ui.SwaggerIndexTransformer; +import org.springdoc.webmvc.ui.SwaggerWelcomeCommon; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -37,6 +44,27 @@ public class SwaggerConfig { private static final String SECURITY_SCHEME_NAME = "BearerAuth"; + /** + * The springdoc beans this one is built from only exist while both switches are on: + * its own ui configuration is conditional on {@code SpringDocConfiguration}, which + * {@code springdoc.api-docs.enabled} gates in turn. Matching both switches keeps a + * deployment that turns the document off from failing to start. + * + * @return the swagger ui index transformer, replacing the springdoc default + */ + @Bean + @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) + @ConditionalOnProperty(name = {"springdoc.api-docs.enabled", "springdoc.swagger-ui.enabled"}, + havingValue = "true", matchIfMissing = true) + public SwaggerIndexTransformer authorizedSwaggerIndexTransformer( + SwaggerUiConfigProperties swaggerUiConfig, + SwaggerUiOAuthProperties swaggerUiOauthProperties, + SwaggerWelcomeCommon swaggerWelcomeCommon, + ObjectMapperProvider objectMapperProvider) { + return new AuthorizedSwaggerIndexTransformer(swaggerUiConfig, swaggerUiOauthProperties, + swaggerWelcomeCommon, objectMapperProvider); + } + @Bean public OpenAPI springOpenApi() { return new OpenAPI() diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformerTest.java new file mode 100644 index 0000000000..985e740788 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/AuthorizedSwaggerIndexTransformerTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * Test case for {@link AuthorizedSwaggerIndexTransformer}. + */ +class AuthorizedSwaggerIndexTransformerTest { + + @Test + void shouldAttachTheHertzBeatTokenToSameOriginRequests() { + final String initializer = """ + window.ui = SwaggerUIBundle({ + configUrl: "/v3/api-docs/swagger-config", + presets: [SwaggerUIBundle.presets.apis] + }); + """; + + final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer); + + assertTrue(transformed.contains("window.localStorage.getItem('Authorization')")); + assertTrue(transformed.contains("request.headers['Authorization'] = `Bearer ${token}`")); + assertTrue(transformed.contains("sameOrigin && token")); + assertTrue(transformed.contains("presets: [SwaggerUIBundle.presets.apis]")); + } + + @Test + void shouldFailClosedWhenTheSwaggerInitializerShapeChanges() { + assertThrows(IllegalStateException.class, + () -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor("window.ui = {};")); + } + + /** + * Springdoc writes an interceptor of its own when csrf support is turned on, and it + * can be configured to write to the {@code Authorization} header too. The last write + * wins, so ours has to come after the one already there. + */ + @Test + void shouldComposeWithAnExistingRequestInterceptor() { + final String initializer = """ + window.ui = SwaggerUIBundle({ + requestInterceptor: (request) => { + request.headers['Authorization'] = 'csrf'; + return request; + }, + presets: [SwaggerUIBundle.presets.apis] + }); + """; + + final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer); + + assertTrue(transformed.contains("request.headers['Authorization'] = 'csrf'")); + assertTrue(transformed.indexOf("request.headers['Authorization'] = `Bearer ${token}`") + > transformed.indexOf("request.headers['Authorization'] = 'csrf'"), + "the token has to be written after the interceptor that was already there"); + assertEquals(1, countOf(transformed, "requestInterceptor:"), + "a second key of the same name would drop one of the two interceptors"); + } + + @Test + void shouldFailClosedWhenTheExistingInterceptorDoesNotReturnTheRequest() { + final String initializer = """ + window.ui = SwaggerUIBundle({ + requestInterceptor: (request) => { + presets: [SwaggerUIBundle.presets.apis] + """; + + assertThrows(IllegalStateException.class, + () -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer)); + } + + /** + * @param initializer the transformed script + * @param token the substring to count + * @return how many times the substring occurs + */ + private static int countOf(String initializer, String token) { + int count = 0; + for (int index = initializer.indexOf(token); index >= 0; index = initializer.indexOf(token, index + 1)) { + count++; + } + return count; + } +} diff --git a/hertzbeat-manager/src/test/resources/sureness.yml b/hertzbeat-manager/src/test/resources/sureness.yml index 1763868f1e..dff4e65966 100644 --- a/hertzbeat-manager/src/test/resources/sureness.yml +++ b/hertzbeat-manager/src/test/resources/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -86,9 +86,16 @@ resourceRole: - /api/ingestion/otlp/**===get===[admin,user,guest] - /api/logs/**===get===[admin,user,guest] - /api/traces/**===get===[admin,user,guest] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -121,10 +128,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index fa6cda6e10..425dd4e1f6 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -71,6 +71,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index f63bee0e8f..8cdaeec049 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -116,11 +116,18 @@ resourceRole: - /api/account/token===get===[admin] - /api/account/token/**===post===[admin] - /api/account/token/**===delete===[admin] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -157,10 +164,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/OpenApiDocumentDisabledByDefaultTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/OpenApiDocumentDisabledByDefaultTest.java new file mode 100644 index 0000000000..33af10aeff --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/OpenApiDocumentDisabledByDefaultTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import jakarta.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +/** + * Guards the springdoc switches that keep the openapi document off by default. + * + *

Scoping the document to the admin role is only half the story. The document is a map + * of every route, http method, parameter and model, so a deployment that has no use for it + * should not serve it at all: both switches are off unless a deployment opts in, and the + * rbac rules stay as the second line of defence for deployments that do. The swagger ui + * page itself is reachable anonymously through the {@code /**}{@code /*.html===get} + * exclusion, but it renders nothing until the caller proves it holds the admin role. + */ +class OpenApiDocumentDisabledByDefaultTest { + + private static final Path SCRIPT_DIR = Path.of("..", "script"); + + private static final List> SWITCHES = List.of( + List.of("springdoc", "api-docs", "enabled"), + List.of("springdoc", "swagger-ui", "enabled")); + + @Test + void shouldDisableTheOpenApiDocumentInThePackagedConfig() throws IOException { + try (InputStream in = OpenApiDocumentDisabledByDefaultTest.class.getResourceAsStream("/application.yml")) { + assertNotNull(in, "application.yml must be on the classpath"); + assertDisabled(documentsOf(in), "application.yml"); + } + } + + /** + * The deployment scripts ship their own copies of {@code application.yml} and mount + * them over the packaged one, so a switch flipped only in the packaged file would + * still leave every container deployment serving the document. + */ + @Test + void shouldDisableTheOpenApiDocumentInDeploymentCopies() throws IOException { + for (Path copy : deploymentCopies()) { + try (InputStream in = Files.newInputStream(copy)) { + assertDisabled(documentsOf(in), copy.toString()); + } + } + } + + /** + * Asserts that every declaration of the springdoc switches across a multi document + * yaml turns the endpoint off, and that at least one declaration exists - a file that + * simply omits them falls back to the springdoc default, which is enabled. + * + * @param documents the yaml documents, in the order spring applies them + * @param source the file the documents came from, for the failure message + */ + private static void assertDisabled(List> documents, String source) { + for (List path : SWITCHES) { + List declarations = documents.stream() + .map(document -> valueAt(document, path)) + .filter(Objects::nonNull) + .toList(); + assertFalse(declarations.isEmpty(), + String.join(".", path) + " is unset in " + source + ", so it falls back to enabled"); + declarations.forEach(declared -> assertEquals(Boolean.FALSE, declared, + String.join(".", path) + " is enabled in " + source)); + } + } + + /** + * @param document the parsed yaml document + * @param path the key path to walk, outermost first + * @return the value at that path, or null when any segment is missing + */ + @Nullable + private static Object valueAt(Map document, List path) { + Object current = document; + for (String key : path) { + if (!(current instanceof Map map)) { + return null; + } + current = map.get(key); + } + return current; + } + + @SuppressWarnings("unchecked") + private static List> documentsOf(InputStream in) { + List> documents = new ArrayList<>(); + for (Object document : new Yaml().loadAll(in)) { + if (document instanceof Map map) { + documents.add((Map) map); + } + } + return documents; + } + + /** + * @return the {@code application.yml} copies shipped by the deployment scripts + */ + private static Set deploymentCopies() throws IOException { + assumeTrue(Files.isDirectory(SCRIPT_DIR), + "running outside the source tree, the packaged file asserted above is all we can see"); + try (Stream paths = Files.walk(SCRIPT_DIR)) { + Set copies = paths.filter(path -> path.getFileName().toString().equals("application.yml")) + .collect(Collectors.toCollection(LinkedHashSet::new)); + assertFalse(copies.isEmpty(), "expected the deployment scripts to ship application.yml copies"); + return copies; + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessOpenApiDocRuleTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessOpenApiDocRuleTest.java new file mode 100644 index 0000000000..76ff76b2e3 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessOpenApiDocRuleTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import com.usthe.sureness.matcher.util.TirePathTree; +import jakarta.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +/** + * Guards the rbac rules covering the generated openapi document. + * + *

These paths used to sit in {@code excludedResource}. Sureness checks the exclusion + * tree before any credential check, so an anonymous request returned the full document: + * every route, http method, parameter name and type, and every request and response + * model. That is a ready made map of the attack surface, so it is scoped like any other + * administrative resource. + */ +class SurenessOpenApiDocRuleTest { + + private static final String SEPARATOR = "==="; + + /** + * {@code excludedResource} entries are written as {@code api===method}, but + * {@code TirePathTree} only accepts the three segment {@code api===method===roles} + * shape and silently drops anything else. {@code DefaultPathRoleMatcher} therefore + * appends this marker to every excluded rule before it builds the exclusion tree, and + * a test that skips the same step builds an empty tree that matches nothing - every + * {@code assertNull} against it then passes no matter what the yaml says. + */ + private static final String EXCLUDE_ROLE = SEPARATOR + "[exclude]"; + + private static final Path SCRIPT_DIR = Path.of("..", "script"); + + private static TirePathTree roleTree; + + private static TirePathTree excludeTree; + + @BeforeAll + @SuppressWarnings("unchecked") + static void loadSurenessConfig() throws IOException { + List resourceRole; + List excludedResource; + try (InputStream in = SurenessOpenApiDocRuleTest.class.getResourceAsStream("/sureness.yml")) { + assertNotNull(in, "sureness.yml must be on the classpath"); + Map document = new Yaml().load(in); + resourceRole = (List) document.get("resourceRole"); + excludedResource = (List) document.get("excludedResource"); + } + assertNotNull(resourceRole, "resourceRole must be present"); + assertNotNull(excludedResource, "excludedResource must be present"); + roleTree = new TirePathTree(); + roleTree.buildTree(new LinkedHashSet<>(resourceRole)); + excludeTree = excludeTreeOf(excludedResource); + } + + @Test + void shouldRestrictTheOpenApiDocumentToAdmin() { + assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs")); + assertEquals("[admin]", rolesFor(roleTree, "/v2/api-docs")); + assertEquals("[admin]", rolesFor(roleTree, "/swagger-resources/configuration/ui")); + } + + /** + * Springdoc also serves the grouped documents and its own config under the same + * prefix; a rule bound to the bare path would leave those anonymous. + */ + @Test + void shouldCoverTheGroupedDocumentsToo() { + assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/swagger-config")); + assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/default")); + assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml/default")); + } + + /** + * {@code OpenApiWebMvcResource} maps the document twice, at the configured path and at + * that path with a {@code .yaml} suffix. The suffixed form is a sibling path segment + * rather than a child, so {@code /v3/api-docs/**} does not reach it and it needs its + * own rule. Without one the yaml document carries no role requirement at all, which + * sureness treats as no restriction for any authenticated caller including guest. + */ + @Test + void shouldRestrictTheYamlDocumentToAdmin() { + assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml")); + } + + @Test + void shouldStopTreatingTheOpenApiDocumentAsAnonymous() { + assertNull(rolesFor(excludeTree, "/v3/api-docs")); + assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml")); + assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml/default")); + assertNull(rolesFor(excludeTree, "/v3/api-docs/swagger-config")); + assertNull(rolesFor(excludeTree, "/v2/api-docs")); + assertNull(rolesFor(excludeTree, "/swagger-resources/configuration/ui")); + } + + /** + * The assertions above are all {@code assertNull}, so they only mean something while + * the exclusion tree is capable of matching at all. This pins a path that is meant to + * stay anonymous and fails if the tree was built in a shape sureness would have + * rejected. + */ + @Test + void shouldKeepMatchingIntentionallyExcludedResources() { + assertEquals("[exclude]", rolesFor(excludeTree, "/api/i18n/lang")); + } + + /** + * The deployment scripts ship their own copies of {@code sureness.yml}; a rule fixed + * only in the packaged file would still leave every container deployment serving the + * document to anonymous callers. + */ + @Test + void deploymentCopiesRestrictTheOpenApiDocument() throws IOException { + for (Path copy : deploymentCopies()) { + TirePathTree copyRoleTree = new TirePathTree(); + copyRoleTree.buildTree(new LinkedHashSet<>(sectionOf(copy, "resourceRole"))); + assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs"), + "the openapi document is unruled or over-granted in " + copy); + assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml"), + "the yaml openapi document is unruled or over-granted in " + copy); + assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml/default"), + "the grouped yaml openapi document is unruled or over-granted in " + copy); + assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs/swagger-config"), + "the springdoc ui config is unruled or over-granted in " + copy); + + TirePathTree copyExcludeTree = excludeTreeOf(sectionOf(copy, "excludedResource")); + assertNull(rolesFor(copyExcludeTree, "/v3/api-docs"), + "the openapi document is still anonymous in " + copy); + assertEquals("[exclude]", rolesFor(copyExcludeTree, "/api/i18n/lang"), + "the exclusion tree matches nothing in " + copy + ", the assertion above proves nothing"); + } + } + + @Nullable + private static String rolesFor(TirePathTree tree, String path) { + return tree.searchPathFilterRoles(path + SEPARATOR + "get"); + } + + /** + * @param excludedResource the {@code excludedResource} rules, as written in the yaml + * @return the exclusion tree, built the way {@code DefaultPathRoleMatcher} builds it + */ + private static TirePathTree excludeTreeOf(List excludedResource) { + TirePathTree tree = new TirePathTree(); + tree.buildTree(excludedResource.stream() + .map(rule -> rule + EXCLUDE_ROLE) + .collect(Collectors.toCollection(LinkedHashSet::new))); + return tree; + } + + /** + * @return the {@code sureness.yml} copies shipped by the deployment scripts + */ + private static Set deploymentCopies() throws IOException { + assumeTrue(Files.isDirectory(SCRIPT_DIR), + "running outside the source tree, the packaged file asserted above is all we can see"); + try (Stream paths = Files.walk(SCRIPT_DIR)) { + Set copies = paths.filter(path -> path.getFileName().toString().equals("sureness.yml")) + .collect(Collectors.toCollection(LinkedHashSet::new)); + assertFalse(copies.isEmpty(), "expected the deployment scripts to ship sureness.yml copies"); + return copies; + } + } + + @SuppressWarnings("unchecked") + private static List sectionOf(Path copy, String section) throws IOException { + Map document; + try (InputStream in = Files.newInputStream(copy)) { + document = new Yaml().load(in); + } + List rules = (List) document.get(section); + assertNotNull(rules, section + " must be present in " + copy); + return rules; + } +} diff --git a/home/docs/start/account-modify.md b/home/docs/start/account-modify.md index ea72ca06a1..8fdba1e21d 100644 --- a/home/docs/start/account-modify.md +++ b/home/docs/start/account-modify.md @@ -51,6 +51,13 @@ resourceRole: - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] - /api/status/page/**===delete===[admin] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # config the resource restful api that need bypass auth protection # rule: api===method @@ -82,10 +89,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* @@ -140,6 +143,32 @@ account: role: [user] ``` +## OpenAPI Document And Swagger UI + +The generated OpenAPI document lists every route, http method, parameter name and type, and every request and response model. It is a ready made map of the attack surface, so HertzBeat does not serve it by default: `springdoc.api-docs.enabled` and `springdoc.swagger-ui.enabled` are both `false` in the shipped `application.yml`, which makes `/v3/api-docs` and `/swagger-ui/index.html` return 404. + +If you need the document, opt in by updating the `application.yml` file in the `config` directory: + +```yaml +springdoc: + api-docs: + enabled: true + swagger-ui: + enabled: true +``` + +Once enabled, the document endpoints are still scoped to the `admin` role by the `resourceRole` rules above. Sign in to the HertzBeat web application as an administrator before opening `/swagger-ui/index.html`; the Swagger UI attaches the stored HertzBeat token to its same-origin document and try-it-out requests, and the page loads without asking for anything. + +Without that session the document is not exposed, but the page does not fail silently either. `/swagger-ui/index.html` is a static file and still loads; its request for `/v3/api-docs/swagger-config` is answered with `401` and a `WWW-Authenticate: Digest` challenge, so the browser asks for a username and password. Administrator credentials entered there let the document through, and an account without the `admin` role is answered with `403`. + +The document can also be fetched directly with an administrator token: + +```shell +curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs +``` + +> ⚠️ Do not move the OpenAPI paths into `excludedResource`; doing so makes the complete document anonymous again. + ## Update Security Secret > This secret is the key for account security encryption management and needs to be updated to your custom key string of the same length. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/account-modify.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/account-modify.md index 9818ff8620..24f82b02e7 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/account-modify.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/account-modify.md @@ -52,6 +52,13 @@ resourceRole: - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] - /api/status/page/**===delete===[admin] + # OpenAPI 文档包含全部路由、参数与数据模型,等同于一份接口地图, + # 因此按普通管理类资源收敛到 admin,不再匿名开放 + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # 需要被过滤保护的资源,不认证鉴权直接访问 # /api/v1/source3===get 表示 /api/v1/source3===get 可以被任何人访问 无需登录认证鉴权 @@ -82,10 +89,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* @@ -141,6 +144,32 @@ account: role: [user] ``` +## OpenAPI 文档与 Swagger UI + +生成的 OpenAPI 文档会列出全部路由、HTTP 方法、参数名与类型，以及所有请求和响应模型，等同于一份现成的攻击面地图，因此 HertzBeat 默认不对外提供：随包发布的 `application.yml` 中 `springdoc.api-docs.enabled` 与 `springdoc.swagger-ui.enabled` 均为 `false`，此时 `/v3/api-docs` 和 `/swagger-ui/index.html` 返回 404。 + +如果确实需要该文档，更新 `config` 目录下的 `application.yml` 文件显式开启： + +```yaml +springdoc: + api-docs: + enabled: true + swagger-ui: + enabled: true +``` + +开启之后，文档接口仍然被上面的 `resourceRole` 规则收敛在 `admin` 角色。请先以管理员身份登录 HertzBeat Web 应用，再打开 `/swagger-ui/index.html`；Swagger UI 会在同源的文档请求与 try-it-out 请求中附带 HertzBeat 保存的令牌，页面不会再要求任何输入。 + +没有该会话时文档同样不会泄露，但页面并非静默失败：`/swagger-ui/index.html` 是静态文件，仍然可以打开，而它请求 `/v3/api-docs/swagger-config` 会得到 `401` 与 `WWW-Authenticate: Digest` 挑战，浏览器因此弹出用户名密码框。在该弹框中输入管理员账号可以正常加载文档；没有 `admin` 角色的账号则会收到 `403`。 + +也可以直接携带管理员令牌获取文档： + +```shell +curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs +``` + +> ⚠️ 不要把 OpenAPI 路径放回 `excludedResource`，否则完整接口文档会再次匿名开放。 + ## 更新安全密钥 > 此密钥为账户安全加密管理的密钥，需要更新为相同长度的你自定义密钥串。 diff --git a/script/application.yml b/script/application.yml index fa6cda6e10..425dd4e1f6 100644 --- a/script/application.yml +++ b/script/application.yml @@ -71,6 +71,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index 7695271d03..f141461fb4 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -74,6 +74,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml index 069fba3069..1815529be7 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -109,11 +109,18 @@ resourceRole: - /api/ai/**===delete===[admin] - /api/logs/sse/**===get===[admin,user,guest] - /api/logs/ingest/**===post===[admin,user] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -147,10 +154,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 90220d0669..f6aa53215a 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -74,6 +74,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml index 069fba3069..1815529be7 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -109,11 +109,18 @@ resourceRole: - /api/ai/**===delete===[admin] - /api/logs/sse/**===get===[admin,user,guest] - /api/logs/ingest/**===post===[admin,user] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -147,10 +154,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index 0074a9dd3f..413bdb3718 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -74,6 +74,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml index 069fba3069..1815529be7 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -109,11 +109,18 @@ resourceRole: - /api/ai/**===delete===[admin] - /api/logs/sse/**===get===[admin,user,guest] - /api/logs/ingest/**===post===[admin,user] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -147,10 +154,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index c25b533a48..2f59761b65 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -74,6 +74,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml index d15a6ac5d2..6281d0927d 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -113,11 +113,18 @@ resourceRole: - /api/ingestion/otlp/**===get===[admin,user,guest] - /api/logs/**===get===[admin,user,guest] - /api/traces/**===get===[admin,user,guest] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -154,10 +161,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index 71d7fb2f58..9189445de0 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -74,6 +74,16 @@ management: export: enabled: true +# The generated OpenAPI document is a map of every route, HTTP method, parameter +# and model, so it is not served by default. A deployment that wants the Swagger +# UI opts in by turning both switches on; the document endpoints stay scoped to +# the admin role in sureness.yml either way. +springdoc: + api-docs: + enabled: false + swagger-ui: + enabled: false + sureness: container: jakarta_servlet auths: diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml index 069fba3069..1815529be7 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -109,11 +109,18 @@ resourceRole: - /api/ai/**===delete===[admin] - /api/logs/sse/**===get===[admin,user,guest] - /api/logs/ingest/**===post===[admin,user] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -147,10 +154,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* diff --git a/script/sureness.yml b/script/sureness.yml index d15a6ac5d2..6281d0927d 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -16,7 +16,7 @@ ## -- sureness.yml account source -- ## # config the resource restful api that need auth protection, base rbac -# rule: api===method===role +# rule: api===method===role # eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access. # eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role. resourceRole: @@ -113,11 +113,18 @@ resourceRole: - /api/ingestion/otlp/**===get===[admin,user,guest] - /api/logs/**===get===[admin,user,guest] - /api/traces/**===get===[admin,user,guest] + # The OpenAPI document is a map of every route, parameter and model, so it is + # scoped like any other administrative resource instead of being anonymous + - /v3/api-docs/**===get===[admin] + - /v3/api-docs.yaml===get===[admin] + - /v3/api-docs.yaml/**===get===[admin] + - /v2/api-docs/**===get===[admin] + - /swagger-resources/**===get===[admin] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] # config the resource restful api that need bypass auth protection -# rule: api===method +# rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - /api/alert/sse/**===* @@ -154,10 +161,6 @@ excludedResource: - /**/*.json===get - /**/*.woff===get - /**/*.eot===get - # swagger ui resource - - /swagger-resources/**===get - - /v2/api-docs===get - - /v3/api-docs===get # h2 database - /h2-console/**===* From f952d47eb42b6c31c39cf1cd929d7a25741b326b Mon Sep 17 00:00:00 2001 From: Bhavya Sonigra Date: Fri, 14 Aug 2026 23:14:43 +0530 Subject: [PATCH 12/18] [security] Restrict plugin upload endpoint to admin role (#4149) (#4257) Signed-off-by: Bhavya Sonigra Co-authored-by: Tomsun28 --- .../src/main/resources/sureness.yml | 4 ++++ .../.env.example | 18 ++++++++++++++++++ .../.gitignore | 16 ++++++++++++++++ .../docker-compose.yaml | 6 ++++-- 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 script/docker-compose/hertzbeat-postgresql-victoria-metrics/.env.example create mode 100644 script/docker-compose/hertzbeat-postgresql-victoria-metrics/.gitignore diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index 8cdaeec049..5aa2f8a26e 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -60,6 +60,10 @@ resourceRole: - /api/collector/**===post===[admin,user] - /api/collector/**===put===[admin,user] - /api/collector/**===delete===[admin] + - /api/plugin/**===get===[admin] + - /api/plugin/**===post===[admin] + - /api/plugin/**===put===[admin] + - /api/plugin/**===delete===[admin] # the secret config holds the jwt signing key and the aes key protecting stored # credentials, so it stays admin only and is additionally refused by the controller - /api/config/secret===get===[admin] diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.env.example b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.env.example new file mode 100644 index 0000000000..77aa295e58 --- /dev/null +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.env.example @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copy this file to .env and set your own strong password before running docker compose up +POSTGRES_USER=root +POSTGRES_PASSWORD= diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.gitignore b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.gitignore new file mode 100644 index 0000000000..51384e37c9 --- /dev/null +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/.gitignore @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.env diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml index 4a581e1afc..60fe377d0d 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml @@ -34,8 +34,8 @@ services: ports: - '15432:5432' environment: - POSTGRES_USER: root - POSTGRES_PASSWORD: 123456 + POSTGRES_USER: ${POSTGRES_USER:-root} + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD (e.g., in .env)}" TZ: Asia/Shanghai PGDATA: /var/lib/postgresql/data/pgdata volumes: @@ -73,6 +73,8 @@ services: HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 + SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-root} + SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in your .env file} depends_on: postgres: condition: service_healthy From b61dbfcacf900e55863ff6fc0a8cc8d9075de181 Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:18:42 -0700 Subject: [PATCH 13/18] fix(collector): correct jsonpath alias parsing for rows missing path (#4265) Co-authored-by: Tomsun28 --- .../collect/http/HttpCollectImpl.java | 12 ++- .../collect/http/HttpCollectImplTest.java | 43 ++++++++++ .../collector/dispatch/MetricsCollect.java | 12 ++- .../dispatch/MetricsCollectTest.java | 73 +++++++++++++++++ .../collector/util/JsonPathParser.java | 18 +++++ .../collector/util/JsonPathParserTest.java | 81 +++++++++++++++++++ 6 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/dispatch/MetricsCollectTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/util/JsonPathParserTest.java diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java index ab9f2dca91..408d32c0ce 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java @@ -704,13 +704,11 @@ public class HttpCollectImpl extends AbstractCollect { valueRowBuilder.addColumn(String.valueOf(value)); } else { if (alias.startsWith("$.")) { - List subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1)); - if (subResults != null && subResults.size() > i) { - Object resultValue = subResults.get(i); - valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue)); - } else { - valueRowBuilder.addColumn(CommonConstants.NULL_VALUE); - } + // per-row evaluation, a global "parseScript + alias" query would misalign rows missing the path + List aliasValues = JsonPathParser.parseRowWithJsonPath(objectValue, alias); + // a wildcard alias matching multiple values is kept whole and rendered as "[v1, v2]" + Object resultValue = aliasValues.size() == 1 ? aliasValues.get(0) : (aliasValues.isEmpty() ? null : aliasValues); + valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue)); } else { addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias); } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImplTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImplTest.java index 05199a2feb..f69a95a32e 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImplTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImplTest.java @@ -20,6 +20,7 @@ package org.apache.hertzbeat.collector.collect.http; import com.google.common.collect.Lists; import com.sun.net.httpserver.HttpServer; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; @@ -383,6 +384,48 @@ class HttpCollectImplTest { assertEquals("0.268751364291017", firstRow.getColumns(0)); } + @Test + void parseResponseByJsonPathKeepsRowAlignmentWhenAliasPathMissing() throws Exception { + String jsonResponse = "{\"items\": [" + + "{\"metadata\": {\"name\": \"pod-a\"}, \"status\": {\"phase\": \"Running\"," + + " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}}," + + "{\"metadata\": {\"name\": \"pod-b-pending\"}, \"status\": {\"phase\": \"Pending\"}}," + + "{\"metadata\": {\"name\": \"pod-c\"}, \"status\": {\"phase\": \"Running\"," + + " \"containerStatuses\": [{\"name\": \"c3\", \"ready\": true, \"restartCount\": 2}]}}" + + "]}"; + HttpProtocol http = HttpProtocol.builder() + .parseType(DispatchConstants.PARSE_JSON_PATH) + .parseScript("$.items.*") + .build(); + List capturedRows = new ArrayList<>(); + CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() { + @Override + public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) { + capturedRows.add(valueRow); + return super.addValueRow(valueRow); + } + }; + Method parseMethod = HttpCollectImpl.class.getDeclaredMethod( + "parseResponseByJsonPath", + String.class, + List.class, + HttpProtocol.class, + CollectRep.MetricsData.Builder.class, + Long.class); + parseMethod.setAccessible(true); + + parseMethod.invoke(httpCollectImpl, jsonResponse, + Lists.newArrayList("$.metadata.name", "$.status.containerStatuses[0].restartCount"), http, builder, 100L); + + assertEquals(3, capturedRows.size()); + assertEquals("pod-a", capturedRows.get(0).getColumns(0)); + assertEquals("5", capturedRows.get(0).getColumns(1)); + assertEquals("pod-b-pending", capturedRows.get(1).getColumns(0)); + assertEquals(CommonConstants.NULL_VALUE, capturedRows.get(1).getColumns(1)); + assertEquals("pod-c", capturedRows.get(2).getColumns(0)); + assertEquals("2", capturedRows.get(2).getColumns(1)); + } + @Test void testParsePromQlLabelValue() throws Exception { // Create Prometheus format test data diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java index 94b2022599..d896e1404b 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java @@ -252,11 +252,12 @@ public class MetricsCollect implements Runnable, Comparable { if (metrics.getCalculates() == null) { metrics.setCalculates(Collections.emptyList()); } + List aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList); // eg: database_pages=Database pages unconventional mapping Map fieldAliasMap = new HashMap<>(8); Map fieldExpressionMap = metrics.getCalculates() .stream() - .map(cal -> transformCal(cal, fieldAliasMap)) + .map(cal -> transformCal(cal, fieldAliasMap, aliasFields)) .filter(Objects::nonNull) .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue)); @@ -270,7 +271,6 @@ public class MetricsCollect implements Runnable, Comparable { .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair) arr[1], (oldValue, newValue) -> newValue)); List fields = metrics.getFields(); - List aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList); Map aliasFieldValueMap = new HashMap<>(8); Map fieldValueMap = new HashMap<>(8); Map stringTypefieldValueMap = new HashMap<>(8); @@ -420,13 +420,19 @@ public class MetricsCollect implements Runnable, Comparable { * @param fieldAliasMap field alias map * @return expr */ - private Object[] transformCal(String cal, Map fieldAliasMap) { + private Object[] transformCal(String cal, Map fieldAliasMap, List aliasFields) { int splitIndex = cal.indexOf("="); if (splitIndex < 0) { return null; } String field = cal.substring(0, splitIndex).trim(); String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#"); + // a direct alias reference (RHS must exactly equal an aliasField, no whitespace/case tolerance) is not a formula, + // JEXL parses "[0]" in such paths as array access and silently returns null + if (aliasFields.contains(expressionStr)) { + fieldAliasMap.put(field, expressionStr); + return null; + } JexlExpression expression; try { expression = JexlExpressionRunner.compile(expressionStr); diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/dispatch/MetricsCollectTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/dispatch/MetricsCollectTest.java new file mode 100644 index 0000000000..6bc9ec8914 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/dispatch/MetricsCollectTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.dispatch; + +import java.util.List; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test case for {@link MetricsCollect} + */ +class MetricsCollectTest { + + @Test + void calculateFieldsMapsIndexedJsonPathAlias() { + Metrics metrics = Metrics.builder() + .name("pods") + .priority((byte) 0) + .fields(List.of( + Metrics.Field.builder().field("pod").type(CommonConstants.TYPE_STRING).build(), + Metrics.Field.builder().field("rc").type(CommonConstants.TYPE_STRING).build())) + .aliasFields(List.of("$.metadata.name", "$.status.containerStatuses[0].restartCount")) + .calculates(List.of( + "pod=$.metadata.name", + "rc=$.status.containerStatuses[0].restartCount")) + .build(); + + Timeout timeout = mock(Timeout.class); + WheelTimerTask timerTask = mock(WheelTimerTask.class); + when(timeout.task()).thenReturn(timerTask); + when(timerTask.getJob()).thenReturn(Job.builder().build()); + MetricsCollect metricsCollect = new MetricsCollect(metrics, timeout, null, "test", List.of()); + + CollectRep.MetricsData.Builder collectData = CollectRep.MetricsData.newBuilder(); + collectData.addValueRow(CollectRep.ValueRow.newBuilder() + .addColumn("pod-a").addColumn("5").build()); + collectData.addValueRow(CollectRep.ValueRow.newBuilder() + .addColumn("pod-b-pending").addColumn(CommonConstants.NULL_VALUE).build()); + + metricsCollect.calculateFields(metrics, collectData); + + List rows = collectData.getValuesList(); + assertEquals(2, rows.size()); + assertEquals("pod-a", rows.get(0).getColumns(0)); + assertEquals("5", rows.get(0).getColumns(1)); + assertEquals("pod-b-pending", rows.get(1).getColumns(0)); + assertEquals(CommonConstants.NULL_VALUE, rows.get(1).getColumns(1)); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/JsonPathParser.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/JsonPathParser.java index 5deb848e58..97a8345b04 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/JsonPathParser.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/JsonPathParser.java @@ -36,12 +36,16 @@ public final class JsonPathParser { private static final ParseContext PARSER; + private static final ParseContext ROW_PARSER; + static { Configuration conf = Configuration.defaultConfiguration() .addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL) .addOptions(Option.ALWAYS_RETURN_LIST); CacheProvider.setCache(new LRUCache(128)); PARSER = JsonPath.using(conf); + // a single row legitimately may not contain the queried path + ROW_PARSER = JsonPath.using(conf.addOptions(Option.SUPPRESS_EXCEPTIONS)); } private JsonPathParser() { @@ -73,4 +77,18 @@ public final class JsonPathParser { return PARSER.parse(content).read(jsonPath, typeRef); } + /** + * use json path to parse one already-parsed row object, missing paths yield an empty list + * @param document parsed json object of a single row + * @param jsonPath jsonPath relative to the row root + * @return matched values, empty list when the path does not exist in this row + */ + public static List parseRowWithJsonPath(Object document, String jsonPath) { + if (document == null || StringUtils.isEmpty(jsonPath)) { + return Collections.emptyList(); + } + List values = ROW_PARSER.parse(document).read(jsonPath); + return values == null ? Collections.emptyList() : values; + } + } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/util/JsonPathParserTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/util/JsonPathParserTest.java new file mode 100644 index 0000000000..b1d4ecc7f2 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/util/JsonPathParserTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.util; + +import com.jayway.jsonpath.PathNotFoundException; +import java.util.List; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test case for {@link JsonPathParser} + */ +class JsonPathParserTest { + + private static final String ROW_JSON = "{\"metadata\": {\"name\": \"pod-a\"}," + + " \"status\": {\"phase\": \"Running\"," + + " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}}"; + + private Object row() { + return JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$").get(0); + } + + @Test + void parseRowWithJsonPathReturnsExistingValue() { + List values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].restartCount"); + + assertEquals(1, values.size()); + assertEquals(5, values.get(0)); + } + + @Test + void parseRowWithJsonPathReturnsEmptyListWhenPathMissing() { + Object pendingRow = JsonPathParser + .parseContentWithJsonPath("{\"metadata\": {\"name\": \"pod-b\"}, \"status\": {\"phase\": \"Pending\"}}", "$") + .get(0); + + List values = JsonPathParser.parseRowWithJsonPath(pendingRow, "$.status.containerStatuses[0].restartCount"); + + assertTrue(values.isEmpty()); + } + + @Test + void parseRowWithJsonPathReturnsAllValuesForWildcard() { + List values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].*"); + + assertEquals(3, values.size()); + assertTrue(values.contains("c1")); + assertTrue(values.contains(true)); + assertTrue(values.contains(5)); + } + + @Test + void parseContentWithJsonPathStillThrowsWhenPathMissing() { + assertThrows(PathNotFoundException.class, + () -> JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$.spec.nodeName")); + } + + @Test + void parseRowWithJsonPathHandlesNullDocumentAndEmptyPath() { + assertTrue(JsonPathParser.parseRowWithJsonPath(null, "$.status").isEmpty()); + assertTrue(JsonPathParser.parseRowWithJsonPath(row(), "").isEmpty()); + } +} From e12a1ddead349ca6aa39f7a9c6bb648d81589a42 Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:45:54 -0700 Subject: [PATCH 14/18] [fix] keep one-row metric data when command output is short (#4308) Co-authored-by: aias00 --- .../collect/common/OneRowResponseSupport.java | 113 +++++++++++++++ .../collect/script/ScriptCollectImpl.java | 79 +++++----- .../collector/collect/ssh/SshCollectImpl.java | 49 +++---- .../common/OneRowResponseSupportTest.java | 123 ++++++++++++++++ .../collect/script/ScriptCollectImplTest.java | 82 +++++++++++ .../collect/ssh/SshCollectImplTest.java | 136 ++++++++++++++++++ .../entity/job/protocol/SshProtocol.java | 5 + 7 files changed, 526 insertions(+), 61 deletions(-) create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupport.java create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupportTest.java diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupport.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupport.java new file mode 100644 index 0000000000..537105377b --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupport.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common; + +import java.util.Collections; +import java.util.List; +import org.apache.hertzbeat.collector.constants.CollectorConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.springframework.util.StringUtils; + +/** + * Shared one-row response handling for command-based collectors. + */ +public final class OneRowResponseSupport { + + /** + * Parse type where each output line maps to one alias field of a single result row. + */ + public static final String PARSE_TYPE_ONE_ROW = "oneRow"; + + private OneRowResponseSupport() { + } + + /** + * Treat blank stdout without an error signal (no stderr, exit status present and <= 1, + * grep-style no match) as valid empty one-row data: append a row of null placeholders so the + * metric stays visible and alertable. + * + * @return true if handled as empty success, false if the caller should report a failure + */ + public static boolean tryAppendEmptyOneRow(String parseType, String stdErr, Integer exitStatus, + List aliasFields, CollectRep.MetricsData.Builder builder, + Long responseTime) { + if (PARSE_TYPE_ONE_ROW.equals(parseType) + && !StringUtils.hasText(stdErr) + && exitStatus != null && exitStatus <= 1) { + appendEmptyValues(aliasFields, builder, responseTime); + return true; + } + return false; + } + + /** + * Build the failure message for a command that produced no usable stdout: prefer the captured + * stderr, then a non-trivial exit status, otherwise the generic null-data message. + */ + public static String buildBlankFailureMessage(String stdErr, Integer exitStatus, + String exitCodePrefix, String nullMessage) { + if (StringUtils.hasText(stdErr)) { + return stdErr.trim(); + } + if (exitStatus != null && exitStatus > 1) { + return exitCodePrefix + exitStatus; + } + return nullMessage; + } + + /** + * Map each output line to one alias field of a single row; missing trailing lines become + * NULL_VALUE columns so a partial result keeps its values and the gap stays alertable. + */ + public static void appendResponseValues(String result, List aliasFields, + CollectRep.MetricsData.Builder builder, Long responseTime) { + List safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields; + String[] lines = result.split("\n"); + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + int aliasIndex = 0; + int lineIndex = 0; + while (aliasIndex < safeAliasFields.size()) { + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(safeAliasFields.get(aliasIndex))) { + valueRowBuilder.addColumn(responseTime.toString()); + } else { + if (lineIndex < lines.length) { + valueRowBuilder.addColumn(lines[lineIndex].trim()); + } else { + valueRowBuilder.addColumn(CommonConstants.NULL_VALUE); + } + lineIndex++; + } + aliasIndex++; + } + builder.addValueRow(valueRowBuilder.build()); + } + + public static void appendEmptyValues(List aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) { + List safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields; + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String aliasField : safeAliasFields) { + if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasField)) { + valueRowBuilder.addColumn(responseTime.toString()); + } else { + valueRowBuilder.addColumn(CommonConstants.NULL_VALUE); + } + } + builder.addValueRow(valueRowBuilder.build()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImpl.java index 3d797b53a1..25fbc061ef 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImpl.java @@ -30,6 +30,7 @@ import java.util.Objects; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; +import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport; import org.apache.hertzbeat.collector.constants.CollectorConstants; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; import org.apache.hertzbeat.common.constants.CommonConstants; @@ -52,7 +53,6 @@ public class ScriptCollectImpl extends AbstractCollect { private static final String BASH_C = "-c"; private static final String POWERSHELL_C = "-Command"; private static final String POWERSHELL_FILE = "-File"; - private static final String PARSE_TYPE_ONE_ROW = "oneRow"; private static final String PARSE_TYPE_MULTI_ROW = "multiRow"; private static final String PARSE_TYPE_NETCAT = "netcat"; private static final String PARSE_TYPE_LOG = "log"; @@ -113,25 +113,48 @@ public class ScriptCollectImpl extends AbstractCollect { try { Process process = processBuilder.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName(scriptProtocol.getCharset()))); - StringBuilder response = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - if (StringUtils.hasText(line)) { - response.append(line).append("\n"); + BufferedReader errorReader = new BufferedReader( + new InputStreamReader(process.getErrorStream(), Charset.forName(scriptProtocol.getCharset()))); + // drain stderr on its own thread: a full stderr pipe would deadlock the stdout read; + // StringBuffer because the drainer may still be writing when the buffer is read + StringBuffer errorBuffer = new StringBuffer(); + Thread errorDrainer = new Thread(() -> { + try { + String errorLine; + while ((errorLine = errorReader.readLine()) != null) { + if (StringUtils.hasText(errorLine)) { + errorBuffer.append(errorLine).append("\n"); + } + } + } catch (IOException e) { + log.warn("read script error stream failed: {}", e.getMessage()); } - } - process.waitFor(); + }); + errorDrainer.setDaemon(true); + errorDrainer.start(); + String result = readResponse(reader); + int exitCode = process.waitFor(); + // bounded: a lingering grandchild can keep the stderr pipe open + errorDrainer.join(1000); Long responseTime = System.currentTimeMillis() - startTime; - String result = String.valueOf(response); + String errorResult = errorBuffer.toString(); if (!StringUtils.hasText(result)) { + if (OneRowResponseSupport.tryAppendEmptyOneRow(scriptProtocol.getParseType(), errorResult, + exitCode, metrics.getAliasFields(), builder, responseTime)) { + return; + } builder.setCode(CollectRep.Code.FAIL); - builder.setMsg("Script response data is null"); + builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitCode, + "Script exited with code: ", "Script response data is null")); return; } + if (StringUtils.hasText(errorResult)) { + log.warn("script command succeeded but wrote to stderr: {}", errorResult.trim()); + } switch (scriptProtocol.getParseType()) { case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime); case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime); - case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime); + case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime); case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime); default -> { builder.setCode(CollectRep.Code.FAIL); @@ -207,28 +230,7 @@ public class ScriptCollectImpl extends AbstractCollect { } private void parseResponseDataByOne(String result, List aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) { - String[] lines = result.split("\n"); - if (lines.length + 1 < aliasFields.size()) { - log.error("ssh response data not enough: {}", result); - return; - } - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - int aliasIndex = 0; - int lineIndex = 0; - while (aliasIndex < aliasFields.size()) { - if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) { - valueRowBuilder.addColumn(responseTime.toString()); - } else { - if (lineIndex < lines.length) { - valueRowBuilder.addColumn(lines[lineIndex].trim()); - } else { - valueRowBuilder.addColumn(CommonConstants.NULL_VALUE); - } - lineIndex++; - } - aliasIndex++; - } - builder.addValueRow(valueRowBuilder.build()); + OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime); } private void parseResponseDataByMulti(String result, List aliasFields, @@ -261,4 +263,15 @@ public class ScriptCollectImpl extends AbstractCollect { builder.addValueRow(valueRowBuilder.build()); } } + + private String readResponse(BufferedReader reader) throws IOException { + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + if (StringUtils.hasText(line)) { + response.append(line).append("\n"); + } + } + return response.toString(); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImpl.java index 41b2987446..26c8408fc7 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImpl.java @@ -21,6 +21,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InterruptedIOException; import java.net.ConnectException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.net.SocketTimeoutException; import java.security.GeneralSecurityException; import java.util.ArrayList; @@ -33,6 +35,7 @@ import java.util.Objects; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; +import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport; import org.apache.hertzbeat.collector.collect.common.ssh.CommonSshBlacklist; import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper; import org.apache.hertzbeat.collector.constants.CollectorConstants; @@ -49,7 +52,6 @@ import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.SshException; import org.apache.sshd.common.channel.exception.SshChannelOpenException; import org.apache.sshd.common.future.CloseFuture; -import org.apache.sshd.common.util.io.output.NoCloseOutputStream; import org.springframework.util.StringUtils; /** @@ -58,7 +60,6 @@ import org.springframework.util.StringUtils; @Slf4j public class SshCollectImpl extends AbstractCollect { - private static final String PARSE_TYPE_ONE_ROW = "oneRow"; private static final String PARSE_TYPE_MULTI_ROW = "multiRow"; private static final String PARSE_TYPE_NETCAT = "netcat"; private static final String PARSE_TYPE_LOG = "log"; @@ -93,8 +94,9 @@ public class SshCollectImpl extends AbstractCollect { } channel = clientSession.createExecChannel(sshProtocol.getScript()); ByteArrayOutputStream response = new ByteArrayOutputStream(); + ByteArrayOutputStream errorResponse = new ByteArrayOutputStream(); channel.setOut(response); - channel.setErr(new NoCloseOutputStream(System.err)); + channel.setErr(errorResponse); channel.open().verify(timeout); List list = new ArrayList<>(); list.add(ClientChannelEvent.CLOSED); @@ -107,16 +109,28 @@ public class SshCollectImpl extends AbstractCollect { throw new SocketTimeoutException("Failed to retrieve command result in time: " + sshProtocol.getScript()); } Long responseTime = System.currentTimeMillis() - startTime; - String result = response.toString(); + Charset charset = StringUtils.hasText(sshProtocol.getCharset()) + ? Charset.forName(sshProtocol.getCharset()) : StandardCharsets.UTF_8; + String result = response.toString(charset); + String errorResult = errorResponse.toString(charset); + Integer exitStatus = channel.getExitStatus(); if (!StringUtils.hasText(result)) { + if (OneRowResponseSupport.tryAppendEmptyOneRow(sshProtocol.getParseType(), errorResult, + exitStatus, metrics.getAliasFields(), builder, responseTime)) { + return; + } builder.setCode(CollectRep.Code.FAIL); - builder.setMsg("ssh shell response data is null"); + builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitStatus, + "ssh command exited with code: ", "ssh shell response data is null")); return; } + if (StringUtils.hasText(errorResult)) { + log.warn("ssh command succeeded but wrote to stderr: {}", errorResult.trim()); + } switch (sshProtocol.getParseType()) { case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime); case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime); - case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime); + case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime); case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime); default -> { builder.setCode(CollectRep.Code.FAIL); @@ -244,28 +258,7 @@ public class SshCollectImpl extends AbstractCollect { } private void parseResponseDataByOne(String result, List aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) { - String[] lines = result.split("\n"); - if (lines.length + 1 < aliasFields.size()) { - log.error("ssh response data not enough: {}", result); - return; - } - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - int aliasIndex = 0; - int lineIndex = 0; - while (aliasIndex < aliasFields.size()) { - if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) { - valueRowBuilder.addColumn(responseTime.toString()); - } else { - if (lineIndex < lines.length) { - valueRowBuilder.addColumn(lines[lineIndex].trim()); - } else { - valueRowBuilder.addColumn(CommonConstants.NULL_VALUE); - } - lineIndex++; - } - aliasIndex++; - } - builder.addValueRow(valueRowBuilder.build()); + OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime); } private void parseResponseDataByMulti(String result, List aliasFields, diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupportTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupportTest.java new file mode 100644 index 0000000000..a504a820fb --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/common/OneRowResponseSupportTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.hertzbeat.collector.constants.CollectorConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.Test; + +class OneRowResponseSupportTest { + + @Test + void appendResponseValuesShouldMapColumnsInOrder() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + OneRowResponseSupport.appendResponseValues( + "pod-a\n5\n", List.of("pod", "restart", CollectorConstants.RESPONSE_TIME), builder, 18L); + + assertEquals(1, builder.getValuesCount()); + assertEquals("pod-a", builder.getValues(0).getColumns(0)); + assertEquals("5", builder.getValues(0).getColumns(1)); + assertEquals("18", builder.getValues(0).getColumns(2)); + } + + @Test + void appendResponseValuesShouldPadMissingTrailingLines() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + OneRowResponseSupport.appendResponseValues( + "52\n35.8033\n5%", + List.of("cpu", "memory", "disk", "nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 18L); + + assertEquals(1, builder.getValuesCount()); + assertEquals("52", builder.getValues(0).getColumns(0)); + assertEquals("35.8033", builder.getValues(0).getColumns(1)); + assertEquals("5%", builder.getValues(0).getColumns(2)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3)); + assertEquals("18", builder.getValues(0).getColumns(4)); + } + + @Test + void appendEmptyValuesShouldFillNullPlaceholders() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + OneRowResponseSupport.appendEmptyValues( + List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 12L); + + assertEquals(1, builder.getValuesCount()); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0)); + assertEquals("12", builder.getValues(0).getColumns(1)); + } + + @Test + void tryAppendEmptyOneRowShouldAcceptGrepNoMatchExitOne() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + // grep with no match exits 1 and writes nothing: treat as valid empty data, not a failure + boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow( + OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", 1, + List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 9L); + + assertTrue(handled); + assertEquals(1, builder.getValuesCount()); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0)); + } + + @Test + void tryAppendEmptyOneRowShouldRejectNullExitStatus() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + // an absent exit status (e.g. dropped ssh channel) must be treated as a failure + boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow( + OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", null, + List.of("nfs_mount"), builder, 9L); + + assertFalse(handled); + assertEquals(0, builder.getValuesCount()); + } + + @Test + void tryAppendEmptyOneRowShouldRejectNonEmptyStderr() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + + boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow( + OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "permission denied", 1, + List.of("nfs_mount"), builder, 9L); + + assertFalse(handled); + assertEquals(0, builder.getValuesCount()); + } + + @Test + void buildBlankFailureMessageShouldPreferStderrThenExitCode() { + assertEquals("permission denied", OneRowResponseSupport.buildBlankFailureMessage( + "permission denied\n", 2, "cmd exited with code: ", "null data")); + assertEquals("cmd exited with code: 2", OneRowResponseSupport.buildBlankFailureMessage( + "", 2, "cmd exited with code: ", "null data")); + assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage( + "", 1, "cmd exited with code: ", "null data")); + assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage( + "", null, "cmd exited with code: ", "null data")); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImplTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImplTest.java index f51fbd5c74..5af641d271 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImplTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/script/ScriptCollectImplTest.java @@ -19,9 +19,12 @@ package org.apache.hertzbeat.collector.collect.script; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.ScriptProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; @@ -138,6 +141,85 @@ public class ScriptCollectImplTest { scriptCollect.collect(builder, metrics); assertEquals(CollectRep.Code.FAIL, builder.getCode()); }); + + // empty stdout without stderr should be treated as empty one-row data + assertDoesNotThrow(() -> { + ScriptProtocol scriptProtocol = ScriptProtocol.builder() + .charset("utf-8") + .parseType("oneRow") + .scriptTool("bash") + .scriptCommand("grep -o 'centos-hermitlv' /dev/null") + .build(); + Metrics metrics = new Metrics(); + metrics.setScript(scriptProtocol); + metrics.setAliasFields(List.of("nfs_mount")); + + builder = CollectRep.MetricsData.newBuilder(); + scriptCollect.collect(builder, metrics); + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals(1, builder.getValuesCount()); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0)); + }); + + // partial output missing more than one trailing field: the old length check + // (lines + 1 < aliases) dropped the whole row here, losing the collected values + assertDoesNotThrow(() -> { + ScriptProtocol scriptProtocol = ScriptProtocol.builder() + .charset("utf-8") + .parseType("oneRow") + .scriptTool("bash") + .scriptCommand("echo 52; echo 35.8033; grep -o 'centos-hermitlv' /dev/null") + .build(); + Metrics metrics = new Metrics(); + metrics.setScript(scriptProtocol); + metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount")); + + builder = CollectRep.MetricsData.newBuilder(); + scriptCollect.collect(builder, metrics); + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals(1, builder.getValuesCount()); + assertEquals("52", builder.getValues(0).getColumns(0)); + assertEquals("35.8033", builder.getValues(0).getColumns(1)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3)); + }); + + // a command that silently exits 1 with no output is indistinguishable from a + // grep no-match, so it is deliberately accepted as an empty success + assertDoesNotThrow(() -> { + ScriptProtocol scriptProtocol = ScriptProtocol.builder() + .charset("utf-8") + .parseType("oneRow") + .scriptTool("bash") + .scriptCommand("exit 1") + .build(); + Metrics metrics = new Metrics(); + metrics.setScript(scriptProtocol); + metrics.setAliasFields(List.of("nfs_mount")); + + builder = CollectRep.MetricsData.newBuilder(); + scriptCollect.collect(builder, metrics); + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0)); + }); + + // non-empty exit code without stderr should still fail when it is not the grep-style no-match case + assertDoesNotThrow(() -> { + ScriptProtocol scriptProtocol = ScriptProtocol.builder() + .charset("utf-8") + .parseType("oneRow") + .scriptTool("bash") + .scriptCommand("exit 2") + .build(); + Metrics metrics = new Metrics(); + metrics.setScript(scriptProtocol); + metrics.setAliasFields(List.of("nfs_mount")); + + builder = CollectRep.MetricsData.newBuilder(); + scriptCollect.collect(builder, metrics); + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + assertTrue(builder.getMsg().contains("code: 2")); + }); } @Test diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImplTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImplTest.java index 2810bb354d..bdb088edd1 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImplTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/ssh/SshCollectImplTest.java @@ -22,6 +22,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; @@ -30,14 +35,21 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.SshProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.sshd.client.channel.ChannelExec; import org.apache.sshd.client.channel.ClientChannel; +import org.apache.sshd.client.channel.ClientChannelEvent; import org.apache.sshd.client.future.OpenFuture; import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.SshException; @@ -229,6 +241,130 @@ class SshCollectImplTest { verify(clientSession).close(); } + @Test + void collectPadsPartialOneRowOutput() throws Exception { + ChannelExec channel = oneRowChannel("52\n35.8033\n5%", "", 0); + Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build(); + metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount")); + + ClientSession clientSession = channelSession(channel); + try (MockedStatic sshHelper = mockStatic(SshHelper.class)) { + sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean())) + .thenReturn(clientSession); + sshCollect.collect(builder, metrics); + } + + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals(1, builder.getValuesCount()); + assertEquals("52", builder.getValues(0).getColumns(0)); + assertEquals("35.8033", builder.getValues(0).getColumns(1)); + assertEquals("5%", builder.getValues(0).getColumns(2)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3)); + } + + @Test + void collectTreatsSilentEmptyOneRowOutputAsEmptyRow() throws Exception { + ChannelExec channel = oneRowChannel("", "", 1); + Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build(); + metrics.setAliasFields(List.of("nfs_mount")); + + ClientSession clientSession = channelSession(channel); + try (MockedStatic sshHelper = mockStatic(SshHelper.class)) { + sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean())) + .thenReturn(clientSession); + sshCollect.collect(builder, metrics); + } + + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0)); + } + + @Test + void collectFailsOnEmptyOutputWithStderr() throws Exception { + ChannelExec channel = oneRowChannel("", "boom: permission denied", 1); + Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build(); + metrics.setAliasFields(List.of("nfs_mount")); + + ClientSession clientSession = channelSession(channel); + try (MockedStatic sshHelper = mockStatic(SshHelper.class)) { + sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean())) + .thenReturn(clientSession); + sshCollect.collect(builder, metrics); + } + + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + assertEquals("boom: permission denied", builder.getMsg()); + } + + @Test + void collectDecodesOutputWithConfiguredCharset() throws Exception { + ChannelExec channel = oneRowChannel("挂载正常".getBytes(Charset.forName("GBK")), new byte[0], 0); + SshProtocol protocol = oneRowProtocol(); + protocol.setCharset("GBK"); + Metrics metrics = Metrics.builder().ssh(protocol).build(); + metrics.setAliasFields(List.of("nfs_mount")); + + ClientSession clientSession = channelSession(channel); + try (MockedStatic sshHelper = mockStatic(SshHelper.class)) { + sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean())) + .thenReturn(clientSession); + sshCollect.collect(builder, metrics); + } + + assertEquals(CollectRep.Code.SUCCESS, builder.getCode()); + assertEquals("挂载正常", builder.getValues(0).getColumns(0)); + } + + private SshProtocol oneRowProtocol() { + return SshProtocol.builder() + .host("target.example.com") + .port("22") + .username("root") + .password("password") + .timeout("1000") + .reuseConnection("true") + .useProxy("false") + .script("echo ok") + .parseType("oneRow") + .build(); + } + + private ClientSession channelSession(ChannelExec channel) throws IOException { + ClientSession clientSession = mock(ClientSession.class); + when(clientSession.createExecChannel("echo ok")).thenReturn(channel); + return clientSession; + } + + private ChannelExec oneRowChannel(String stdout, String stderr, int exitStatus) throws IOException { + return oneRowChannel(stdout.getBytes(StandardCharsets.UTF_8), stderr.getBytes(StandardCharsets.UTF_8), exitStatus); + } + + private ChannelExec oneRowChannel(byte[] stdout, byte[] stderr, int exitStatus) throws IOException { + ChannelExec channel = mock(ChannelExec.class); + OpenFuture openFuture = mock(OpenFuture.class); + CloseFuture closeFuture = mock(CloseFuture.class); + AtomicReference out = new AtomicReference<>(); + AtomicReference err = new AtomicReference<>(); + doAnswer(inv -> { + out.set(inv.getArgument(0)); + return null; + }).when(channel).setOut(any()); + doAnswer(inv -> { + err.set(inv.getArgument(0)); + return null; + }).when(channel).setErr(any()); + when(channel.open()).thenReturn(openFuture); + when(channel.waitFor(any(), anyLong())).thenAnswer(inv -> { + out.get().write(stdout); + err.get().write(stderr); + return Set.of(ClientChannelEvent.CLOSED); + }); + when(channel.getExitStatus()).thenReturn(exitStatus); + when(channel.close(false)).thenReturn(closeFuture); + when(closeFuture.await(anyLong())).thenReturn(true); + return channel; + } + private SshProtocol protocol(int timeout) { return SshProtocol.builder() .host("target.example.com") diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/protocol/SshProtocol.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/protocol/SshProtocol.java index 1b79e45528..b2f98a3e05 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/protocol/SshProtocol.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/protocol/SshProtocol.java @@ -86,6 +86,11 @@ public class SshProtocol implements CommonRequestProtocol, Protocol { */ private String parseType; + /** + * Charset of the remote command output, default UTF-8 + */ + private String charset; + /** * IP ADDRESS OR DOMAIN NAME OF THE PEER PROXY HOST */ From 1ebfe76c37ae4b8536c9f914384e5184c5e9ed7f Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:05:32 -0700 Subject: [PATCH 15/18] [fix] evaluate threshold rules with empty field values (#4307) Co-authored-by: aias00 --- .../MetricsRealTimeAlertCalculator.java | 16 ++-- ...tricsRealTimeAlertCalculatorMatchTest.java | 88 +++++++++++++++++++ 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java index f28a48a5d2..126041ca69 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java @@ -251,20 +251,14 @@ public class MetricsRealTimeAlertCalculator { } final int fieldType = field.getType(); + // strict jexl aborts the whole rule on undefined variables, + // so define every field even when its value is empty or unparseable if (fieldType == CommonConstants.TYPE_NUMBER) { - final Double doubleValue; - if ((doubleValue = CommonUtil.parseStrDouble(valueStr)) != null) { - fieldValueMap.put(field.getName(), doubleValue); - } + fieldValueMap.put(field.getName(), CommonUtil.parseStrDouble(valueStr)); } else if (fieldType == CommonConstants.TYPE_TIME) { - final Integer integerValue; - if ((integerValue = CommonUtil.parseStrInteger(valueStr)) != null) { - fieldValueMap.put(field.getName(), integerValue); - } + fieldValueMap.put(field.getName(), CommonUtil.parseStrInteger(valueStr)); } else { - if (StringUtils.isNotEmpty(valueStr)) { - fieldValueMap.put(field.getName(), valueStr); - } + fieldValueMap.put(field.getName(), valueStr); } if (field.getLabel()) { diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java index 6844bb1bcb..44929a5d29 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java @@ -27,11 +27,13 @@ import org.apache.hertzbeat.alert.service.AlertDefineService; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -43,6 +45,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -254,4 +257,89 @@ public class MetricsRealTimeAlertCalculatorMatchTest { verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any()); } + @Test + void testEmptyStringFieldStillTriggersAlert() throws InterruptedException { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + builder.setId(518679137103104L) + .setApp("fullsite") + .setMetrics("summary") + .setPriority(1) + .setCode(CollectRep.Code.SUCCESS); + + CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build(); + CollectRep.Field statusCode = CollectRep.Field.newBuilder().setName("statusCode").setType(CommonConstants.TYPE_STRING).build(); + CollectRep.Field errorMsg = CollectRep.Field.newBuilder().setName("errorMsg").setType(CommonConstants.TYPE_STRING).build(); + + Map meta = new HashMap<>(); + meta.put(MetricDataConstants.INSTANCE_NAME, "site"); + meta.put(MetricDataConstants.INSTANCE, "127.0.0.1"); + + builder.addMetadataAll(meta); + builder.addAllFields(Lists.newArrayList(url, statusCode, errorMsg)); + builder.addValueRow(CollectRep.ValueRow.newBuilder() + .addColumn("https://example.com/broken").addColumn("404").addColumn("").build()); + + CollectRep.MetricsData metricsData = builder.build(); + + AlertDefine matchDefine = new AlertDefine(); + matchDefine.setId(1L); + matchDefine.setName("sitemap-status"); + matchDefine.setExpr("equals(__app__,\"fullsite\") && !matches(statusCode,\"^2[0-9]+\") && !contains(errorMsg,\"timed out\")"); + matchDefine.setTemplate("site down: ${url}"); + matchDefine.setTimes(1); + + when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(matchDefine)); + when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException()); + + metricsRealTimeAlertCalculator.startCalculate(); + + Thread.sleep(3000); + + ArgumentCaptor alertCaptor = ArgumentCaptor.forClass(SingleAlert.class); + verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(alertCaptor.capture()); + assertEquals("site down: https://example.com/broken", alertCaptor.getValue().getContent()); + } + + @Test + void testUnparseableNumberFieldDoesNotAbortRule() throws InterruptedException { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + builder.setId(518679137103105L) + .setApp("fullsite") + .setMetrics("summary") + .setPriority(1) + .setCode(CollectRep.Code.SUCCESS); + + CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build(); + CollectRep.Field responseTime = CollectRep.Field.newBuilder().setName("responseTime").setType(CommonConstants.TYPE_NUMBER).build(); + + Map meta = new HashMap<>(); + meta.put(MetricDataConstants.INSTANCE_NAME, "site"); + meta.put(MetricDataConstants.INSTANCE, "127.0.0.1"); + + builder.addMetadataAll(meta); + builder.addAllFields(Lists.newArrayList(url, responseTime)); + builder.addValueRow(CollectRep.ValueRow.newBuilder() + .addColumn("https://example.com/a").addColumn("").build()); + + CollectRep.MetricsData metricsData = builder.build(); + + AlertDefine guardedDefine = new AlertDefine(); + guardedDefine.setId(2L); + guardedDefine.setName("slow-site"); + guardedDefine.setExpr("equals(__app__,\"fullsite\") && exists(responseTime) && responseTime > 100"); + guardedDefine.setTemplate("slow: ${url}"); + guardedDefine.setTimes(1); + + when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(guardedDefine)); + when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException()); + + metricsRealTimeAlertCalculator.startCalculate(); + + Thread.sleep(3000); + + // unparseable number is defined as null: exists() short-circuits to false, no alarm and no abort + verify(alarmCommonReduce, never()).reduceAndSendAlarm(any()); + verify(alarmCacheManager, times(1)).removeFiring(any(), any()); + } + } From a08a36595273fa2f60f19d5f88757c02cd28b27f Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:33:06 -0700 Subject: [PATCH 16/18] fix(collector): keep partial telnet metrics, surface raw reply (#4311) Co-authored-by: aias00 --- .../collect/telnet/TelnetCollectImpl.java | 48 +++++--- .../collect/telnet/TelnetCollectImplTest.java | 110 ++++++++++++++++++ 2 files changed, 140 insertions(+), 18 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImpl.java index 20602caced..e10860cef0 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImpl.java @@ -67,14 +67,20 @@ public class TelnetCollectImpl extends AbstractCollect { long responseTime = System.currentTimeMillis() - startTime; List aliasFields = metrics.getAliasFields(); String app = builder.getApp(); - Map resultMap = execCmdAndParseResult(telnetClient, telnet.getCmd(), app); - resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime)); - if (resultMap.size() < aliasFields.size()) { - log.error("telnet response data not enough: {}", resultMap); + CmdResult cmdResult = execCmdAndParseResult(telnetClient, telnet.getCmd(), app); + Map resultMap = cmdResult.values(); + boolean expectsCmdMetrics = StringUtils.isNotBlank(telnet.getCmd()) + && aliasFields.stream().anyMatch(field -> !CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(field)); + boolean hasExpectedMetric = aliasFields.stream().anyMatch(resultMap::containsKey); + if (expectsCmdMetrics && !hasExpectedMetric) { + // e.g. zookeeper refusing a 4lw command not in its 4lw.commands.whitelist + String reply = sanitizeReply(cmdResult.rawResponse()); + log.warn("telnet cmd [{}] returned no expected metrics: {}", telnet.getCmd(), reply); builder.setCode(CollectRep.Code.FAIL); - builder.setMsg("The cmd execution results do not match the expected number of metrics."); + builder.setMsg("Cmd [" + telnet.getCmd() + "] returned no expected metrics. Response: " + reply); return; } + resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime)); CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); for (String field : aliasFields) { String fieldValue = resultMap.get(field); @@ -118,30 +124,36 @@ public class TelnetCollectImpl extends AbstractCollect { return DispatchConstants.PROTOCOL_TELNET; } - private static Map execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException { + record CmdResult(Map values, String rawResponse) { + } + + private static String sanitizeReply(String raw) { + return StringUtils.abbreviate(raw.trim().replaceAll("[\\p{Cntrl}]+", " "), 300); + } + + private static CmdResult execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException { if (cmd == null || StringUtils.isEmpty(cmd.trim())) { - return new HashMap<>(16); + return new CmdResult(new HashMap<>(16), ""); } OutputStream outputStream = telnetClient.getOutputStream(); outputStream.write(cmd.getBytes(StandardCharsets.UTF_8)); outputStream.flush(); String result = new String(telnetClient.getInputStream().readAllBytes()); String[] lines = result.split("\n"); - if (CollectorConstants.ZOOKEEPER_APP.equals(app) && CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) { + if (lines.length > 0 && CollectorConstants.ZOOKEEPER_APP.equals(app) + && CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) { lines = Arrays.stream(lines) .skip(1) .toArray(String[]::new); } - boolean contains = lines[0].contains("="); - return Arrays.stream(lines) - .map(item -> { - if (contains) { - return item.split("="); - } else { - return item.split("\t"); - } - }) + if (lines.length == 0) { + return new CmdResult(new HashMap<>(16), result); + } + String separator = lines[0].contains("=") ? "=" : "\t"; + Map values = Arrays.stream(lines) + .map(item -> item.split(separator, 2)) .filter(item -> item.length == 2) - .collect(Collectors.toMap(x -> x[0], x -> x[1])); + .collect(Collectors.toMap(x -> x[0], x -> x[1], (first, second) -> first, HashMap::new)); + return new CmdResult(values, result); } } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImplTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImplTest.java index 0c1b724a8e..d824abe42a 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImplTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/telnet/TelnetCollectImplTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -30,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.net.telnet.TelnetClient; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.TelnetProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; @@ -155,6 +157,114 @@ class TelnetCollectImplTest { mocked.close(); } + @Test + void testCollectPadsMissingMetrics() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a", "b", "c")); + try (MockedConstruction mocked = mockTelnetReply("a=1")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(1, builder.getValuesCount()); + assertEquals("1", builder.getValues(0).getColumns(1)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2)); + assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3)); + } + + @Test + void testCollectFailsWithRawReplyWhenNothingParsed() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a")); + try (MockedConstruction mocked = + mockTelnetReply("conf is not executed because it is not in the whitelist.")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + assertTrue(builder.getMsg().contains("not in the whitelist")); + } + + @Test + void testCollectKeepsFirstOnDuplicateKeys() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("mntr", List.of("a", "b")); + try (MockedConstruction mocked = mockTelnetReply("a=1\na=2\nb=3")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(1, builder.getValuesCount()); + assertEquals("1", builder.getValues(0).getColumns(0)); + assertEquals("3", builder.getValues(0).getColumns(1)); + } + + @Test + void testCollectFailsWhenReplyHasOnlyUnrelatedPairs() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a")); + try (MockedConstruction mocked = mockTelnetReply("error=conf disabled")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + } + + @Test + void testCollectSurvivesHeaderOnlyEnviReply() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper"); + Metrics metrics = telnetMetrics("envi", List.of("responseTime", "a")); + try (MockedConstruction mocked = mockTelnetReply("Environment:")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + } + + @Test + void testCollectFailsCleanlyOnNewlineOnlyReply() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper"); + Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a")); + try (MockedConstruction mocked = mockTelnetReply("\n")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(CollectRep.Code.FAIL, builder.getCode()); + assertTrue(builder.getMsg().contains("returned no expected metrics")); + } + + @Test + void testCollectKeepsValueContainingSeparator() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("conf", List.of("dataDir", "secureClientPort")); + try (MockedConstruction mocked = mockTelnetReply("dataDir=/data/zk=a\nsecureClientPort=")) { + telnetCollect.collect(builder, metrics); + } + assertEquals(1, builder.getValuesCount()); + assertEquals("/data/zk=a", builder.getValues(0).getColumns(0)); + assertEquals("", builder.getValues(0).getColumns(1)); + } + + @Test + void testCollectBlankCmdKeepsResponseTimeRow() { + CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder(); + Metrics metrics = telnetMetrics("", List.of("responseTime")); + try (MockedConstruction mocked = + Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) -> + Mockito.when(telnetClient.isConnected()).thenReturn(true))) { + telnetCollect.collect(builder, metrics); + } + assertEquals(1, builder.getValuesCount()); + } + + private static Metrics telnetMetrics(String cmd, List aliasFields) { + Metrics metrics = new Metrics(); + metrics.setTelnet(TelnetProtocol.builder().timeout("10").port("2181").cmd(cmd).build()); + metrics.setAliasFields(aliasFields); + return metrics; + } + + private static MockedConstruction mockTelnetReply(String reply) { + InputStream inputStream = new ByteArrayInputStream(reply.getBytes(StandardCharsets.UTF_8)); + return Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) -> { + Mockito.when(telnetClient.isConnected()).thenReturn(true); + Mockito.when(telnetClient.getOutputStream()).thenReturn(Mockito.mock(OutputStream.class)); + Mockito.when(telnetClient.getInputStream()).thenReturn(inputStream); + }); + } + @Test void preCheck() throws IllegalArgumentException { // metrics is null From 1573236b153e7e08f48d88059e8561b42695d700 Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:41:56 -0700 Subject: [PATCH 17/18] fix(deps): replace bouncycastle jdk15on with jdk18on 1.85 (#4312) Co-authored-by: aias00 --- .../hertzbeat-collector-basic/pom.xml | 4 +- .../collect/mqtt/MqttSslFactoryTest.java | 88 +++++++++++++++++++ .../hertzbeat-collector-nebulagraph/pom.xml | 13 +++ .../VesoftSslBouncyCastleSmokeTest.java | 67 ++++++++++++++ material/licenses/LICENSE | 6 +- material/licenses/backend/LICENSE | 6 +- ...jdk15on.txt => LICENSE-bcpkix-jdk18on.txt} | 0 ...jdk15on.txt => LICENSE-bcprov-jdk18on.txt} | 0 ...jdk15on.txt => LICENSE-bcutil-jdk18on.txt} | 0 material/licenses/collector/LICENSE | 6 +- ...jdk15on.txt => LICENSE-bcpkix-jdk18on.txt} | 0 ...jdk15on.txt => LICENSE-bcprov-jdk18on.txt} | 0 ...jdk15on.txt => LICENSE-bcutil-jdk18on.txt} | 0 pom.xml | 1 + 14 files changed, 180 insertions(+), 11 deletions(-) create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/mqtt/MqttSslFactoryTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-nebulagraph/src/test/java/org/apache/hertzbeat/collector/collect/nebulagraph/VesoftSslBouncyCastleSmokeTest.java rename material/licenses/backend/{LICENSE-bcpkix-jdk15on.txt => LICENSE-bcpkix-jdk18on.txt} (100%) rename material/licenses/backend/{LICENSE-bcprov-jdk15on.txt => LICENSE-bcprov-jdk18on.txt} (100%) rename material/licenses/backend/{LICENSE-bcutil-jdk15on.txt => LICENSE-bcutil-jdk18on.txt} (100%) rename material/licenses/collector/{LICENSE-bcpkix-jdk15on.txt => LICENSE-bcpkix-jdk18on.txt} (100%) rename material/licenses/collector/{LICENSE-bcprov-jdk15on.txt => LICENSE-bcprov-jdk18on.txt} (100%) rename material/licenses/collector/{LICENSE-bcutil-jdk15on.txt => LICENSE-bcutil-jdk18on.txt} (100%) diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index 63c88e262d..a59700956f 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -151,8 +151,8 @@ org.bouncycastle - bcpkix-jdk15on - 1.68 + bcpkix-jdk18on + ${bouncycastle.version} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/mqtt/MqttSslFactoryTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/mqtt/MqttSslFactoryTest.java new file mode 100644 index 0000000000..51a98f988a --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/collect/mqtt/MqttSslFactoryTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mqtt; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.StringWriter; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.cert.X509Certificate; +import java.util.Date; +import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.openssl.jcajce.JcaPKCS8Generator; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class MqttSslFactoryTest { + + private static String certPem; + private static String pkcs1KeyPem; + private static String pkcs8KeyPem; + + @BeforeAll + static void generateCertAndKeys() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + KeyPair keyPair = generator.generateKeyPair(); + X500Name subject = new X500Name("CN=hb-3540-mqtt"); + JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder( + subject, BigInteger.ONE, + new Date(System.currentTimeMillis() - 60_000), + new Date(System.currentTimeMillis() + 3_600_000), + subject, keyPair.getPublic()); + X509Certificate cert = new JcaX509CertificateConverter() + .getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate()))); + + certPem = writePem(cert); + pkcs1KeyPem = writePem(keyPair.getPrivate()); + pkcs8KeyPem = writePem(new JcaPKCS8Generator(keyPair.getPrivate(), null)); + } + + @Test + void parsesPkcs1ClientKey() { + assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs1KeyPem), true)); + } + + @Test + void parsesPkcs8ClientKey() { + assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs8KeyPem), true)); + } + + private static MqttProtocol mqttProtocol(String clientKey) { + return MqttProtocol.builder() + .tlsVersion("TLSv1.2") + .clientCert(certPem) + .clientKey(clientKey) + .build(); + } + + private static String writePem(Object object) throws Exception { + StringWriter out = new StringWriter(); + try (JcaPEMWriter writer = new JcaPEMWriter(out)) { + writer.writeObject(object); + } + return out.toString(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml index 9ea2e9d914..d4e5dedd92 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml @@ -54,7 +54,20 @@ com.alibaba fastjson + + org.bouncycastle + bcpkix-jdk15on + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/test/java/org/apache/hertzbeat/collector/collect/nebulagraph/VesoftSslBouncyCastleSmokeTest.java b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/test/java/org/apache/hertzbeat/collector/collect/nebulagraph/VesoftSslBouncyCastleSmokeTest.java new file mode 100644 index 0000000000..7a0e49c709 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/test/java/org/apache/hertzbeat/collector/collect/nebulagraph/VesoftSslBouncyCastleSmokeTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.nebulagraph; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.vesoft.nebula.client.graph.data.CASignedSSLParam; +import com.vesoft.nebula.util.SslUtil; +import java.io.FileWriter; +import java.math.BigInteger; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.cert.X509Certificate; +import java.util.Date; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class VesoftSslBouncyCastleSmokeTest { + + @Test + void vesoftSslUtilWorksWithBouncyCastleJdk18on(@TempDir Path dir) throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + KeyPair keyPair = generator.generateKeyPair(); + X500Name subject = new X500Name("CN=hb-3540-smoke"); + JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder( + subject, BigInteger.ONE, + new Date(System.currentTimeMillis() - 60_000), + new Date(System.currentTimeMillis() + 3_600_000), + subject, keyPair.getPublic()); + X509Certificate cert = new JcaX509CertificateConverter() + .getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate()))); + + Path crt = dir.resolve("smoke.crt"); + Path key = dir.resolve("smoke.key"); + try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(crt.toFile()))) { + writer.writeObject(cert); + } + try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(key.toFile()))) { + writer.writeObject(keyPair.getPrivate()); + } + + CASignedSSLParam param = new CASignedSSLParam(crt.toString(), crt.toString(), key.toString()); + assertNotNull(SslUtil.getSSLSocketFactoryWithCA(param)); + } +} diff --git a/material/licenses/LICENSE b/material/licenses/LICENSE index dd1c37687e..93891a34d2 100644 --- a/material/licenses/LICENSE +++ b/material/licenses/LICENSE @@ -525,9 +525,9 @@ The following components are provided under the MIT License. See project link fo The text of each license is also included in licenses/LICENSE-[project].txt. https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT diff --git a/material/licenses/backend/LICENSE b/material/licenses/backend/LICENSE index 3440fc8c31..a15eb9a0d0 100644 --- a/material/licenses/backend/LICENSE +++ b/material/licenses/backend/LICENSE @@ -524,9 +524,9 @@ The following components are provided under the MIT License. See project link fo The text of each license is also included in licenses/LICENSE-[project].txt. https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT diff --git a/material/licenses/backend/LICENSE-bcpkix-jdk15on.txt b/material/licenses/backend/LICENSE-bcpkix-jdk18on.txt similarity index 100% rename from material/licenses/backend/LICENSE-bcpkix-jdk15on.txt rename to material/licenses/backend/LICENSE-bcpkix-jdk18on.txt diff --git a/material/licenses/backend/LICENSE-bcprov-jdk15on.txt b/material/licenses/backend/LICENSE-bcprov-jdk18on.txt similarity index 100% rename from material/licenses/backend/LICENSE-bcprov-jdk15on.txt rename to material/licenses/backend/LICENSE-bcprov-jdk18on.txt diff --git a/material/licenses/backend/LICENSE-bcutil-jdk15on.txt b/material/licenses/backend/LICENSE-bcutil-jdk18on.txt similarity index 100% rename from material/licenses/backend/LICENSE-bcutil-jdk15on.txt rename to material/licenses/backend/LICENSE-bcutil-jdk18on.txt diff --git a/material/licenses/collector/LICENSE b/material/licenses/collector/LICENSE index ea272bc156..6c87257830 100644 --- a/material/licenses/collector/LICENSE +++ b/material/licenses/collector/LICENSE @@ -396,9 +396,9 @@ The following components are provided under the MIT License. See project link fo The text of each license is also included in licenses/LICENSE-[project].txt. https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT - https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT + https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT https://mvnrepository.com/artifact/org.slf4j/jcl-over-slf4j/2.0.9 MIT https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/2.0.9 MIT diff --git a/material/licenses/collector/LICENSE-bcpkix-jdk15on.txt b/material/licenses/collector/LICENSE-bcpkix-jdk18on.txt similarity index 100% rename from material/licenses/collector/LICENSE-bcpkix-jdk15on.txt rename to material/licenses/collector/LICENSE-bcpkix-jdk18on.txt diff --git a/material/licenses/collector/LICENSE-bcprov-jdk15on.txt b/material/licenses/collector/LICENSE-bcprov-jdk18on.txt similarity index 100% rename from material/licenses/collector/LICENSE-bcprov-jdk15on.txt rename to material/licenses/collector/LICENSE-bcprov-jdk18on.txt diff --git a/material/licenses/collector/LICENSE-bcutil-jdk15on.txt b/material/licenses/collector/LICENSE-bcutil-jdk18on.txt similarity index 100% rename from material/licenses/collector/LICENSE-bcutil-jdk15on.txt rename to material/licenses/collector/LICENSE-bcutil-jdk18on.txt diff --git a/pom.xml b/pom.xml index 087c7119a7..1def3df5ae 100644 --- a/pom.xml +++ b/pom.xml @@ -165,6 +165,7 @@ 1.4.5 3.1.1 3.6.0 + 1.85 1.0.0 3.1.37 3.23.5 From 33284bb95553719782b8084161c23c8ecdaa89ae Mon Sep 17 00:00:00 2001 From: NekoPunch <95899648+orangeCatDeveloper@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:46:55 -0700 Subject: [PATCH 18/18] [fix] strip NUL bytes before tdengine history insert (#4310) Co-authored-by: aias00 --- .../tsdb/tdengine/TdEngineDataStorage.java | 10 ++++--- .../store/TdEngineDataStorageTest.java | 29 +++++++++++++++++-- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java index 361d45dbbb..94eb19413d 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java @@ -66,6 +66,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://"; private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')"); + private static final Pattern NUL_CHAR_PATTERN = Pattern.compile("\\u0000"); private static final String INSTANCE_NULL = "''"; private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s"; private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s"; @@ -346,12 +347,13 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } private String formatStringValue(String value) { - String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0"); - // bugfix Argument list too long - if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) { + // snmp octet strings may carry NUL padding that breaks the insert sql + String formatValue = NUL_CHAR_PATTERN.matcher(value).replaceAll(""); + // truncate the logical value before escaping so the cut cannot split an escape sequence + if (formatValue.length() > tableStrColumnDefineMaxLength) { formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength); } - return formatValue; + return SQL_SPECIAL_STRING_PATTERN.matcher(formatValue).replaceAll("\\\\$0"); } @Override diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java index 97f94090c6..2c3327f87e 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java @@ -115,6 +115,25 @@ class TdEngineDataStorageTest { assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7"); } + @Test + void testSaveDataStripsControlCharacters() throws Exception { + tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties); + setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource); + setParentPrivateField(tdEngineDataStorage, "serverAvailable", true); + + // snmp octet strings can carry NUL bytes (issue #1481); tabs are legitimate data and stay + CollectRep.MetricsData metricsData = generateMockedMetricsData("Loopback\tInterface 1\u0000"); + tdEngineDataStorage.saveData(metricsData); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture()); + String executedSql = sqlCaptor.getValue(); + + assertTrue(executedSql.contains("Loopback\tInterface 1'"), "NUL stripped, tab preserved"); + assertTrue(executedSql.indexOf('\u0000') < 0, "no raw NUL byte may reach the sql text"); + assertTrue(!executedSql.contains("\\u0000"), "no textual NUL escape may survive in the labels json"); + } + @Test void destroy() { } @@ -146,6 +165,10 @@ class TdEngineDataStorageTest { } public static CollectRep.MetricsData generateMockedMetricsData() { + return generateMockedMetricsData("test-%server-01"); + } + + public static CollectRep.MetricsData generateMockedMetricsData(String instanceValue) { CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class); when(mockMetricsData.getId()).thenReturn(0L); @@ -156,9 +179,9 @@ class TdEngineDataStorageTest { when(mockMetricsData.getInstance()).thenReturn("test-%server-01"); CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class); - List columnValues = List.of("test-%server-01", "68.7"); + List columnValues = List.of(instanceValue, "68.7"); when(mockValueRow.getColumnsList()).thenReturn(columnValues); - when(mockValueRow.getColumns(0)).thenReturn("test-%server-01"); + when(mockValueRow.getColumns(0)).thenReturn(instanceValue); when(mockValueRow.getColumns(1)).thenReturn("68.7"); List mockValueRowsList = List.of(mockValueRow); when(mockMetricsData.getValues()).thenReturn(mockValueRowsList); @@ -177,7 +200,7 @@ class TdEngineDataStorageTest { Field instanceArrowField = new Field("instance", instanceFieldType, null); ArrowCell instanceCell = Mockito.mock(ArrowCell.class); when(instanceCell.getField()).thenReturn(instanceArrowField); - when(instanceCell.getValue()).thenReturn("test-%server-01"); + when(instanceCell.getValue()).thenReturn(instanceValue); when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true); when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);