Compare commits

...
Author SHA1 Message Date
LogicandDuansg b9f1d65cb5 [refactor] Apache Fory (#3873)
Signed-off-by: Logic <zqr10159@dromara.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-27 17:28:44 +08:00
Logic 1dea5467fc [refactor]: springboot4.0.0 (#3872) 2025-11-27 17:22:07 +08:00
DeleiGuo 2f122dfb05 [docs] fix the missing documents references in the sidebar (#3867) 2025-11-26 16:52:43 +08:00
f7b0010180 fix: Remove redundant metrics data send in KafkaCommonDataQueue (#3854)
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-24 22:11:49 +08:00
2c0fecaa62 [improve] Added simple exponential backoff strategy (#3860)
Signed-off-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yang Chen <1597081640@qq.com>
2025-11-21 16:28:21 +08:00
548966906c [bugfix] align tooltip position for mute icon with other icons (#3862)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-11-20 22:22:56 +08:00
6473546f56 [feat] Add DB2 database monitoring support (#3857)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-20 22:16:52 +08:00
035d63c880 [feature] Support FeiShu/Lark App Alert Notification (#3856)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Calvin <zhengqiwei@apache.org>
2025-11-20 00:14:57 +08:00
Yang ChenandCalvin d56176dfa5 feat: add new docker-compose deployment solution (#3859)
Co-authored-by: Calvin <zhengqiwei@apache.org>
2025-11-19 23:49:13 +08:00
Yang ChenandCalvin c1815f5107 fix: allow access to /log/** path to resolve unauthorized error on refresh (#3858)
Co-authored-by: Calvin <zhengqiwei@apache.org>
2025-11-18 01:33:29 +08:00
7ac175e726 [Improve]Unify UI border-radius to 6px for a more professional look (#3853)
Co-authored-by: Duansg <siguoduan@gmail.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-11-17 23:02:21 +08:00
eb5e843251 [feat] Add webhook authentication support for alert notifications (#3797)
Co-authored-by: Sherlock Yin <sherlock.yin1994@gmail.com>
Co-authored-by: yunfan24 <yunfan24@outlook.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: kangli <likang@apache.org>
Co-authored-by: 刘俊辉 <liu.junhui3@iwhalecloud.com>
2025-11-17 22:59:35 +08:00
starryCoderandDuansg bc098aa4d2 [bugfix]manager: Avoid NPE when metrics data queue is empty (#3848)
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-14 16:09:49 +08:00
Tomsun28 bcd394b47f [docs] update doc fix wrong manager (#3850)
Signed-off-by: tomsun28 <tomsun28@outlook.com>
2025-11-08 20:33:42 +08:00
Duansg fd2202fcaf [doc] add new committer blog (#3847) 2025-11-06 21:37:17 +08:00
a7629e317a Modify the issue of incorrect signature in Alibaba's SMS sending (#3788)
Co-authored-by: Logic <zqr10159@dromara.org>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: shown <yuluo08290126@gmail.com>
Co-authored-by: yunfan24 <yunfan24@outlook.com>
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-06 10:38:48 +08:00
194 changed files with 5059 additions and 1909 deletions
+10 -10
View File
@@ -90,24 +90,24 @@ e2e:
backend:
- changed-files:
- any-glob-to-any-file:
- "alerter/**/*"
- "collector/**/*"
- "common/**/*"
- "grafana/**/*"
- "manager/**/*"
- "push/**/*"
- "remoting/**/*"
- "warehouse/**/*"
- "hertzbeat-alerter/**/*"
- "hertzbeat-collector/**/*"
- "hertzbeat-common/**/*"
- "hertzbeat-grafana/**/*"
- "hertzbeat-manager/**/*"
- "hertzbeat-push/**/*"
- "hertzbeat-remoting/**/*"
- "hertzbeat-warehouse/**/*"
collector:
- changed-files:
- any-glob-to-any-file:
- "collector/**/*"
- "hertzbeat-collector/**/*"
monitoring-template:
- changed-files:
- any-glob-to-any-file:
- "manager/src/main/resources/define/**/*"
- "hertzbeat-manager/src/main/resources/define/**/*"
HIP:
- changed-files:
+3 -3
View File
@@ -4,7 +4,7 @@ image:
tasks:
- name: Run backend
before: cd hertzbeat-manager
before: cd hertzbeat-startup
command: |
gp sync-await setup-backend
mvn spring-boot:run
@@ -13,7 +13,7 @@ tasks:
before: cd web-app
command: |
gp sync-await setup-frontend
yarn start --public-host "`gp url 4200`"
pnpm start --public-host "`gp url 4200`"
openMode: split-right
- name: Setup backend
@@ -25,7 +25,7 @@ tasks:
- name: Setup frontend
init: |
cd web-app && yarn install
cd web-app && pnpm install
command: |
gp sync-done setup-frontend
exit 0
+8 -30
View File
@@ -20,27 +20,27 @@
- 💬**Suggestion**: You can also make some suggestions to the project or community to promote the healthy development of the community;
More see [Contribution Types](https://allcontributors.org/docs/en/emoji-key)
More see [Contribution Types](https://allcontributors.org/emoji-key)
Even small corrections to typos are very welcome :)
### Getting HertzBeat up and running
> To get HertzBeat code running on your development tools, and able to debug with breakpoints.
> This is a front-end and back-end separation project. To start the local code, the back-end [manager](manager) and the front-end [web-app](web-app) must be started separately.
> This is a front-end and back-end separation project. To start the local code, the back-end [startup](hertzbeat-startup) and the front-end [web-app](web-app) must be started separately.
#### Backend start
1. Requires `maven3+`, `java17` and `lombok` environments
2. (Optional) Modify the configuration file: `manager/src/main/resources/application.yml`
2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml`
3. Execute under the project root directory: `mvn clean install -DskipTests`
4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`
5. Start `springboot manager` service: `manager/src/main/java/org/apache/hertzbeat/hertzbeat-manager/Manager.java`
5. Start `springboot startup` service: `hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java`
#### Frontend start
@@ -131,17 +131,6 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
## 🥐 Architecture
- **[manager](https://github.com/apache/hertzbeat/tree/master/hertzbeat-manager)** Provide monitoring management, system management basic services.
> Provides monitoring management, monitoring configuration management, system user management, etc.
- **[collector](https://github.com/apache/hertzbeat/tree/master/collector)** Provide metrics data collection services.
> Use common protocols to remotely collect and obtain peer-to-peer metrics data.
- **[warehouse](https://github.com/apache/hertzbeat/tree/master/warehouse)** Provide monitoring data warehousing services.
> Metrics data management, data query, calculation and statistics.
- **[alerter](https://github.com/apache/hertzbeat/tree/master/hertzbeat-alerter)** Provide alert service.
> Alarm calculation trigger, monitoring status linkage, alarm configuration, and alarm notification.
- **[web-app](https://github.com/apache/hertzbeat/tree/master/web-app)** Provide web ui.
> Angular Web UI.
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
<br>
@@ -172,26 +161,26 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
- 💬**建议**:也可以对项目或者社区提出一些建议,促进社区的良性发展;
更多贡献方式参见 [Contribution Types](https://allcontributors.org/docs/en/emoji-key)
更多贡献方式参见 [Contribution Types](https://allcontributors.org/emoji-key)
即便是小到错别字的修正我们也都非常欢迎 :)
### 让 HertzBeat 运行起来
> 让 HertzBeat 的代码在您的开发工具上运行起来,并且能够断点调试。
> 此为前后端分离项目,本地代码启动需将后端[manager](manager)和前端[web-app](web-app)分别启动生效。
> 此为前后端分离项目,本地代码启动需将后端[hertzbeat-startup](hertzbeat-startup)和前端[web-app](web-app)分别启动生效。
#### 后端启动
1. 需要 `maven3+`, `java17` 和 `lombok` 环境
2. (可选)修改配置文件配置信息-`manager/src/main/resources/application.yml`
2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml`
3. 在项目根目录运行编译: `mvn clean install -DskipTests`
4. 在 `jvm` 加入参数 `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`
5. 启动`springboot manager`服务 `manager/src/main/java/org/apache/hertzbeat/hertzbeat-manager/Manager.java`
5. 启动`springboot startup`服务 `hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/HertzBeatApplication.java`
#### 前端启动
@@ -282,15 +271,4 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group.
### 模块
- **[manager](https://github.com/apache/hertzbeat/tree/master/hertzbeat-manager)** 提供监控管理,系统管理基础服务
> 提供对监控的管理,监控应用配置的管理,系统用户租户后台管理等。
- **[collector](https://github.com/apache/hertzbeat/tree/master/collector)** 提供监控数据采集服务
> 使用通用协议远程采集获取对端指标数据。
- **[warehouse](https://github.com/apache/hertzbeat/tree/master/warehouse)** 提供监控数据仓储服务
> 采集指标结果数据管理,数据落盘,查询,计算统计。
- **[alerter](https://github.com/apache/hertzbeat/tree/master/hertzbeat-alerter)** 提供告警服务
> 告警计算触发,任务状态联动,告警配置,告警通知。
- **[web-app](https://github.com/apache/hertzbeat/tree/master/web-app)** 提供可视化控制台页面
> 监控告警系统可视化控制台前端
![hertzBeat](home/static/img/docs/hertzbeat-arch.png)
+2 -2
View File
@@ -155,8 +155,8 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach
##### 3Start via source code
1. Local source code debugging needs to start the back-end project `manager` and the front-end project `web-app`.
2. Backendneed `maven3+`, `java17`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `manager` service.
1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`.
2. Backendneed `maven3+`, `java17`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service.
3. Webneed `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup.
4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat`
+2 -2
View File
@@ -152,8 +152,8 @@
##### 方式三:本地代码启动
1. 此为前后端分离项目,本地代码调试需要分别启动后端工程 `manager` 和前端工程 `web-app`
2. 后端:需要 `maven3+`, `java17` 和 `lombok` 环境,修改 `YML` 配置信息,添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `manager` 服务即可。
1. 此为前后端分离项目,本地代码调试需要分别启动后端工程 `hertzbeat-startup` 和前端工程 `web-app`
2. 后端:需要 `maven3+`, `java17` 和 `lombok` 环境,修改 `YML` 配置信息,添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。
3. 前端:需要 `nodejs npm angular-cli`环境,待本地后端启动后,在 `web-app` 目录下启动 `ng serve --open`
4. 浏览器访问 `http://localhost:4200` 即可开始,默认账号密码 `admin/hertzbeat`
+2 -2
View File
@@ -155,8 +155,8 @@
##### 方式3:ローカルの実行
1. ローカルの実行には、バックエンドのプロジェクト`manager`とフロントエンドのプロジェクト`web-app`を起動する必要があります。
2. バックエンド:`maven3+`、`java17`と`lombok`の環境は必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し、`manager` を起動します。
1. ローカルの実行には、バックエンドのプロジェクト`hertzbeat-startup`とフロントエンドのプロジェクト`web-app`を起動する必要があります。
2. バックエンド:`maven3+`、`java17`と`lombok`の環境は必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し、`hertzbeat-startup` を起動します。
3. フロントエンド:`nodejs npm angular-cli`の環境は必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `ng serve --open` というコマンドを実行します。
4. スタート:`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード:`admin/hertzbeat`。
+1 -1
View File
@@ -27,7 +27,7 @@
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.3</spring-ai.version>
<java.version>17</java.version>
<java.version>21</java.version>
</properties>
<dependencies>
@@ -17,25 +17,23 @@
package org.apache.hertzbeat.alert;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* alarm module thread pool
* alarm module thread pool with Virtual Threads
*/
@Component
@Slf4j
public class AlerterWorkerPool {
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor notifyExecutor;
private ThreadPoolExecutor logWorkerExecutor;
private ExecutorService workerExecutor;
private ExecutorService notifyExecutor;
private ExecutorService logWorkerExecutor;
public AlerterWorkerPool() {
initWorkExecutor();
@@ -44,62 +42,31 @@ public class AlerterWorkerPool {
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("alerter-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(10,
10,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("alerter-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
private void initNotifyExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter notifyExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("notify-worker-%d")
.build();
notifyExecutor = new ThreadPoolExecutor(6,
6,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("notify-worker-", 0)
.factory();
notifyExecutor = Executors.newThreadPerTaskExecutor(factory);
}
private void initLogWorkerExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter logWorkerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("log-worker-%d")
.build();
logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(1000),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("log-worker-", 0)
.factory();
logWorkerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
/**
* Run the alerter task
* @param runnable task
* @throws RejectedExecutionException when The thread pool is full of
*/
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable){
workerExecutor.execute(runnable);
}
@@ -107,9 +74,8 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the notifyExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeNotify(Runnable runnable) throws RejectedExecutionException {
public void executeNotify(Runnable runnable){
notifyExecutor.execute(runnable);
}
@@ -117,9 +83,8 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the logWorkerExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeLogJob(Runnable runnable) throws RejectedExecutionException {
public void executeLogJob(Runnable runnable){
logWorkerExecutor.execute(runnable);
}
}
}
@@ -17,15 +17,6 @@
package org.apache.hertzbeat.alert.calculate.realtime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.AlerterWorkerPool;
@@ -41,11 +32,24 @@ import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Calculate alarms based on the alarm definition rules and collected data
*/
@@ -122,13 +126,22 @@ public class MetricsRealTimeAlertCalculator {
*/
public void startCalculate() {
Runnable runnable = () -> {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
while (!Thread.currentThread().isInterrupted()) {
try {
CollectRep.MetricsData metricsData = dataQueue.pollMetricsDataToAlerter();
if (metricsData == null) {
continue;
}
backoff.reset();
calculate(metricsData);
dataQueue.sendMetricsDataToStorage(metricsData);
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception e) {
log.error("calculate alarm error: {}.", e.getMessage(), e);
}
@@ -17,13 +17,17 @@
package org.apache.hertzbeat.alert.calculate.realtime;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.calculate.realtime.window.LogWorker;
import org.apache.hertzbeat.alert.calculate.realtime.window.TimeService;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
import org.springframework.stereotype.Component;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.concurrent.LinkedBlockingQueue;
@@ -58,16 +62,23 @@ public class WindowedLogRealTimeAlertCalculator implements Runnable {
@Override
public void run() {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
while (!Thread.currentThread().isInterrupted()) {
try {
LogEntry logEntry = dataQueue.pollLogEntry();
if (logEntry != null) {
processLogEntry(logEntry);
dataQueue.sendLogEntryToStorage(logEntry);
if (logEntry == null) {
continue;
}
backoff.reset();
processLogEntry(logEntry);
dataQueue.sendLogEntryToStorage(logEntry);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception e) {
log.error("Error in log dispatch loop: {}", e.getMessage(), e);
}
@@ -0,0 +1,487 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.notice.impl;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.notice.AlertNoticeException;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Collectors;
/**
* FeiShu app alert notify impl
*/
@Component
@RequiredArgsConstructor
@Slf4j
public class FeiShuAppAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl {
/**
* get tenant access_token url
*/
private static final String TENANT_ACCESS_TOKEN_URL = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal";
/**
* get FeiShu app employee url
*/
private static final String EMPLOYEE_URL = "https://open.feishu.cn/open-apis/ehr/v1/employees?status=2&status=4&user_id_type=user_id&page_size=100";
/**
* send FeiShu app message url
*/
private static final String APP_MESSAGE_URL = "https://open.feishu.cn/open-apis/im/v1/messages";
/**
* send FeiShu app batch message url
*/
private static final String APP_BATCH_MESSAGE_URL = "https://open.feishu.cn/open-apis/message/v4/batch_send/";
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final byte USER_RECEIVE_TYPE = 0;
private static final byte CHAT_RECEIVE_TYPE = 1;
private static final byte PART_RECEIVE_TYPE = 2;
private static final byte ALL_RECEIVE_TYPE = 3;
private final RestTemplate restTemplate;
private final AlerterProperties alerterProperties;
@Override
public void send(NoticeReceiver receiver, NoticeTemplate noticeTemplate, GroupAlert alert) throws AlertNoticeException {
var appId = receiver.getAppId();
var appSecret = receiver.getAppSecret();
var larkReceiveIdType = receiver.getLarkReceiveType();
try {
var accessToken = getAccessToken(appId, appSecret);
var notificationContent = JsonUtil.toJson(renderContent(noticeTemplate, alert));
JsonNode messageContent = createLarkMessage(receiver, notificationContent);
switch (larkReceiveIdType) {
case USER_RECEIVE_TYPE -> {
String[] userIds = receiver.getUserId().split(",");
if (userIds.length == 1) {
sendLarkMessage(accessToken, "user_id", userIds[0], messageContent);
} else {
sendLarkUserBatchMessage(accessToken, userIds, messageContent);
}
}
case CHAT_RECEIVE_TYPE -> sendLarkMessage(accessToken, "chat_id", receiver.getChatId(), messageContent);
case PART_RECEIVE_TYPE ->
sendLarkDepartmentBatchMessage(accessToken, receiver.getPartyId().split(","), messageContent);
case ALL_RECEIVE_TYPE -> {
List<String> userIds = new ArrayList<>();
getLarkEmployeeUserIds(accessToken, null, userIds);
sendLarkUserBatchMessage(accessToken, userIds.toArray(new String[0]), messageContent);
}
default -> throw new AlertNoticeException("Invalid larkReceiveIdType: " + larkReceiveIdType);
}
} catch (Exception e) {
throw new AlertNoticeException("[FeiShu App Notify Error] " + e.getMessage());
}
}
@Override
public byte type() {
return 14;
}
/**
* Send FeiShu app message to chat or designated personnel
*
* @param accessToken Tenant access token
* @param receiverIdType FeiShu app send message receiver id type: user_id, chat_id
* @param receiverId FeiShu app user id or chat id
* @param messageContent Message content
* @see <a href="https://open.feishu.cn/document/server-docs/im-v1/batch_message/send-messages-in-batches">send message</a>
*/
private void sendLarkMessage(String accessToken, String receiverIdType, String receiverId, JsonNode messageContent) throws JsonProcessingException {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setBearerAuth(accessToken);
FeiShuAppMessageDto messageDto = FeiShuAppMessageDto.builder()
.receiveId(receiverId)
.content(escapedCompactJson(messageContent))
.build();
HttpEntity<FeiShuAppMessageDto> request = new HttpEntity<>(messageDto, headers);
call(APP_MESSAGE_URL + "?receive_id_type=" + receiverIdType, request, HttpMethod.POST, FeiShuAppResponse.class);
}
/**
* Send FeiShu app department batch message
*
* @param accessToken Tenant access token
* @param partyIds FeiShu app department ids
* @param messageContent Message content
* @see <a href="https://open.feishu.cn/document/server-docs/im-v1/batch_message/send-messages-in-batches">send batch message</a>
*/
private void sendLarkDepartmentBatchMessage(String accessToken, String[] partyIds, JsonNode messageContent) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setBearerAuth(accessToken);
FeiShuAppBatchMessageDto batchMessageDto = FeiShuAppBatchMessageDto.builder()
.departmentIds(partyIds)
.card(messageContent)
.build();
HttpEntity<FeiShuAppBatchMessageDto> request = new HttpEntity<>(batchMessageDto, headers);
call(APP_BATCH_MESSAGE_URL, request, HttpMethod.POST, FeiShuAppResponse.class);
}
/**
* Send FeiShu app batch message to user
*
* @param accessToken Tenant access token
* @param userIds FeiShu app user ids
* @param messageContent Message content
* @see <a href="https://open.feishu.cn/document/server-docs/im-v1/batch_message/send-messages-in-batches">send batch message</a>
*/
private void sendLarkUserBatchMessage(String accessToken, String[] userIds, JsonNode messageContent) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setBearerAuth(accessToken);
FeiShuAppBatchMessageDto batchMessageDto = FeiShuAppBatchMessageDto.builder()
.userIds(userIds)
.card(messageContent)
.build();
HttpEntity<FeiShuAppBatchMessageDto> request = new HttpEntity<>(batchMessageDto, headers);
FeiShuAppResponse call = call(APP_BATCH_MESSAGE_URL, request, HttpMethod.POST, FeiShuAppResponse.class);
}
/**
* Get FeiShu app tenant access token
*
* @param appId Unique identifier for the application, obtained after creating the application
* @param appSecret Application key, obtained after creating the application
* @return Tenant access token
* @see <a href="https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal">tenant_access_token</a>
*/
private String getAccessToken(String appId, String appSecret) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
FeiShuAppAccessTokenDto accessTokenRequest = FeiShuAppAccessTokenDto.builder()
.appId(appId)
.appSecret(appSecret)
.build();
HttpEntity<FeiShuAppAccessTokenDto> request = new HttpEntity<>(accessTokenRequest, headers);
FeiShuAppAccessTokenResponse data = call(TENANT_ACCESS_TOKEN_URL, request, HttpMethod.POST, FeiShuAppAccessTokenResponse.class);
return data.getTenantAccessToken();
}
/**
* Get FeiShu app employee user ids
*
* @param accessToken FeiShu app tenant access token
* @param pageToken Paging marker, left blank for the first request, indicating traversal from scratch;
* When there are more items in the pagination query result, a new page_token will be returned at the same time.
* The next iteration can use this page_token to obtain the query result
* @param userIds Collection for recursive padding
* @see <a href="https://open.feishu.cn/document/server-docs/ehr-v1/list?appId=cli_a999532b1f52900b">https://open.feishu.cn/open-apis/ehr/v1/employees</a>
*/
private void getLarkEmployeeUserIds(String accessToken, String pageToken, List<String> userIds) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setBearerAuth(accessToken);
HttpEntity<String> request = new HttpEntity<>(headers);
var url = StringUtils.isNotBlank(pageToken) ? EMPLOYEE_URL + "&page_token=" + pageToken : EMPLOYEE_URL;
FeiShuAppEmployeeResponse employeeResponse = call(url,
request,
HttpMethod.GET,
FeiShuAppEmployeeResponse.class);
if (Objects.equals(employeeResponse.getCode(), 0)) {
userIds.addAll(employeeResponse.getData().getItems().stream().map(FeiShuAppEmployeeResponse.Employee::getUserId).toList());
var hasMore = employeeResponse.getData().getHasMore();
if (Boolean.TRUE.equals(hasMore)) {
getLarkEmployeeUserIds(accessToken, employeeResponse.getData().getPageToken(), userIds);
}
}
}
private <R extends FeiShuAppResponse, E> R call(String url, HttpEntity<E> request, HttpMethod httpMethod, Class<R> responseType) {
ResponseEntity<R> response = restTemplate.exchange(url, httpMethod, request, responseType);
if (Objects.nonNull(response.getBody()) && !Objects.equals(response.getBody().getCode(), 0)) {
log.warn("Send FeiShu App Error: {}", response.getBody().getMsg());
throw new AlertNoticeException("Http StatusCode " + response.getStatusCode() + " Error: " + response.getBody().getMsg());
}
return response.getBody();
}
private JsonNode createLarkMessage(NoticeReceiver receiver, String notificationContent) throws JsonProcessingException {
String larkCardMessage = """
{
"schema": "2.0",
"config": {
"update_multi": true,
"locales": [
"en_us",
"zh_cn"
],
"style": {
"text_size": {
"normal_v2": {
"default": "normal",
"pc": "normal",
"mobile": "heading"
}
}
}
},
"body": {
"direction": "vertical",
"padding": "12px 12px 12px 12px",
"elements": [
{
"tag": "markdown",
"content": "%s",
"i18n_content": {
"en_us": ""
},
"text_align": "left",
"text_size": "normal_v2",
"margin": "0px 0px 0px 0px"
},
{
"tag": "hr",
"margin": "0px 0px 0px 0px"
},
{
"tag": "column_set",
"horizontal_align": "left",
"columns": [
{
"tag": "column",
"width": "weighted",
"elements": [
{
"tag": "button",
"text": {
"tag": "plain_text",
"content": "登入控制台",
"i18n_content": {
"en_us": "Login In"
}
},
"type": "default",
"width": "default",
"size": "medium",
"behaviors": [
{
"type": "open_url",
"default_url": "%s",
"pc_url": "",
"ios_url": "",
"android_url": ""
}
]
}
],
"direction": "horizontal",
"vertical_spacing": "8px",
"horizontal_align": "left",
"vertical_align": "top",
"weight": 1
}
],
"margin": "0px 0px 0px 0px"
}
]
},
"header": {
"title": {
"tag": "plain_text",
"content": "HertzBeat 告警",
"i18n_content": {
"en_us": "HertzBeat Alarm"
}
},
"subtitle": {
"tag": "plain_text",
"content": ""
},
"template": "red",
"padding": "12px 12px 12px 12px"
}
}
""";
Byte larkReceiveIdType = receiver.getLarkReceiveType();
String userId = receiver.getUserId();
String atUserElement = "";
if (Objects.equals(larkReceiveIdType, CHAT_RECEIVE_TYPE) && StringUtils.isNotBlank(userId)) {
atUserElement = "\\n" + Arrays.stream(userId.split(","))
.map(id -> "<at id=" + id + "></at>")
.collect(Collectors.joining(" "));
}
if (notificationContent.startsWith("\"") && notificationContent.endsWith("\"")) {
notificationContent = StringUtils.removeStart(notificationContent, "\"");
notificationContent = StringUtils.removeEnd(notificationContent, "\"");
}
String jsonStr = String.format(larkCardMessage,
notificationContent.replace("\"", "\\\"") + atUserElement,
alerterProperties.getConsoleUrl());
return OBJECT_MAPPER.readTree(jsonStr);
}
private String escapedCompactJson(JsonNode json) throws JsonProcessingException {
return OBJECT_MAPPER.writeValueAsString(json);
}
/**
* feiShu app response
*/
@Data
protected static class FeiShuAppResponse {
private Integer code;
private String msg;
}
/**
* FeiShu app message get tenant access token request
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
protected static class FeiShuAppAccessTokenDto {
@JsonProperty("app_id")
private String appId;
@JsonProperty("app_secret")
private String appSecret;
}
/**
* FeiShu app message get tenant access token response
*/
@EqualsAndHashCode(callSuper = true)
@Data
protected static class FeiShuAppAccessTokenResponse extends FeiShuAppResponse {
@JsonProperty("tenant_access_token")
private String tenantAccessToken;
}
/**
* FeiShu app employee response
*/
@EqualsAndHashCode(callSuper = true)
@Data
protected static class FeiShuAppEmployeeResponse extends FeiShuAppResponse {
private EmployeeResponseData data;
@Data
private static class EmployeeResponseData {
@JsonProperty("page_token")
private String pageToken;
@JsonProperty("has_more")
private Boolean hasMore;
private List<Employee> items;
}
@Data
private static class Employee {
@JsonProperty("user_id")
private String userId;
}
}
/**
* FeiShu app message request
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
protected static class FeiShuAppMessageDto {
@JsonProperty("receive_id")
private String receiveId;
@Builder.Default
@JsonProperty("msg_type")
private String msgType = "interactive";
@JsonProperty("content")
private String content;
@Builder.Default
private final String uuid = UUID.randomUUID().toString();
}
/**
* FeiShu app batch message request
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
protected static class FeiShuAppBatchMessageDto {
@Builder.Default
@JsonProperty("msg_type")
private final String msgType = "interactive";
private JsonNode card;
@JsonProperty("department_ids")
private String[] departmentIds;
@JsonProperty("user_ids")
private String[] userIds;
}
}
@@ -48,6 +48,11 @@ final class WebHookAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
}
HttpHeaders headers = new HttpHeaders();
if ("Basic".equalsIgnoreCase(receiver.getHookAuthType())) {
headers.setBasicAuth(receiver.getHookAuthToken());
} else if ("Bearer".equalsIgnoreCase(receiver.getHookAuthType())) {
headers.setBearerAuth(receiver.getHookAuthToken());
}
headers.setContentType(MediaType.APPLICATION_JSON);
// alert.setContent(escapeJsonStr(alert.getContent()));
String webhookJson = renderContent(noticeTemplate, alert);
@@ -88,7 +88,7 @@ public class CryptoUtils {
* @return lowercase hexadecimal string
*/
public static String hmacSha256Hex(String key, String data) {
return hmacSha256Hex(hmac256(key.getBytes(UTF8), data), data);
return hmacSha256Hex(key.getBytes(UTF8), data);
}
/**
@@ -0,0 +1,333 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.notice.impl;
import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.notice.AlertNoticeException;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.NoticeReceiver;
import org.apache.hertzbeat.common.entity.alerter.NoticeTemplate;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ResourceBundle;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
/**
* Test case for FeiShu App Alert Notify
*/
@ExtendWith(MockitoExtension.class)
class FeiShuAppAlertNotifyHandlerImplTest {
@Mock
private RestTemplate restTemplate;
@Mock
private ResourceBundle bundle;
@Mock
private AlerterProperties alerterProperties;
@InjectMocks
private FeiShuAppAlertNotifyHandlerImpl feiShuAppAlertNotifyHandler;
private NoticeReceiver receiver;
private GroupAlert groupAlert;
private NoticeTemplate template;
@BeforeEach
public void setUp() {
receiver = new NoticeReceiver();
receiver.setId(1L);
receiver.setName("test-receiver");
receiver.setType((byte) 14);
receiver.setAppId("cli-test-app-id");
receiver.setAppSecret("test-app-secret");
groupAlert = new GroupAlert();
SingleAlert singleAlert = new SingleAlert();
singleAlert.setLabels(new HashMap<>());
singleAlert.getLabels().put("severity", "critical");
singleAlert.getLabels().put("alertname", "Test Alert");
List<SingleAlert> alerts = new ArrayList<>();
alerts.add(singleAlert);
groupAlert.setAlerts(alerts);
template = new NoticeTemplate();
template.setId(1L);
template.setName("test-template");
template.setContent("test content");
lenient().when(bundle.getString("alerter.notify.title")).thenReturn("Alert Notification");
lenient().when(alerterProperties.getConsoleUrl()).thenReturn("https://console.hertzbeat.com");
}
/**
* Test successful notification to single user
*/
@Test
public void testNotifyAlertSuccessSingleUser() {
// Setup receiver for single user
receiver.setLarkReceiveType((byte) 0);
receiver.setUserId("user-001");
// Mock access token response
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
// Mock message send response
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse messageResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse();
messageResp.setCode(0);
messageResp.setMsg("success");
// Mock restTemplate calls
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse.class)))
.thenReturn(new ResponseEntity<>(messageResp, HttpStatus.OK));
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
}
/**
* Test successful notification to multiple users
*/
@Test
public void testNotifyAlertSuccessMultipleUsers() {
receiver.setLarkReceiveType((byte) 0);
receiver.setUserId("user-001,user-002,user-003");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse messageResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse();
messageResp.setCode(0);
messageResp.setMsg("success");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse.class)))
.thenReturn(new ResponseEntity<>(messageResp, HttpStatus.OK));
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
}
/**
* Test successful notification to chat
*/
@Test
public void testNotifyAlertSuccessChat() {
receiver.setLarkReceiveType((byte) 1);
receiver.setChatId("chat-001");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse messageResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse();
messageResp.setCode(0);
messageResp.setMsg("success");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse.class)))
.thenReturn(new ResponseEntity<>(messageResp, HttpStatus.OK));
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
}
/**
* Test successful notification to departments
*/
@Test
public void testNotifyAlertSuccessDepartments() {
receiver.setLarkReceiveType((byte) 2);
receiver.setPartyId("dept-001,dept-002");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse messageResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse();
messageResp.setCode(0);
messageResp.setMsg("success");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse.class)))
.thenReturn(new ResponseEntity<>(messageResp, HttpStatus.OK));
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
}
/**
* Test notification failure due to access token error
*/
@Test
public void testNotifyAlertFailureAccessToken() {
receiver.setLarkReceiveType((byte) 0);
receiver.setUserId("user-001");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(999);
accessTokenResp.setMsg("Invalid app credentials");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
assertThrows(AlertNoticeException.class, () -> {
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
});
}
/**
* Test notification failure due to message send error
*/
@Test
public void testNotifyAlertFailureMessageSend() {
receiver.setLarkReceiveType((byte) 0);
receiver.setUserId("user-001");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse messageResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse();
messageResp.setCode(999);
messageResp.setMsg("User not found");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppResponse.class)))
.thenReturn(new ResponseEntity<>(messageResp, HttpStatus.OK));
assertThrows(AlertNoticeException.class, () -> {
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
});
}
/**
* Test invalid larkReceiveType
*/
@Test
public void testInvalidLarkReceiveType() {
receiver.setLarkReceiveType((byte) 99);
receiver.setUserId("user-001");
FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse accessTokenResp =
new FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse();
accessTokenResp.setCode(0);
accessTokenResp.setMsg("success");
accessTokenResp.setTenantAccessToken("test-access-token");
when(restTemplate.exchange(
anyString(),
eq(org.springframework.http.HttpMethod.POST),
any(),
eq(FeiShuAppAlertNotifyHandlerImpl.FeiShuAppAccessTokenResponse.class)))
.thenReturn(new ResponseEntity<>(accessTokenResp, HttpStatus.OK));
assertThrows(AlertNoticeException.class, () -> {
feiShuAppAlertNotifyHandler.send(receiver, template, groupAlert);
});
}
}
@@ -42,7 +42,7 @@ public class CryptoUtilsTest {
@Test
void testHmacSha256Hex() {
String signature = CryptoUtils.hmacSha256Hex("your-real-key", "your-real-data");;
assertEquals("41878ccd7ecd795a2dd7ec39be7f33fed4be3ec75f5307689e39dd6f41fdbaac", signature);
assertEquals("f09adf5f4bf94eddecf0f7c8f39a3a8dc7f95ccdc2faf2e5330bc5a78e4bba95", signature);
}
}
@@ -109,6 +109,12 @@
<artifactId>orai18n</artifactId>
<scope>provided</scope>
</dependency>
<!-- db2 -->
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>provided</scope>
</dependency>
<!--redis-->
<dependency>
<groupId>io.lettuce</groupId>
@@ -152,7 +158,6 @@
</dependency>
<!--plc-->
<dependency>
<groupId>org.apache.plc4x</groupId>
@@ -172,6 +172,14 @@ public class JdbcCommonCollect extends AbstractCollect {
".*en\\s*([/\\\\]|\\\\n|/n|\\n)\\s*crypt.*",
".*ci\\s*([/\\\\]|\\\\n|/n|\\n)\\s*pher.*"
});
// DB2 bypass mode
PLATFORM_BYPASS_PATTERNS.put("db2", new String[]{
".*security\\s*([/\\\\]|\\\\n|/n|\\n)\\s*mechanism.*",
".*client\\s*([/\\\\]|\\\\n|/n|\\n)\\s*accounting.*",
".*ssl\\s*([/\\\\]|\\\\n|/n|\\n)\\s*connection.*",
".*use\\s*([/\\\\]|\\\\n|/n|\\n)\\s*client\\s*([/\\\\]|\\\\n|/n|\\n)\\s*information.*"
});
}
private final GlobalConnectionCache connectionCommonCache = GlobalConnectionCache.getInstance();
@@ -531,6 +539,8 @@ public class JdbcCommonCollect extends AbstractCollect {
case "oracle" -> "jdbc:oracle:thin:@" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase());
case "dm" -> "jdbc:dm://" + host + ":" + port;
case "db2" -> "jdbc:db2://" + host + ":" + port
+ "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase());
case "testcontainers" -> "jdbc:tc:" + host + ":" + port
+ ":///" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase()) + "?user=root&password=root";
default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform());
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,79 +15,80 @@
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.http;
package org.apache.hertzbeat.collector.collect. http;
import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
import static org.apache.hertzbeat. common.constants.SignConstants.RIGHT_DASH;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com. google.gson.JsonParser;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java. io.InterruptedIOException;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util. List;
import java.util. Map;
import java.util. Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.net.ssl.SSLException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml. parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax. xml.xpath.XPathConstants;
import javax.xml.xpath. XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import lombok.extern.slf4j.Slf4j;
import lombok.extern.slf4j. Slf4j;
import org. apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.collect.http.promethus.AbstractPrometheusParse;
import org.apache.hertzbeat.collector.collect.http.promethus.PrometheusParseCreator;
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
import org.apache. hertzbeat.collector.collect. prometheus.parser.MetricFamily;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache. hertzbeat.collector.dispatch. DispatchConstants;
import org. apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.collector.util.JsonPathParser;
import org.apache.hertzbeat.collector.util.TimeExpressionUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache. hertzbeat.common.constants. NetworkConstants;
import org.apache. hertzbeat.common.constants. SignConstants;
import org.apache.hertzbeat.common.entity.job. Metrics;
import org.apache. hertzbeat.common.entity. job.protocol.HttpProtocol;
import org.apache.hertzbeat. common.entity.message.CollectRep;
import org.apache.hertzbeat. common.util.Base64Util;
import org. apache.hertzbeat.common. util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.HttpMethod;
import org.apache. hc.core5.http.ClassicHttpResponse;
import org. apache.hc.core5. http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache. hc.core5.http. HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache. hc.client5.http.auth.AuthScope;
import org.apache. hc.client5.http. auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.ClientProtocolException;
import org. apache.hc.client5. http.config.RequestConfig;
import org. apache.hc.client5. http.classic.methods.HttpGet;
import org. apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org. apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache. hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc. core5.http.protocol.HttpContext;
import org. apache.hc.core5. http.io.entity.EntityUtils;
import org.apache. hc.core5.util.Timeout;
import org. springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import org. springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.util.UriUtils;
import org.xml.sax.InputSource;
@@ -96,9 +97,9 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java. util.regex.Pattern;
import java.util.HashSet;
import java.util.ArrayList;
import java.util. ArrayList;
import java.util.Collections;
/**
@@ -106,6 +107,28 @@ import java.util.Collections;
*/
@Slf4j
public class HttpCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Set.of(
HttpStatus.SC_OK,
HttpStatus.SC_CREATED,
@@ -122,28 +145,37 @@ public class HttpCollectImpl extends AbstractCollect {
}
@Override
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
public void collect(CollectRep. MetricsData.Builder builder, Metrics metrics) {
long startTime = System.currentTimeMillis();
HttpProtocol httpProtocol = metrics.getHttp();
String url = httpProtocol.getUrl();
if (!StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
if (! StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
}
if (CollectionUtils.isEmpty(httpProtocol.getSuccessCodes())) {
httpProtocol.setSuccessCodes(List.of(HttpStatus.SC_OK + ""));
httpProtocol. setSuccessCodes(List. of(HttpStatus.SC_OK + ""));
}
HttpContext httpContext = createHttpContext(metrics.getHttp());
HttpUriRequest request = createHttpRequest(metrics.getHttp());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig. getRequest();
HttpContext httpContext = createHttpContext(metrics. getHttp(), requestWithConfig. getRequestConfig());
// Use HttpClientResponseHandler to handle the response and avoid deprecated execute method
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
boolean isSuccessInvoke = checkSuccessInvoke(metrics, statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return;
return null;
}
long responseTime = System.currentTimeMillis() - startTime;
@@ -153,25 +185,20 @@ public class HttpCollectImpl extends AbstractCollect {
try {
if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
if (entity != null) {
parseResponseByPrometheusExporter(entity.getContent(), metrics.getAliasFields(), builder);
parseResponseByPrometheusExporter(entity. getContent(), metrics.getAliasFields(), builder);
}
} else if (DispatchConstants.PARSE_HEADER.equals(parseType)) {
parseResponseByHeader(builder, metrics.getAliasFields(), response);
// Consume entity to release connection
EntityUtils.consumeQuietly(entity);
} else {
/*
this could create large objects, potentially impacting JVM memory space significantly.
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8);
String resp = entity == null ? "" : EntityUtils. toString(entity, StandardCharsets. UTF_8);
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
switch (parseType) {
case DispatchConstants.PARSE_JSON_PATH ->
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
parseResponseByJsonPath(resp, metrics. getAliasFields(), metrics. getHttp(), builder, responseTime);
case DispatchConstants.PARSE_PROM_QL ->
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_XML_PATH ->
@@ -179,7 +206,7 @@ public class HttpCollectImpl extends AbstractCollect {
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, statusCode);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
parseResponseBySiteMap(resp, metrics. getAliasFields(), builder);
case DispatchConstants.PARSE_CONFIG ->
parseResponseByConfig(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
default ->
@@ -191,15 +218,20 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder. setCode(CollectRep. Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (UnknownHostException e2) {
String errorMsg = CommonUtil.getMessageFromThrowable(e2);
log.info(errorMsg);
builder.setCode(CollectRep.Code.UN_REACHABLE);
builder. setCode(CollectRep. Code.UN_REACHABLE);
builder.setMsg("unknown host:" + errorMsg);
} catch (InterruptedIOException | ConnectException | SSLException e3) {
String errorMsg = CommonUtil.getMessageFromThrowable(e3);
@@ -208,7 +240,7 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setMsg(errorMsg);
} catch (IOException e4) {
String errorMsg = CommonUtil.getMessageFromThrowable(e4);
log.info(errorMsg);
log. info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
@@ -216,17 +248,13 @@ public class HttpCollectImpl extends AbstractCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, CloseableHttpResponse response) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, ClassicHttpResponse response) {
CollectRep.ValueRow. Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliases) {
if (!StringUtils.hasText(alias)) {
if (! StringUtils.hasText(alias)) {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
continue;
}
@@ -238,7 +266,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(firstHeader.getValue());
}
builder.addValueRow(valueRowBuilder.build());
builder. addValueRow(valueRowBuilder.build());
}
@Override
@@ -263,7 +291,7 @@ public class HttpCollectImpl extends AbstractCollect {
private void addColumnForSummary(Long responseTime, CollectRep.ValueRow.Builder valueRowBuilder, int keywordNum, String alias) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -282,13 +310,13 @@ public class HttpCollectImpl extends AbstractCollect {
DocumentBuilder db = dbf.newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
NodeList urlList = document.getElementsByTagName("url");
for (int i = 0; i < urlList.getLength(); i++) {
for (int i = 0; i < urlList. getLength(); i++) {
Node urlNode = urlList.item(i);
NodeList childNodes = urlNode.getChildNodes();
for (int k = 0; k < childNodes.getLength(); k++) {
Node currentNode = childNodes.item(k);
// distinguish between text nodes and element nodes
if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
if (currentNode.getNodeType() == Node. ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
// retrieves the value of the loc node
siteUrls.add(currentNode.getFirstChild().getNodeValue());
break;
@@ -318,10 +346,13 @@ public class HttpCollectImpl extends AbstractCollect {
long startTime = System.currentTimeMillis();
try {
HttpGet httpGet = new HttpGet(siteUrl);
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Integer> statusHandler = response -> {
EntityUtils.consume(response.getEntity());
}
return response.getCode();
};
statusCode = CommonHttpClient.getHttpClient().execute(httpGet, statusHandler);
} catch (ClientProtocolException e1) {
if (e1.getCause() != null) {
errorMsg = e1.getCause().getMessage();
@@ -345,7 +376,7 @@ public class HttpCollectImpl extends AbstractCollect {
} else if (NetworkConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(statusCode == null
? CommonConstants.NULL_VALUE : String.valueOf(statusCode));
} else if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
} else if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(responseTime));
} else if (NetworkConstants.ERROR_MSG.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(errorMsg);
@@ -362,13 +393,13 @@ public class HttpCollectImpl extends AbstractCollect {
HttpProtocol http = metrics.getHttp();
List<String> aliasFields = metrics.getAliasFields();
String xpathExpression = http.getParseScript();
if (!StringUtils.hasText(xpathExpression)) {
if (! StringUtils.hasText(xpathExpression)) {
log.warn("Http collect parse type is xmlPath, but the xpath expression is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("XPath expression is empty");
return;
}
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
@@ -378,13 +409,13 @@ public class HttpCollectImpl extends AbstractCollect {
dbf.setXIncludeAware(false);
dbf.setExpandEntityReferences(false);
DocumentBuilder db = dbf.newDocumentBuilder();
DocumentBuilder db = dbf. newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
XPathFactory xpathFactory = XPathFactory.newInstance();
XPath xpath = xpathFactory.newXPath();
NodeList nodeList = (NodeList) xpath.evaluate(xpathExpression, document, XPathConstants.NODESET);
NodeList nodeList = (NodeList) xpath. evaluate(xpathExpression, document, XPathConstants.NODESET);
if (nodeList == null || nodeList.getLength() == 0) {
log.debug("XPath expression '{}' returned no nodes.", xpathExpression);
@@ -397,10 +428,10 @@ public class HttpCollectImpl extends AbstractCollect {
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
valueRowBuilder.addColumn(CommonConstants. NULL_VALUE);
}
}
builder.addValueRow(valueRowBuilder.build());
@@ -413,14 +444,14 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
try {
String value = (String) xpath.evaluate(alias, node, XPathConstants.STRING);
valueRowBuilder.addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
} catch (XPathExpressionException e) {
log.warn("Failed to evaluate XPath '{}' for node [{}]: {}", alias, node.getNodeName(), e.getMessage());
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -432,7 +463,7 @@ public class HttpCollectImpl extends AbstractCollect {
} catch (Exception e) {
log.warn("Failed to parse XML response with XPath '{}': {}", xpathExpression, e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg("Failed to parse XML response: " + e.getMessage());
}
}
@@ -441,25 +472,12 @@ public class HttpCollectImpl extends AbstractCollect {
/**
* Parses the response body in Properties/Config format.
* Two modes are supported:
* 1. single-object mode: if http.parseScript is null, aliasFields are treated as indicator names.
* - If there is a locator in the indicator definition, use the locator as the key of the Properties.
* - Otherwise, use aliasField (metric name) as the key for Properties.
* Generate a single row of data.
* 2. array mode: if http.parseScript is not empty (e.g. “users”), treat it as an array base path.
* Treat aliasFields as the attribute name of an array element, and generate a single row of data for each array index. locator is invalid in this mode.
*
* @param resp Response body string
* @param aliasFields List of metrics aliases (i.e., the list of fields in metrics.fields).
* @param http http protocol configuration
* @param builder The metrics data builder.
* @param responseTime response time
*/
private void parseResponseByConfig(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
if (!StringUtils.hasText(resp)) {
if (! StringUtils.hasText(resp)) {
log.warn("Http collect parse type is config, but response body is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder. setCode(CollectRep. Code.FAIL);
builder.setMsg("Response body is empty");
return;
}
@@ -469,14 +487,14 @@ public class HttpCollectImpl extends AbstractCollect {
properties.load(reader);
} catch (IOException e) {
log.warn("Failed to parse config response: {}", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg("Failed to parse config response: " + e.getMessage());
return;
}
String arrayBasePath = http.getParseScript();
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
if (!StringUtils.hasText(arrayBasePath)) {
if (! StringUtils.hasText(arrayBasePath)) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
@@ -485,7 +503,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
String value = properties.getProperty(alias);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(value != null ? value : CommonConstants. NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -518,13 +536,13 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
valueRowBuilder.addColumn(responseTime. toString());
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
valueRowBuilder. addColumn(Integer.toString(keywordNum));
} else {
String currentKey = arrayBasePath + "[" + i + "]." + alias;
String value = properties.getProperty(currentKey);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(value != null ? value : CommonConstants.NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -549,24 +567,24 @@ public class HttpCollectImpl extends AbstractCollect {
for (int i = 0; i < valueRow.getColumnsCount(); i++) {
String columnValue = valueRow.getColumns(i);
String alias = aliasFields.get(i);
if (!CommonConstants.NULL_VALUE.equals(columnValue) && (!NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD.equalsIgnoreCase(alias))) {
if (! CommonConstants.NULL_VALUE.equals(columnValue) && (! NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD. equalsIgnoreCase(alias))) {
hasMeaningfulData = true;
break;
}
if ((NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) || CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) && !CommonConstants.NULL_VALUE.equals(columnValue)) {
if ((NetworkConstants.RESPONSE_TIME. equalsIgnoreCase(alias) || CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) && ! CommonConstants.NULL_VALUE.equals(columnValue)) {
hasMeaningfulData = true;
}
}
return hasMeaningfulData;
}
@SuppressWarnings("unchecked")
private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
List<Object> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
for (int i = 0; i < results.size(); i++) {
Object objectValue = results.get(i);
// if a property is missing or empty due to target version issues, filter it. Refer to the app-elasticsearch.yml configuration under name: nodes
if (objectValue == null) {
continue;
}
@@ -578,8 +596,8 @@ public class HttpCollectImpl extends AbstractCollect {
if (value != null) {
valueRowBuilder.addColumn(String.valueOf(value));
} else {
if (alias.startsWith("$.")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
if (alias.startsWith("$. ")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias. substring(1));
if (subResults != null && subResults.size() > i) {
Object resultValue = subResults.get(i);
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
@@ -633,7 +651,7 @@ public class HttpCollectImpl extends AbstractCollect {
return;
}
MetricFamily metricFamily = metricFamilyMap.get(metrics);
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily.getMetricList())) {
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily. getMetricList())) {
return;
}
for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
@@ -645,7 +663,7 @@ public class HttpCollectImpl extends AbstractCollect {
String columnValue = labelMap.get(aliasField);
if (columnValue != null) {
valueRowBuilder.addColumn(columnValue);
} else if (CommonConstants.PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
} else if (CommonConstants. PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
valueRowBuilder.addColumn(String.valueOf(metric.getValue()));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -657,9 +675,9 @@ public class HttpCollectImpl extends AbstractCollect {
private void parseResponseByDefault(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
CollectRep.MetricsData. Builder builder, Long responseTime) {
JsonElement element = JsonParser.parseString(resp);
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
if (element.isJsonArray()) {
JsonArray array = element.getAsJsonArray();
for (JsonElement jsonElement : array) {
@@ -691,46 +709,53 @@ public class HttpCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpProtocol http protocol
* @param requestConfig request config (can be null)
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpProtocol) {
HttpProtocol.Authorization auth = httpProtocol.getAuthorization();
public HttpContext createHttpContext(HttpProtocol httpProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
// Set request config if provided
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
HttpProtocol. Authorization auth = httpProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword());
auth.getDigestAuthPassword(). toCharArray());
AuthScope authScope = new AuthScope(httpProtocol.getHost(), Integer.parseInt(httpProtocol.getPort()));
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
*
* @param httpProtocol http params
* @return http uri request
* @return http request with config
*/
public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
RequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod().toUpperCase();
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpProtocol) {
ClassicRequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod(). toUpperCase();
if (HttpMethod.GET.matches(httpMethod)) {
requestBuilder = RequestBuilder.get();
requestBuilder = ClassicRequestBuilder.get();
} else if (HttpMethod.POST.matches(httpMethod)) {
requestBuilder = RequestBuilder.post();
requestBuilder = ClassicRequestBuilder.post();
} else if (HttpMethod.PUT.matches(httpMethod)) {
requestBuilder = RequestBuilder.put();
requestBuilder = ClassicRequestBuilder.put();
} else if (HttpMethod.DELETE.matches(httpMethod)) {
requestBuilder = RequestBuilder.delete();
} else if (HttpMethod.PATCH.matches(httpMethod)) {
requestBuilder = RequestBuilder.patch();
requestBuilder = ClassicRequestBuilder.delete();
} else if (HttpMethod. PATCH.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.patch();
} else {
// not support the method
log.error("not support the http method: {}.", httpProtocol.getMethod());
@@ -741,17 +766,17 @@ public class HttpCollectImpl extends AbstractCollect {
boolean enableUrlEncoding = Boolean.parseBoolean(httpProtocol.getEnableUrlEncoding());
StringBuilder queryParams = new StringBuilder();
if (params != null && !params.isEmpty()) {
if (params != null && ! params.isEmpty()) {
for (Map.Entry<String, String> param : params.entrySet()) {
String key = param.getKey();
String value = param.getValue();
if (!StringUtils.hasText(key)) {
if (! StringUtils.hasText(key)) {
continue;
}
if (!queryParams.isEmpty()) {
queryParams.append("&");
if (! queryParams.isEmpty()) {
queryParams. append("&");
}
if (enableUrlEncoding) {
@@ -762,9 +787,9 @@ public class HttpCollectImpl extends AbstractCollect {
if (StringUtils.hasText(value)) {
String calculatedValue = TimeExpressionUtil.calculate(value);
if (enableUrlEncoding) {
calculatedValue = UriUtils.encodeQueryParam(calculatedValue, "UTF-8");
calculatedValue = UriUtils. encodeQueryParam(calculatedValue, "UTF-8");
}
queryParams.append("=").append(calculatedValue);
queryParams. append("=").append(calculatedValue);
}
}
}
@@ -775,7 +800,7 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
// headers The custom request header is overwritten here
Map<String, String> headers = httpProtocol.getHeaders();
if (headers != null && !headers.isEmpty()) {
if (headers != null && ! headers.isEmpty()) {
for (Map.Entry<String, String> header : headers.entrySet()) {
if (StringUtils.hasText(header.getValue())) {
requestBuilder.addHeader(header.getKey(), header.getValue());
@@ -783,7 +808,7 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
// add accept
if (DispatchConstants.PARSE_DEFAULT.equals(httpProtocol.getParseType())
if (DispatchConstants.PARSE_DEFAULT. equals(httpProtocol.getParseType())
|| DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
} else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) {
@@ -799,10 +824,10 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization. getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + SignConstants.DOUBLE_MARK + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
requestBuilder.addHeader(HttpHeaders. AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
}
}
@@ -817,14 +842,14 @@ public class HttpCollectImpl extends AbstractCollect {
if (enableUrlEncoding) {
// if the url contains parameters directly
if (httpProtocol.getUrl().contains("?")) {
String path = httpProtocol.getUrl().substring(0, httpProtocol.getUrl().indexOf("?"));
String path = httpProtocol.getUrl(). substring(0, httpProtocol.getUrl().indexOf("?"));
String query = httpProtocol.getUrl().substring(httpProtocol.getUrl().indexOf("?") + 1);
uri = UriUtils.encodePath(path, "UTF-8") + "?" + UriUtils.encodeQuery(query, "UTF-8");
} else {
uri = UriUtils.encodePath(httpProtocol.getUrl(), "UTF-8");
}
} else {
uri = httpProtocol.getUrl();
uri = httpProtocol. getUrl();
}
// append query params
@@ -834,10 +859,10 @@ public class HttpCollectImpl extends AbstractCollect {
String finalUri;
if (IpDomainUtil.isHasSchema(httpProtocol.getHost())) {
finalUri = httpProtocol.getHost() + ":" + httpProtocol.getPort() + uri;
finalUri = httpProtocol.getHost() + ":" + httpProtocol. getPort() + uri;
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
String ipAddressType = IpDomainUtil. checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6. equals(ipAddressType)
? String.format("[%s]:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri)
: String.format("%s:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(httpProtocol.getSsl());
@@ -855,21 +880,23 @@ public class HttpCollectImpl extends AbstractCollect {
throw e;
}
// custom timeout
// custom timeout - create RequestConfig separately
RequestConfig requestConfig = null;
int timeout = CollectUtil.getTimeout(httpProtocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
// build the request and return with config
return new HttpRequestWithConfig(requestBuilder. build(), requestConfig);
}
private boolean checkSuccessInvoke(Metrics metrics, int statusCode) {
List<String> successCodes = metrics.getHttp().getSuccessCodes();
List<String> successCodes = metrics.getHttp(). getSuccessCodes();
Set<Integer> successCodeSet = successCodes != null ? successCodes.stream().map(code -> {
try {
return Integer.valueOf(code);
@@ -117,11 +117,11 @@ public class SslCertificateCollectImpl extends AbstractCollect {
} else if (NAME_EXPIRED.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Boolean.toString(expired));
} else if (NAME_START_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toLocaleString());
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toString());
} else if (NAME_START_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotBefore().getTime()));
} else if (NAME_END_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toLocaleString());
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toString());
} else if (NAME_END_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotAfter().getTime()));
} else {
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -44,16 +44,17 @@ import org.apache.hertzbeat.common.entity.job.protocol.NginxProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.MediaType;
/**
@@ -62,6 +63,27 @@ import org.springframework.http.MediaType;
@Slf4j
public class NginxCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final String NGINX_STATUS_NAME = "nginx_status";
private static final String REQ_STATUS_NAME = "req_status";
private static final String AVAILABLE = "available";
@@ -93,34 +115,36 @@ public class NginxCollectImpl extends AbstractCollect {
nginxProtocol.setUrl(url == null ? RIGHT_DASH : RIGHT_DASH + url.trim());
}
HttpContext httpContext = createHttpContext(metrics.getNginx());
HttpUriRequest request = createHttpRequest(metrics.getNginx());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)){
// send an HTTP request and get the response data
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getNginx());
HttpContext httpContext = createHttpContext(metrics.getNginx(), requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + statusCode);
return;
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
Long responseTime = System.currentTimeMillis() - startTime;
// call different parsing methods based on the metrics name
if (StringUtils.equalsAny(metrics.getName(), NGINX_STATUS_NAME, AVAILABLE)) {
if (NGINX_STATUS_NAME.equals(metrics.getName()) || AVAILABLE.equals(metrics.getName())) {
parseNginxStatusResponse(builder, resp, metrics, responseTime);
} else if (REQ_STATUS_NAME.equals(metrics.getName())) {
parseReqStatusResponse(builder, resp, metrics, responseTime);
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
@@ -130,18 +154,21 @@ public class NginxCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NGINX;
}
private HttpContext createHttpContext(NginxProtocol nginxProtocol) {
private HttpContext createHttpContext(NginxProtocol nginxProtocol, RequestConfig requestConfig) {
HttpHost host = new HttpHost(nginxProtocol.getHost(), Integer.parseInt(nginxProtocol.getPort()));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
return httpClientContext;
}
private HttpUriRequest createHttpRequest(NginxProtocol nginxProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(NginxProtocol nginxProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
String portWithUri = nginxProtocol.getPort() + CollectUtil.replaceUriSpecialChar(nginxProtocol.getUrl());
String host = nginxProtocol.getHost();
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + portWithUri);
} else {
@@ -162,16 +189,16 @@ public class NginxCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(nginxProtocol.getTimeout());
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -335,4 +362,4 @@ public class NginxCollectImpl extends AbstractCollect {
return GET + result;
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,24 +47,24 @@ import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -74,11 +74,32 @@ import org.springframework.util.StringUtils;
*/
@Slf4j
public class PrometheusAutoCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder,
Metrics metrics) {
@@ -89,11 +110,13 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
builder.setMsg(e.getMessage());
return null;
}
HttpContext httpContext = createHttpContext(metrics.getPrometheus());
HttpUriRequest request = createHttpRequest(metrics.getPrometheus());
try (CloseableHttpResponse response =
CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getPrometheus());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getPrometheus(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<List<CollectRep.MetricsData>> responseHandler = response -> {
int statusCode = response.getCode();
boolean isSuccessInvoke = defaultSuccessStatusCodes.contains(statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
@@ -102,12 +125,18 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
return null;
}
try {
// Parse directly from the entity stream inside the handler loop while connection is open
return parseResponseByPrometheusExporter(response.getEntity().getContent(), builder);
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
return null;
}
};
try {
return CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
@@ -133,31 +162,27 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_PROMETHEUS;
}
private void validateParams(Metrics metrics) throws Exception {
if (metrics == null || metrics.getPrometheus() == null) {
throw new Exception("Prometheus collect must has prometheus params");
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (protocol.getPath() == null
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
private List<CollectRep.MetricsData> parseResponseByPrometheusExporter(InputStream inputStream, CollectRep.MetricsData.Builder builder) throws IOException {
long endTime = System.currentTimeMillis();
builder.setTime(endTime);
@@ -201,40 +226,49 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
return metricsDataList;
}
/**
* create httpContext
*
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol) {
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
AuthCache authCache = new BasicAuthCache();
authCache.put(new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort())), new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
* @param protocol http params
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -260,7 +294,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
// add accept
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -268,7 +302,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -280,17 +314,17 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
if (StringUtils.hasLength(protocol.getPayload()) && (HttpMethod.POST.matches(protocol.getMethod()) || HttpMethod.PUT.matches(protocol.getMethod()))) {
requestBuilder.setEntity(new StringEntity(protocol.getPayload(), StandardCharsets.UTF_8));
}
// uri
String uri = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -298,18 +332,19 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -319,11 +354,11 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
public static PrometheusAutoCollectImpl getInstance() {
return PrometheusAutoCollectImpl.SingleInstance.INSTANCE;
}
/**
* static instance
*/
private static class SingleInstance {
private static final PrometheusAutoCollectImpl INSTANCE = new PrometheusAutoCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,25 +30,25 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -71,6 +71,27 @@ import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
@Slf4j
public class PrometheusProxyCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@@ -80,7 +101,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
PrometheusProtocol prometheusProtocol = metrics.getPrometheus();
HttpUriRequest request;
try {
validateParams(metrics);
} catch (Exception e) {
@@ -89,17 +110,19 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
return Collections.singletonList(builder.build());
}
HttpContext httpContext = createHttpContext(prometheusProtocol);
request = createHttpRequest(prometheusProtocol);
HttpRequestWithConfig requestWithConfig = createHttpRequest(prometheusProtocol);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(prometheusProtocol, requestWithConfig.getRequestConfig());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
// Use HttpClientResponseHandler to avoid deprecated execute method and manual resource closing
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
log.debug("Prometheus proxy collect, response status: {}", statusCode);
if (!defaultSuccessStatusCodes.contains(statusCode)) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return Collections.singletonList(builder.build());
return null;
}
String rawTextContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
@@ -118,6 +141,11 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
builder.addValueRow(valueRowBuilder.build());
builder.setCode(CollectRep.Code.SUCCESS);
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error("Prometheus proxy collect error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e1);
@@ -143,10 +171,6 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
log.error("Prometheus proxy collect unknown error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@@ -162,7 +186,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (!StringUtils.hasText(protocol.getHost())
|| !StringUtils.hasText(protocol.getPort())) {
|| !StringUtils.hasText(protocol.getPort())) {
throw new Exception("Prometheus collect must has host and port params");
}
if (protocol.getPath() == null
@@ -171,42 +195,51 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
/**
* create httpContext
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol) {
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
AuthCache authCache = new BasicAuthCache();
HttpHost targetHost = new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort()));
authCache.put(targetHost, new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol http params
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -231,7 +264,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
if (headers == null || headers.keySet().stream().noneMatch(HttpHeaders.ACCEPT::equalsIgnoreCase)) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE + ";version=0.0.4,*/*;q=0.1");
}
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -241,7 +274,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -255,15 +288,15 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN_VALUE);
}
}
String uriPath = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uriPath);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -271,24 +304,22 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout());
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
} else {
RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -305,4 +336,4 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
private static class SingleInstance {
private static final PrometheusProxyCollectImpl INSTANCE = new PrometheusProxyCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -24,6 +24,15 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -38,15 +47,6 @@ import org.apache.hertzbeat.common.entity.push.PushMetricsDto;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
/**
@@ -55,6 +55,27 @@ import org.springframework.http.MediaType;
@Slf4j
public class PushCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final Map<Long, Long> timeMap = new ConcurrentHashMap<>();
// ms
@@ -83,20 +104,25 @@ public class PushCollectImpl extends AbstractCollect {
Long time = timeMap.getOrDefault(monitorId, curTime - FIRST_COLLECT_INTERVAL);
timeMap.put(monitorId, curTime);
HttpContext httpContext = createHttpContext(pushProtocol);
HttpUriRequest request = createHttpRequest(pushProtocol, monitorId, time);
HttpRequestWithConfig requestWithConfig = createHttpRequest(pushProtocol, monitorId, time);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(pushProtocol, requestWithConfig.getRequestConfig());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return;
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
parseResponse(builder, resp, metrics);
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
String resp = CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
if (resp != null) {
parseResponse(builder, resp, metrics);
}
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error(errorMsg, e);
@@ -111,16 +137,17 @@ public class PushCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_PUSH;
}
private HttpContext createHttpContext(PushProtocol pushProtocol) {
HttpHost host = new HttpHost(pushProtocol.getHost(), Integer.parseInt(pushProtocol.getPort()));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
private HttpContext createHttpContext(PushProtocol pushProtocol, RequestConfig requestConfig) {
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
return httpClientContext;
}
private HttpUriRequest createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// uri
String uri = CollectUtil.replaceUriSpecialChar(pushProtocol.getUri());
@@ -142,19 +169,17 @@ public class PushCollectImpl extends AbstractCollect {
requestBuilder.addParameter("time", String.valueOf(startTime));
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
//requestBuilder.setUri(pushProtocol.getUri());
RequestConfig requestConfig = null;
if (DEFAULT_TIMEOUT > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(DEFAULT_TIMEOUT)
.setSocketTimeout(DEFAULT_TIMEOUT)
// Using deprecated setConnectTimeout for request-level override
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
.setResponseTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
private void parseResponse(CollectRep.MetricsData.Builder builder, String resp, Metrics metric) {
@@ -180,4 +205,4 @@ public class PushCollectImpl extends AbstractCollect {
}
builder.setTime(System.currentTimeMillis());
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,20 +18,20 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.protocol.RedfishProtocol;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.springframework.http.MediaType;
/**
@@ -58,11 +58,21 @@ public class RedfishClient {
redfishProtocol.getUsername(), redfishProtocol.getPassword(), Integer.parseInt(redfishProtocol.getTimeout()));
}
@SuppressWarnings("deprecation")
public ConnectSession connect() throws Exception {
HttpHost host = new HttpHost(this.host, this.port);
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
RequestBuilder requestBuilder = RequestBuilder.post();
HttpClientContext httpClientContext = HttpClientContext.create();
// Configure RequestConfig if timeout is set
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(this.timeout))
.setResponseTimeout(Timeout.ofMilliseconds(this.timeout))
.setRedirectsEnabled(true)
.build();
httpClientContext.setRequestConfig(requestConfig);
}
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.post();
String uri = REDFISH_SESSION_SERVICE;
if (IpDomainUtil.isHasSchema(this.host)) {
@@ -79,37 +89,31 @@ public class RedfishClient {
requestBuilder.addHeader(HttpHeaders.CONNECTION, NetworkConstants.KEEP_ALIVE);
requestBuilder.addHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE);
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
// Content-Encoding header is usually for compression, setting charset is done in Content-Type usually,
// but keeping original logic of setting it explicitly if that was the intent, though standard is usually empty or gzip.
// original: requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
final String json = "{\"UserName\": \"" + this.username + "\", \"Password\": \"" + this.password + "\"}";
StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8);
requestBuilder.setEntity(entity);
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(this.timeout)
.setSocketTimeout(this.timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
ClassicHttpRequest request = requestBuilder.build();
HttpUriRequest request = requestBuilder.build();
Session session;
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpClientContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Session> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_CREATED) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
String location = response.getFirstHeader(NetworkConstants.LOCATION).getValue();
String auth = response.getFirstHeader(NetworkConstants.X_AUTH_TOKEN).getValue();
session = new Session(auth, location, this.host, this.port);
return new Session(auth, location, this.host, this.port);
};
try {
Session session = CommonHttpClient.getHttpClient().execute(request, httpClientContext, responseHandler);
return new RedfishConnectSession(session);
} catch (Exception e) {
throw new Exception("Redfish session create error: " + e.getMessage());
} finally {
request.abort();
throw new Exception("Redfish session create error: " + e.getMessage(), e);
}
return new RedfishConnectSession(session);
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,15 +18,15 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
/**
* Redfish connect session
@@ -54,15 +54,20 @@ public class RedfishConnectSession implements ConnectSession {
HttpDelete httpDelete = new HttpDelete(url);
httpDelete.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpDelete.setHeader(NetworkConstants.LOCATION, session.location());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpDelete)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
EntityUtils.consume(response.getEntity());
return null;
};
try {
CommonHttpClient.getHttpClient().execute(httpDelete, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session close error:" + e.getMessage());
} finally {
httpDelete.abort();
throw new Exception("Redfish session close error:" + e.getMessage(), e);
}
}
@@ -84,16 +89,19 @@ public class RedfishConnectSession implements ConnectSession {
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpGet.setHeader(NetworkConstants.LOCATION, session.location());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
return CommonHttpClient.getHttpClient().execute(httpGet, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session get resource error:" + e.getMessage());
} finally {
httpGet.abort();
throw new Exception("Redfish session get resource error:" + e.getMessage(), e);
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,6 +22,10 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -32,10 +36,6 @@ import org.apache.hertzbeat.common.entity.sd.ConnectionConfig;
import org.apache.hertzbeat.common.entity.sd.EurekaDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.XmlUtil;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.util.EntityUtils;
import org.springframework.util.CollectionUtils;
/**
@@ -57,38 +57,55 @@ public class EurekaSdCollectImpl extends AbstractCollect {
@Override
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
HttpUriRequest request = RequestBuilder.get().setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH).build();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request)) {
int statusCode = response.getStatusLine().getStatusCode();
ClassicHttpRequest request = ClassicRequestBuilder.get()
.setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH)
.build();
// Use ResponseHandler to avoid manual resource management and deprecated execute methods
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != 200) {
log.warn("Failed to fetch eureka sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return;
return null;
}
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
String responseBody = CommonHttpClient.getHttpClient().execute(request, responseHandler);
if (responseBody != null) {
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -108,4 +125,4 @@ public class EurekaSdCollectImpl extends AbstractCollect {
public String supportProtocol() {
return DispatchConstants.PROTOCOL_EUREKA_SD;
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -41,18 +41,18 @@ import org.apache.hertzbeat.common.entity.sd.ServiceDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import org.springframework.web.util.UriUtils;
@@ -63,6 +63,28 @@ import org.springframework.web.util.UriUtils;
*/
@Slf4j
public class HttpSdCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
}
@@ -71,22 +93,30 @@ public class HttpSdCollectImpl extends AbstractCollect {
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
HttpUriRequest request = createHttpRequest(metrics.getHttp_sd());
HttpContext httpContext = createHttpContext(metrics.getHttp_sd());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp_sd());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getHttp_sd(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != 200) {
log.warn("Failed to fetch sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return;
return null;
}
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
TypeReference<List<ServiceDiscoveryResponseEntity>> typeReference = new TypeReference<>() {};
final List<ServiceDiscoveryResponseEntity> responseEntityList = JsonUtil.fromJson(responseBody, typeReference);
if (CollectionUtils.isEmpty(responseEntityList)) {
return;
return null;
}
responseEntityList.stream()
@@ -100,11 +130,21 @@ public class HttpSdCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -137,17 +177,23 @@ public class HttpSdCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpSdProtocol http sd protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpSdProtocol) {
public HttpContext createHttpContext(HttpProtocol httpSdProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
HttpProtocol.Authorization auth = httpSdProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (org.springframework.util.StringUtils.hasText(auth.getDigestAuthUsername())
&& org.springframework.util.StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword());
auth.getDigestAuthPassword().toCharArray());
URL url;
try {
url = new URL(httpSdProtocol.getUrl());
@@ -159,20 +205,20 @@ public class HttpSdCollectImpl extends AbstractCollect {
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
*
* @param httpSdProtocol http request set
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(HttpProtocol httpSdProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpSdProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// The default request header can be overridden if customized
// keep-alive
@@ -227,16 +273,17 @@ public class HttpSdCollectImpl extends AbstractCollect {
throw e;
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(httpSdProtocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
}
}
@@ -29,7 +29,7 @@ import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingClient;
import org.junit.jupiter.api.BeforeEach;
@@ -113,7 +113,7 @@ class CollectServerTest {
RemotingClient remotingClient = mock(RemotingClient.class);
ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient);
ClusterMsg.Message message = mock(ClusterMsg.Message.class);
ClusterMessage message = mock(ClusterMessage.class);
collectServer.sendMsg(message);
@@ -135,7 +135,7 @@ class CollectServerTest {
collectNettyEventListener.onChannelActive(channel);
verify(timerDispatch, times(1)).goOnline();
verify(remotingClient, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(remotingClient, times(1)).sendMsg(any(ClusterMessage.class));
ScheduledExecutorService scheduledExecutor =
(ScheduledExecutorService) ReflectionTestUtils.getField(collectServer, "scheduledExecutor");
@@ -34,7 +34,7 @@ import org.apache.hertzbeat.collector.dispatch.WorkerPool;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -116,7 +116,7 @@ class CollectJobServiceTest {
collectJobService.collectSyncOneTimeJobData(job);
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
}
@Test
@@ -137,7 +137,7 @@ class CollectJobServiceTest {
.build();
collectJobService.sendAsyncCollectData(metricsData);
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
}
@Test
@@ -100,6 +100,35 @@
</build>
<profiles>
<profile>
<id>dev-driver</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>inner</id>
<build>
@@ -28,8 +28,6 @@ spring:
jackson:
default-property-inclusion: ALWAYS
# need to disable spring boot mongodb auto config, or default mongodb connection tried and failed...
autoconfigure:
exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
management:
endpoints:
web:
@@ -1,50 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true">
<springProperty scope="context" name="application_name" source="spring.application.name" defaultValue="collector"/>
<!-- Output logs to ConsoleAppender -->
<appender name="ConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
<appender name="CONSOLE_RAW" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<!--<pattern>%d %p (%file:%line\)- %m%n</pattern>-->
<!-- Format output: %d for date, %thread for thread name, %-5level for level with 5 character width, %msg for log message, %n for new line -->
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="ConsoleAppender" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>0</discardingThreshold>
<queueSize>512</queueSize>
<includeCallerData>true</includeCallerData>
<appender-ref ref="CONSOLE_RAW"/>
</appender>
<appender name="SystemOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- Rolling policy for loggers, by date and by size -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- Archive log file path. %d{yyyy-MM-dd} specifies date format, %i specifies index -->
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}.%i.log.zip</fileNamePattern>
<!-- Log retention duration -->
<maxFileSize>50MB</maxFileSize>
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<!-- Besides logging by day, log files cannot exceed 200M, if exceeded, log files will start from index 0 -->
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
@@ -52,32 +32,23 @@
</appender>
<appender name="ErrOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}-error.%i.log.zip</fileNamePattern>
<!-- Log retention duration -->
<maxFileSize>50MB</maxFileSize>
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!-- This log file records error and above levels -->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
</appender>
<!-- Settings for this logger: for example, all output logs under the org.springframework package must be at level info or above to be output! -->
<!-- This can avoid outputting many common debug information of the spring framework! -->
<logger name="org.springframework" level="info" />
<logger name="org.json" level="error"/>
<logger name="io.netty" level="info"/>
@@ -91,9 +62,9 @@
<logger name="org.mongodb" level="warn"/>
<logger name="io.greptime" level="warn"/>
<!-- Production environment configuration -->
<springProfile name="prod">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
@@ -101,12 +72,12 @@
<springProfile name="cluster">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="dev">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -115,7 +86,6 @@
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="mysql">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -124,4 +94,4 @@
</root>
</springProfile>
</configuration>
</configuration>
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,33 +17,30 @@
package org.apache.hertzbeat.collector.collect.common.http;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.X509Certificate;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContexts;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.reactor.ssl.SSLBufferMode;
import org.apache.hc.core5.ssl.SSLContexts;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
/**
* common http client
* common http client for HttpComponents Client 5.x
*/
@Slf4j
public class CommonHttpClient {
@@ -84,11 +81,13 @@ public class CommonHttpClient {
/**
* ssl supported version
* Note: SSLv3 is often disabled in modern JDKs
*/
private static final String[] SUPPORTED_SSL = {"TLSv1", "TLSv1.1", "TLSv1.2", "SSLv3"};
static {
try {
// 1. SSL Context Configuration
SSLContext sslContext = SSLContexts.createDefault();
X509TrustManager x509TrustManager = new X509TrustManager() {
@Override
@@ -102,7 +101,7 @@ public class CommonHttpClient {
for (X509Certificate certificate : x509Certificates) {
Date deadline = certificate.getNotAfter();
if (deadline != null && now.after(deadline)) {
throw new CertificateExpiredException();
throw new CertificateExpiredException("Server certificate expired at " + deadline);
}
}
}
@@ -112,57 +111,63 @@ public class CommonHttpClient {
public X509Certificate[] getAcceptedIssuers() { return null; }
};
sslContext.init(null, new TrustManager[]{x509TrustManager}, null);
SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, SUPPORTED_SSL, null, new NoopHostnameVerifier());
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", sslFactory)
// 2. TlsStrategy Configuration (Replaces deprecated SSLConnectionSocketFactory/Registry)
DefaultClientTlsStrategy tlsStrategy = new DefaultClientTlsStrategy(
sslContext,
SUPPORTED_SSL,
null, // Supported cipher suites (null = default)
SSLBufferMode.STATIC,
NoopHostnameVerifier.INSTANCE
);
// 3. Connection Manager Configuration (Using Builder)
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(CONNECT_TIMEOUT))
.setSocketTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT))
.setValidateAfterInactivity(TimeValue.ofMilliseconds(INACTIVITY_VALIDATED_TIME))
.build();
connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(tlsStrategy)
.setMaxConnTotal(MAX_TOTAL_CONNECTIONS)
.setMaxConnPerRoute(MAX_PER_ROUTE_CONNECTIONS)
.setDefaultConnectionConfig(connectionConfig)
.build();
// 4. Request Config (Connection Request Timeout & Redirects)
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(REQUIRE_CONNECT_TIMEOUT)
.setConnectTimeout(CONNECT_TIMEOUT)
.setSocketTimeout(SOCKET_TIMEOUT)
// auto redirect when 301 302 response status
.setConnectionRequestTimeout(Timeout.ofMilliseconds(REQUIRE_CONNECT_TIMEOUT))
.setRedirectsEnabled(true)
.build();
// connection pool
connectionManager = new PoolingHttpClientConnectionManager(registry);
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS);
connectionManager.setValidateAfterInactivity(INACTIVITY_VALIDATED_TIME);
// 5. Build HttpClient
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
// clean up unavailable expired connections
.evictExpiredConnections()
// clean up available but idle connections
.evictIdleConnections(100, TimeUnit.SECONDS)
.evictIdleConnections(TimeValue.ofSeconds(100))
.build();
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("http-connection-pool-cleaner-%d")
.setDaemon(true)
.build();
ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory);
scheduledExecutor.scheduleWithFixedDelay(() -> {
connectionManager.closeExpiredConnections();
connectionManager.closeIdleConnections(40, TimeUnit.SECONDS);
}, 40L, 40L, TimeUnit.SECONDS);
// shutdown hook
// Shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close));
} catch (Exception ignored) {}
} catch (Exception e) {
log.error("Initialize CommonHttpClient error", e);
}
}
public static CloseableHttpClient getHttpClient() {
return httpClient;
}
public static void close() {
try {
httpClient.close();
if (httpClient != null) {
httpClient.close();
}
} catch (Exception e) {
log.error("close http client error", e);
}
}
}
}
@@ -17,65 +17,48 @@
package org.apache.hertzbeat.collector.dispatch;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Collection task worker thread pool
* Collection task worker thread pool with Virtual Threads
*/
@Component
@Slf4j
public class WorkerPool implements DisposableBean {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public WorkerPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
// thread factory
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable);
log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("collect-worker-%d")
.build();
int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors());
int maxSize = Runtime.getRuntime().availableProcessors() * 16;
workerExecutor = new ThreadPoolExecutor(coreSize,
maxSize,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory virtualThreadFactory = Thread.ofVirtual()
.name("collect-vt-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(virtualThreadFactory);
log.info("WorkerPool initialized with JDK 25 Virtual Threads successfully.");
}
/**
* Run the collection task thread
*
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
* Run the collection task
* @param runnable Task
*/
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable) {
workerExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
public void destroy() {
if (workerExecutor != null) {
workerExecutor.shutdownNow();
workerExecutor.close();
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,7 +18,6 @@
package org.apache.hertzbeat.collector.dispatch.entrance;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
import io.netty.channel.Channel;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
@@ -33,7 +32,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProces
import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.RemotingClient;
@@ -65,7 +64,7 @@ public class CollectServer implements CommandLineRunner {
private final TimerDispatch timerDispatch;
private final CollectorInfoProperties infoProperties;
private RemotingClient remotingClient;
private ScheduledExecutorService scheduledExecutor;
@@ -97,13 +96,13 @@ public class CollectServer implements CommandLineRunner {
nettyClientConfig.setServerPort(nettyProperties.getManagerPort());
this.remotingClient = new NettyRemotingClient(nettyClientConfig, new CollectNettyEventListener(), threadPool);
this.remotingClient.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_CLOSE, new GoCloseProcessor(this));
}
public void shutdown() {
@@ -116,7 +115,7 @@ public class CollectServer implements CommandLineRunner {
return collectJobService;
}
public void sendMsg(final ClusterMsg.Message message) {
public void sendMsg(final ClusterMessage message) {
this.remotingClient.sendMsg(message);
}
@@ -143,10 +142,10 @@ public class CollectServer implements CommandLineRunner {
.build();
timerDispatch.goOnline();
// send online message
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(collectorInfo)))
ClusterMessage message = ClusterMessage.builder()
.identity(identity)
.type(ClusterMessage.MessageType.GO_ONLINE)
.msg(JsonUtil.toJsonBytes(collectorInfo))
.build();
CollectServer.this.sendMsg(message);
@@ -163,13 +162,13 @@ public class CollectServer implements CommandLineRunner {
// schedule send heartbeat message
scheduledExecutor.scheduleAtFixedRate(() -> {
try {
ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
ClusterMessage heartbeat = ClusterMessage.builder()
.identity(identity)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
.build();
CollectServer.this.sendMsg(heartbeat);
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
} catch (Exception e) {
log.error("schedule send heartbeat to server error.{}", e.getMessage());
}
@@ -182,4 +181,4 @@ public class CollectServer implements CommandLineRunner {
log.info("handle idle event triggered. collector is going offline.");
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.collector.dispatch.entrance.internal;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.WorkerPool;
@@ -25,7 +24,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.ArrowUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
@@ -117,10 +116,10 @@ public class CollectJobService {
workerPool.executeJob(() -> {
List<CollectRep.MetricsData> metricsDataList = this.collectSyncJobData(oneTimeJob);
byte[] msg = ArrowUtil.serializeMetricsData(metricsDataList);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
ClusterMessage message = ClusterMessage.builder()
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
});
@@ -153,22 +152,22 @@ public class CollectJobService {
*/
public void sendAsyncCollectData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_DATA)
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
}
public void sendAsyncServiceDiscoveryData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
.build();
this.collectServer.sendMsg(message);
}
@@ -184,4 +183,4 @@ public class CollectJobService {
public void setCollectServer(CollectServer collectServer) {
this.collectServer = collectServer;
}
}
}
@@ -21,7 +21,7 @@ import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -37,8 +37,8 @@ public class CollectCyclicDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job job = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job job = JsonUtil.fromJson(message.getMsg(), Job.class);
if (job == null) {
log.error("collector receive cyclic task job is null");
return null;
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -35,8 +35,8 @@ public class CollectOneTimeDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg(), Job.class);
collectServer.getCollectJobService().collectSyncOneTimeJobData(oneTimeJob);
return null;
}
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -39,9 +39,9 @@ public class DeleteCyclicTaskProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
TypeReference<List<Long>> typeReference = new TypeReference<>() {};
List<Long> jobIds = JsonUtil.fromJson(message.getMsg().toStringUtf8(), typeReference);
List<Long> jobIds = JsonUtil.fromJson(message.getMsgString(), typeReference);
if (jobIds == null || jobIds.isEmpty()) {
log.error("collector receive delete cyclic task job ids is null");
return null;
@@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
import org.springframework.boot.SpringApplication;
@@ -41,12 +41,12 @@ public class GoCloseProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg());
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsgString());
}
this.timerDispatch.goOffline();
this.collectServer.shutdown();
@@ -17,12 +17,12 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import java.nio.charset.StandardCharsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -36,20 +36,21 @@ public class GoOfflineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
timerDispatch.goOffline();
log.info("receive offline message and handle success");
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsg());
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsgString());
return null;
}
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
return ClusterMessage.builder()
.identity(message.getIdentity())
.direction(ClusterMessage.Direction.RESPONSE)
.type(ClusterMessage.MessageType.GO_OFFLINE)
.msg(String.valueOf(CommonConstants.SUCCESS_CODE).getBytes(StandardCharsets.UTF_8))
.build();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,13 +17,11 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
@@ -35,18 +33,20 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
*/
@Slf4j
public class GoOnlineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
if (message.getMsg().isEmpty()) {
String msgString = message.getMsgString();
if (message.getMsg() == null || msgString == null || msgString.isEmpty()) {
log.warn("The message that server response to collector is empty, please upgrade server");
} else {
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg().toStringUtf8(), ServerInfo.class);
// Use the new JsonUtil.fromJson(byte[], Class) method
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg(), ServerInfo.class);
if (serverInfo == null || serverInfo.getAesSecret() == null) {
log.warn("The message that server response to collector has not secret empty, please check");
} else {
@@ -55,10 +55,8 @@ public class GoOnlineProcessor implements NettyRemotingProcessor {
}
timerDispatch.goOnline();
log.info("receive online message and handle success");
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
.build();
// Return null to stop the ping-pong loop.
// The collector should not reply to the server's confirmation response.
return null;
}
}
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
/**
@@ -28,7 +28,7 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@Slf4j
public class HeartbeatProcessor implements NettyRemotingProcessor {
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
log.info("collector receive manager server response heartbeat, time: {}. ", System.currentTimeMillis());
return null;
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.collector.collect.nebulagraph;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
@@ -25,7 +24,17 @@ import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -37,18 +46,9 @@ import org.apache.hertzbeat.common.entity.job.protocol.NebulaGraphProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/**
* NebulaGraph collect
* NebulaGraph collect
*/
@Slf4j
public class NebulaGraphCollectImpl extends AbstractCollect {
@@ -66,6 +66,27 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
private static final String STORAGE_API = "/rocksdb_stats";
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
if (metrics == null || metrics.getNebulaGraph() == null) {
@@ -91,37 +112,50 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return;
}
String resp;
long responseTime;
HashMap<String, String> resultMap = new HashMap<>(64);
HttpContext httpContext = createHttpContext(nebulaGraph.getHost(), nebulaGraph.getPort());
HttpUriRequest request = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
// Create Request and Config
HttpRequestWithConfig requestWithConfig = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
nebulaGraph.getUrl(), nebulaGraph.getTimeout());
// Create Context and set Config
HttpContext httpContext = createHttpContext(requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
// Send an HTTP request to obtain response data
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return;
}
resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
responseTime = System.currentTimeMillis() - startTime;
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
} catch (IOException e) {
try {
// Use ResponseHandler to avoid manual resource management and CloseableHttpResponse deprecation issues
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
long responseTime = System.currentTimeMillis() - startTime;
HashMap<String, String> resultMap = new HashMap<>(64);
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
return null;
};
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
@@ -134,43 +168,47 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NEBULAGRAPH;
}
private HttpContext createHttpContext(String host, String port) {
HttpHost httpHost = new HttpHost(host, Integer.parseInt(port));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(httpHost);
return httpClientContext;
private HttpContext createHttpContext(RequestConfig requestConfig) {
HttpClientContext context = HttpClientContext.create();
if (requestConfig != null) {
context.setRequestConfig(requestConfig);
}
return context;
}
private HttpUriRequest createHttpRequest(String host, String port, String url, String timeoutStr) {
RequestBuilder requestBuilder = RequestBuilder.get();
// uri
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(String host, String port, String url, String timeoutStr) {
// HttpClient 5 uses ClassicRequestBuilder
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// uri construction
String uri = CollectUtil.replaceUriSpecialChar(url);
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + port + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(host);
String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s", host, port + uri)
: String.format("%s:%s", host, port + uri);
? String.format("[%s]:%s%s", host, port, uri)
: String.format("%s:%s%s", host, port, uri);
requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri);
}
requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain");
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(timeoutStr);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
// Use setConnectTimeout despite deprecation to allow per-request connection timeout override on a shared client
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -178,6 +216,7 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
*
* @param responseBody response body
* @param timePeriod time period
* @param resultMap result map
*/
private void parseStatsResponse(String responseBody, String timePeriod, HashMap<String, String> resultMap) {
// Set up regular expression matching
@@ -188,22 +227,26 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
Matcher matcher = pattern.matcher(str);
if (matcher.find()) {
String[] split = str.split(timeRegex);
resultMap.put(split[0], split[1]);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
}
}
}
/**
* Parse the Storage response and filter by time period
*
* @param responseBody response body
* @param resultMap result map
*/
private void parseStorageResponse(String responseBody, HashMap<String, String> resultMap) {
String[] strArray = responseBody.split(STR_SPLIT);
for (String str : strArray) {
String[] split = str.split(STORAGE_SPLIT_KEY_VALUE);
resultMap.put(split[0], split[1]);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
}
}
}
}
+1 -1
View File
@@ -28,7 +28,7 @@
<name>${project.artifactId}</name>
<packaging>pom</packaging>
<properties>
<java.version>17</java.version>
<java.version>21</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
+19 -16
View File
@@ -46,20 +46,9 @@
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- jpa -->
<dependency>
<groupId>org.eclipse.persistence</groupId>
<artifactId>org.eclipse.persistence.jpa</artifactId>
<version>${eclipse-jpa.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<exclusions>
<exclusion>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- validation -->
<dependency>
@@ -76,6 +65,10 @@
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-jackson2</artifactId>
</dependency>
<!-- swagger -->
<dependency>
<groupId>org.springdoc</groupId>
@@ -95,11 +88,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
<!-- Use apache.http.conn.util instead of sun.net.util -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<!-- caffeine-->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
@@ -183,6 +171,21 @@
<version>${javaparser.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-core</artifactId>
<version>${fory.version}</version>
</dependency>
<!-- row/arrow format support -->
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-format</artifactId>
<version>${fory.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
</dependencies>
</project>
@@ -125,6 +125,11 @@ public class CommonProperties {
*/
private String logEntryToStorageQueueName;
/**
* Timeout for blocking wait in seconds (defaults to 1 second if not configured)
*/
private Long waitTimeout;
}
/**
@@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.persistence.config.SessionCustomizer;
import org.eclipse.persistence.descriptors.ClassDescriptor;
import org.eclipse.persistence.mappings.DatabaseMapping;
import org.eclipse.persistence.sessions.Session;
/**
* change column name that from upper-case to lower-case-with-underscore
*/
@Slf4j
public class EclipseLinkCustomizer implements SessionCustomizer {
@Override
public void customize(Session session) throws Exception {
for (ClassDescriptor descriptor : session.getDescriptors().values()) {
for (DatabaseMapping mapping : descriptor.getMappings()) {
if (mapping.isDirectToFieldMapping()) {
// update the column name to lower case with underscore
if (!mapping.getField().getName().equalsIgnoreCase(mapping.getAttributeName())) {
// already custom define the column name, ignore
continue;
}
// todo here i try to change column name that from upper-case to lower-case-with-underscore
// but failed, when update name, something relate exception happen more
mapping.getField().setName(convertCamelCase(mapping.getAttributeName()));
}
}
}
}
private String convertCamelCase(String camelCase) {
StringBuilder result = new StringBuilder();
if (camelCase != null && !camelCase.isEmpty()) {
result.append(Character.toLowerCase(camelCase.charAt(0)));
for (int i = 1; i < camelCase.length(); i++) {
char currentChar = camelCase.charAt(i);
if (Character.isUpperCase(currentChar)) {
result.append('_').append(Character.toLowerCase(currentChar));
} else {
result.append(currentChar);
}
}
}
return result.toString();
}
}
@@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.eclipse.persistence.config.PersistenceUnitProperties;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.orm.jpa.JpaBaseConfiguration;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.orm.jpa.vendor.AbstractJpaVendorAdapter;
import org.springframework.orm.jpa.vendor.EclipseLinkJpaVendorAdapter;
import org.springframework.transaction.jta.JtaTransactionManager;
/**
* jpa eclipselink impl config
*/
@Configuration
@ConditionalOnProperty(prefix = "spring.datasource", name = "url")
public class EclipseLinkJpaConfiguration extends JpaBaseConfiguration {
protected EclipseLinkJpaConfiguration(DataSource dataSource, JpaProperties properties,
ObjectProvider<JtaTransactionManager> jtaTransactionManager) {
super(dataSource, properties, jtaTransactionManager);
}
@Override
protected AbstractJpaVendorAdapter createJpaVendorAdapter() {
return new EclipseLinkJpaVendorAdapter();
}
@Override
protected Map<String, Object> getVendorProperties() {
HashMap<String, Object> map = new HashMap<>(8);
map.put(PersistenceUnitProperties.DDL_GENERATION, "create-or-extend-tables");
map.put(PersistenceUnitProperties.SESSION_CUSTOMIZER, "org.apache.hertzbeat.common.config.EclipseLinkCustomizer");
map.put(PersistenceUnitProperties.ALLOW_NATIVE_SQL_QUERIES, "true");
map.put(PersistenceUnitProperties.WEAVING, "true");
return map;
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -55,7 +55,7 @@ public class ChatConversation {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation title")
private String title;
@@ -79,6 +79,6 @@ public class ChatConversation {
* List of messages in this conversation (one-to-many relationship)
*/
@OneToMany
@JoinColumn(name = "conversation_id")
@JoinColumn(name = "conversation_id", insertable = false, updatable = false)
private List<ChatMessage> messages;
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.ai;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.GeneratedValue;
@@ -48,7 +49,7 @@ import lombok.NoArgsConstructor;
@Entity
@EntityListeners(AuditingEntityListener.class)
@Table(name = "hzb_ai_message", indexes = {
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
})
@AllArgsConstructor
@NoArgsConstructor
@@ -57,8 +58,9 @@ public class ChatMessage {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation id")
@Column(name = "conversation_id")
private Long conversationId;
@Schema(title = "message content")
@@ -84,5 +86,5 @@ public class ChatMessage {
@Schema(title = "Record modify time", example = "1612198444000", accessMode = READ_ONLY)
@LastModifiedDate
private LocalDateTime gmtUpdate;
}
}
@@ -101,6 +101,7 @@ public class AlertDefine {
private String datasource;
@Schema(title = "Is Enabled", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "The creator of this record", example = "tom", accessMode = READ_ONLY)
@@ -69,9 +69,11 @@ public class AlertSilence {
private String name;
@Schema(title = "Whether to enable this policy", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to match all", example = "true")
@Builder.Default
private boolean matchAll = true;
@Schema(title = "Silence type 0: once, 1:cyc", example = "1")
@@ -68,12 +68,13 @@ public class NoticeReceiver {
private String name;
@Schema(title = "Notification information method: 0-SMS 1-Email 2-webhook 3-WeChat Official Account 4-Enterprise WeChat Robot "
+ "5-DingTalk Robot 6-FeiShu Robot 7-Telegram Bot 8-SlackWebHook 9-Discord Bot 10-Enterprise WeChat app message",
+ "5-DingTalk Robot 6-FeiShu Robot 7-Telegram Bot 8-SlackWebHook 9-Discord Bot 10-Enterprise WeChat app message "
+ "11-Slack 12-Discord 13-Gotify 14-FeiShu app message",
description = "Notification information method: "
+ "0-SMS 1-Email 2-webhook 3-WeChat Official Account "
+ "4-Enterprise WeChat Robot 5-DingTalk Robot 6-FeiShu Robot "
+ "7-Telegram Bot 8-SlackWebHook 9-Discord Bot 10-Enterprise "
+ "WeChat app message",
+ "WeChat app message 11-Slack 12-Discord 13-Gotify 14-FeiShu app message",
accessMode = READ_WRITE)
@Min(0)
@NotNull(message = "type can not null")
@@ -98,6 +99,20 @@ public class NoticeReceiver {
@Column(length = 1000)
private String hookUrl;
@Schema(title = "Auth Type: WebHook Authorization Type",
description = "Auth Type: WebHook Authorization Type, one of the 'None', 'Basic' and 'Bearer'",
example = "None", accessMode = READ_ONLY)
@Size(max = 300)
@Column(length = 300)
private String hookAuthType;
@Schema(title = "Auth Token: WebHook Authorization Token",
description = "Auth Token: WebHook Authorization Token",
example = "YWxpY2U6c3VwZXJtYW4", accessMode = READ_WRITE)
@Size(max = 300)
@Column(length = 300)
private String hookAuthToken;
@Schema(title = "openId : The notification method is valid for WeChat official account, enterprise WeChat robot or FlyBook robot",
description = "openId : The notification method is valid for WeChat official account, enterprise WeChat robot or FlyBook robot",
example = "343432", accessMode = READ_WRITE)
@@ -105,6 +120,12 @@ public class NoticeReceiver {
@Column(length = 300)
private String wechatId;
@Schema(title = "FeiShu app id : The notification method is valid for FeiShu app message",
description = "FeiShu app id : The notification method is valid for FeiShu app message",
example = "34823984635647", accessMode = READ_WRITE)
@Size(max = 255)
private String appId;
@Schema(title = "Access token : The notification method is valid for DingTalk robot",
description = "Access token : The notification method is valid for DingTalk robot",
example = "34823984635647", accessMode = READ_WRITE)
@@ -127,11 +148,21 @@ public class NoticeReceiver {
example = "779294123", accessMode = READ_WRITE)
private String tgMessageThreadId;
@Schema(title = "FeiShu app message receiveType: 0-user 1-chat 2-party 3-all",
description = "FeiShu app message receiveType: 0-user 1-chat 2-party 3-all",
example = "1", accessMode = READ_WRITE)
private Byte larkReceiveType;
@Schema(title = "DingTalk,FeiShu,WeWork user id: The notification method is valid for DingTalk,FeiShu,WeWork Bot",
description = "DingTalk,FeiShu,WeWork user id: The notification method is valid for DingTalk,FeiShu,WeWork Bot",
example = "779294123", accessMode = READ_WRITE)
private String userId;
@Schema(title = "FeiShu app message chatId: The notification method is valid for FeiShu app message",
description = "FeiShu app message chatId: The notification method is valid for FeiShu app message",
example = "779294123", accessMode = READ_WRITE)
private String chatId;
@Schema(title = "URL address: The notification method is valid for Slack",
description = "URL address: The notification method is valid for Slack",
example = "https://hooks.slack.com/services/XXXX/XXXX/XXXX", accessMode = READ_WRITE)
@@ -103,11 +103,13 @@ public class NoticeRule {
@Schema(title = "Whether to enable this policy",
description = "Whether to enable this policy",
example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to forward all",
description = "Whether to forward all",
example = "false", accessMode = READ_WRITE)
@Builder.Default
private boolean filterAll = true;
@Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}")
@@ -85,6 +85,7 @@ public class NoticeTemplate {
description = "Is it a preset template: true- preset template false- custom template.",
accessMode = READ_WRITE)
@Column(columnDefinition = "boolean default false")
@Builder.Default
private boolean preset = false;
@Schema(title = "Template content",
@@ -43,5 +43,6 @@ public class CollectorInfo {
private String version;
@NotNull
@Builder.Default
private String mode = CommonConstants.MODE_PUBLIC;
}
@@ -23,7 +23,7 @@ import lombok.EqualsAndHashCode;
import org.apache.hertzbeat.common.constants.ImportTaskStatusEnum;
import org.apache.hertzbeat.common.constants.ManagerEventTypeEnum;
import org.apache.hertzbeat.common.constants.NotifyLevelEnum;
import org.springframework.lang.Nullable;
import org.jspecify.annotations.Nullable;
/**
* Import task message
@@ -50,5 +50,6 @@ public class Configmap implements Serializable {
* 0: number 1: string 2: encrypted string 3: json string mapped by map
* number,string,secret
*/
@Builder.Default
private byte type = 1;
}
@@ -57,6 +57,7 @@ public class Job {
/**
* Tenant id
*/
@Builder.Default
private long tenantId = 0;
/**
* Monitoring Task ID
@@ -78,6 +79,7 @@ public class Job {
/**
* Is hide this app in main menus layout, only for app type, default true.
*/
@Builder.Default
private boolean hide = true;
/**
* Large categories of monitoring
@@ -111,6 +113,7 @@ public class Job {
/**
* Default task collection time interval (unit: second) eg: 30,60,600
*/
@Builder.Default
private long defaultInterval = 600L;
/**
* Refresh time list for one cycle of the job
@@ -119,6 +122,7 @@ public class Job {
/**
* Whether it is a recurring periodic task true is yes, false is no
*/
@Builder.Default
private boolean isCyclic = false;
/**
* monitor input need params
@@ -136,21 +140,25 @@ public class Job {
/**
* Whether it is a service discovery job, true is yes, false is no
*/
@Builder.Default
private boolean isSd = false;
/**
* Whether to use the Prometheus proxy
*/
@Builder.Default
private boolean prometheusProxyMode = false;
/**
* Scheduling type: interval or cron
*/
@Builder.Default
private String scheduleType = "interval";
/**
* Cron expression for scheduling, used when scheduleType is "cron"
*/
@Builder.Default
private String cronExpression = null;
/**
@@ -115,6 +115,7 @@ public class Metrics {
* Is it visible true or false
* if false, web ui will not see this metrics.
*/
@Builder.Default
private boolean visible = true;
/**
* Public attribute - collection and monitoring final result attribute set eg: speed | times | size
@@ -393,10 +394,12 @@ public class Metrics {
/**
* Metric type 0-number: number 1-string: string
*/
@Builder.Default
private byte type = 1;
/**
* Whether this field is the label
*/
@Builder.Default
private boolean label = false;
/**
* Metric unit
@@ -36,6 +36,7 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* enable ssh tunnel
*/
@Builder.Default
private String enable = "false";
/**
@@ -46,11 +47,13 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* Peer host port
*/
@Builder.Default
private String port = "22";
/**
* TIME OUT PERIOD
*/
@Builder.Default
private String timeout = "6000";
/**
@@ -76,5 +79,6 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* share connection session
*/
@Builder.Default
private String shareConnection = "true";
}
@@ -64,5 +64,6 @@ public class FtpProtocol implements CommonRequestProtocol, Protocol {
* Whether ftp uses link encryption ssl/tls, i.e. ftp or sftp
*
*/
@Builder.Default
private String ssl = "false";
}
@@ -51,6 +51,7 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether http uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
* http request methods: get, post, put, delete, patch
@@ -99,6 +100,7 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
* Whether to enable URL encoding for the path. Default is true.
* When true, the URL path will be encoded. When false, the URL path will not be encoded.
*/
@Builder.Default
private String enableUrlEncoding = "true";
/**
@@ -48,6 +48,7 @@ public class ImapProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -74,6 +74,7 @@ public class JdbcProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
* ssh tunnel
@@ -43,6 +43,7 @@ public class JmxProtocol implements CommonRequestProtocol, Protocol {
/**
* enable ssl?
*/
@Builder.Default
private String ssl = "false";
/**
@@ -54,5 +54,6 @@ public class KafkaProtocol implements CommonRequestProtocol, Protocol {
/**
* Monitor internal topic
*/
@Builder.Default
private String monitorInternalTopic = "false";
}
@@ -54,6 +54,7 @@ public class NginxProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether nginx uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
@@ -49,6 +49,7 @@ public class Pop3Protocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -50,6 +50,7 @@ public class PrometheusProtocol implements CommonRequestProtocol, Protocol {
/**
* http Whether to use link-encrypted ssl/tls, that is, http or https
*/
@Builder.Default
private String ssl = "false";
/**
* HTTP REQUEST METHOD: get, post, put, delete, patch
@@ -34,6 +34,7 @@ import org.apache.hertzbeat.common.entity.dto.Field;
public class PushProtocol implements CommonRequestProtocol, Protocol {
private String host;
private String port;
@Builder.Default
private String uri = "/api/push";
private List<Field> fields;
}
@@ -69,6 +69,7 @@ public class SnmpProtocol implements CommonRequestProtocol, Protocol {
/**
* operation: get, walk
*/
@Builder.Default
private String operation = "get";
/**
* oid map
@@ -69,6 +69,7 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
@@ -104,6 +105,7 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* flag of use proxy
*/
@Builder.Default
private String useProxy = "false";
/**
@@ -96,6 +96,7 @@ public class ParamDefine {
* Is it mandatory true-required false-optional
*/
@Schema(title = "Is it mandatory true-required false-optional", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean required = false;
/**
@@ -156,6 +157,7 @@ public class ParamDefine {
* Is it an advanced hidden parameter true-yes false-no
*/
@Schema(title = "Is it an advanced hidden parameter true-yes false-no", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean hide = false;
/**
@@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.entity.message;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.nio.charset.StandardCharsets;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
/**
* cluster message entity for fury serialization
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ClusterMessage implements Serializable {
/**
* collector identity
*/
@Builder.Default
private String identity = "";
/**
* message direction
*/
@Builder.Default
private Direction direction = Direction.REQUEST;
/**
* message type
*/
private MessageType type;
/**
* message content
* Use byte[] to ensure data integrity for both JSON strings (UTF-8) and Binary data (Arrow).
* Avoids String encoding/decoding issues (Mojibake) across different JVMs or Languages.
*/
private byte[] msg;
@JsonIgnore
public String getMsgString() {
if (this.msg == null) {
return null;
}
return new String(this.msg, StandardCharsets.UTF_8);
}
@JsonIgnore
public void setMsgString(String jsonString) {
if (jsonString != null) {
this.msg = jsonString.getBytes(StandardCharsets.UTF_8);
} else {
this.msg = null;
}
}
/**
* Message Type Enum
*/
public enum MessageType {
// heartbeat message
HEARTBEAT,
// collector go online to master message
GO_ONLINE,
// collector go offline to master message
GO_OFFLINE,
// collector go close to master
GO_CLOSE,
// issue cyclic collect task
ISSUE_CYCLIC_TASK,
// delete cyclic collect task
DELETE_CYCLIC_TASK,
// issue one-time collect task
ISSUE_ONE_TIME_TASK,
// response one-time collect data
RESPONSE_ONE_TIME_TASK_DATA,
// response cyclic collect data
RESPONSE_CYCLIC_TASK_DATA,
// response cyclic service discovery data
RESPONSE_CYCLIC_TASK_SD_DATA
}
/**
* Direction Enum
*/
public enum Direction {
// request message
REQUEST,
// request response
RESPONSE
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.common.entity.sd;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
@@ -33,6 +33,7 @@ import org.apache.hertzbeat.common.serialize.KafkaLogEntryDeserializer;
import org.apache.hertzbeat.common.serialize.KafkaLogEntrySerializer;
import org.apache.hertzbeat.common.serialize.KafkaMetricsDataDeserializer;
import org.apache.hertzbeat.common.serialize.KafkaMetricsDataSerializer;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -117,7 +118,7 @@ public class KafkaCommonDataQueue implements CommonDataQueue, DisposableBean {
Map<String, Object> metricsToStorageConsumerConfig = new HashMap<>(consumerConfig);
metricsToStorageConsumerConfig.put("group.id", "metrics-persistent-consumer");
metricsDataToStorageConsumer = new KafkaConsumer<>(metricsToStorageConsumerConfig, new LongDeserializer(), new KafkaMetricsDataDeserializer());
metricsDataToStorageConsumer.subscribe(Collections.singletonList(kafka.getMetricsDataTopic()));
metricsDataToStorageConsumer.subscribe(Collections.singletonList(kafka.getMetricsDataToStorageTopic()));
Map<String, Object> serviceDiscoveryDataConsumerConfig = new HashMap<>(consumerConfig);
serviceDiscoveryDataConsumerConfig.put("group.id", "service-discovery-data-consumer");
@@ -176,6 +177,7 @@ public class KafkaCommonDataQueue implements CommonDataQueue, DisposableBean {
dataConsumer.commitAsync();
} catch (Exception e) {
log.error(e.getMessage());
throw new CommonDataQueueUnknownException(e.getMessage(), e);
} finally {
lock.unlock();
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.common.queue.impl;
import io.lettuce.core.KeyValue;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
@@ -29,10 +30,13 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.serialize.RedisLogEntryCodec;
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import java.util.Objects;
/**
* common data queue implement redis.
*/
@@ -56,6 +60,7 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
private final String logEntryQueueName;
private final String logEntryToStorageQueueName;
private final CommonProperties.RedisProperties redisProperties;
private final Long waitTimeout;
public RedisCommonDataQueue(CommonProperties properties) {
@@ -83,36 +88,23 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
this.metricsDataQueueNameToAlerter = redisProperties.getMetricsDataQueueNameToAlerter();
this.logEntryQueueName = redisProperties.getLogEntryQueueName();
this.logEntryToStorageQueueName = redisProperties.getLogEntryToStorageQueueName();
this.waitTimeout = Objects.requireNonNullElse(redisProperties.getWaitTimeout(), 1L);
}
@Override
public CollectRep.MetricsData pollMetricsDataToAlerter() {
try {
return syncCommands.rpop(metricsDataQueueNameToAlerter);
} catch (Exception e) {
log.error(e.getMessage());
return null;
}
public CollectRep.MetricsData pollMetricsDataToAlerter() throws InterruptedException {
return genericBlockingPollFunction(metricsDataQueueNameToAlerter, syncCommands);
}
@Override
public CollectRep.MetricsData pollMetricsDataToStorage() throws InterruptedException {
try {
return syncCommands.rpop(metricsDataQueueNameToStorage);
} catch (Exception e) {
log.error(e.getMessage());
return null;
}
return genericBlockingPollFunction(metricsDataQueueNameToStorage, syncCommands);
}
@Override
public CollectRep.MetricsData pollServiceDiscoveryData() throws InterruptedException {
try {
return syncCommands.rpop(metricsDataQueueNameForServiceDiscovery);
} catch (Exception e) {
log.error(e.getMessage());
return null;
}
return genericBlockingPollFunction(metricsDataQueueNameForServiceDiscovery, syncCommands);
}
@Override
@@ -153,12 +145,7 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
@Override
public LogEntry pollLogEntry() throws InterruptedException {
try {
return logEntrySyncCommands.rpop(logEntryQueueName);
} catch (Exception e) {
log.error("Failed to poll LogEntry from Redis: {}", e.getMessage());
throw new InterruptedException("Failed to poll LogEntry from Redis");
}
return genericBlockingPollFunction(logEntryQueueName, logEntrySyncCommands);
}
@Override
@@ -172,12 +159,7 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
@Override
public LogEntry pollLogEntryToStorage() throws InterruptedException {
try {
return logEntrySyncCommands.rpop(logEntryToStorageQueueName);
} catch (Exception e) {
log.error("Failed to poll LogEntry from storage via Redis: {}", e.getMessage());
throw new InterruptedException("Failed to poll LogEntry from storage via Redis");
}
return genericBlockingPollFunction(logEntryToStorageQueueName, logEntrySyncCommands);
}
@Override
@@ -187,4 +169,23 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
redisClient.shutdown();
}
private <T> T genericBlockingPollFunction(String key, RedisCommands<String, T> commands) throws InterruptedException {
try {
// Use BRPOP for blocking pop with the configured timeout.
// If data arrives, it returns immediately; if it times out, it returns null.
KeyValue<String, T> keyData = commands.brpop(waitTimeout, key);
if (keyData != null) {
return keyData.getValue();
} else {
// Returns null on timeout
return null;
}
} catch (Exception e) {
log.error("Redis BRPOP failed: {}", e.getMessage());
throw new CommonDataQueueUnknownException(e.getMessage(), e);
}
}
}
@@ -17,60 +17,49 @@
package org.apache.hertzbeat.common.support;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* common task worker thread pool
* common task worker thread pool with Virtual Threads
*/
@Component
@Slf4j
public class CommonThreadPool implements DisposableBean {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public CommonThreadPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("common executor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("common-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(1,
Integer.MAX_VALUE,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("common-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
log.info("CommonThreadPool initialized with Virtual Threads.");
}
/**
* Run the task thread
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
* @param runnable Task
*/
public void execute(Runnable runnable) throws RejectedExecutionException {
public void execute(Runnable runnable) {
workerExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
if (workerExecutor != null) {
workerExecutor.shutdownNow();
workerExecutor.close();
}
}
}
}
@@ -21,7 +21,7 @@ import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
import org.springframework.stereotype.Component;
/**
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.support.exception;
/**
* Common data queue unknown exception
*/
public class CommonDataQueueUnknownException extends RuntimeException {
public CommonDataQueueUnknownException() {
}
public CommonDataQueueUnknownException(String message) {
super(message);
}
public CommonDataQueueUnknownException(String message, Throwable cause) {
super(message, cause);
}
public CommonDataQueueUnknownException(Throwable cause) {
super(cause);
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -37,7 +37,7 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
*/
@Slf4j
public final class ArrowUtil {
private ArrowUtil() {
}
@@ -53,7 +53,7 @@ public final class ArrowUtil {
public static byte[] serializeMultipleRoots(List<VectorSchemaRoot> roots) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(out)) {
dataOut.writeInt(roots.size());
for (VectorSchemaRoot root : roots) {
ArrowStreamWriter writer = new ArrowStreamWriter(
@@ -87,10 +87,10 @@ public final class ArrowUtil {
List<VectorSchemaRoot> roots = new ArrayList<>();
try (ByteArrayInputStream in = new ByteArrayInputStream(data);
DataInputStream dataIn = new DataInputStream(in)) {
int rootCount = dataIn.readInt();
RootAllocator allocator = new RootAllocator();
for (int i = 0; i < rootCount; i++) {
ArrowStreamReader reader = new ArrowStreamReader(
Channels.newChannel(in),
@@ -153,5 +153,4 @@ public final class ArrowUtil {
}
return serializeMultipleRoots(roots);
}
}
}
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
import java.util.concurrent.TimeUnit;
/**
* Backoff utility class.
*/
public final class BackoffUtils {
/** Private constructor to prevent instantiation */
private BackoffUtils() {}
/**
* Sleeps for the next delay specified by the ExponentialBackoff instance.
* If the thread is interrupted during sleep, it resets the interrupt status
* and returns false to indicate that the operation should not continue.
*
* @param backoff the ExponentialBackoff instance to get the next delay from
* @return true if the sleep completed without interruption, false otherwise
*/
public static boolean shouldContinueAfterBackoff(ExponentialBackoff backoff) {
if (Thread.currentThread().isInterrupted()) {
return false;
}
try {
TimeUnit.MILLISECONDS.sleep(backoff.nextDelay());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
return true;
}
}
@@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
/**
* Exponential backoff utility class.
*
* <p>Provides exponentially increasing delays starting from an initial value,
* doubling with each call to {@link #nextDelay()} until reaching the maximum value.
* Call {@link #reset()} to restart from the initial delay.</p>
*
* <p>Note: This class is <b>not</b> thread-safe. Each thread should use its own instance.</p>
*/
public final class ExponentialBackoff {
private final long initial;
private final long max;
private long current;
public ExponentialBackoff(long initial, long max) {
if (initial <= 0 || max < initial) {
throw new IllegalArgumentException("Invalid exponential backoff params");
}
this.initial = initial;
this.max = max;
this.current = initial;
}
/**
* Returns the current delay value and advances to the next delay.
* The delay doubles on each call until it reaches the maximum value.
*
* @return the delay in milliseconds to wait before the next retry
*/
public long nextDelay() {
long delay = this.current;
if (this.current <= this.max / 2) {
this.current = this.current * 2;
} else {
this.current = this.max;
}
return delay;
}
/**
* Resets the backoff to the initial delay value.
* Should be called when an operation succeeds after previous failures.
*/
public void reset() {
this.current = this.initial;
}
}
@@ -24,8 +24,8 @@ import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.net.InetAddressUtils;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.http.conn.util.InetAddressUtils;
import org.springframework.util.StringUtils;
/**
@@ -60,10 +60,10 @@ public final class IpDomainUtil {
if (LOCALHOST.equalsIgnoreCase(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv4Address(ipDomain)) {
if (InetAddressUtils.isIPv4(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv6Address(ipDomain)) {
if (InetAddressUtils.isIPv6(ipDomain)) {
return true;
}
return DOMAIN_PATTERN.matcher(ipDomain).matches();
@@ -75,7 +75,7 @@ public final class IpDomainUtil {
* @return true or false
*/
public static boolean isHasSchema(String domainIp) {
if (domainIp == null || !StringUtils.hasText(domainIp)) {
if (!StringUtils.hasText(domainIp)) {
return false;
}
return DOMAIN_SCHEMA.matcher(domainIp).matches();
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -62,6 +62,23 @@ public final class JsonUtil {
}
}
/**
* Object to byte array
* @param source object
* @return byte array
*/
public static byte[] toJsonBytes(Object source) {
if (source == null) {
return null;
}
try {
return OBJECT_MAPPER.writeValueAsBytes(source);
} catch (JsonProcessingException e) {
log.error(e.getMessage(), e);
return null;
}
}
public static <T> T fromJson(String jsonStr, Class<T> clazz) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -74,6 +91,25 @@ public final class JsonUtil {
}
}
/**
* byte array to Object
* @param jsonBytes json byte array
* @param clazz object class
* @param <T> object type
* @return object
*/
public static <T> T fromJson(byte[] jsonBytes, Class<T> clazz) {
if (jsonBytes == null || jsonBytes.length == 0) {
return null;
}
try {
return OBJECT_MAPPER.readValue(jsonBytes, clazz);
} catch (Exception e) {
log.error(e.getMessage(), e);
return null;
}
}
public static <T> T fromJson(String jsonStr, TypeReference<T> type) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -85,7 +121,7 @@ public final class JsonUtil {
return null;
}
}
public static JsonNode fromJson(String jsonStr) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -143,4 +179,4 @@ public final class JsonUtil {
char end = jsonStr.charAt(jsonStr.length() - 1);
return (start == '{' && end == '}') || (start == '[' && end == ']');
}
}
}
@@ -27,6 +27,7 @@ import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands;
import io.lettuce.core.KeyValue;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
@@ -78,7 +79,7 @@ class RedisCommonDataQueueTest {
when(redisProperties.getLogEntryToStorageQueueName()).thenReturn("logEntryToStorageQueue");
when(redisProperties.getRedisHost()).thenReturn("localhost");
when(redisProperties.getRedisPort()).thenReturn(6379);
when(redisProperties.getWaitTimeout()).thenReturn(1L);
try (MockedStatic<RedisClient> mockedRedisClient = mockStatic(RedisClient.class)) {
mockedRedisClient.when(() -> RedisClient.create(any(RedisURI.class))).thenReturn(redisClient);
when(redisClient.connect(any(RedisMetricsDataCodec.class))).thenReturn(connection);
@@ -95,11 +96,13 @@ class RedisCommonDataQueueTest {
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
.setMetrics("test metrics")
.build();
String queueName = "metricsDataQueueToAlerter";
when(syncCommands.rpop("metricsDataQueueToAlerter")).thenReturn(metricsData);
when(syncCommands.brpop(1L, queueName)).thenReturn(KeyValue.just(queueName, metricsData));
CollectRep.MetricsData actualMetricsData = redisCommonDataQueue.pollMetricsDataToAlerter();
assertEquals(metricsData, actualMetricsData);
verify(syncCommands).brpop(1L, queueName);
}
@Test
@@ -163,13 +166,14 @@ class RedisCommonDataQueueTest {
.severityText("WARN")
.body("Test warning log message")
.build();
String queueName = "logEntryQueue";
when(logEntrySyncCommands.rpop("logEntryQueue")).thenReturn(expectedLogEntry);
when(logEntrySyncCommands.brpop(1L, queueName)).thenReturn(KeyValue.just(queueName, expectedLogEntry));
LogEntry result = redisCommonDataQueue.pollLogEntry();
assertEquals(expectedLogEntry, result);
verify(logEntrySyncCommands).rpop("logEntryQueue");
verify(logEntrySyncCommands).brpop(1L, "logEntryQueue");
}
@Test
@@ -181,13 +185,13 @@ class RedisCommonDataQueueTest {
.severityText("FATAL")
.body("Critical error log for storage")
.build();
String queueName = "logEntryToStorageQueue";
when(logEntrySyncCommands.rpop("logEntryToStorageQueue")).thenReturn(expectedLogEntry);
when(logEntrySyncCommands.brpop(1L, queueName)).thenReturn(KeyValue.just(queueName, expectedLogEntry));
LogEntry result = redisCommonDataQueue.pollLogEntryToStorage();
assertEquals(expectedLogEntry, result);
verify(logEntrySyncCommands).rpop("logEntryToStorageQueue");
verify(logEntrySyncCommands).brpop(1L, "logEntryToStorageQueue");
}
@Test
@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Test case for {@link BackoffUtils}
*/
class BackoffUtilsTest {
@Test
void shouldContinueAfterBackoff() {
ExponentialBackoff backoff = new ExponentialBackoff(10L, 100L);
boolean shouldContinue = BackoffUtils.shouldContinueAfterBackoff(backoff);
assertTrue(shouldContinue);
}
@Test
void shouldNotContinueWhenInterrupted() {
Thread.currentThread().interrupt();
ExponentialBackoff backoff = new ExponentialBackoff(10L, 100L);
boolean shouldContinue = BackoffUtils.shouldContinueAfterBackoff(backoff);
assertFalse(shouldContinue);
}
@Test
void shouldHandleInterruptedException() throws InterruptedException {
final Thread mainThread = Thread.currentThread();
ExponentialBackoff backoff = new ExponentialBackoff(1000L, 2000L);
Thread interruptingThread = new Thread(() -> {
try {
// Give the main thread some time to enter the sleep
Thread.sleep(200);
mainThread.interrupt();
} catch (InterruptedException ignored) {
}
});
interruptingThread.start();
boolean shouldContinue = BackoffUtils.shouldContinueAfterBackoff(backoff);
interruptingThread.join();
assertFalse(shouldContinue);
assertTrue(Thread.currentThread().isInterrupted());
}
}
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
* Test case for {@link ExponentialBackoff}
*/
class ExponentialBackoffTest {
@Test
void testProgressionAndCap() {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
Assertions.assertEquals(50L, backoff.nextDelay());
Assertions.assertEquals(100L, backoff.nextDelay());
Assertions.assertEquals(200L, backoff.nextDelay());
Assertions.assertEquals(400L, backoff.nextDelay());
Assertions.assertEquals(800L, backoff.nextDelay());
Assertions.assertEquals(1000L, backoff.nextDelay());
Assertions.assertEquals(1000L, backoff.nextDelay());
}
@Test
void testReset() {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
Assertions.assertEquals(50L, backoff.nextDelay());
Assertions.assertEquals(100L, backoff.nextDelay());
backoff.reset();
Assertions.assertEquals(50L, backoff.nextDelay());
}
@Test
void testInvalidParams() {
Assertions.assertThrows(IllegalArgumentException.class, () -> new ExponentialBackoff(0L, 1000L));
Assertions.assertThrows(IllegalArgumentException.class, () -> new ExponentialBackoff(50L, 10L));
}
}
+6 -14
View File
@@ -99,7 +99,7 @@
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<artifactId>spring-boot-starter-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -139,16 +139,8 @@
</dependency>
<!-- database migration -->
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-database-postgresql</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-flyway</artifactId>
</dependency>
<!-- mysql -->
<dependency>
@@ -176,10 +168,10 @@
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
<!-- okhttp -->
<!-- httpclient -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
<!-- h2 database-->
<dependency>
@@ -18,13 +18,6 @@
package org.apache.hertzbeat.manager.component.sd;
import com.google.common.collect.Maps;
import java.time.LocalDateTime;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
@@ -34,6 +27,9 @@ import org.apache.hertzbeat.common.entity.manager.MonitorBind;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorDao;
@@ -43,6 +39,14 @@ import org.apache.hertzbeat.manager.service.MonitorService;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Service Discovery Worker
*/
@@ -80,8 +84,13 @@ public class ServiceDiscoveryWorker implements InitializingBean {
private class SdUpdateTask implements Runnable {
@Override
public void run() {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
while (!Thread.currentThread().isInterrupted()) {
try (final CollectRep.MetricsData metricsData = dataQueue.pollServiceDiscoveryData()) {
if (metricsData == null) {
continue;
}
backoff.reset();
Long monitorId = metricsData.getId();
final Monitor mainMonitor = monitorDao.findById(monitorId).orElse(null);
if (mainMonitor == null) {
@@ -153,6 +162,10 @@ public class ServiceDiscoveryWorker implements InitializingBean {
final Set<Long> needCancelMonitorIdSet = subMonitorBindMap.values().stream()
.map(MonitorBind::getMonitorId).collect(Collectors.toSet());
monitorService.deleteMonitors(needCancelMonitorIdSet);
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception exception) {
log.error(exception.getMessage(), exception);
}
@@ -26,14 +26,14 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.template.TemplateAvailabilityProvider;
import org.springframework.boot.autoconfigure.template.TemplateAvailabilityProviders;
import org.springframework.boot.autoconfigure.web.WebProperties;
import org.springframework.boot.autoconfigure.web.servlet.error.ErrorViewResolver;
import org.springframework.boot.webmvc.autoconfigure.error.ErrorViewResolver;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.Ordered;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.servlet.ModelAndView;
@@ -19,8 +19,8 @@ package org.apache.hertzbeat.manager.config;
import org.flywaydb.core.Flyway;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.flyway.FlywayMigrationInitializer;
import org.springframework.boot.autoconfigure.flyway.FlywayProperties;
import org.springframework.boot.flyway.autoconfigure.FlywayMigrationInitializer;
import org.springframework.boot.flyway.autoconfigure.FlywayProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -24,7 +24,7 @@ import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestExecution;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
/**
* Rest Template interceptor adds request header information
@@ -22,10 +22,12 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.text.SimpleDateFormat;
import java.util.TimeZone;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import tools.jackson.databind.JacksonModule;
import tools.jackson.databind.json.JsonMapper;
import tools.jackson.databind.module.SimpleModule;
/**
* jackson config
@@ -35,21 +37,17 @@ import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
public class JacksonConfig {
@Bean
public Jackson2ObjectMapperBuilderCustomizer jacksonCustomizer() {
public JsonMapperBuilderCustomizer jacksonCustomizer() {
return builder -> {
JavaTimeModule javaTimeModule = new JavaTimeModule();
final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
builder.defaultTimeZone(TimeZone.getDefault());
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
simpleDateFormat.setTimeZone(TimeZone.getDefault());
builder.modules(javaTimeModule)
.timeZone(TimeZone.getDefault())
.dateFormat(simpleDateFormat);
builder.defaultDateFormat(simpleDateFormat);
};
}
@Bean
public ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) {
public JsonMapper jsonMapper(JsonMapper.Builder builder) {
return builder.build();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,13 +19,18 @@ package org.apache.hertzbeat.manager.config;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.OkHttp3ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
/**
@@ -44,18 +49,24 @@ public class RestTemplateConfig {
@Bean
public ClientHttpRequestFactory simpleClientHttpRequestFactory() {
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS);
return new OkHttp3ClientHttpRequestFactory(
new OkHttpClient.Builder()
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
.connectionPool(new ConnectionPool(
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
TimeUnit.SECONDS)
).build()
);
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS))
.setSocketTimeout(Timeout.of(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS))
.build();
connectionManager.setDefaultConnectionConfig(connectionConfig);
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.evictIdleConnections(TimeValue.of(NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT, TimeUnit.SECONDS))
.disableAutomaticRetries()
.build();
return new HttpComponentsClientHttpRequestFactory(httpClient);
}
}
}
@@ -28,12 +28,12 @@ import org.apache.sshd.common.io.nio2.Nio2ServiceFactoryFactory;
import org.apache.sshd.common.session.SessionListener;
import org.apache.sshd.common.util.security.bouncycastle.BouncyCastleSecurityProviderRegistrar;
import org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar;
import org.jspecify.annotations.NonNull;
import org.springframework.aot.hint.ExecutableMode;
import org.springframework.aot.hint.MemberCategory;
import org.springframework.aot.hint.RuntimeHints;
import org.springframework.aot.hint.RuntimeHintsRegistrar;
import org.springframework.aot.hint.TypeReference;
import org.springframework.lang.NonNull;
import org.springframework.util.ClassUtils;
/**
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.manager.scheduler;
import com.google.protobuf.ByteString;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
@@ -44,7 +43,7 @@ import org.apache.hertzbeat.common.entity.manager.CollectorMonitorBind;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
@@ -155,9 +154,9 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
List<Param> params = paramDao.findParamsByMonitorId(monitor.getId());
List<Configmap> configmaps = params.stream()
.map(param -> Configmap.builder()
.key(param.getField())
.value(param.getParamValue())
.type(param.getType()).build()).collect(Collectors.toList());
.key(param.getField())
.value(param.getParamValue())
.type(param.getType()).build()).collect(Collectors.toList());
List<ParamDefine> paramDefaultValue = appDefine.getParams().stream()
.filter(item -> StringUtils.isNotBlank(item.getDefaultValue()))
.toList();
@@ -214,10 +213,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK)
.msg(JsonUtil.toJsonBytes(job))
.build();
this.manageServer.sendMsg(collectorName, message);
}
@@ -229,10 +228,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
assignJobs.getRemovingJobs().forEach(jobId -> collectJobService.cancelAsyncCollectJob(jobId));
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(assignJobs.getRemovingJobs())))
ClusterMessage message = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.DELETE_CYCLIC_TASK)
.msg(JsonUtil.toJsonBytes(assignJobs.getRemovingJobs()))
.build();
this.manageServer.sendMsg(collectorName, message);
}
@@ -243,13 +242,13 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
@Override
public boolean offlineCollector(String identity) {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_OFFLINE)
.setDirection(ClusterMsg.Direction.REQUEST)
.setIdentity(identity)
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.GO_OFFLINE)
.direction(ClusterMessage.Direction.REQUEST)
.identity(identity)
.build();
ClusterMsg.Message response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsg().toStringUtf8())) {
ClusterMessage response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsgString())) {
return false;
}
log.info("send offline collector message to {} success", identity);
@@ -264,14 +263,14 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
return false;
}
ServerInfo serverInfo = ServerInfo.builder().aesSecret(AesUtil.getDefaultSecretKey()).build();
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(serverInfo)))
.setIdentity(identity)
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.GO_ONLINE)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(serverInfo))
.identity(identity)
.build();
ClusterMsg.Message response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsg().toStringUtf8())) {
ClusterMessage response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsgString())) {
return false;
}
log.info("send online collector message to {} success", identity);
@@ -304,10 +303,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
CountDownLatch countDownLatch = new CountDownLatch(1);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
boolean result = this.manageServer.sendMsg(node.getIdentity(), message);
@@ -347,10 +346,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
return collectJobService.collectSyncJobData(job);
}
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
boolean result = this.manageServer.sendMsg(node.getIdentity(), message);
if (result) {
@@ -399,10 +398,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
this.manageServer.sendMsg(node.getIdentity(), message);
}
@@ -443,10 +442,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
collectJobService.cancelAsyncCollectJob(jobId);
} else {
ClusterMsg.Message deleteMessage = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(List.of(jobId))))
ClusterMessage deleteMessage = ClusterMessage.builder()
.type(ClusterMessage.MessageType.DELETE_CYCLIC_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(List.of(jobId)))
.build();
this.manageServer.sendMsg(node.getIdentity(), deleteMessage);
}
@@ -471,4 +470,4 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
public void setManageServer(ManageServer manageServer) {
this.manageServer = manageServer;
}
}
}
@@ -17,46 +17,34 @@
package org.apache.hertzbeat.manager.scheduler;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* manager module thread pool
* manager module thread pool with Virtual Threads
*/
@Slf4j
@Component
public class ManagerWorkerPool {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public ManagerWorkerPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("manager-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(6,
10,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("manager-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable) {
workerExecutor.execute(runnable);
}
}

Some files were not shown because too many files have changed in this diff Show More