Compare commits

..
Author SHA1 Message Date
Tomsun28 d0c6933a88 Merge branch 'master' into feat/refactor_monitor 2025-12-05 22:03:13 +08:00
Duansg 3e567e5be6 [improve]Improve issues related to tag conflicts (#3889) 2025-12-05 22:02:43 +08:00
Tomsun28 a995dd7daf Merge branch 'master' into feat/refactor_monitor 2025-12-04 23:56:29 +08:00
Tomsun28andaias00 4ae6ba9a3c feat: update add monitors pop selected menus modal (#3878)
Signed-off-by: tomsun28 <tomsun28@outlook.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
2025-12-04 23:55:10 +08:00
360abcc61d [feature] Using duckdb instead of jpa to store history data (#3883)
Signed-off-by: Logic <zqr10159@126.com>
Co-authored-by: aias00 <liuhongyu@apache.org>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-04 23:51:04 +08:00
liuhy c8739f530e [add] Add parameter validators and their corresponding tests 2025-12-04 17:02:17 +08:00
aias00andCopilot 7b2c822802 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/validator/impl/ArrayParamValidator.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:01:28 +08:00
aias00andCopilot 339c9775e3 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/validator/ParamValidatorManager.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:01:19 +08:00
aias00andCopilot 8efbba1693 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/validator/impl/ArrayParamValidator.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:01:10 +08:00
aias00andCopilot 20e016dff7 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:01:00 +08:00
aias00andCopilot f5bd371114 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/validator/impl/BooleanParamValidator.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:00:49 +08:00
aias00andgithub-actions[bot] 172a279082 Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/TextParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:00:21 +08:00
aias00andCopilot f4d19a2f7f Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/ArrayParamValidatorTest.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:00:11 +08:00
aias00andCopilot 47f2d5be6c Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/HostParamValidatorTest.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 17:00:01 +08:00
aias00andgithub-actions[bot] c48abad4cf Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/helper/MonitorImExportHelperTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:59:34 +08:00
aias00andgithub-actions[bot] 6c6b253f00 Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/PasswordParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:59:17 +08:00
aias00andgithub-actions[bot] 6878bf84ae Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/OptionParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:59:04 +08:00
aias00andgithub-actions[bot] 926a399105 Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/NumberParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:50 +08:00
aias00andgithub-actions[bot] 29e1e9cb40 Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/JsonParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:39 +08:00
aias00andgithub-actions[bot] 1397bb15ec Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/HostParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:31 +08:00
aias00andgithub-actions[bot] 8a9765c99a Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/HostParamValidatorAdapterTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:23 +08:00
aias00andgithub-actions[bot] 6f935357b8 Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/impl/BooleanParamValidatorTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:11 +08:00
aias00andgithub-actions[bot] 9faead5aee Update hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/validator/ParamValidatorManagerTest.java
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: aias00 <liuhongyu@apache.org>
2025-12-04 16:58:01 +08:00
liuhy a56aa09ed6 [add] Add parameter validators and their corresponding tests 2025-12-04 16:52:12 +08:00
P_PeacefulandTomsun28 41e57db032 [bugfix] notice policy lost Gotify receiver type (#3886)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-03 22:44:20 +08:00
DuansgandTomsun28 b62137d185 [improve]Improve performance issues caused by too many metrics (#3879)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-03 22:30:26 +08:00
DuansgandTomsun28 2bd635e1d4 [fix]Fix some systemic issues (#3884)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-12-03 22:25:02 +08:00
Yang Chen 8e8ab80aa0 Revert "[refractor] Using Spring jdbc to query Greptime log" (#3885) 2025-12-03 12:32:45 +08:00
LogicandYang Chen 7f4c2e0487 [refractor] Using Spring jdbc to query Greptime log (#3880)
Signed-off-by: Yang Chen <1597081640@qq.com>
Co-authored-by: Yang Chen <1597081640@qq.com>
2025-12-03 12:11:08 +08:00
e52e715a53 improvement: optimize log stream for high TPS scenarios (#3876)
Signed-off-by: Yang Chen <1597081640@qq.com>
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-12-03 11:18:49 +08:00
e3e7377d86 [refactor] Refactor Monitor host field to instance and update related logic (#3863)
Signed-off-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-12-01 21:45:59 +08:00
P_PeacefulandTomsun28 625dbe9624 [bugfix] redis cluster monitor master-slave relationship is inconsistent (#3874)
Co-authored-by: Tomsun28 <tomsun28@outlook.com>
2025-11-30 19:39:40 +08:00
Duansgandaias00 42f0a74edf [fix] adding trusted domain verification (#3866)
Co-authored-by: aias00 <liuhongyu@apache.org>
2025-11-27 20:38:00 +08:00
Tomsun28 cc9f53d71a chore: update sureness yml (#3870) 2025-11-27 20:10:46 +08:00
249 changed files with 5756 additions and 3265 deletions
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "ftp",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorFTP}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "api",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorHTTP}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "ping",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorPing}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "port",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorPort}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "fullsite",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorSiteMap}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "ssl_cert",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorSSL}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "udp_port",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorUDP}}"
},
"params": [
+1 -1
View File
@@ -5,7 +5,7 @@
"intervals": 60,
"tags": [],
"app": "website",
"host": "127.0.0.1",
"instance": "127.0.0.1",
"name": "{{.param.monitorWebsite}}"
},
"params": [
+1 -1
View File
@@ -4,7 +4,7 @@
"intervals": 10,
"tags": [],
"app": "kubernetes",
"host": "172.29.0.11",
"instance": "172.29.0.11",
"name": "Brave_Stingray_55yR"
},
"collector": "",
+1 -1
View File
@@ -27,7 +27,7 @@
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.3</spring-ai.version>
<java.version>21</java.version>
<java.version>17</java.version>
</properties>
<dependencies>
@@ -33,16 +33,17 @@ public interface MetricsTools {
/**
* Get historical metrics data for a monitor
* @param monitorId Monitor ID
* @param app Monitor type (e.g., "linux", "mysql", "http")
* @param metrics Metrics name (e.g., "system", "cpu", "memory")
* @param metric Specific metric field (e.g., "usage", "used", "available")
* @param label Label filter for specific instances
* @param history Time range (e.g., "1h", "6h", "24h", "7d")
*
* @param instance Instance identifier (e.g., "ip:port", "ip", or "domain")
* @param app Monitor type (e.g., "linux", "mysql", "http")
* @param metrics Metrics name (e.g., "system", "cpu", "memory")
* @param metric Specific metric field (e.g., "usage", "used", "available")
* @param label Label filter for specific instances
* @param history Time range (e.g., "1h", "6h", "24h", "7d")
* @param interval Whether to aggregate data with intervals
* @return Historical metrics data formatted for display
*/
String getHistoricalMetrics(Long monitorId, String app, String metrics, String metric, String label, String history, Boolean interval);
String getHistoricalMetrics(String instance, String app, String metrics, String metric, String label, String history, Boolean interval);
/**
* Check warehouse storage server status
@@ -152,7 +152,7 @@ public class MetricsToolsImpl implements MetricsTools {
Ask user to provide the filters for labels, history and interval aggregation
""")
public String getHistoricalMetrics(
@ToolParam(description = "Monitor ID", required = true) Long monitorId,
@ToolParam(description = "Instance identifier (e.g., 'ip:port', 'ip', or 'domain')") String instance,
@ToolParam(description = "Monitor type (e.g., 'linux', 'mysql', 'http')", required = true) String app,
@ToolParam(description = "Metrics name (e.g., 'target', 'cpu', 'memory')", required = true) String metrics,
@ToolParam(description = "Field Parameter (e.g., 'usage', 'used', 'available')", required = false) String fieldParameter,
@@ -161,7 +161,7 @@ public class MetricsToolsImpl implements MetricsTools {
@ToolParam(description = "Whether to aggregate data with intervals", required = false) Boolean interval) {
try {
log.info("Getting historical metrics for monitor {} and metrics {}", monitorId, metrics);
log.info("Getting historical metrics for monitor instance {} and metrics {}", instance, metrics);
if (history == null || history.trim().isEmpty()) {
history = "24h";
@@ -170,15 +170,15 @@ public class MetricsToolsImpl implements MetricsTools {
interval = true;
}
MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(
monitorId, app, metrics, fieldParameter, label, history, interval);
MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(instance,
app, metrics, fieldParameter, history, interval);
if (historyData == null) {
return String.format("No historical metrics data found for monitor ID %d and metrics '%s'", monitorId, metrics);
return String.format("No historical metrics data found for monitor %s and metrics '%s'", instance, metrics);
}
StringBuilder response = new StringBuilder();
response.append("HISTORICAL METRICS: ").append(metrics).append(" (Monitor ID: ").append(monitorId).append(")\n");
response.append("HISTORICAL METRICS: ").append(metrics).append(" (Monitor ID: ").append(instance).append(")\n");
response.append("Time Range: ").append(history).append(" | Interval Aggregation: ").append(interval).append("\n");
response.append("=".repeat(60)).append("\n\n");
@@ -36,6 +36,7 @@ import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Implementation of Monitoring Tools functionality
@@ -171,7 +172,7 @@ public class MonitorToolsImpl implements MonitorTools {
response.append("ID: ").append(monitor.getId())
.append(" | Name: ").append(monitor.getName())
.append(" | Type: ").append(monitor.getApp())
.append(" | Host: ").append(monitor.getHost())
.append(" | Instance: ").append(monitor.getInstance())
.append(" | Status: ").append(UtilityClass.getStatusText(monitor.getStatus()));
// Add creation date for better context
@@ -257,12 +258,14 @@ public class MonitorToolsImpl implements MonitorTools {
if (intervals == null || intervals < 10) {
intervals = 600;
}
String instance = Objects.nonNull(port) ? host.trim() + ":" + port : host.trim();
// Create Monitor entity
Monitor monitor = Monitor.builder()
.name(name.trim())
.app(app.toLowerCase().trim())
.host(host.trim())
.instance(instance)
.intervals(intervals)
.status((byte) 1)
.type((byte) 0)
@@ -17,23 +17,25 @@
package org.apache.hertzbeat.alert;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* alarm module thread pool with Virtual Threads
* alarm module thread pool
*/
@Component
@Slf4j
public class AlerterWorkerPool {
private ExecutorService workerExecutor;
private ExecutorService notifyExecutor;
private ExecutorService logWorkerExecutor;
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor notifyExecutor;
private ThreadPoolExecutor logWorkerExecutor;
public AlerterWorkerPool() {
initWorkExecutor();
@@ -42,31 +44,62 @@ public class AlerterWorkerPool {
}
private void initWorkExecutor() {
ThreadFactory factory = Thread.ofVirtual()
.name("alerter-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("alerter-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(10,
10,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
private void initNotifyExecutor() {
ThreadFactory factory = Thread.ofVirtual()
.name("notify-worker-", 0)
.factory();
notifyExecutor = Executors.newThreadPerTaskExecutor(factory);
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter notifyExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("notify-worker-%d")
.build();
notifyExecutor = new ThreadPoolExecutor(6,
6,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
private void initLogWorkerExecutor() {
ThreadFactory factory = Thread.ofVirtual()
.name("log-worker-", 0)
.factory();
logWorkerExecutor = Executors.newThreadPerTaskExecutor(factory);
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter logWorkerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("log-worker-%d")
.build();
logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(1000),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
/**
* Run the alerter task
* @param runnable task
* @throws RejectedExecutionException when The thread pool is full of
*/
public void executeJob(Runnable runnable){
public void executeJob(Runnable runnable) throws RejectedExecutionException {
workerExecutor.execute(runnable);
}
@@ -74,8 +107,9 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the notifyExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeNotify(Runnable runnable){
public void executeNotify(Runnable runnable) throws RejectedExecutionException {
notifyExecutor.execute(runnable);
}
@@ -83,8 +117,9 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the logWorkerExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeLogJob(Runnable runnable){
public void executeLogJob(Runnable runnable) throws RejectedExecutionException {
logWorkerExecutor.execute(runnable);
}
}
}
@@ -156,7 +156,7 @@ public class MetricsRealTimeAlertCalculator {
long currentTimeMilli = System.currentTimeMillis();
String instance = String.valueOf(metricsData.getId());
String instanceName = metricsData.getInstanceName();
String instanceHost = metricsData.getInstanceHost();
String instanceHost = metricsData.getInstance();
String app = metricsData.getApp();
String metrics = metricsData.getMetrics();
if ((CommonConstants.PROMETHEUS_APP_PREFIX + instanceName).equals(metricsData.getApp())) {
@@ -205,12 +205,11 @@ public class MetricsRealTimeAlertCalculator {
}
Long defineId = define.getId();
Map<String, String> commonFingerPrints = new HashMap<>(8);
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE, instance);
// here use the alert name as finger, not care the alert name may be changed
commonFingerPrints.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(define.getId()));
commonFingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName());
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_NAME, instanceName);
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_HOST, instanceHost);
commonFingerPrints.put(CommonConstants.LABEL_INSTANCE, instanceHost);
commonFingerPrints.putAll(define.getLabels());
if (labels != null) {
commonFingerPrints.putAll(labels);
@@ -71,7 +71,6 @@ public class WindowedLogRealTimeAlertCalculator implements Runnable {
}
backoff.reset();
processLogEntry(logEntry);
dataQueue.sendLogEntryToStorage(logEntry);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.alert.service.impl;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.dto.HuaweiCloudExternAlert;
@@ -43,6 +44,7 @@ import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.text.MessageFormat;
import java.time.Instant;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
@@ -70,6 +72,8 @@ public class HuaweiCloudExternAlertService implements ExternAlertService {
private static final String CHARSET_UTF8 = StandardCharsets.UTF_8.name();
private static final String SUBSCRIBE_URL_PREFIX = "https://console.huaweicloud.com/smn/subscription/confirm";
private final AlarmCommonReduce alarmCommonReduce;
public HuaweiCloudExternAlertService(AlarmCommonReduce alarmCommonReduce) {
@@ -197,6 +201,9 @@ public class HuaweiCloudExternAlertService implements ExternAlertService {
if (StringUtils.isBlank(subscribeUrl)) {
return;
}
if (!subscribeUrl.startsWith(SUBSCRIBE_URL_PREFIX)) {
throw new SecurityException("Untrusted domain: " + subscribeUrl);
}
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpGet httpGet = new HttpGet(subscribeUrl);
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
@@ -303,6 +310,13 @@ public class HuaweiCloudExternAlertService implements ExternAlertService {
*/
private X509Certificate getCertificate(String signCertUrl) throws Exception {
URL url = new URL(signCertUrl);
if (!"https".equalsIgnoreCase(url.getProtocol())) {
throw new SecurityException("Only HTTPS is allowed");
}
boolean trusted = Arrays.stream(Region.values()).anyMatch(ep -> ep.getEndpoint().equals(url.getHost()));
if (!trusted) {
throw new SecurityException("Untrusted domain: " + url.getHost());
}
try (InputStream in = url.openStream()) {
CertificateFactory cf = CertificateFactory.getInstance(CERTIFICATE_TYPE);
return (X509Certificate) cf.generateCertificate(in);
@@ -340,4 +354,51 @@ public class HuaweiCloudExternAlertService implements ExternAlertService {
return "huaweicloud-ces";
}
/**
* doc: <a href="https://console.huaweicloud.com/apiexplorer/#/endpoint/SMN">SMN API</a>
*/
@Getter
private enum Region {
AE_AD_1("ae-ad-1", "smn.ae-ad-1.myhuaweicloud.com"),
AF_SOUTH_1("af-south-1", "smn.af-south-1.myhuaweicloud.com"),
AP_SOUTHEAST_1("ap-southeast-1", "smn.ap-southeast-1.myhuaweicloud.com"),
AP_SOUTHEAST_2("ap-southeast-2", "smn.ap-southeast-2.myhuaweicloud.com"),
AP_SOUTHEAST_3("ap-southeast-3", "smn.ap-southeast-3.myhuaweicloud.com"),
AP_SOUTHEAST_4("ap-southeast-4", "smn.ap-southeast-4.myhuaweicloud.com"),
CN_EAST_2("cn-east-2", "smn.cn-east-2.myhuaweicloud.com"),
CN_EAST_3("cn-east-3", "smn.cn-east-3.myhuaweicloud.com"),
CN_EAST_4("cn-east-4", "smn.cn-east-4.myhuaweicloud.com"),
CN_EAST_5("cn-east-5", "smn.cn-east-5.myhuaweicloud.com"),
CN_NORTH_1("cn-north-1", "smn.cn-north-1.myhuaweicloud.com"),
CN_NORTH_11("cn-north-11", "smn.cn-north-11.myhuaweicloud.com"),
CN_NORTH_12("cn-north-12", "smn.cn-north-12.myhuaweicloud.com"),
CN_NORTH_2("cn-north-2", "smn.cn-north-2.myhuaweicloud.cn"),
CN_NORTH_4("cn-north-4", "smn.cn-north-4.myhuaweicloud.com"),
CN_NORTH_9("cn-north-9", "smn.cn-north-9.myhuaweicloud.com"),
CN_SOUTH_1("cn-south-1", "smn.cn-south-1.myhuaweicloud.com"),
CN_SOUTH_2("cn-south-2", "smn.cn-south-2.myhuaweicloud.com"),
CN_SOUTH_4("cn-south-4", "smn.cn-south-4.myhuaweicloud.com"),
CN_SOUTHWEST_2("cn-southwest-2", "smn.cn-southwest-2.myhuaweicloud.com"),
CN_SOUTHWEST_3("cn-southwest-3", "smn.cn-southwest-3.myhuaweicloud.com"),
EU_WEST_0("eu-west-0", "smn.eu-west-0.myhuaweicloud.com"),
LA_NORTH_2("la-north-2", "smn.la-north-2.myhuaweicloud.com"),
LA_SOUTH_2("la-south-2", "smn.la-south-2.myhuaweicloud.com"),
MY_KUALALUMPUR_1("my-kualalumpur-1", "smn.my-kualalumpur-1.myhuaweicloud.com"),
NA_MEXICO_1("na-mexico-1", "smn.na-mexico-1.myhuaweicloud.com"),
RU_MOSCOW_1("ru-moscow-1", "smn.ru-moscow-1.myhuaweicloud.com"),
SA_BRAZIL_1("sa-brazil-1", "smn.sa-brazil-1.myhuaweicloud.com"),
TR_WEST_1("tr-west-1", "smn.tr-west-1.myhuaweicloud.com"),
EU_WEST_101("eu-west-101", "smn.eu-west-101.myhuaweicloud.eu");
private final String id;
private final String endpoint;
Region(String id, String endpoint) {
this.id = id;
this.endpoint = endpoint;
}
}
}
@@ -303,9 +303,8 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
@Override
public boolean sendTestMsg(NoticeReceiver noticeReceiver) {
Map<String, String> labels = new HashMap<>(8);
labels.put(CommonConstants.LABEL_INSTANCE, "1000000");
labels.put(CommonConstants.LABEL_INSTANCE, "127.0.0.1");
labels.put(CommonConstants.LABEL_ALERT_NAME, "CPU Usage Alert");
labels.put(CommonConstants.LABEL_INSTANCE_HOST, "127.0.0.1");
Map<String, String> annotations = new HashMap<>(8);
annotations.put("suggest", "Please check the CPU usage of the server");
SingleAlert singleAlert1 = SingleAlert.builder()
@@ -125,7 +125,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
Map<String, String> meta = new HashMap<>();
meta.put(MetricDataConstants.INSTANCE_NAME, "Cool_Stingray_34Nj_copy");
meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1");
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
builder.addMetadataAll(meta);
builder.addAllFields(Lists.newArrayList(destination, mode, metricValue));
@@ -175,7 +175,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
Map<String, String> meta = new HashMap<>();
meta.put(MetricDataConstants.INSTANCE_NAME, "Cool_Stingray_34Nj");
meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1");
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
builder.addMetadataAll(meta);
builder.addAllFields(Lists.newArrayList(destination, mode, metricValue));
@@ -225,7 +225,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
Map<String, String> meta = new HashMap<>();
meta.put(MetricDataConstants.INSTANCE_NAME, "Vibrant_Gazelle_83vJ");
meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1");
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
builder.addMetadataAll(meta);
builder.addAllFields(Lists.newArrayList(responseTime));
@@ -108,7 +108,6 @@ class WindowedLogRealTimeAlertCalculatorTest {
verify(timeService).isLateData(anyLong());
verify(timeService).updateMaxTimestamp(anyLong());
verify(logWorker).reduceAndSendLogTask(validLogEntry);
verify(dataQueue).sendLogEntryToStorage(validLogEntry);
}
@Test
@@ -134,7 +133,6 @@ class WindowedLogRealTimeAlertCalculatorTest {
verify(timeService, never()).isLateData(anyLong());
verify(timeService, never()).updateMaxTimestamp(anyLong());
verify(logWorker, never()).reduceAndSendLogTask(any());
verify(dataQueue).sendLogEntryToStorage(invalidTimestampLogEntry);
}
@Test
@@ -162,7 +160,6 @@ class WindowedLogRealTimeAlertCalculatorTest {
verify(timeService).isLateData(anyLong());
verify(timeService, never()).updateMaxTimestamp(anyLong());
verify(logWorker, never()).reduceAndSendLogTask(any());
verify(dataQueue).sendLogEntryToStorage(lateDataLogEntry);
}
@Test
@@ -88,7 +88,7 @@ class AlertDefineControllerTest {
Monitor.builder()
.id(1L)
.app("app")
.host("localhost")
.instance("localhost")
.name("monitor")
.build()
)
@@ -146,7 +146,42 @@ public class HuaweiCloudExternAlertServiceTest {
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class));
}
@Test
void testSubscriptionUrlFail() {
HuaweiCloudExternAlert externAlert = new HuaweiCloudExternAlert();
externAlert.setSubscribeUrl("https://console.huaweicloud.com/smn/subscription/confirm?token=477a784601d744e4ab9ab83986502d31c4b938"
+ "0ec0b64392b134e517c3aa17eb7b3a12dc9f3b4ab495e61c4dee654b435d7223ea934345bf8ae8901cef912b1d&topic_urn=urn:smn:cn-north-4"
+ ":477a784601d744e4ab9ab83986502d31:CES_notification_group_bngJ2aMpX&region=cn-north-4");
externAlert.setSignature("ottf37C/2RdDgqimRQMIBU6i7XjUfPPMU760jJn71wwP3825YPoIT22uw2A9399rkm9Jrt1qUEFrDLuA5yHFLd5n/XoM4FghIgyFn7VIfgpuVM31a+co78s"
+ "YBiZ1egOCE/AwFm2oygRhfIceUj9Kw9vmc06el9TXY6RtE5tAEF6qEmICtTh45KwtCO/WRs3DY72dQi5hm0w7/tktS4WFZ1iP4LHt5eCwFvnH0u29Y96cJNI0fLUQxI5MkhgjK"
+ "77JkFK7UT6ZYJZhzgSp/B7OQGStOQx+3Duvx4T4CzccZQM3sca81Z0B0GFGWeVXuEHyCPLsayY/Iz+5Tco51elT8w==");
externAlert.setTopicUrn("urn:smn:cn-north-4:477a784601d744e4ab9ab83986502d31:CES_notification_group_bngJ2aMpX");
externAlert.setMessageId("242fac183d3a4936b5ead6c725a32ed0");
externAlert.setType("SubscriptionConfirmation");
externAlert.setMessage("You are invited to subscribe to topic: urn:smn:cn-north-4:477a784601d744e4ab9ab83986502d31:"
+ "CES_notification_group_bngJ2aMpX. To confirm this subscription, please visit the subscribe_url included in this message. The subscribe_url is valid only within 48 hours.");
externAlert.setSigningCertUrl("https://xxxx.myhuaweicloud.com/smn/SMN_cn-north-4_b98100ca131b4116ab8ee7ccedbaae99.pem");
externAlert.setTimestamp("2025-06-07T15:07:14Z");
externAlertService.addExternAlert(JsonUtil.toJson(externAlert));
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class));
}
@Test
void testCertUrlFailed() {
HuaweiCloudExternAlert externAlert = new HuaweiCloudExternAlert();
externAlert.setSignature("TImrLoeb0tV1JZJSPyA0rpC9mNqH3MmhwQ4tgpuHHa+JztfGVZFvkU//OthKKhzpDAoYiXOYG9DbzXCLb"
+ "vaGePIRITakoynYyYr9zZIpdx9jXhQNlgF8np1+t0JxNeoIq0DYWgH52tsodwqOm+OnmkcHwCRo/1rFv85KrKAaX2gy3sNwX"
+ "w1hKnAwAw0mJlxHHSf/N3+7j6GoxCNV7fN9K4CpJiLMGNvUa7zVmG0U9mPvt/7Lac155kPPQ9lYyeL7vVI0e4sfRbuQruz3E"
+ "0ZP40TKx0afoeR0/Bx/IoZzRP1La7pKlbEISvkcM7TqW/IOGQTkhVsQ32RFRxZWO2snw==");
externAlert.setSubject("DCS Redis “dcs-h4tv” ");
externAlert.setTopicUrn("urn:smn:cn-north-4:477a784601d744e4ab9ab83986502d31:CES_notification_group_bngJ2aMpX");
externAlert.setMessageId("d3672d737bb742cf8c2aa3f0fd72d4d1");
externAlert.setType("Notification");
externAlert.setMessage("{}");
externAlert.setSigningCertUrl("https://xxx.myhuaweicloud.com/failedUrl");
externAlert.setTimestamp("2025-06-07T15:12:09Z");
externAlertService.addExternAlert(JsonUtil.toJson(externAlert));
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class));
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,80 +15,79 @@
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect. http;
package org.apache.hertzbeat.collector.collect.http;
import static org.apache.hertzbeat. common.constants.SignConstants.RIGHT_DASH;
import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com. google.gson.JsonParser;
import com.google.gson.JsonParser;
import java.io.IOException;
import java.io.InputStream;
import java. io.InterruptedIOException;
import java.io.InterruptedIOException;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.LinkedList;
import java.util. List;
import java.util. Map;
import java.util. Objects;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.net.ssl.SSLException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml. parsers.DocumentBuilderFactory;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax. xml.xpath.XPathConstants;
import javax.xml.xpath. XPathExpressionException;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import lombok.extern.slf4j. Slf4j;
import org. apache.hc.core5.http.ClassicHttpRequest;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.collect.http.promethus.AbstractPrometheusParse;
import org.apache.hertzbeat.collector.collect.http.promethus.PrometheusParseCreator;
import org.apache. hertzbeat.collector.collect. prometheus.parser.MetricFamily;
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache. hertzbeat.collector.dispatch. DispatchConstants;
import org. apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.collector.util.JsonPathParser;
import org.apache.hertzbeat.collector.util.TimeExpressionUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache. hertzbeat.common.constants. NetworkConstants;
import org.apache. hertzbeat.common.constants. SignConstants;
import org.apache.hertzbeat.common.entity.job. Metrics;
import org.apache. hertzbeat.common.entity. job.protocol.HttpProtocol;
import org.apache.hertzbeat. common.entity.message.CollectRep;
import org.apache.hertzbeat. common.util.Base64Util;
import org. apache.hertzbeat.common. util.CommonUtil;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache. hc.core5.http.ClassicHttpResponse;
import org. apache.hc.core5. http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache. hc.core5.http. HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache. hc.client5.http.auth.AuthScope;
import org.apache. hc.client5.http. auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.ClientProtocolException;
import org. apache.hc.client5. http.config.RequestConfig;
import org. apache.hc.client5. http.classic.methods.HttpGet;
import org. apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org. apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache. hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc. core5.http.protocol.HttpContext;
import org. apache.hc.core5. http.io.entity.EntityUtils;
import org.apache. hc.core5.util.Timeout;
import org. springframework.http.HttpMethod;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org. springframework.util.CollectionUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.util.UriUtils;
import org.xml.sax.InputSource;
@@ -97,9 +96,9 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.Properties;
import java.util.regex.Matcher;
import java. util.regex.Pattern;
import java.util.regex.Pattern;
import java.util.HashSet;
import java.util. ArrayList;
import java.util.ArrayList;
import java.util.Collections;
/**
@@ -107,28 +106,6 @@ import java.util.Collections;
*/
@Slf4j
public class HttpCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Set.of(
HttpStatus.SC_OK,
HttpStatus.SC_CREATED,
@@ -145,37 +122,28 @@ public class HttpCollectImpl extends AbstractCollect {
}
@Override
public void collect(CollectRep. MetricsData.Builder builder, Metrics metrics) {
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
long startTime = System.currentTimeMillis();
HttpProtocol httpProtocol = metrics.getHttp();
String url = httpProtocol.getUrl();
if (! StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
if (!StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
}
if (CollectionUtils.isEmpty(httpProtocol.getSuccessCodes())) {
httpProtocol. setSuccessCodes(List. of(HttpStatus.SC_OK + ""));
httpProtocol.setSuccessCodes(List.of(HttpStatus.SC_OK + ""));
}
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig. getRequest();
HttpContext httpContext = createHttpContext(metrics. getHttp(), requestWithConfig. getRequestConfig());
// Use HttpClientResponseHandler to handle the response and avoid deprecated execute method
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
HttpContext httpContext = createHttpContext(metrics.getHttp());
HttpUriRequest request = createHttpRequest(metrics.getHttp());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
boolean isSuccessInvoke = checkSuccessInvoke(metrics, statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
builder.setCode(CollectRep.Code. FAIL);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return null;
return;
}
long responseTime = System.currentTimeMillis() - startTime;
@@ -185,20 +153,25 @@ public class HttpCollectImpl extends AbstractCollect {
try {
if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
if (entity != null) {
parseResponseByPrometheusExporter(entity. getContent(), metrics.getAliasFields(), builder);
parseResponseByPrometheusExporter(entity.getContent(), metrics.getAliasFields(), builder);
}
} else if (DispatchConstants.PARSE_HEADER.equals(parseType)) {
parseResponseByHeader(builder, metrics.getAliasFields(), response);
// Consume entity to release connection
EntityUtils.consumeQuietly(entity);
} else {
String resp = entity == null ? "" : EntityUtils. toString(entity, StandardCharsets. UTF_8);
/*
this could create large objects, potentially impacting JVM memory space significantly.
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8);
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
switch (parseType) {
case DispatchConstants.PARSE_JSON_PATH ->
parseResponseByJsonPath(resp, metrics. getAliasFields(), metrics. getHttp(), builder, responseTime);
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
case DispatchConstants.PARSE_PROM_QL ->
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_XML_PATH ->
@@ -206,7 +179,7 @@ public class HttpCollectImpl extends AbstractCollect {
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, statusCode);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics. getAliasFields(), builder);
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
case DispatchConstants.PARSE_CONFIG ->
parseResponseByConfig(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
default ->
@@ -218,20 +191,15 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
builder. setCode(CollectRep. Code.UN_CONNECTABLE);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (UnknownHostException e2) {
String errorMsg = CommonUtil.getMessageFromThrowable(e2);
log.info(errorMsg);
builder. setCode(CollectRep. Code.UN_REACHABLE);
builder.setCode(CollectRep.Code.UN_REACHABLE);
builder.setMsg("unknown host:" + errorMsg);
} catch (InterruptedIOException | ConnectException | SSLException e3) {
String errorMsg = CommonUtil.getMessageFromThrowable(e3);
@@ -240,7 +208,7 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setMsg(errorMsg);
} catch (IOException e4) {
String errorMsg = CommonUtil.getMessageFromThrowable(e4);
log. info(errorMsg);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
@@ -248,13 +216,17 @@ public class HttpCollectImpl extends AbstractCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, ClassicHttpResponse response) {
CollectRep.ValueRow. Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, CloseableHttpResponse response) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliases) {
if (! StringUtils.hasText(alias)) {
if (!StringUtils.hasText(alias)) {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
continue;
}
@@ -266,7 +238,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(firstHeader.getValue());
}
builder. addValueRow(valueRowBuilder.build());
builder.addValueRow(valueRowBuilder.build());
}
@Override
@@ -291,7 +263,7 @@ public class HttpCollectImpl extends AbstractCollect {
private void addColumnForSummary(Long responseTime, CollectRep.ValueRow.Builder valueRowBuilder, int keywordNum, String alias) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -310,13 +282,13 @@ public class HttpCollectImpl extends AbstractCollect {
DocumentBuilder db = dbf.newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
NodeList urlList = document.getElementsByTagName("url");
for (int i = 0; i < urlList. getLength(); i++) {
for (int i = 0; i < urlList.getLength(); i++) {
Node urlNode = urlList.item(i);
NodeList childNodes = urlNode.getChildNodes();
for (int k = 0; k < childNodes.getLength(); k++) {
Node currentNode = childNodes.item(k);
// distinguish between text nodes and element nodes
if (currentNode.getNodeType() == Node. ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
// retrieves the value of the loc node
siteUrls.add(currentNode.getFirstChild().getNodeValue());
break;
@@ -346,13 +318,10 @@ public class HttpCollectImpl extends AbstractCollect {
long startTime = System.currentTimeMillis();
try {
HttpGet httpGet = new HttpGet(siteUrl);
HttpClientResponseHandler<Integer> statusHandler = response -> {
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
statusCode = response.getStatusLine().getStatusCode();
EntityUtils.consume(response.getEntity());
return response.getCode();
};
statusCode = CommonHttpClient.getHttpClient().execute(httpGet, statusHandler);
}
} catch (ClientProtocolException e1) {
if (e1.getCause() != null) {
errorMsg = e1.getCause().getMessage();
@@ -376,7 +345,7 @@ public class HttpCollectImpl extends AbstractCollect {
} else if (NetworkConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(statusCode == null
? CommonConstants.NULL_VALUE : String.valueOf(statusCode));
} else if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
} else if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(responseTime));
} else if (NetworkConstants.ERROR_MSG.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(errorMsg);
@@ -393,13 +362,13 @@ public class HttpCollectImpl extends AbstractCollect {
HttpProtocol http = metrics.getHttp();
List<String> aliasFields = metrics.getAliasFields();
String xpathExpression = http.getParseScript();
if (! StringUtils.hasText(xpathExpression)) {
if (!StringUtils.hasText(xpathExpression)) {
log.warn("Http collect parse type is xmlPath, but the xpath expression is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("XPath expression is empty");
return;
}
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
@@ -409,13 +378,13 @@ public class HttpCollectImpl extends AbstractCollect {
dbf.setXIncludeAware(false);
dbf.setExpandEntityReferences(false);
DocumentBuilder db = dbf. newDocumentBuilder();
DocumentBuilder db = dbf.newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
XPathFactory xpathFactory = XPathFactory.newInstance();
XPath xpath = xpathFactory.newXPath();
NodeList nodeList = (NodeList) xpath. evaluate(xpathExpression, document, XPathConstants.NODESET);
NodeList nodeList = (NodeList) xpath.evaluate(xpathExpression, document, XPathConstants.NODESET);
if (nodeList == null || nodeList.getLength() == 0) {
log.debug("XPath expression '{}' returned no nodes.", xpathExpression);
@@ -428,10 +397,10 @@ public class HttpCollectImpl extends AbstractCollect {
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants. NULL_VALUE);
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
}
builder.addValueRow(valueRowBuilder.build());
@@ -444,14 +413,14 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
try {
String value = (String) xpath.evaluate(alias, node, XPathConstants.STRING);
valueRowBuilder. addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
valueRowBuilder.addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
} catch (XPathExpressionException e) {
log.warn("Failed to evaluate XPath '{}' for node [{}]: {}", alias, node.getNodeName(), e.getMessage());
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -463,7 +432,7 @@ public class HttpCollectImpl extends AbstractCollect {
} catch (Exception e) {
log.warn("Failed to parse XML response with XPath '{}': {}", xpathExpression, e.getMessage(), e);
builder.setCode(CollectRep.Code. FAIL);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to parse XML response: " + e.getMessage());
}
}
@@ -472,12 +441,25 @@ public class HttpCollectImpl extends AbstractCollect {
/**
* Parses the response body in Properties/Config format.
* Two modes are supported:
* 1. single-object mode: if http.parseScript is null, aliasFields are treated as indicator names.
* - If there is a locator in the indicator definition, use the locator as the key of the Properties.
* - Otherwise, use aliasField (metric name) as the key for Properties.
* Generate a single row of data.
* 2. array mode: if http.parseScript is not empty (e.g. “users”), treat it as an array base path.
* Treat aliasFields as the attribute name of an array element, and generate a single row of data for each array index. locator is invalid in this mode.
*
* @param resp Response body string
* @param aliasFields List of metrics aliases (i.e., the list of fields in metrics.fields).
* @param http http protocol configuration
* @param builder The metrics data builder.
* @param responseTime response time
*/
private void parseResponseByConfig(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
if (! StringUtils.hasText(resp)) {
if (!StringUtils.hasText(resp)) {
log.warn("Http collect parse type is config, but response body is empty.");
builder. setCode(CollectRep. Code.FAIL);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Response body is empty");
return;
}
@@ -487,14 +469,14 @@ public class HttpCollectImpl extends AbstractCollect {
properties.load(reader);
} catch (IOException e) {
log.warn("Failed to parse config response: {}", e.getMessage(), e);
builder.setCode(CollectRep.Code. FAIL);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to parse config response: " + e.getMessage());
return;
}
String arrayBasePath = http.getParseScript();
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
if (! StringUtils.hasText(arrayBasePath)) {
if (!StringUtils.hasText(arrayBasePath)) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
@@ -503,7 +485,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
String value = properties.getProperty(alias);
valueRowBuilder. addColumn(value != null ? value : CommonConstants. NULL_VALUE);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -536,13 +518,13 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime. toString());
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
valueRowBuilder. addColumn(Integer.toString(keywordNum));
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
String currentKey = arrayBasePath + "[" + i + "]." + alias;
String value = properties.getProperty(currentKey);
valueRowBuilder. addColumn(value != null ? value : CommonConstants.NULL_VALUE);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -567,24 +549,24 @@ public class HttpCollectImpl extends AbstractCollect {
for (int i = 0; i < valueRow.getColumnsCount(); i++) {
String columnValue = valueRow.getColumns(i);
String alias = aliasFields.get(i);
if (! CommonConstants.NULL_VALUE.equals(columnValue) && (! NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD. equalsIgnoreCase(alias))) {
if (!CommonConstants.NULL_VALUE.equals(columnValue) && (!NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD.equalsIgnoreCase(alias))) {
hasMeaningfulData = true;
break;
}
if ((NetworkConstants.RESPONSE_TIME. equalsIgnoreCase(alias) || CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) && ! CommonConstants.NULL_VALUE.equals(columnValue)) {
if ((NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) || CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) && !CommonConstants.NULL_VALUE.equals(columnValue)) {
hasMeaningfulData = true;
}
}
return hasMeaningfulData;
}
@SuppressWarnings("unchecked")
private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
List<Object> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
for (int i = 0; i < results.size(); i++) {
Object objectValue = results.get(i);
// if a property is missing or empty due to target version issues, filter it. Refer to the app-elasticsearch.yml configuration under name: nodes
if (objectValue == null) {
continue;
}
@@ -596,8 +578,8 @@ public class HttpCollectImpl extends AbstractCollect {
if (value != null) {
valueRowBuilder.addColumn(String.valueOf(value));
} else {
if (alias.startsWith("$. ")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias. substring(1));
if (alias.startsWith("$.")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
if (subResults != null && subResults.size() > i) {
Object resultValue = subResults.get(i);
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
@@ -651,7 +633,7 @@ public class HttpCollectImpl extends AbstractCollect {
return;
}
MetricFamily metricFamily = metricFamilyMap.get(metrics);
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily. getMetricList())) {
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily.getMetricList())) {
return;
}
for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
@@ -663,7 +645,7 @@ public class HttpCollectImpl extends AbstractCollect {
String columnValue = labelMap.get(aliasField);
if (columnValue != null) {
valueRowBuilder.addColumn(columnValue);
} else if (CommonConstants. PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
} else if (CommonConstants.PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
valueRowBuilder.addColumn(String.valueOf(metric.getValue()));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -675,9 +657,9 @@ public class HttpCollectImpl extends AbstractCollect {
private void parseResponseByDefault(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData. Builder builder, Long responseTime) {
CollectRep.MetricsData.Builder builder, Long responseTime) {
JsonElement element = JsonParser.parseString(resp);
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
if (element.isJsonArray()) {
JsonArray array = element.getAsJsonArray();
for (JsonElement jsonElement : array) {
@@ -709,53 +691,46 @@ public class HttpCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpProtocol http protocol
* @param requestConfig request config (can be null)
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
// Set request config if provided
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
HttpProtocol. Authorization auth = httpProtocol.getAuthorization();
public HttpContext createHttpContext(HttpProtocol httpProtocol) {
HttpProtocol.Authorization auth = httpProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword(). toCharArray());
auth.getDigestAuthPassword());
AuthScope authScope = new AuthScope(httpProtocol.getHost(), Integer.parseInt(httpProtocol.getPort()));
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return clientContext;
return null;
}
/**
* create http request
*
* @param httpProtocol http params
* @return http request with config
* @return http uri request
*/
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpProtocol) {
ClassicRequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod(). toUpperCase();
public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
RequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod().toUpperCase();
if (HttpMethod.GET.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.get();
requestBuilder = RequestBuilder.get();
} else if (HttpMethod.POST.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.post();
requestBuilder = RequestBuilder.post();
} else if (HttpMethod.PUT.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.put();
requestBuilder = RequestBuilder.put();
} else if (HttpMethod.DELETE.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.delete();
} else if (HttpMethod. PATCH.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.patch();
requestBuilder = RequestBuilder.delete();
} else if (HttpMethod.PATCH.matches(httpMethod)) {
requestBuilder = RequestBuilder.patch();
} else {
// not support the method
log.error("not support the http method: {}.", httpProtocol.getMethod());
@@ -766,17 +741,17 @@ public class HttpCollectImpl extends AbstractCollect {
boolean enableUrlEncoding = Boolean.parseBoolean(httpProtocol.getEnableUrlEncoding());
StringBuilder queryParams = new StringBuilder();
if (params != null && ! params.isEmpty()) {
if (params != null && !params.isEmpty()) {
for (Map.Entry<String, String> param : params.entrySet()) {
String key = param.getKey();
String value = param.getValue();
if (! StringUtils.hasText(key)) {
if (!StringUtils.hasText(key)) {
continue;
}
if (! queryParams.isEmpty()) {
queryParams. append("&");
if (!queryParams.isEmpty()) {
queryParams.append("&");
}
if (enableUrlEncoding) {
@@ -787,9 +762,9 @@ public class HttpCollectImpl extends AbstractCollect {
if (StringUtils.hasText(value)) {
String calculatedValue = TimeExpressionUtil.calculate(value);
if (enableUrlEncoding) {
calculatedValue = UriUtils. encodeQueryParam(calculatedValue, "UTF-8");
calculatedValue = UriUtils.encodeQueryParam(calculatedValue, "UTF-8");
}
queryParams. append("=").append(calculatedValue);
queryParams.append("=").append(calculatedValue);
}
}
}
@@ -800,7 +775,7 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
// headers The custom request header is overwritten here
Map<String, String> headers = httpProtocol.getHeaders();
if (headers != null && ! headers.isEmpty()) {
if (headers != null && !headers.isEmpty()) {
for (Map.Entry<String, String> header : headers.entrySet()) {
if (StringUtils.hasText(header.getValue())) {
requestBuilder.addHeader(header.getKey(), header.getValue());
@@ -808,7 +783,7 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
// add accept
if (DispatchConstants.PARSE_DEFAULT. equals(httpProtocol.getParseType())
if (DispatchConstants.PARSE_DEFAULT.equals(httpProtocol.getParseType())
|| DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
} else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) {
@@ -824,10 +799,10 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization. getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + SignConstants.DOUBLE_MARK + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders. AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
}
}
@@ -842,14 +817,14 @@ public class HttpCollectImpl extends AbstractCollect {
if (enableUrlEncoding) {
// if the url contains parameters directly
if (httpProtocol.getUrl().contains("?")) {
String path = httpProtocol.getUrl(). substring(0, httpProtocol.getUrl().indexOf("?"));
String path = httpProtocol.getUrl().substring(0, httpProtocol.getUrl().indexOf("?"));
String query = httpProtocol.getUrl().substring(httpProtocol.getUrl().indexOf("?") + 1);
uri = UriUtils.encodePath(path, "UTF-8") + "?" + UriUtils.encodeQuery(query, "UTF-8");
} else {
uri = UriUtils.encodePath(httpProtocol.getUrl(), "UTF-8");
}
} else {
uri = httpProtocol. getUrl();
uri = httpProtocol.getUrl();
}
// append query params
@@ -859,10 +834,10 @@ public class HttpCollectImpl extends AbstractCollect {
String finalUri;
if (IpDomainUtil.isHasSchema(httpProtocol.getHost())) {
finalUri = httpProtocol.getHost() + ":" + httpProtocol. getPort() + uri;
finalUri = httpProtocol.getHost() + ":" + httpProtocol.getPort() + uri;
} else {
String ipAddressType = IpDomainUtil. checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6. equals(ipAddressType)
String ipAddressType = IpDomainUtil.checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri)
: String.format("%s:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(httpProtocol.getSsl());
@@ -880,23 +855,21 @@ public class HttpCollectImpl extends AbstractCollect {
throw e;
}
// custom timeout - create RequestConfig separately
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(httpProtocol.getTimeout(), 0);
if (timeout > 0) {
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
// build the request and return with config
return new HttpRequestWithConfig(requestBuilder. build(), requestConfig);
return requestBuilder.build();
}
private boolean checkSuccessInvoke(Metrics metrics, int statusCode) {
List<String> successCodes = metrics.getHttp(). getSuccessCodes();
List<String> successCodes = metrics.getHttp().getSuccessCodes();
Set<Integer> successCodeSet = successCodes != null ? successCodes.stream().map(code -> {
try {
return Integer.valueOf(code);
@@ -117,11 +117,11 @@ public class SslCertificateCollectImpl extends AbstractCollect {
} else if (NAME_EXPIRED.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Boolean.toString(expired));
} else if (NAME_START_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toString());
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toLocaleString());
} else if (NAME_START_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotBefore().getTime()));
} else if (NAME_END_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toString());
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toLocaleString());
} else if (NAME_END_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotAfter().getTime()));
} else {
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -44,17 +44,16 @@ import org.apache.hertzbeat.common.entity.job.protocol.NginxProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
/**
@@ -63,27 +62,6 @@ import org.springframework.http.MediaType;
@Slf4j
public class NginxCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final String NGINX_STATUS_NAME = "nginx_status";
private static final String REQ_STATUS_NAME = "req_status";
private static final String AVAILABLE = "available";
@@ -115,36 +93,34 @@ public class NginxCollectImpl extends AbstractCollect {
nginxProtocol.setUrl(url == null ? RIGHT_DASH : RIGHT_DASH + url.trim());
}
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getNginx());
HttpContext httpContext = createHttpContext(metrics.getNginx(), requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
HttpContext httpContext = createHttpContext(metrics.getNginx());
HttpUriRequest request = createHttpRequest(metrics.getNginx());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)){
// send an HTTP request and get the response data
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + statusCode);
return null;
return;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
Long responseTime = System.currentTimeMillis() - startTime;
// call different parsing methods based on the metrics name
if (NGINX_STATUS_NAME.equals(metrics.getName()) || AVAILABLE.equals(metrics.getName())) {
if (StringUtils.equalsAny(metrics.getName(), NGINX_STATUS_NAME, AVAILABLE)) {
parseNginxStatusResponse(builder, resp, metrics, responseTime);
} else if (REQ_STATUS_NAME.equals(metrics.getName())) {
parseReqStatusResponse(builder, resp, metrics, responseTime);
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
@@ -154,21 +130,18 @@ public class NginxCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NGINX;
}
private HttpContext createHttpContext(NginxProtocol nginxProtocol, RequestConfig requestConfig) {
private HttpContext createHttpContext(NginxProtocol nginxProtocol) {
HttpHost host = new HttpHost(nginxProtocol.getHost(), Integer.parseInt(nginxProtocol.getPort()));
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
return httpClientContext;
}
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(NginxProtocol nginxProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
private HttpUriRequest createHttpRequest(NginxProtocol nginxProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
String portWithUri = nginxProtocol.getPort() + CollectUtil.replaceUriSpecialChar(nginxProtocol.getUrl());
String host = nginxProtocol.getHost();
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + portWithUri);
} else {
@@ -189,16 +162,16 @@ public class NginxCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(nginxProtocol.getTimeout());
if (timeout > 0) {
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
/**
@@ -362,4 +335,4 @@ public class NginxCollectImpl extends AbstractCollect {
return GET + result;
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,24 +47,24 @@ import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -74,32 +74,11 @@ import org.springframework.util.StringUtils;
*/
@Slf4j
public class PrometheusAutoCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder,
Metrics metrics) {
@@ -110,13 +89,11 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
builder.setMsg(e.getMessage());
return null;
}
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getPrometheus());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getPrometheus(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<List<CollectRep.MetricsData>> responseHandler = response -> {
int statusCode = response.getCode();
HttpContext httpContext = createHttpContext(metrics.getPrometheus());
HttpUriRequest request = createHttpRequest(metrics.getPrometheus());
try (CloseableHttpResponse response =
CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
boolean isSuccessInvoke = defaultSuccessStatusCodes.contains(statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
@@ -125,18 +102,12 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
return null;
}
try {
// Parse directly from the entity stream inside the handler loop while connection is open
return parseResponseByPrometheusExporter(response.getEntity().getContent(), builder);
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
return null;
}
};
try {
return CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
@@ -162,27 +133,31 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_PROMETHEUS;
}
private void validateParams(Metrics metrics) throws Exception {
if (metrics == null || metrics.getPrometheus() == null) {
throw new Exception("Prometheus collect must has prometheus params");
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (protocol.getPath() == null
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
private List<CollectRep.MetricsData> parseResponseByPrometheusExporter(InputStream inputStream, CollectRep.MetricsData.Builder builder) throws IOException {
long endTime = System.currentTimeMillis();
builder.setTime(endTime);
@@ -226,49 +201,40 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
return metricsDataList;
}
/**
* create httpContext
*
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
public HttpContext createHttpContext(PrometheusProtocol protocol) {
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
AuthCache authCache = new BasicAuthCache();
authCache.put(new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort())), new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return clientContext;
return null;
}
/**
* create http request
* @param protocol http params
* @return http uri request wrapper
* @return http uri request
*/
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -294,7 +260,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
// add accept
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -302,7 +268,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -314,17 +280,17 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
if (StringUtils.hasLength(protocol.getPayload()) && (HttpMethod.POST.matches(protocol.getMethod()) || HttpMethod.PUT.matches(protocol.getMethod()))) {
requestBuilder.setEntity(new StringEntity(protocol.getPayload(), StandardCharsets.UTF_8));
}
// uri
String uri = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -332,19 +298,18 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout(), 0);
if (timeout > 0) {
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
/**
@@ -354,11 +319,11 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
public static PrometheusAutoCollectImpl getInstance() {
return PrometheusAutoCollectImpl.SingleInstance.INSTANCE;
}
/**
* static instance
*/
private static class SingleInstance {
private static final PrometheusAutoCollectImpl INSTANCE = new PrometheusAutoCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,25 +30,25 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -71,27 +71,6 @@ import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
@Slf4j
public class PrometheusProxyCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@@ -101,7 +80,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
PrometheusProtocol prometheusProtocol = metrics.getPrometheus();
HttpUriRequest request;
try {
validateParams(metrics);
} catch (Exception e) {
@@ -110,19 +89,17 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
return Collections.singletonList(builder.build());
}
HttpRequestWithConfig requestWithConfig = createHttpRequest(prometheusProtocol);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(prometheusProtocol, requestWithConfig.getRequestConfig());
HttpContext httpContext = createHttpContext(prometheusProtocol);
request = createHttpRequest(prometheusProtocol);
// Use HttpClientResponseHandler to avoid deprecated execute method and manual resource closing
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
log.debug("Prometheus proxy collect, response status: {}", statusCode);
if (!defaultSuccessStatusCodes.contains(statusCode)) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return null;
return Collections.singletonList(builder.build());
}
String rawTextContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
@@ -141,11 +118,6 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
builder.addValueRow(valueRowBuilder.build());
builder.setCode(CollectRep.Code.SUCCESS);
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error("Prometheus proxy collect error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e1);
@@ -171,6 +143,10 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
log.error("Prometheus proxy collect unknown error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@@ -186,7 +162,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (!StringUtils.hasText(protocol.getHost())
|| !StringUtils.hasText(protocol.getPort())) {
|| !StringUtils.hasText(protocol.getPort())) {
throw new Exception("Prometheus collect must has host and port params");
}
if (protocol.getPath() == null
@@ -195,51 +171,42 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
/**
* create httpContext
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
public HttpContext createHttpContext(PrometheusProtocol protocol) {
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
AuthCache authCache = new BasicAuthCache();
HttpHost targetHost = new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort()));
authCache.put(targetHost, new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return clientContext;
return null;
}
/**
* create http request
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol http params
* @return http uri request wrapper
* @return http uri request
*/
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -264,7 +231,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
if (headers == null || headers.keySet().stream().noneMatch(HttpHeaders.ACCEPT::equalsIgnoreCase)) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE + ";version=0.0.4,*/*;q=0.1");
}
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -274,7 +241,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -288,15 +255,15 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN_VALUE);
}
}
String uriPath = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uriPath);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -304,22 +271,24 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout());
if (timeout > 0) {
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
} else {
requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
/**
@@ -336,4 +305,4 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
private static class SingleInstance {
private static final PrometheusProxyCollectImpl INSTANCE = new PrometheusProxyCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -24,15 +24,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -47,6 +38,15 @@ import org.apache.hertzbeat.common.entity.push.PushMetricsDto;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
/**
@@ -55,27 +55,6 @@ import org.springframework.http.MediaType;
@Slf4j
public class PushCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final Map<Long, Long> timeMap = new ConcurrentHashMap<>();
// ms
@@ -104,25 +83,20 @@ public class PushCollectImpl extends AbstractCollect {
Long time = timeMap.getOrDefault(monitorId, curTime - FIRST_COLLECT_INTERVAL);
timeMap.put(monitorId, curTime);
HttpRequestWithConfig requestWithConfig = createHttpRequest(pushProtocol, monitorId, time);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(pushProtocol, requestWithConfig.getRequestConfig());
HttpContext httpContext = createHttpContext(pushProtocol);
HttpUriRequest request = createHttpRequest(pushProtocol, monitorId, time);
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return null;
return;
}
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
parseResponse(builder, resp, metrics);
try {
String resp = CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
if (resp != null) {
parseResponse(builder, resp, metrics);
}
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error(errorMsg, e);
@@ -137,17 +111,16 @@ public class PushCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_PUSH;
}
private HttpContext createHttpContext(PushProtocol pushProtocol, RequestConfig requestConfig) {
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
private HttpContext createHttpContext(PushProtocol pushProtocol) {
HttpHost host = new HttpHost(pushProtocol.getHost(), Integer.parseInt(pushProtocol.getPort()));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
return httpClientContext;
}
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
private HttpUriRequest createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
RequestBuilder requestBuilder = RequestBuilder.get();
// uri
String uri = CollectUtil.replaceUriSpecialChar(pushProtocol.getUri());
@@ -169,17 +142,19 @@ public class PushCollectImpl extends AbstractCollect {
requestBuilder.addParameter("time", String.valueOf(startTime));
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
RequestConfig requestConfig = null;
//requestBuilder.setUri(pushProtocol.getUri());
if (DEFAULT_TIMEOUT > 0) {
// Using deprecated setConnectTimeout for request-level override
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
.setResponseTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(DEFAULT_TIMEOUT)
.setSocketTimeout(DEFAULT_TIMEOUT)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
private void parseResponse(CollectRep.MetricsData.Builder builder, String resp, Metrics metric) {
@@ -205,4 +180,4 @@ public class PushCollectImpl extends AbstractCollect {
}
builder.setTime(System.currentTimeMillis());
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,20 +18,20 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.protocol.RedfishProtocol;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.springframework.http.MediaType;
/**
@@ -58,21 +58,11 @@ public class RedfishClient {
redfishProtocol.getUsername(), redfishProtocol.getPassword(), Integer.parseInt(redfishProtocol.getTimeout()));
}
@SuppressWarnings("deprecation")
public ConnectSession connect() throws Exception {
HttpClientContext httpClientContext = HttpClientContext.create();
// Configure RequestConfig if timeout is set
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(this.timeout))
.setResponseTimeout(Timeout.ofMilliseconds(this.timeout))
.setRedirectsEnabled(true)
.build();
httpClientContext.setRequestConfig(requestConfig);
}
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.post();
HttpHost host = new HttpHost(this.host, this.port);
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
RequestBuilder requestBuilder = RequestBuilder.post();
String uri = REDFISH_SESSION_SERVICE;
if (IpDomainUtil.isHasSchema(this.host)) {
@@ -89,31 +79,37 @@ public class RedfishClient {
requestBuilder.addHeader(HttpHeaders.CONNECTION, NetworkConstants.KEEP_ALIVE);
requestBuilder.addHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE);
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
// Content-Encoding header is usually for compression, setting charset is done in Content-Type usually,
// but keeping original logic of setting it explicitly if that was the intent, though standard is usually empty or gzip.
// original: requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
final String json = "{\"UserName\": \"" + this.username + "\", \"Password\": \"" + this.password + "\"}";
StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8);
requestBuilder.setEntity(entity);
ClassicHttpRequest request = requestBuilder.build();
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(this.timeout)
.setSocketTimeout(this.timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
HttpClientResponseHandler<Session> responseHandler = response -> {
int statusCode = response.getCode();
HttpUriRequest request = requestBuilder.build();
Session session;
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpClientContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_CREATED) {
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
String location = response.getFirstHeader(NetworkConstants.LOCATION).getValue();
String auth = response.getFirstHeader(NetworkConstants.X_AUTH_TOKEN).getValue();
return new Session(auth, location, this.host, this.port);
};
try {
Session session = CommonHttpClient.getHttpClient().execute(request, httpClientContext, responseHandler);
return new RedfishConnectSession(session);
session = new Session(auth, location, this.host, this.port);
} catch (Exception e) {
throw new Exception("Redfish session create error: " + e.getMessage(), e);
throw new Exception("Redfish session create error: " + e.getMessage());
} finally {
request.abort();
}
return new RedfishConnectSession(session);
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,15 +18,15 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
/**
* Redfish connect session
@@ -54,20 +54,15 @@ public class RedfishConnectSession implements ConnectSession {
HttpDelete httpDelete = new HttpDelete(url);
httpDelete.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpDelete.setHeader(NetworkConstants.LOCATION, session.location());
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpDelete)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
EntityUtils.consume(response.getEntity());
return null;
};
try {
CommonHttpClient.getHttpClient().execute(httpDelete, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session close error:" + e.getMessage(), e);
throw new Exception("Redfish session close error:" + e.getMessage());
} finally {
httpDelete.abort();
}
}
@@ -89,19 +84,16 @@ public class RedfishConnectSession implements ConnectSession {
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpGet.setHeader(NetworkConstants.LOCATION, session.location());
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
return CommonHttpClient.getHttpClient().execute(httpGet, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session get resource error:" + e.getMessage(), e);
throw new Exception("Redfish session get resource error:" + e.getMessage());
} finally {
httpGet.abort();
}
}
}
}
@@ -67,6 +67,8 @@ public class RedisCommonCollectImpl extends AbstractCollect {
private static final String CLUSTER = "3";
private static final String SINGLE = "1";
private static final String CLUSTER_INFO = "cluster";
private static final String UNIQUE_IDENTITY = "identity";
@@ -140,7 +142,7 @@ public class RedisCommonCollectImpl extends AbstractCollect {
* @return data
*/
private List<Map<String, String>> getClusterRedisInfo(Metrics metrics) throws GeneralSecurityException, IOException {
Map<String, StatefulRedisClusterConnection<String, String>> connectionMap = getConnectionList(metrics.getRedis());
Map<String, StatefulRedisConnection<String, String>> connectionMap = getConnectionList(metrics.getRedis());
List<Map<String, String>> list = new ArrayList<>(connectionMap.size());
connectionMap.forEach((identity, connection) ->{
String info = connection.sync().info(metrics.getName());
@@ -214,16 +216,23 @@ public class RedisCommonCollectImpl extends AbstractCollect {
* @param redisProtocol protocol
* @return connection map
*/
private Map<String, StatefulRedisClusterConnection<String, String>> getConnectionList(RedisProtocol redisProtocol) throws GeneralSecurityException, IOException {
private Map<String, StatefulRedisConnection<String, String>> getConnectionList(RedisProtocol redisProtocol) throws GeneralSecurityException, IOException {
// first connection
StatefulRedisClusterConnection<String, String> connection = getClusterConnection(redisProtocol);
Partitions partitions = connection.getPartitions();
Map<String, StatefulRedisClusterConnection<String, String>> clusterConnectionMap = new HashMap<>(partitions.size());
Map<String, StatefulRedisConnection<String, String>> clusterConnectionMap = new HashMap<>(partitions.size());
for (RedisClusterNode partition : partitions) {
RedisURI uri = partition.getUri();
redisProtocol.setHost(uri.getHost());
redisProtocol.setPort(String.valueOf(uri.getPort()));
StatefulRedisClusterConnection<String, String> clusterConnection = getClusterConnection(redisProtocol);
RedisProtocol singleRedisProtocol = RedisProtocol.builder()
.host(uri.getHost())
.port(String.valueOf(uri.getPort()))
.username(redisProtocol.getUsername())
.password(redisProtocol.getPassword())
.pattern(SINGLE)
.timeout(redisProtocol.getTimeout())
.sshTunnel(redisProtocol.getSshTunnel())
.build();
StatefulRedisConnection<String, String> clusterConnection = getSingleConnection(singleRedisProtocol);
clusterConnectionMap.put(doUri(uri.getHost(), uri.getPort()), clusterConnection);
}
return clusterConnectionMap;
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,10 +22,6 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -36,6 +32,10 @@ import org.apache.hertzbeat.common.entity.sd.ConnectionConfig;
import org.apache.hertzbeat.common.entity.sd.EurekaDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.XmlUtil;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.util.EntityUtils;
import org.springframework.util.CollectionUtils;
/**
@@ -57,55 +57,38 @@ public class EurekaSdCollectImpl extends AbstractCollect {
@Override
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
ClassicHttpRequest request = ClassicRequestBuilder.get()
.setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH)
.build();
// Use ResponseHandler to avoid manual resource management and deprecated execute methods
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
HttpUriRequest request = RequestBuilder.get().setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH).build();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
log.warn("Failed to fetch eureka sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return null;
return;
}
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
String responseBody = CommonHttpClient.getHttpClient().execute(request, responseHandler);
if (responseBody != null) {
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -125,4 +108,4 @@ public class EurekaSdCollectImpl extends AbstractCollect {
public String supportProtocol() {
return DispatchConstants.PROTOCOL_EUREKA_SD;
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -41,18 +41,18 @@ import org.apache.hertzbeat.common.entity.sd.ServiceDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.apache.http.HttpHeaders;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import org.springframework.web.util.UriUtils;
@@ -63,28 +63,6 @@ import org.springframework.web.util.UriUtils;
*/
@Slf4j
public class HttpSdCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
}
@@ -93,30 +71,22 @@ public class HttpSdCollectImpl extends AbstractCollect {
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp_sd());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getHttp_sd(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
HttpUriRequest request = createHttpRequest(metrics.getHttp_sd());
HttpContext httpContext = createHttpContext(metrics.getHttp_sd());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
log.warn("Failed to fetch sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return null;
return;
}
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
TypeReference<List<ServiceDiscoveryResponseEntity>> typeReference = new TypeReference<>() {};
final List<ServiceDiscoveryResponseEntity> responseEntityList = JsonUtil.fromJson(responseBody, typeReference);
if (CollectionUtils.isEmpty(responseEntityList)) {
return null;
return;
}
responseEntityList.stream()
@@ -130,21 +100,11 @@ public class HttpSdCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -177,23 +137,17 @@ public class HttpSdCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpSdProtocol http sd protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpSdProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
public HttpContext createHttpContext(HttpProtocol httpSdProtocol) {
HttpProtocol.Authorization auth = httpSdProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (org.springframework.util.StringUtils.hasText(auth.getDigestAuthUsername())
&& org.springframework.util.StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword().toCharArray());
auth.getDigestAuthPassword());
URL url;
try {
url = new URL(httpSdProtocol.getUrl());
@@ -205,20 +159,20 @@ public class HttpSdCollectImpl extends AbstractCollect {
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return clientContext;
return null;
}
/**
* create http request
*
* @param httpSdProtocol http request set
* @return http uri request wrapper
* @return http uri request
*/
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpSdProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
public HttpUriRequest createHttpRequest(HttpProtocol httpSdProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
// The default request header can be overridden if customized
// keep-alive
@@ -273,17 +227,16 @@ public class HttpSdCollectImpl extends AbstractCollect {
throw e;
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(httpSdProtocol.getTimeout(), 0);
if (timeout > 0) {
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
}
}
@@ -19,10 +19,13 @@ package org.apache.hertzbeat.collector.collect.redis;
import static org.apache.hertzbeat.common.constants.CommonConstants.TYPE_STRING;
import static org.junit.jupiter.api.Assertions.assertEquals;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands;
import io.lettuce.core.cluster.RedisClusterClient;
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import io.lettuce.core.cluster.models.partitions.Partitions;
import io.lettuce.core.cluster.models.partitions.RedisClusterNode;
import io.lettuce.core.resource.ClientResources;
@@ -37,6 +40,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -51,13 +55,19 @@ public class RedisClusterCollectImplTest {
@Mock
private StatefulRedisClusterConnection<String, String> connection;
private StatefulRedisClusterConnection<String, String> clusterConnection;
@Mock
private RedisAdvancedClusterCommands<String, String> cmd;
private StatefulRedisConnection<String, String> singleConnection;
@Mock
private RedisClusterClient client;
private RedisCommands<String, String> cmd;
@Mock
private RedisClusterClient clusterClient;
@Mock
private RedisClient singleClient;
@BeforeEach
void setUp() {
@@ -65,8 +75,10 @@ public class RedisClusterCollectImplTest {
@AfterEach
void setDown() {
connection.close();
client.shutdown();
clusterConnection.close();
singleConnection.close();
clusterClient.shutdown();
singleClient.shutdown();
}
@Test
@@ -110,10 +122,14 @@ public class RedisClusterCollectImplTest {
metrics.setAliasFields(aliasField);
metrics.setFields(fields);
Mockito.mockStatic(RedisClusterClient.class).when(() -> RedisClusterClient.create(Mockito.any(ClientResources.class),
Mockito.any(RedisURI.class))).thenReturn(client);
Mockito.when(client.connect()).thenReturn(connection);
MockedStatic<RedisClusterClient> redisClusterClientMockedStatic = Mockito.mockStatic(RedisClusterClient.class);
redisClusterClientMockedStatic.when(() -> RedisClusterClient.create(Mockito.any(ClientResources.class),
Mockito.any(RedisURI.class))).thenReturn(clusterClient);
Mockito.when(clusterClient.connect()).thenReturn(clusterConnection);
MockedStatic<RedisClient> redisClientMockedStatic = Mockito.mockStatic(RedisClient.class);
redisClientMockedStatic.when(() -> RedisClient.create(Mockito.any(ClientResources.class),
Mockito.any(RedisURI.class))).thenReturn(singleClient);
Mockito.when(singleClient.connect()).thenReturn(singleConnection);
Partitions partitions = new Partitions();
RedisClusterNode node = new RedisClusterNode();
@@ -125,9 +141,9 @@ public class RedisClusterCollectImplTest {
node2.setUri(RedisURI.create("redis://" + uri2));
partitions.add(node2);
Mockito.when(connection.getPartitions()).thenReturn(partitions);
Mockito.when(clusterConnection.getPartitions()).thenReturn(partitions);
Mockito.when(connection.sync()).thenReturn(cmd);
Mockito.when(singleConnection.sync()).thenReturn(cmd);
Mockito.when(cmd.info(metrics.getName())).thenReturn(info);
Mockito.when(cmd.clusterInfo()).thenReturn(clusterInfo);
@@ -147,6 +163,8 @@ public class RedisClusterCollectImplTest {
assertEquals(row.getColumns(2), uri2);
}
}
redisClusterClientMockedStatic.close();
redisClientMockedStatic.close();
}
}
@@ -29,7 +29,7 @@ import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingClient;
import org.junit.jupiter.api.BeforeEach;
@@ -113,7 +113,7 @@ class CollectServerTest {
RemotingClient remotingClient = mock(RemotingClient.class);
ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient);
ClusterMessage message = mock(ClusterMessage.class);
ClusterMsg.Message message = mock(ClusterMsg.Message.class);
collectServer.sendMsg(message);
@@ -135,7 +135,7 @@ class CollectServerTest {
collectNettyEventListener.onChannelActive(channel);
verify(timerDispatch, times(1)).goOnline();
verify(remotingClient, times(1)).sendMsg(any(ClusterMessage.class));
verify(remotingClient, times(1)).sendMsg(any(ClusterMsg.Message.class));
ScheduledExecutorService scheduledExecutor =
(ScheduledExecutorService) ReflectionTestUtils.getField(collectServer, "scheduledExecutor");
@@ -34,7 +34,7 @@ import org.apache.hertzbeat.collector.dispatch.WorkerPool;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -116,7 +116,7 @@ class CollectJobServiceTest {
collectJobService.collectSyncOneTimeJobData(job);
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
}
@Test
@@ -137,7 +137,7 @@ class CollectJobServiceTest {
.build();
collectJobService.sendAsyncCollectData(metricsData);
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
}
@Test
@@ -100,35 +100,6 @@
</build>
<profiles>
<profile>
<id>dev-driver</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>inner</id>
<build>
@@ -28,6 +28,8 @@ spring:
jackson:
default-property-inclusion: ALWAYS
# need to disable spring boot mongodb auto config, or default mongodb connection tried and failed...
autoconfigure:
exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
management:
endpoints:
web:
@@ -1,30 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true">
<springProperty scope="context" name="application_name" source="spring.application.name" defaultValue="collector"/>
<appender name="CONSOLE_RAW" class="ch.qos.logback.core.ConsoleAppender">
<!-- Output logs to ConsoleAppender -->
<appender name="ConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<!--<pattern>%d %p (%file:%line\)- %m%n</pattern>-->
<!-- Format output: %d for date, %thread for thread name, %-5level for level with 5 character width, %msg for log message, %n for new line -->
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="ConsoleAppender" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>0</discardingThreshold>
<queueSize>512</queueSize>
<includeCallerData>true</includeCallerData>
<appender-ref ref="CONSOLE_RAW"/>
</appender>
<appender name="SystemOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- Rolling policy for loggers, by date and by size -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- Archive log file path. %d{yyyy-MM-dd} specifies date format, %i specifies index -->
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}.%i.log.zip</fileNamePattern>
<maxFileSize>50MB</maxFileSize>
<!-- Log retention duration -->
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<!-- Besides logging by day, log files cannot exceed 200M, if exceeded, log files will start from index 0 -->
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
@@ -32,23 +52,32 @@
</appender>
<appender name="ErrOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}-error.%i.log.zip</fileNamePattern>
<maxFileSize>50MB</maxFileSize>
<!-- Log retention duration -->
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!-- This log file records error and above levels -->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
</appender>
<!-- Settings for this logger: for example, all output logs under the org.springframework package must be at level info or above to be output! -->
<!-- This can avoid outputting many common debug information of the spring framework! -->
<logger name="org.springframework" level="info" />
<logger name="org.json" level="error"/>
<logger name="io.netty" level="info"/>
@@ -62,9 +91,9 @@
<logger name="org.mongodb" level="warn"/>
<logger name="io.greptime" level="warn"/>
<!-- Production environment configuration -->
<springProfile name="prod">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
@@ -72,12 +101,12 @@
<springProfile name="cluster">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="dev">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -86,6 +115,7 @@
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="mysql">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -94,4 +124,4 @@
</root>
</springProfile>
</configuration>
</configuration>
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,30 +17,33 @@
package org.apache.hertzbeat.collector.collect.common.http;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.X509Certificate;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.reactor.ssl.SSLBufferMode;
import org.apache.hc.core5.ssl.SSLContexts;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContexts;
/**
* common http client for HttpComponents Client 5.x
* common http client
*/
@Slf4j
public class CommonHttpClient {
@@ -81,13 +84,11 @@ public class CommonHttpClient {
/**
* ssl supported version
* Note: SSLv3 is often disabled in modern JDKs
*/
private static final String[] SUPPORTED_SSL = {"TLSv1", "TLSv1.1", "TLSv1.2", "SSLv3"};
static {
try {
// 1. SSL Context Configuration
SSLContext sslContext = SSLContexts.createDefault();
X509TrustManager x509TrustManager = new X509TrustManager() {
@Override
@@ -101,7 +102,7 @@ public class CommonHttpClient {
for (X509Certificate certificate : x509Certificates) {
Date deadline = certificate.getNotAfter();
if (deadline != null && now.after(deadline)) {
throw new CertificateExpiredException("Server certificate expired at " + deadline);
throw new CertificateExpiredException();
}
}
}
@@ -111,63 +112,57 @@ public class CommonHttpClient {
public X509Certificate[] getAcceptedIssuers() { return null; }
};
sslContext.init(null, new TrustManager[]{x509TrustManager}, null);
// 2. TlsStrategy Configuration (Replaces deprecated SSLConnectionSocketFactory/Registry)
DefaultClientTlsStrategy tlsStrategy = new DefaultClientTlsStrategy(
sslContext,
SUPPORTED_SSL,
null, // Supported cipher suites (null = default)
SSLBufferMode.STATIC,
NoopHostnameVerifier.INSTANCE
);
// 3. Connection Manager Configuration (Using Builder)
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(CONNECT_TIMEOUT))
.setSocketTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT))
.setValidateAfterInactivity(TimeValue.ofMilliseconds(INACTIVITY_VALIDATED_TIME))
SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, SUPPORTED_SSL, null, new NoopHostnameVerifier());
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", sslFactory)
.build();
connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(tlsStrategy)
.setMaxConnTotal(MAX_TOTAL_CONNECTIONS)
.setMaxConnPerRoute(MAX_PER_ROUTE_CONNECTIONS)
.setDefaultConnectionConfig(connectionConfig)
.build();
// 4. Request Config (Connection Request Timeout & Redirects)
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(Timeout.ofMilliseconds(REQUIRE_CONNECT_TIMEOUT))
.setConnectionRequestTimeout(REQUIRE_CONNECT_TIMEOUT)
.setConnectTimeout(CONNECT_TIMEOUT)
.setSocketTimeout(SOCKET_TIMEOUT)
// auto redirect when 301 302 response status
.setRedirectsEnabled(true)
.build();
// 5. Build HttpClient
// connection pool
connectionManager = new PoolingHttpClientConnectionManager(registry);
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS);
connectionManager.setValidateAfterInactivity(INACTIVITY_VALIDATED_TIME);
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
// clean up unavailable expired connections
.evictExpiredConnections()
.evictIdleConnections(TimeValue.ofSeconds(100))
// clean up available but idle connections
.evictIdleConnections(100, TimeUnit.SECONDS)
.build();
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("http-connection-pool-cleaner-%d")
.setDaemon(true)
.build();
ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory);
scheduledExecutor.scheduleWithFixedDelay(() -> {
connectionManager.closeExpiredConnections();
connectionManager.closeIdleConnections(40, TimeUnit.SECONDS);
}, 40L, 40L, TimeUnit.SECONDS);
// Shutdown hook
// shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close));
} catch (Exception e) {
log.error("Initialize CommonHttpClient error", e);
}
} catch (Exception ignored) {}
}
public static CloseableHttpClient getHttpClient() {
return httpClient;
}
public static void close() {
try {
if (httpClient != null) {
httpClient.close();
}
httpClient.close();
} catch (Exception e) {
log.error("close http client error", e);
}
}
}
}
@@ -17,48 +17,65 @@
package org.apache.hertzbeat.collector.dispatch;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Collection task worker thread pool with Virtual Threads
* Collection task worker thread pool
*/
@Component
@Slf4j
public class WorkerPool implements DisposableBean {
private ExecutorService workerExecutor;
private ThreadPoolExecutor workerExecutor;
public WorkerPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory virtualThreadFactory = Thread.ofVirtual()
.name("collect-vt-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(virtualThreadFactory);
log.info("WorkerPool initialized with JDK 25 Virtual Threads successfully.");
// thread factory
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable);
log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("collect-worker-%d")
.build();
int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors());
int maxSize = Runtime.getRuntime().availableProcessors() * 16;
workerExecutor = new ThreadPoolExecutor(coreSize,
maxSize,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
/**
* Run the collection task
* @param runnable Task
* Run the collection task thread
*
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
*/
public void executeJob(Runnable runnable) {
public void executeJob(Runnable runnable) throws RejectedExecutionException {
workerExecutor.execute(runnable);
}
@Override
public void destroy() {
public void destroy() throws Exception {
if (workerExecutor != null) {
workerExecutor.close();
workerExecutor.shutdownNow();
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.collector.dispatch.entrance;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
import io.netty.channel.Channel;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
@@ -32,7 +33,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProces
import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.RemotingClient;
@@ -64,7 +65,7 @@ public class CollectServer implements CommandLineRunner {
private final TimerDispatch timerDispatch;
private final CollectorInfoProperties infoProperties;
private RemotingClient remotingClient;
private ScheduledExecutorService scheduledExecutor;
@@ -96,13 +97,13 @@ public class CollectServer implements CommandLineRunner {
nettyClientConfig.setServerPort(nettyProperties.getManagerPort());
this.remotingClient = new NettyRemotingClient(nettyClientConfig, new CollectNettyEventListener(), threadPool);
this.remotingClient.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_CLOSE, new GoCloseProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this));
}
public void shutdown() {
@@ -115,7 +116,7 @@ public class CollectServer implements CommandLineRunner {
return collectJobService;
}
public void sendMsg(final ClusterMessage message) {
public void sendMsg(final ClusterMsg.Message message) {
this.remotingClient.sendMsg(message);
}
@@ -142,10 +143,10 @@ public class CollectServer implements CommandLineRunner {
.build();
timerDispatch.goOnline();
// send online message
ClusterMessage message = ClusterMessage.builder()
.identity(identity)
.type(ClusterMessage.MessageType.GO_ONLINE)
.msg(JsonUtil.toJsonBytes(collectorInfo))
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(collectorInfo)))
.build();
CollectServer.this.sendMsg(message);
@@ -162,13 +163,13 @@ public class CollectServer implements CommandLineRunner {
// schedule send heartbeat message
scheduledExecutor.scheduleAtFixedRate(() -> {
try {
ClusterMessage heartbeat = ClusterMessage.builder()
.identity(identity)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
.build();
CollectServer.this.sendMsg(heartbeat);
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
} catch (Exception e) {
log.error("schedule send heartbeat to server error.{}", e.getMessage());
}
@@ -181,4 +182,4 @@ public class CollectServer implements CommandLineRunner {
log.info("handle idle event triggered. collector is going offline.");
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.collector.dispatch.entrance.internal;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.WorkerPool;
@@ -24,7 +25,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.ArrowUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
@@ -116,10 +117,10 @@ public class CollectJobService {
workerPool.executeJob(() -> {
List<CollectRep.MetricsData> metricsDataList = this.collectSyncJobData(oneTimeJob);
byte[] msg = ArrowUtil.serializeMetricsData(metricsDataList);
ClusterMessage message = ClusterMessage.builder()
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
});
@@ -152,22 +153,22 @@ public class CollectJobService {
*/
public void sendAsyncCollectData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_DATA)
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
}
public void sendAsyncServiceDiscoveryData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
.build();
this.collectServer.sendMsg(message);
}
@@ -183,4 +184,4 @@ public class CollectJobService {
public void setCollectServer(CollectServer collectServer) {
this.collectServer = collectServer;
}
}
}
@@ -21,7 +21,7 @@ import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -37,8 +37,8 @@ public class CollectCyclicDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job job = JsonUtil.fromJson(message.getMsg(), Job.class);
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job job = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
if (job == null) {
log.error("collector receive cyclic task job is null");
return null;
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -35,8 +35,8 @@ public class CollectOneTimeDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg(), Job.class);
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
collectServer.getCollectJobService().collectSyncOneTimeJobData(oneTimeJob);
return null;
}
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -39,9 +39,9 @@ public class DeleteCyclicTaskProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
TypeReference<List<Long>> typeReference = new TypeReference<>() {};
List<Long> jobIds = JsonUtil.fromJson(message.getMsgString(), typeReference);
List<Long> jobIds = JsonUtil.fromJson(message.getMsg().toStringUtf8(), typeReference);
if (jobIds == null || jobIds.isEmpty()) {
log.error("collector receive delete cyclic task job ids is null");
return null;
@@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
import org.springframework.boot.SpringApplication;
@@ -41,12 +41,12 @@ public class GoCloseProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsgString());
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg());
}
this.timerDispatch.goOffline();
this.collectServer.shutdown();
@@ -17,12 +17,12 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import java.nio.charset.StandardCharsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -36,21 +36,20 @@ public class GoOfflineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
timerDispatch.goOffline();
log.info("receive offline message and handle success");
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsgString());
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsg());
return null;
}
return ClusterMessage.builder()
.identity(message.getIdentity())
.direction(ClusterMessage.Direction.RESPONSE)
.type(ClusterMessage.MessageType.GO_OFFLINE)
.msg(String.valueOf(CommonConstants.SUCCESS_CODE).getBytes(StandardCharsets.UTF_8))
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
.build();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,11 +17,13 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
@@ -33,20 +35,18 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
*/
@Slf4j
public class GoOnlineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
String msgString = message.getMsgString();
if (message.getMsg() == null || msgString == null || msgString.isEmpty()) {
if (message.getMsg().isEmpty()) {
log.warn("The message that server response to collector is empty, please upgrade server");
} else {
// Use the new JsonUtil.fromJson(byte[], Class) method
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg(), ServerInfo.class);
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg().toStringUtf8(), ServerInfo.class);
if (serverInfo == null || serverInfo.getAesSecret() == null) {
log.warn("The message that server response to collector has not secret empty, please check");
} else {
@@ -55,8 +55,10 @@ public class GoOnlineProcessor implements NettyRemotingProcessor {
}
timerDispatch.goOnline();
log.info("receive online message and handle success");
// Return null to stop the ping-pong loop.
// The collector should not reply to the server's confirmation response.
return null;
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
.build();
}
}
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
/**
@@ -28,7 +28,7 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@Slf4j
public class HeartbeatProcessor implements NettyRemotingProcessor {
@Override
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
log.info("collector receive manager server response heartbeat, time: {}. ", System.currentTimeMillis());
return null;
}
@@ -25,6 +25,8 @@ import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import java.util.List;
/**
* for collector instance
* send collect response data by netty
@@ -108,4 +110,24 @@ public class NettyDataQueue implements CommonDataQueue {
public LogEntry pollLogEntryToStorage() throws InterruptedException {
return null;
}
@Override
public void sendLogEntryToAlertBatch(List<LogEntry> logEntries) {
}
@Override
public List<LogEntry> pollLogEntryToAlertBatch(int maxBatchSize) throws InterruptedException {
return List.of();
}
@Override
public void sendLogEntryToStorageBatch(List<LogEntry> logEntries) {
}
@Override
public List<LogEntry> pollLogEntryToStorageBatch(int maxBatchSize) throws InterruptedException {
return List.of();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.collector.collect.nebulagraph;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
@@ -24,17 +25,7 @@ import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -46,9 +37,18 @@ import org.apache.hertzbeat.common.entity.job.protocol.NebulaGraphProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/**
* NebulaGraph collect
* NebulaGraph collect
*/
@Slf4j
public class NebulaGraphCollectImpl extends AbstractCollect {
@@ -66,27 +66,6 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
private static final String STORAGE_API = "/rocksdb_stats";
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
if (metrics == null || metrics.getNebulaGraph() == null) {
@@ -112,50 +91,37 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return;
}
// Create Request and Config
HttpRequestWithConfig requestWithConfig = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
String resp;
long responseTime;
HashMap<String, String> resultMap = new HashMap<>(64);
HttpContext httpContext = createHttpContext(nebulaGraph.getHost(), nebulaGraph.getPort());
HttpUriRequest request = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
nebulaGraph.getUrl(), nebulaGraph.getTimeout());
// Create Context and set Config
HttpContext httpContext = createHttpContext(requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
// Send an HTTP request to obtain response data
try {
// Use ResponseHandler to avoid manual resource management and CloseableHttpResponse deprecation issues
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
long responseTime = System.currentTimeMillis() - startTime;
HashMap<String, String> resultMap = new HashMap<>(64);
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
return null;
};
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return;
}
resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
responseTime = System.currentTimeMillis() - startTime;
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
@@ -168,47 +134,43 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NEBULAGRAPH;
}
private HttpContext createHttpContext(RequestConfig requestConfig) {
HttpClientContext context = HttpClientContext.create();
if (requestConfig != null) {
context.setRequestConfig(requestConfig);
}
return context;
private HttpContext createHttpContext(String host, String port) {
HttpHost httpHost = new HttpHost(host, Integer.parseInt(port));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(httpHost);
return httpClientContext;
}
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(String host, String port, String url, String timeoutStr) {
// HttpClient 5 uses ClassicRequestBuilder
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// uri construction
private HttpUriRequest createHttpRequest(String host, String port, String url, String timeoutStr) {
RequestBuilder requestBuilder = RequestBuilder.get();
// uri
String uri = CollectUtil.replaceUriSpecialChar(url);
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + port + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(host);
String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", host, port, uri)
: String.format("%s:%s%s", host, port, uri);
? String.format("[%s]:%s", host, port + uri)
: String.format("%s:%s", host, port + uri);
requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri);
}
requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain");
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(timeoutStr);
if (timeout > 0) {
// Use setConnectTimeout despite deprecation to allow per-request connection timeout override on a shared client
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
return requestBuilder.build();
}
/**
@@ -216,7 +178,6 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
*
* @param responseBody response body
* @param timePeriod time period
* @param resultMap result map
*/
private void parseStatsResponse(String responseBody, String timePeriod, HashMap<String, String> resultMap) {
// Set up regular expression matching
@@ -227,26 +188,22 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
Matcher matcher = pattern.matcher(str);
if (matcher.find()) {
String[] split = str.split(timeRegex);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
resultMap.put(split[0], split[1]);
}
}
}
/**
* Parse the Storage response and filter by time period
*
* @param responseBody response body
* @param resultMap result map
*/
private void parseStorageResponse(String responseBody, HashMap<String, String> resultMap) {
String[] strArray = responseBody.split(STR_SPLIT);
for (String str : strArray) {
String[] split = str.split(STORAGE_SPLIT_KEY_VALUE);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
resultMap.put(split[0], split[1]);
}
}
}
}
+1 -1
View File
@@ -28,7 +28,7 @@
<name>${project.artifactId}</name>
<packaging>pom</packaging>
<properties>
<java.version>21</java.version>
<java.version>17</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
+16 -19
View File
@@ -46,9 +46,20 @@
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- jpa -->
<dependency>
<groupId>org.eclipse.persistence</groupId>
<artifactId>org.eclipse.persistence.jpa</artifactId>
<version>${eclipse-jpa.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<exclusions>
<exclusion>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- validation -->
<dependency>
@@ -65,10 +76,6 @@
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-jackson2</artifactId>
</dependency>
<!-- swagger -->
<dependency>
<groupId>org.springdoc</groupId>
@@ -88,6 +95,11 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
<!-- Use apache.http.conn.util instead of sun.net.util -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<!-- caffeine-->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
@@ -171,21 +183,6 @@
<version>${javaparser.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-core</artifactId>
<version>${fory.version}</version>
</dependency>
<!-- row/arrow format support -->
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-format</artifactId>
<version>${fory.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.persistence.config.SessionCustomizer;
import org.eclipse.persistence.descriptors.ClassDescriptor;
import org.eclipse.persistence.mappings.DatabaseMapping;
import org.eclipse.persistence.sessions.Session;
/**
* change column name that from upper-case to lower-case-with-underscore
*/
@Slf4j
public class EclipseLinkCustomizer implements SessionCustomizer {
@Override
public void customize(Session session) throws Exception {
for (ClassDescriptor descriptor : session.getDescriptors().values()) {
for (DatabaseMapping mapping : descriptor.getMappings()) {
if (mapping.isDirectToFieldMapping()) {
// update the column name to lower case with underscore
if (!mapping.getField().getName().equalsIgnoreCase(mapping.getAttributeName())) {
// already custom define the column name, ignore
continue;
}
// todo here i try to change column name that from upper-case to lower-case-with-underscore
// but failed, when update name, something relate exception happen more
mapping.getField().setName(convertCamelCase(mapping.getAttributeName()));
}
}
}
}
private String convertCamelCase(String camelCase) {
StringBuilder result = new StringBuilder();
if (camelCase != null && !camelCase.isEmpty()) {
result.append(Character.toLowerCase(camelCase.charAt(0)));
for (int i = 1; i < camelCase.length(); i++) {
char currentChar = camelCase.charAt(i);
if (Character.isUpperCase(currentChar)) {
result.append('_').append(Character.toLowerCase(currentChar));
} else {
result.append(currentChar);
}
}
}
return result.toString();
}
}
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.eclipse.persistence.config.PersistenceUnitProperties;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.orm.jpa.JpaBaseConfiguration;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.orm.jpa.vendor.AbstractJpaVendorAdapter;
import org.springframework.orm.jpa.vendor.EclipseLinkJpaVendorAdapter;
import org.springframework.transaction.jta.JtaTransactionManager;
/**
* jpa eclipselink impl config
*/
@Configuration
@ConditionalOnProperty(prefix = "spring.datasource", name = "url")
public class EclipseLinkJpaConfiguration extends JpaBaseConfiguration {
protected EclipseLinkJpaConfiguration(DataSource dataSource, JpaProperties properties,
ObjectProvider<JtaTransactionManager> jtaTransactionManager) {
super(dataSource, properties, jtaTransactionManager);
}
@Override
protected AbstractJpaVendorAdapter createJpaVendorAdapter() {
return new EclipseLinkJpaVendorAdapter();
}
@Override
protected Map<String, Object> getVendorProperties() {
HashMap<String, Object> map = new HashMap<>(8);
map.put(PersistenceUnitProperties.DDL_GENERATION, "create-or-extend-tables");
map.put(PersistenceUnitProperties.SESSION_CUSTOMIZER, "org.apache.hertzbeat.common.config.EclipseLinkCustomizer");
map.put(PersistenceUnitProperties.ALLOW_NATIVE_SQL_QUERIES, "true");
map.put(PersistenceUnitProperties.WEAVING, "true");
return map;
}
}
@@ -97,11 +97,6 @@ public interface CommonConstants {
*/
String LABEL_ALERT_NAME = "alertname";
/**
* label key: instance host
*/
String LABEL_INSTANCE_HOST = "instancehost";
/**
* label key: instance name
*/
@@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.constants;
/**
* HertzBeat Reserved Keywords Enum
*/
public enum HertzBeatKeywordsEnum {
INSTANCE("instance", "metric_instance");
private final String keyword;
private final String alias;
HertzBeatKeywordsEnum(String keyword, String alias) {
this.keyword = keyword;
this.alias = alias;
}
public String getKeyword() {
return keyword;
}
public String getAlias() {
return alias;
}
}
@@ -38,5 +38,5 @@ public interface MetricDataConstants {
String CODE = "code";
String MSG = "msg";
String INSTANCE_NAME = "instancename";
String INSTANCE_HOST = "instancehost";
String INSTANCE = "instance";
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -55,7 +55,7 @@ public class ChatConversation {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation title")
private String title;
@@ -79,6 +79,6 @@ public class ChatConversation {
* List of messages in this conversation (one-to-many relationship)
*/
@OneToMany
@JoinColumn(name = "conversation_id", insertable = false, updatable = false)
@JoinColumn(name = "conversation_id")
private List<ChatMessage> messages;
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,7 +19,6 @@ package org.apache.hertzbeat.common.entity.ai;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.GeneratedValue;
@@ -49,7 +48,7 @@ import lombok.NoArgsConstructor;
@Entity
@EntityListeners(AuditingEntityListener.class)
@Table(name = "hzb_ai_message", indexes = {
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
})
@AllArgsConstructor
@NoArgsConstructor
@@ -58,9 +57,8 @@ public class ChatMessage {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation id")
@Column(name = "conversation_id")
private Long conversationId;
@Schema(title = "message content")
@@ -86,5 +84,5 @@ public class ChatMessage {
@Schema(title = "Record modify time", example = "1612198444000", accessMode = READ_ONLY)
@LastModifiedDate
private LocalDateTime gmtUpdate;
}
}
@@ -101,7 +101,6 @@ public class AlertDefine {
private String datasource;
@Schema(title = "Is Enabled", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "The creator of this record", example = "tom", accessMode = READ_ONLY)
@@ -69,11 +69,9 @@ public class AlertSilence {
private String name;
@Schema(title = "Whether to enable this policy", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to match all", example = "true")
@Builder.Default
private boolean matchAll = true;
@Schema(title = "Silence type 0: once, 1:cyc", example = "1")
@@ -103,13 +103,11 @@ public class NoticeRule {
@Schema(title = "Whether to enable this policy",
description = "Whether to enable this policy",
example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to forward all",
description = "Whether to forward all",
example = "false", accessMode = READ_WRITE)
@Builder.Default
private boolean filterAll = true;
@Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}")
@@ -85,7 +85,6 @@ public class NoticeTemplate {
description = "Is it a preset template: true- preset template false- custom template.",
accessMode = READ_WRITE)
@Column(columnDefinition = "boolean default false")
@Builder.Default
private boolean preset = false;
@Schema(title = "Template content",
@@ -43,6 +43,5 @@ public class CollectorInfo {
private String version;
@NotNull
@Builder.Default
private String mode = CommonConstants.MODE_PUBLIC;
}
@@ -23,7 +23,7 @@ import lombok.EqualsAndHashCode;
import org.apache.hertzbeat.common.constants.ImportTaskStatusEnum;
import org.apache.hertzbeat.common.constants.ManagerEventTypeEnum;
import org.apache.hertzbeat.common.constants.NotifyLevelEnum;
import org.jspecify.annotations.Nullable;
import org.springframework.lang.Nullable;
/**
* Import task message
@@ -18,13 +18,14 @@
package org.apache.hertzbeat.common.entity.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
/**
* Metric History Range Query Data
*/
@@ -35,8 +36,8 @@ import lombok.NoArgsConstructor;
@Schema(description = "Metric History Range Query Data")
public class MetricsHistoryData {
@Schema(title = "Monitoring Task ID")
private Long id;
@Schema(title = "Monitor Instance (e.g., ip:port or domain)")
private String instance;
@Schema(title = "Monitoring Type")
private String app;
@@ -50,6 +50,5 @@ public class Configmap implements Serializable {
* 0: number 1: string 2: encrypted string 3: json string mapped by map
* number,string,secret
*/
@Builder.Default
private byte type = 1;
}
@@ -57,7 +57,6 @@ public class Job {
/**
* Tenant id
*/
@Builder.Default
private long tenantId = 0;
/**
* Monitoring Task ID
@@ -79,7 +78,6 @@ public class Job {
/**
* Is hide this app in main menus layout, only for app type, default true.
*/
@Builder.Default
private boolean hide = true;
/**
* Large categories of monitoring
@@ -113,7 +111,6 @@ public class Job {
/**
* Default task collection time interval (unit: second) eg: 30,60,600
*/
@Builder.Default
private long defaultInterval = 600L;
/**
* Refresh time list for one cycle of the job
@@ -122,7 +119,6 @@ public class Job {
/**
* Whether it is a recurring periodic task true is yes, false is no
*/
@Builder.Default
private boolean isCyclic = false;
/**
* monitor input need params
@@ -140,25 +136,21 @@ public class Job {
/**
* Whether it is a service discovery job, true is yes, false is no
*/
@Builder.Default
private boolean isSd = false;
/**
* Whether to use the Prometheus proxy
*/
@Builder.Default
private boolean prometheusProxyMode = false;
/**
* Scheduling type: interval or cron
*/
@Builder.Default
private String scheduleType = "interval";
/**
* Cron expression for scheduling, used when scheduleType is "cron"
*/
@Builder.Default
private String cronExpression = null;
/**
@@ -115,7 +115,6 @@ public class Metrics {
* Is it visible true or false
* if false, web ui will not see this metrics.
*/
@Builder.Default
private boolean visible = true;
/**
* Public attribute - collection and monitoring final result attribute set eg: speed | times | size
@@ -394,12 +393,10 @@ public class Metrics {
/**
* Metric type 0-number: number 1-string: string
*/
@Builder.Default
private byte type = 1;
/**
* Whether this field is the label
*/
@Builder.Default
private boolean label = false;
/**
* Metric unit
@@ -36,7 +36,6 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* enable ssh tunnel
*/
@Builder.Default
private String enable = "false";
/**
@@ -47,13 +46,11 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* Peer host port
*/
@Builder.Default
private String port = "22";
/**
* TIME OUT PERIOD
*/
@Builder.Default
private String timeout = "6000";
/**
@@ -79,6 +76,5 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* share connection session
*/
@Builder.Default
private String shareConnection = "true";
}
@@ -64,6 +64,5 @@ public class FtpProtocol implements CommonRequestProtocol, Protocol {
* Whether ftp uses link encryption ssl/tls, i.e. ftp or sftp
*
*/
@Builder.Default
private String ssl = "false";
}
@@ -51,7 +51,6 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether http uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
* http request methods: get, post, put, delete, patch
@@ -100,7 +99,6 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
* Whether to enable URL encoding for the path. Default is true.
* When true, the URL path will be encoded. When false, the URL path will not be encoded.
*/
@Builder.Default
private String enableUrlEncoding = "true";
/**
@@ -48,7 +48,6 @@ public class ImapProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -74,7 +74,6 @@ public class JdbcProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
* ssh tunnel
@@ -43,7 +43,6 @@ public class JmxProtocol implements CommonRequestProtocol, Protocol {
/**
* enable ssl?
*/
@Builder.Default
private String ssl = "false";
/**
@@ -54,6 +54,5 @@ public class KafkaProtocol implements CommonRequestProtocol, Protocol {
/**
* Monitor internal topic
*/
@Builder.Default
private String monitorInternalTopic = "false";
}
@@ -54,7 +54,6 @@ public class NginxProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether nginx uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
@@ -49,7 +49,6 @@ public class Pop3Protocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -50,7 +50,6 @@ public class PrometheusProtocol implements CommonRequestProtocol, Protocol {
/**
* http Whether to use link-encrypted ssl/tls, that is, http or https
*/
@Builder.Default
private String ssl = "false";
/**
* HTTP REQUEST METHOD: get, post, put, delete, patch
@@ -34,7 +34,6 @@ import org.apache.hertzbeat.common.entity.dto.Field;
public class PushProtocol implements CommonRequestProtocol, Protocol {
private String host;
private String port;
@Builder.Default
private String uri = "/api/push";
private List<Field> fields;
}
@@ -69,7 +69,6 @@ public class SnmpProtocol implements CommonRequestProtocol, Protocol {
/**
* operation: get, walk
*/
@Builder.Default
private String operation = "get";
/**
* oid map
@@ -69,7 +69,6 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
@@ -105,7 +104,6 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* flag of use proxy
*/
@Builder.Default
private String useProxy = "false";
/**
@@ -52,7 +52,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@Table(name = "hzb_monitor", indexes = {
@Index(name = "monitor_query_index", columnList = "app"),
@Index(name = "monitor_query_index", columnList = "host"),
@Index(name = "monitor_query_index", columnList = "instance"),
@Index(name = "monitor_query_index", columnList = "name")
})
@Data
@@ -82,10 +82,10 @@ public class Monitor {
@Size(max = 100)
private String scrape;
@Schema(title = "peer host: ipv4, ipv6, domain name", example = "192.167.25.11", accessMode = READ_WRITE)
@Schema(title = "the monitor target: ip/domain+port or ip/domain", example = "192.167.25.11:8081", accessMode = READ_WRITE)
@Size(max = 100)
@HostValid
private String host;
private String instance;
@Schema(title = "Monitoring of the acquisition interval time in seconds", example = "600", accessMode = READ_WRITE)
@Min(10)
@@ -96,7 +96,6 @@ public class ParamDefine {
* Is it mandatory true-required false-optional
*/
@Schema(title = "Is it mandatory true-required false-optional", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean required = false;
/**
@@ -157,7 +156,6 @@ public class ParamDefine {
* Is it an advanced hidden parameter true-yes false-no
*/
@Schema(title = "Is it an advanced hidden parameter true-yes false-no", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean hide = false;
/**
@@ -1,114 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.entity.message;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.nio.charset.StandardCharsets;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
/**
* cluster message entity for fury serialization
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ClusterMessage implements Serializable {
/**
* collector identity
*/
@Builder.Default
private String identity = "";
/**
* message direction
*/
@Builder.Default
private Direction direction = Direction.REQUEST;
/**
* message type
*/
private MessageType type;
/**
* message content
* Use byte[] to ensure data integrity for both JSON strings (UTF-8) and Binary data (Arrow).
* Avoids String encoding/decoding issues (Mojibake) across different JVMs or Languages.
*/
private byte[] msg;
@JsonIgnore
public String getMsgString() {
if (this.msg == null) {
return null;
}
return new String(this.msg, StandardCharsets.UTF_8);
}
@JsonIgnore
public void setMsgString(String jsonString) {
if (jsonString != null) {
this.msg = jsonString.getBytes(StandardCharsets.UTF_8);
} else {
this.msg = null;
}
}
/**
* Message Type Enum
*/
public enum MessageType {
// heartbeat message
HEARTBEAT,
// collector go online to master message
GO_ONLINE,
// collector go offline to master message
GO_OFFLINE,
// collector go close to master
GO_CLOSE,
// issue cyclic collect task
ISSUE_CYCLIC_TASK,
// delete cyclic collect task
DELETE_CYCLIC_TASK,
// issue one-time collect task
ISSUE_ONE_TIME_TASK,
// response one-time collect data
RESPONSE_ONE_TIME_TASK_DATA,
// response cyclic collect data
RESPONSE_CYCLIC_TASK_DATA,
// response cyclic service discovery data
RESPONSE_CYCLIC_TASK_SD_DATA
}
/**
* Direction Enum
*/
public enum Direction {
// request message
REQUEST,
// request response
RESPONSE
}
}
@@ -225,9 +225,9 @@ public final class CollectRep {
return metadata.getOrDefault(MetricDataConstants.INSTANCE_NAME, null);
}
public String getInstanceHost() {
public String getInstance() {
Map<String, String> metadata = getMetadata();
return metadata.getOrDefault(MetricDataConstants.INSTANCE_HOST, null);
return metadata.getOrDefault(MetricDataConstants.INSTANCE, null);
}
public Map<String, String> getLabels() {
@@ -501,8 +501,8 @@ public final class CollectRep {
return metadata.getOrDefault(MetricDataConstants.INSTANCE_NAME, null);
}
public String getInstanceHost() {
return metadata.getOrDefault(MetricDataConstants.INSTANCE_HOST, null);
public String getInstance() {
return metadata.getOrDefault(MetricDataConstants.INSTANCE, null);
}
public Map<String, String> getLabels() {
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.common.entity.sd;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
@@ -37,7 +37,7 @@ import lombok.NoArgsConstructor;
*/
@Entity
@Table(name = "hzb_history", indexes = {
@Index(name = "history_query_index", columnList = "monitorId"),
@Index(name = "history_query_index", columnList = "instance"),
@Index(name = "history_query_index", columnList = "app"),
@Index(name = "history_query_index", columnList = "metrics"),
@Index(name = "history_query_index", columnList = "metric")
@@ -54,8 +54,8 @@ public class History {
@Schema(description = "Metric data history entity primary key index ID", example = "87584674384", accessMode = READ_ONLY)
private Long id;
@Schema(title = "Monitoring Id", example = "87432674336", accessMode = READ_WRITE)
private Long monitorId;
@Schema(title = "Monitoring instance", example = "127.0.0.1:8080", accessMode = READ_WRITE)
private String instance;
@Schema(title = "Monitoring Type mysql oracle db2")
private String app;
@@ -67,7 +67,7 @@ public class History {
private String metric;
@Column(length = 5000)
private String instance;
private String metricLabels;
@Schema(title = "Metric Type 0: Number 1String")
private Byte metricType;
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.common.queue;
import java.util.List;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.message.CollectRep;
@@ -91,4 +92,32 @@ public interface CommonDataQueue {
* @throws InterruptedException when poll timeout
*/
LogEntry pollLogEntryToStorage() throws InterruptedException;
/**
* send batch log entries to alert queue
* @param logEntries list of log entry data
*/
void sendLogEntryToAlertBatch(List<LogEntry> logEntries);
/**
* poll batch log entries from alert queue
* @param maxBatchSize maximum number of entries to poll
* @return list of log entry data
* @throws InterruptedException when poll timeout
*/
List<LogEntry> pollLogEntryToAlertBatch(int maxBatchSize) throws InterruptedException;
/**
* send batch log entries to storage queue
* @param logEntries list of log entry data
*/
void sendLogEntryToStorageBatch(List<LogEntry> logEntries);
/**
* poll batch log entries from storage queue
* @param maxBatchSize maximum number of entries to poll
* @return list of log entry data
* @throws InterruptedException when poll timeout
*/
List<LogEntry> pollLogEntryToStorageBatch(int maxBatchSize) throws InterruptedException;
}
@@ -17,9 +17,12 @@
package org.apache.hertzbeat.common.queue.impl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.DataQueueConstants;
import org.apache.hertzbeat.common.entity.log.LogEntry;
@@ -117,6 +120,48 @@ public class InMemoryCommonDataQueue implements CommonDataQueue, DisposableBean
return logEntryToStorageQueue.take();
}
@Override
public void sendLogEntryToAlertBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
for (LogEntry logEntry : logEntries) {
logEntryQueue.offer(logEntry);
}
}
@Override
public List<LogEntry> pollLogEntryToAlertBatch(int maxBatchSize) throws InterruptedException {
List<LogEntry> batch = new ArrayList<>(maxBatchSize);
LogEntry first = logEntryQueue.poll(1, TimeUnit.SECONDS);
if (first != null) {
batch.add(first);
logEntryQueue.drainTo(batch, maxBatchSize - 1);
}
return batch;
}
@Override
public void sendLogEntryToStorageBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
for (LogEntry logEntry : logEntries) {
logEntryToStorageQueue.offer(logEntry);
}
}
@Override
public List<LogEntry> pollLogEntryToStorageBatch(int maxBatchSize) throws InterruptedException {
List<LogEntry> batch = new ArrayList<>(maxBatchSize);
LogEntry first = logEntryToStorageQueue.poll(1, TimeUnit.SECONDS);
if (first != null) {
batch.add(first);
logEntryToStorageQueue.drainTo(batch, maxBatchSize - 1);
}
return batch;
}
@Override
public void destroy() {
metricsDataToAlertQueue.clear();
@@ -18,8 +18,10 @@
package org.apache.hertzbeat.common.queue.impl;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.ReentrantLock;
@@ -261,6 +263,92 @@ public class KafkaCommonDataQueue implements CommonDataQueue, DisposableBean {
return genericPollDataFunction(logEntryToStorageQueue, logEntryToStorageConsumer, logEntryToStorageLock);
}
@Override
public void sendLogEntryToAlertBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
if (logEntryProducer != null) {
try {
for (LogEntry logEntry : logEntries) {
ProducerRecord<Long, LogEntry> record = new ProducerRecord<>(kafka.getLogEntryDataTopic(), logEntry);
logEntryProducer.send(record);
}
} catch (Exception e) {
log.error("Failed to send LogEntry batch to Kafka: {}", e.getMessage());
for (LogEntry logEntry : logEntries) {
logEntryQueue.offer(logEntry);
}
}
} else {
log.warn("logEntryProducer is not enabled, using memory queue");
for (LogEntry logEntry : logEntries) {
logEntryQueue.offer(logEntry);
}
}
}
@Override
public List<LogEntry> pollLogEntryToAlertBatch(int maxBatchSize) throws InterruptedException {
return genericBatchPollDataFunction(logEntryQueue, logEntryConsumer, logEntryLock, maxBatchSize);
}
@Override
public void sendLogEntryToStorageBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
if (logEntryProducer != null) {
try {
for (LogEntry logEntry : logEntries) {
ProducerRecord<Long, LogEntry> record = new ProducerRecord<>(kafka.getLogEntryDataToStorageTopic(), logEntry);
logEntryProducer.send(record);
}
} catch (Exception e) {
log.error("Failed to send LogEntry batch to storage via Kafka: {}", e.getMessage());
for (LogEntry logEntry : logEntries) {
logEntryToStorageQueue.offer(logEntry);
}
}
} else {
log.warn("logEntryProducer is not enabled, using memory queue for storage");
for (LogEntry logEntry : logEntries) {
logEntryToStorageQueue.offer(logEntry);
}
}
}
@Override
public List<LogEntry> pollLogEntryToStorageBatch(int maxBatchSize) throws InterruptedException {
return genericBatchPollDataFunction(logEntryToStorageQueue, logEntryToStorageConsumer, logEntryToStorageLock, maxBatchSize);
}
public <T> List<T> genericBatchPollDataFunction(LinkedBlockingQueue<T> dataQueue, KafkaConsumer<Long, T> dataConsumer,
ReentrantLock lock, int maxBatchSize) throws InterruptedException {
List<T> batch = new ArrayList<>(maxBatchSize);
lock.lockInterruptibly();
try {
dataQueue.drainTo(batch, maxBatchSize);
if (batch.size() >= maxBatchSize) {
return batch;
}
ConsumerRecords<Long, T> records = dataConsumer.poll(Duration.ofSeconds(1));
for (ConsumerRecord<Long, T> record : records) {
if (batch.size() < maxBatchSize) {
batch.add(record.value());
} else {
dataQueue.offer(record.value());
}
}
dataConsumer.commitAsync();
} catch (Exception e) {
log.error(e.getMessage());
} finally {
lock.unlock();
}
return batch;
}
@Override
public void destroy() throws Exception {
if (metricsDataProducer != null) {
@@ -17,6 +17,8 @@
package org.apache.hertzbeat.common.queue.impl;
import java.util.ArrayList;
import java.util.List;
import io.lettuce.core.KeyValue;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
@@ -162,6 +164,42 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
return genericBlockingPollFunction(logEntryToStorageQueueName, logEntrySyncCommands);
}
@Override
@SuppressWarnings("unchecked")
public void sendLogEntryToAlertBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
try {
logEntrySyncCommands.lpush(logEntryQueueName, logEntries.toArray(new LogEntry[0]));
} catch (Exception e) {
log.error("Failed to send LogEntry batch to Redis: {}", e.getMessage());
}
}
@Override
public List<LogEntry> pollLogEntryToAlertBatch(int maxBatchSize) throws InterruptedException {
return genericBatchPollFunction(logEntryQueueName, logEntrySyncCommands, maxBatchSize);
}
@Override
@SuppressWarnings("unchecked")
public void sendLogEntryToStorageBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
try {
logEntrySyncCommands.lpush(logEntryToStorageQueueName, logEntries.toArray(new LogEntry[0]));
} catch (Exception e) {
log.error("Failed to send LogEntry batch to storage via Redis: {}", e.getMessage());
}
}
@Override
public List<LogEntry> pollLogEntryToStorageBatch(int maxBatchSize) throws InterruptedException {
return genericBatchPollFunction(logEntryToStorageQueueName, logEntrySyncCommands, maxBatchSize);
}
@Override
public void destroy() {
connection.close();
@@ -188,4 +226,17 @@ public class RedisCommonDataQueue implements CommonDataQueue, DisposableBean {
}
}
private List<LogEntry> genericBatchPollFunction(String key, RedisCommands<String, LogEntry> commands, int maxBatchSize) {
List<LogEntry> batch = new ArrayList<>(maxBatchSize);
try {
List<LogEntry> elements = commands.rpop(key, maxBatchSize);
if (elements != null) {
batch.addAll(elements);
}
} catch (Exception e) {
log.error("Redis batch poll failed: {}", e.getMessage());
}
return batch;
}
}
@@ -17,49 +17,60 @@
package org.apache.hertzbeat.common.support;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* common task worker thread pool with Virtual Threads
* common task worker thread pool
*/
@Component
@Slf4j
public class CommonThreadPool implements DisposableBean {
private ExecutorService workerExecutor;
private ThreadPoolExecutor workerExecutor;
public CommonThreadPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory factory = Thread.ofVirtual()
.name("common-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
log.info("CommonThreadPool initialized with Virtual Threads.");
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("common executor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("common-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(1,
Integer.MAX_VALUE,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
/**
* Run the task thread
* @param runnable Task
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
*/
public void execute(Runnable runnable) {
public void execute(Runnable runnable) throws RejectedExecutionException {
workerExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
if (workerExecutor != null) {
workerExecutor.close();
workerExecutor.shutdownNow();
}
}
}
}
@@ -21,7 +21,7 @@ import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.jspecify.annotations.NonNull;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
/**
@@ -44,7 +44,21 @@ public class HostParamValidator implements ConstraintValidator<HostValid, String
value = value.replaceFirst(PATTERN_HTTPS, BLANK);
}
return IpDomainUtil.validateIpDomain(value);
String hostPart = value;
if (value.contains(":")) {
// if contains multiple ":", it may be IPv6 with port
if (value.lastIndexOf(":") > value.indexOf(":") && value.contains("[")) {
int portIndex = value.lastIndexOf(":");
hostPart = value.substring(0, portIndex);
} else if (value.split(":").length == 2) {
// it is IPv4 or domain with port
String[] parts = value.split(":");
hostPart = parts[0];
}
}
return IpDomainUtil.validateIpDomain(hostPart);
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -37,7 +37,7 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
*/
@Slf4j
public final class ArrowUtil {
private ArrowUtil() {
}
@@ -53,7 +53,7 @@ public final class ArrowUtil {
public static byte[] serializeMultipleRoots(List<VectorSchemaRoot> roots) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(out)) {
dataOut.writeInt(roots.size());
for (VectorSchemaRoot root : roots) {
ArrowStreamWriter writer = new ArrowStreamWriter(
@@ -87,10 +87,10 @@ public final class ArrowUtil {
List<VectorSchemaRoot> roots = new ArrayList<>();
try (ByteArrayInputStream in = new ByteArrayInputStream(data);
DataInputStream dataIn = new DataInputStream(in)) {
int rootCount = dataIn.readInt();
RootAllocator allocator = new RootAllocator();
for (int i = 0; i < rootCount; i++) {
ArrowStreamReader reader = new ArrowStreamReader(
Channels.newChannel(in),
@@ -153,4 +153,5 @@ public final class ArrowUtil {
}
return serializeMultipleRoots(roots);
}
}
}
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.util;
import org.apache.hertzbeat.common.constants.HertzBeatKeywordsEnum;
import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Validate JEXL rules
*/
public class HertzBeatKeywordsUtil {
private static final Map<String, HertzBeatKeywordsEnum> KEYWORDS =
Arrays.stream(HertzBeatKeywordsEnum.values())
.collect(Collectors.toMap(HertzBeatKeywordsEnum::getKeyword, e -> e));
/**
* Verify if the field matches any reserved keywords
*
* @param field Field name to verify
*/
public static void verifyKeywords(String field) {
HertzBeatKeywordsEnum keyword = KEYWORDS.get(field);
if (null == keyword) {
return;
}
throw new IllegalArgumentException(
String.format("Field matches keyword `%s`, please set alias `%s`.",
keyword.getKeyword(), keyword.getAlias()));
}
}
@@ -24,8 +24,8 @@ import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.net.InetAddressUtils;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.http.conn.util.InetAddressUtils;
import org.springframework.util.StringUtils;
/**
@@ -60,10 +60,10 @@ public final class IpDomainUtil {
if (LOCALHOST.equalsIgnoreCase(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv4(ipDomain)) {
if (InetAddressUtils.isIPv4Address(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv6(ipDomain)) {
if (InetAddressUtils.isIPv6Address(ipDomain)) {
return true;
}
return DOMAIN_PATTERN.matcher(ipDomain).matches();
@@ -75,7 +75,7 @@ public final class IpDomainUtil {
* @return true or false
*/
public static boolean isHasSchema(String domainIp) {
if (!StringUtils.hasText(domainIp)) {
if (domainIp == null || !StringUtils.hasText(domainIp)) {
return false;
}
return DOMAIN_SCHEMA.matcher(domainIp).matches();

Some files were not shown because too many files have changed in this diff Show More