Compare commits

...
Author SHA1 Message Date
LogicandDuansg b9f1d65cb5 [refactor] Apache Fory (#3873)
Signed-off-by: Logic <zqr10159@dromara.org>
Co-authored-by: Duansg <siguoduan@gmail.com>
2025-11-27 17:28:44 +08:00
Logic 1dea5467fc [refactor]: springboot4.0.0 (#3872) 2025-11-27 17:22:07 +08:00
111 changed files with 1997 additions and 1592 deletions
+1 -1
View File
@@ -27,7 +27,7 @@
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.3</spring-ai.version>
<java.version>17</java.version>
<java.version>21</java.version>
</properties>
<dependencies>
@@ -17,25 +17,23 @@
package org.apache.hertzbeat.alert;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* alarm module thread pool
* alarm module thread pool with Virtual Threads
*/
@Component
@Slf4j
public class AlerterWorkerPool {
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor notifyExecutor;
private ThreadPoolExecutor logWorkerExecutor;
private ExecutorService workerExecutor;
private ExecutorService notifyExecutor;
private ExecutorService logWorkerExecutor;
public AlerterWorkerPool() {
initWorkExecutor();
@@ -44,62 +42,31 @@ public class AlerterWorkerPool {
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("alerter-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(10,
10,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("alerter-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
private void initNotifyExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter notifyExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("notify-worker-%d")
.build();
notifyExecutor = new ThreadPoolExecutor(6,
6,
10,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("notify-worker-", 0)
.factory();
notifyExecutor = Executors.newThreadPerTaskExecutor(factory);
}
private void initLogWorkerExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Alerter logWorkerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("log-worker-%d")
.build();
logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(1000),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("log-worker-", 0)
.factory();
logWorkerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
/**
* Run the alerter task
* @param runnable task
* @throws RejectedExecutionException when The thread pool is full of
*/
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable){
workerExecutor.execute(runnable);
}
@@ -107,9 +74,8 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the notifyExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeNotify(Runnable runnable) throws RejectedExecutionException {
public void executeNotify(Runnable runnable){
notifyExecutor.execute(runnable);
}
@@ -117,9 +83,8 @@ public class AlerterWorkerPool {
* Executes the given runnable task using the logWorkerExecutor.
*
* @param runnable the task to be executed
* @throws RejectedExecutionException if the task cannot be accepted for execution
*/
public void executeLogJob(Runnable runnable) throws RejectedExecutionException {
public void executeLogJob(Runnable runnable){
logWorkerExecutor.execute(runnable);
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,79 +15,80 @@
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.http;
package org.apache.hertzbeat.collector.collect. http;
import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
import static org.apache.hertzbeat. common.constants.SignConstants.RIGHT_DASH;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com. google.gson.JsonParser;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java. io.InterruptedIOException;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util. List;
import java.util. Map;
import java.util. Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.net.ssl.SSLException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml. parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax. xml.xpath.XPathConstants;
import javax.xml.xpath. XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import lombok.extern.slf4j.Slf4j;
import lombok.extern.slf4j. Slf4j;
import org. apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.collect.http.promethus.AbstractPrometheusParse;
import org.apache.hertzbeat.collector.collect.http.promethus.PrometheusParseCreator;
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
import org.apache. hertzbeat.collector.collect. prometheus.parser.MetricFamily;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache. hertzbeat.collector.dispatch. DispatchConstants;
import org. apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.collector.util.JsonPathParser;
import org.apache.hertzbeat.collector.util.TimeExpressionUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache. hertzbeat.common.constants. NetworkConstants;
import org.apache. hertzbeat.common.constants. SignConstants;
import org.apache.hertzbeat.common.entity.job. Metrics;
import org.apache. hertzbeat.common.entity. job.protocol.HttpProtocol;
import org.apache.hertzbeat. common.entity.message.CollectRep;
import org.apache.hertzbeat. common.util.Base64Util;
import org. apache.hertzbeat.common. util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.HttpMethod;
import org.apache. hc.core5.http.ClassicHttpResponse;
import org. apache.hc.core5. http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache. hc.core5.http. HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache. hc.client5.http.auth.AuthScope;
import org.apache. hc.client5.http. auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.ClientProtocolException;
import org. apache.hc.client5. http.config.RequestConfig;
import org. apache.hc.client5. http.classic.methods.HttpGet;
import org. apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org. apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache. hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc. core5.http.protocol.HttpContext;
import org. apache.hc.core5. http.io.entity.EntityUtils;
import org.apache. hc.core5.util.Timeout;
import org. springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import org. springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.util.UriUtils;
import org.xml.sax.InputSource;
@@ -96,9 +97,9 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java. util.regex.Pattern;
import java.util.HashSet;
import java.util.ArrayList;
import java.util. ArrayList;
import java.util.Collections;
/**
@@ -106,6 +107,28 @@ import java.util.Collections;
*/
@Slf4j
public class HttpCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Set.of(
HttpStatus.SC_OK,
HttpStatus.SC_CREATED,
@@ -122,28 +145,37 @@ public class HttpCollectImpl extends AbstractCollect {
}
@Override
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
public void collect(CollectRep. MetricsData.Builder builder, Metrics metrics) {
long startTime = System.currentTimeMillis();
HttpProtocol httpProtocol = metrics.getHttp();
String url = httpProtocol.getUrl();
if (!StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
if (! StringUtils.hasText(url) || !url.startsWith(RIGHT_DASH)) {
httpProtocol.setUrl(StringUtils.hasText(url) ? RIGHT_DASH + url.trim() : RIGHT_DASH);
}
if (CollectionUtils.isEmpty(httpProtocol.getSuccessCodes())) {
httpProtocol.setSuccessCodes(List.of(HttpStatus.SC_OK + ""));
httpProtocol. setSuccessCodes(List. of(HttpStatus.SC_OK + ""));
}
HttpContext httpContext = createHttpContext(metrics.getHttp());
HttpUriRequest request = createHttpRequest(metrics.getHttp());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig. getRequest();
HttpContext httpContext = createHttpContext(metrics. getHttp(), requestWithConfig. getRequestConfig());
// Use HttpClientResponseHandler to handle the response and avoid deprecated execute method
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
boolean isSuccessInvoke = checkSuccessInvoke(metrics, statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return;
return null;
}
long responseTime = System.currentTimeMillis() - startTime;
@@ -153,25 +185,20 @@ public class HttpCollectImpl extends AbstractCollect {
try {
if (DispatchConstants.PARSE_PROMETHEUS.equals(parseType)) {
if (entity != null) {
parseResponseByPrometheusExporter(entity.getContent(), metrics.getAliasFields(), builder);
parseResponseByPrometheusExporter(entity. getContent(), metrics.getAliasFields(), builder);
}
} else if (DispatchConstants.PARSE_HEADER.equals(parseType)) {
parseResponseByHeader(builder, metrics.getAliasFields(), response);
// Consume entity to release connection
EntityUtils.consumeQuietly(entity);
} else {
/*
this could create large objects, potentially impacting JVM memory space significantly.
Option 1: Parse using InputStream, but this requires significant code changes;
Option 2: Manually trigger garbage collection, similar to how it's done in Dubbo for large inputs.
*/
String resp = entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8);
String resp = entity == null ? "" : EntityUtils. toString(entity, StandardCharsets. UTF_8);
if (!StringUtils.hasText(resp)) {
log.info("http response entity is empty, status: {}.", statusCode);
}
switch (parseType) {
case DispatchConstants.PARSE_JSON_PATH ->
parseResponseByJsonPath(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
parseResponseByJsonPath(resp, metrics. getAliasFields(), metrics. getHttp(), builder, responseTime);
case DispatchConstants.PARSE_PROM_QL ->
parseResponseByPromQl(resp, metrics.getAliasFields(), metrics.getHttp(), builder);
case DispatchConstants.PARSE_XML_PATH ->
@@ -179,7 +206,7 @@ public class HttpCollectImpl extends AbstractCollect {
case DispatchConstants.PARSE_WEBSITE ->
parseResponseByWebsite(resp, metrics, metrics.getHttp(), builder, responseTime, statusCode);
case DispatchConstants.PARSE_SITE_MAP ->
parseResponseBySiteMap(resp, metrics.getAliasFields(), builder);
parseResponseBySiteMap(resp, metrics. getAliasFields(), builder);
case DispatchConstants.PARSE_CONFIG ->
parseResponseByConfig(resp, metrics.getAliasFields(), metrics.getHttp(), builder, responseTime);
default ->
@@ -191,15 +218,20 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder. setCode(CollectRep. Code.UN_CONNECTABLE);
builder.setMsg(errorMsg);
} catch (UnknownHostException e2) {
String errorMsg = CommonUtil.getMessageFromThrowable(e2);
log.info(errorMsg);
builder.setCode(CollectRep.Code.UN_REACHABLE);
builder. setCode(CollectRep. Code.UN_REACHABLE);
builder.setMsg("unknown host:" + errorMsg);
} catch (InterruptedIOException | ConnectException | SSLException e3) {
String errorMsg = CommonUtil.getMessageFromThrowable(e3);
@@ -208,7 +240,7 @@ public class HttpCollectImpl extends AbstractCollect {
builder.setMsg(errorMsg);
} catch (IOException e4) {
String errorMsg = CommonUtil.getMessageFromThrowable(e4);
log.info(errorMsg);
log. info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
@@ -216,17 +248,13 @@ public class HttpCollectImpl extends AbstractCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, CloseableHttpResponse response) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
private void parseResponseByHeader(CollectRep.MetricsData.Builder builder, List<String> aliases, ClassicHttpResponse response) {
CollectRep.ValueRow. Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliases) {
if (!StringUtils.hasText(alias)) {
if (! StringUtils.hasText(alias)) {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
continue;
}
@@ -238,7 +266,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(firstHeader.getValue());
}
builder.addValueRow(valueRowBuilder.build());
builder. addValueRow(valueRowBuilder.build());
}
@Override
@@ -263,7 +291,7 @@ public class HttpCollectImpl extends AbstractCollect {
private void addColumnForSummary(Long responseTime, CollectRep.ValueRow.Builder valueRowBuilder, int keywordNum, String alias) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -282,13 +310,13 @@ public class HttpCollectImpl extends AbstractCollect {
DocumentBuilder db = dbf.newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
NodeList urlList = document.getElementsByTagName("url");
for (int i = 0; i < urlList.getLength(); i++) {
for (int i = 0; i < urlList. getLength(); i++) {
Node urlNode = urlList.item(i);
NodeList childNodes = urlNode.getChildNodes();
for (int k = 0; k < childNodes.getLength(); k++) {
Node currentNode = childNodes.item(k);
// distinguish between text nodes and element nodes
if (currentNode.getNodeType() == Node.ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
if (currentNode.getNodeType() == Node. ELEMENT_NODE && "loc".equals(currentNode.getNodeName())) {
// retrieves the value of the loc node
siteUrls.add(currentNode.getFirstChild().getNodeValue());
break;
@@ -318,10 +346,13 @@ public class HttpCollectImpl extends AbstractCollect {
long startTime = System.currentTimeMillis();
try {
HttpGet httpGet = new HttpGet(siteUrl);
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Integer> statusHandler = response -> {
EntityUtils.consume(response.getEntity());
}
return response.getCode();
};
statusCode = CommonHttpClient.getHttpClient().execute(httpGet, statusHandler);
} catch (ClientProtocolException e1) {
if (e1.getCause() != null) {
errorMsg = e1.getCause().getMessage();
@@ -345,7 +376,7 @@ public class HttpCollectImpl extends AbstractCollect {
} else if (NetworkConstants.STATUS_CODE.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(statusCode == null
? CommonConstants.NULL_VALUE : String.valueOf(statusCode));
} else if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
} else if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(responseTime));
} else if (NetworkConstants.ERROR_MSG.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(errorMsg);
@@ -362,13 +393,13 @@ public class HttpCollectImpl extends AbstractCollect {
HttpProtocol http = metrics.getHttp();
List<String> aliasFields = metrics.getAliasFields();
String xpathExpression = http.getParseScript();
if (!StringUtils.hasText(xpathExpression)) {
if (! StringUtils.hasText(xpathExpression)) {
log.warn("Http collect parse type is xmlPath, but the xpath expression is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("XPath expression is empty");
return;
}
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
@@ -378,13 +409,13 @@ public class HttpCollectImpl extends AbstractCollect {
dbf.setXIncludeAware(false);
dbf.setExpandEntityReferences(false);
DocumentBuilder db = dbf.newDocumentBuilder();
DocumentBuilder db = dbf. newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(resp)));
XPathFactory xpathFactory = XPathFactory.newInstance();
XPath xpath = xpathFactory.newXPath();
NodeList nodeList = (NodeList) xpath.evaluate(xpathExpression, document, XPathConstants.NODESET);
NodeList nodeList = (NodeList) xpath. evaluate(xpathExpression, document, XPathConstants.NODESET);
if (nodeList == null || nodeList.getLength() == 0) {
log.debug("XPath expression '{}' returned no nodes.", xpathExpression);
@@ -397,10 +428,10 @@ public class HttpCollectImpl extends AbstractCollect {
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
valueRowBuilder.addColumn(CommonConstants. NULL_VALUE);
}
}
builder.addValueRow(valueRowBuilder.build());
@@ -413,14 +444,14 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
if (NetworkConstants. RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
} else if (CollectorConstants. KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
try {
String value = (String) xpath.evaluate(alias, node, XPathConstants.STRING);
valueRowBuilder.addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(StringUtils.hasText(value) ? value : CommonConstants.NULL_VALUE);
} catch (XPathExpressionException e) {
log.warn("Failed to evaluate XPath '{}' for node [{}]: {}", alias, node.getNodeName(), e.getMessage());
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -432,7 +463,7 @@ public class HttpCollectImpl extends AbstractCollect {
} catch (Exception e) {
log.warn("Failed to parse XML response with XPath '{}': {}", xpathExpression, e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg("Failed to parse XML response: " + e.getMessage());
}
}
@@ -441,25 +472,12 @@ public class HttpCollectImpl extends AbstractCollect {
/**
* Parses the response body in Properties/Config format.
* Two modes are supported:
* 1. single-object mode: if http.parseScript is null, aliasFields are treated as indicator names.
* - If there is a locator in the indicator definition, use the locator as the key of the Properties.
* - Otherwise, use aliasField (metric name) as the key for Properties.
* Generate a single row of data.
* 2. array mode: if http.parseScript is not empty (e.g. “users”), treat it as an array base path.
* Treat aliasFields as the attribute name of an array element, and generate a single row of data for each array index. locator is invalid in this mode.
*
* @param resp Response body string
* @param aliasFields List of metrics aliases (i.e., the list of fields in metrics.fields).
* @param http http protocol configuration
* @param builder The metrics data builder.
* @param responseTime response time
*/
private void parseResponseByConfig(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
if (!StringUtils.hasText(resp)) {
if (! StringUtils.hasText(resp)) {
log.warn("Http collect parse type is config, but response body is empty.");
builder.setCode(CollectRep.Code.FAIL);
builder. setCode(CollectRep. Code.FAIL);
builder.setMsg("Response body is empty");
return;
}
@@ -469,14 +487,14 @@ public class HttpCollectImpl extends AbstractCollect {
properties.load(reader);
} catch (IOException e) {
log.warn("Failed to parse config response: {}", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setCode(CollectRep.Code. FAIL);
builder.setMsg("Failed to parse config response: " + e.getMessage());
return;
}
String arrayBasePath = http.getParseScript();
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
if (!StringUtils.hasText(arrayBasePath)) {
if (! StringUtils.hasText(arrayBasePath)) {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
@@ -485,7 +503,7 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
} else {
String value = properties.getProperty(alias);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(value != null ? value : CommonConstants. NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -518,13 +536,13 @@ public class HttpCollectImpl extends AbstractCollect {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String alias : aliasFields) {
if (NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(responseTime.toString());
} else if (CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Integer.toString(keywordNum));
valueRowBuilder.addColumn(responseTime. toString());
} else if (CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) {
valueRowBuilder. addColumn(Integer.toString(keywordNum));
} else {
String currentKey = arrayBasePath + "[" + i + "]." + alias;
String value = properties.getProperty(currentKey);
valueRowBuilder.addColumn(value != null ? value : CommonConstants.NULL_VALUE);
valueRowBuilder. addColumn(value != null ? value : CommonConstants.NULL_VALUE);
}
}
CollectRep.ValueRow valueRow = valueRowBuilder.build();
@@ -549,24 +567,24 @@ public class HttpCollectImpl extends AbstractCollect {
for (int i = 0; i < valueRow.getColumnsCount(); i++) {
String columnValue = valueRow.getColumns(i);
String alias = aliasFields.get(i);
if (!CommonConstants.NULL_VALUE.equals(columnValue) && (!NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD.equalsIgnoreCase(alias))) {
if (! CommonConstants.NULL_VALUE.equals(columnValue) && (! NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) && !CollectorConstants.KEYWORD. equalsIgnoreCase(alias))) {
hasMeaningfulData = true;
break;
}
if ((NetworkConstants.RESPONSE_TIME.equalsIgnoreCase(alias) || CollectorConstants.KEYWORD.equalsIgnoreCase(alias)) && !CommonConstants.NULL_VALUE.equals(columnValue)) {
if ((NetworkConstants.RESPONSE_TIME. equalsIgnoreCase(alias) || CollectorConstants.KEYWORD. equalsIgnoreCase(alias)) && ! CommonConstants.NULL_VALUE.equals(columnValue)) {
hasMeaningfulData = true;
}
}
return hasMeaningfulData;
}
@SuppressWarnings("unchecked")
private void parseResponseByJsonPath(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
List<Object> results = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript());
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
for (int i = 0; i < results.size(); i++) {
Object objectValue = results.get(i);
// if a property is missing or empty due to target version issues, filter it. Refer to the app-elasticsearch.yml configuration under name: nodes
if (objectValue == null) {
continue;
}
@@ -578,8 +596,8 @@ public class HttpCollectImpl extends AbstractCollect {
if (value != null) {
valueRowBuilder.addColumn(String.valueOf(value));
} else {
if (alias.startsWith("$.")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
if (alias.startsWith("$. ")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias. substring(1));
if (subResults != null && subResults.size() > i) {
Object resultValue = subResults.get(i);
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
@@ -633,7 +651,7 @@ public class HttpCollectImpl extends AbstractCollect {
return;
}
MetricFamily metricFamily = metricFamilyMap.get(metrics);
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily.getMetricList())) {
if (null == metricFamily || CollectionUtils.isEmpty(metricFamily. getMetricList())) {
return;
}
for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
@@ -645,7 +663,7 @@ public class HttpCollectImpl extends AbstractCollect {
String columnValue = labelMap.get(aliasField);
if (columnValue != null) {
valueRowBuilder.addColumn(columnValue);
} else if (CommonConstants.PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
} else if (CommonConstants. PROM_VALUE.equals(aliasField) || CommonConstants.PROM_METRIC_VALUE.equals(aliasField)) {
valueRowBuilder.addColumn(String.valueOf(metric.getValue()));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
@@ -657,9 +675,9 @@ public class HttpCollectImpl extends AbstractCollect {
private void parseResponseByDefault(String resp, List<String> aliasFields, HttpProtocol http,
CollectRep.MetricsData.Builder builder, Long responseTime) {
CollectRep.MetricsData. Builder builder, Long responseTime) {
JsonElement element = JsonParser.parseString(resp);
int keywordNum = CollectUtil.countMatchKeyword(resp, http.getKeyword());
int keywordNum = CollectUtil. countMatchKeyword(resp, http.getKeyword());
if (element.isJsonArray()) {
JsonArray array = element.getAsJsonArray();
for (JsonElement jsonElement : array) {
@@ -691,46 +709,53 @@ public class HttpCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpProtocol http protocol
* @param requestConfig request config (can be null)
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpProtocol) {
HttpProtocol.Authorization auth = httpProtocol.getAuthorization();
public HttpContext createHttpContext(HttpProtocol httpProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
// Set request config if provided
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
HttpProtocol. Authorization auth = httpProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword());
auth.getDigestAuthPassword(). toCharArray());
AuthScope authScope = new AuthScope(httpProtocol.getHost(), Integer.parseInt(httpProtocol.getPort()));
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
*
* @param httpProtocol http params
* @return http uri request
* @return http request with config
*/
public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
RequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod().toUpperCase();
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpProtocol) {
ClassicRequestBuilder requestBuilder;
String httpMethod = httpProtocol.getMethod(). toUpperCase();
if (HttpMethod.GET.matches(httpMethod)) {
requestBuilder = RequestBuilder.get();
requestBuilder = ClassicRequestBuilder.get();
} else if (HttpMethod.POST.matches(httpMethod)) {
requestBuilder = RequestBuilder.post();
requestBuilder = ClassicRequestBuilder.post();
} else if (HttpMethod.PUT.matches(httpMethod)) {
requestBuilder = RequestBuilder.put();
requestBuilder = ClassicRequestBuilder.put();
} else if (HttpMethod.DELETE.matches(httpMethod)) {
requestBuilder = RequestBuilder.delete();
} else if (HttpMethod.PATCH.matches(httpMethod)) {
requestBuilder = RequestBuilder.patch();
requestBuilder = ClassicRequestBuilder.delete();
} else if (HttpMethod. PATCH.matches(httpMethod)) {
requestBuilder = ClassicRequestBuilder.patch();
} else {
// not support the method
log.error("not support the http method: {}.", httpProtocol.getMethod());
@@ -741,17 +766,17 @@ public class HttpCollectImpl extends AbstractCollect {
boolean enableUrlEncoding = Boolean.parseBoolean(httpProtocol.getEnableUrlEncoding());
StringBuilder queryParams = new StringBuilder();
if (params != null && !params.isEmpty()) {
if (params != null && ! params.isEmpty()) {
for (Map.Entry<String, String> param : params.entrySet()) {
String key = param.getKey();
String value = param.getValue();
if (!StringUtils.hasText(key)) {
if (! StringUtils.hasText(key)) {
continue;
}
if (!queryParams.isEmpty()) {
queryParams.append("&");
if (! queryParams.isEmpty()) {
queryParams. append("&");
}
if (enableUrlEncoding) {
@@ -762,9 +787,9 @@ public class HttpCollectImpl extends AbstractCollect {
if (StringUtils.hasText(value)) {
String calculatedValue = TimeExpressionUtil.calculate(value);
if (enableUrlEncoding) {
calculatedValue = UriUtils.encodeQueryParam(calculatedValue, "UTF-8");
calculatedValue = UriUtils. encodeQueryParam(calculatedValue, "UTF-8");
}
queryParams.append("=").append(calculatedValue);
queryParams. append("=").append(calculatedValue);
}
}
}
@@ -775,7 +800,7 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
// headers The custom request header is overwritten here
Map<String, String> headers = httpProtocol.getHeaders();
if (headers != null && !headers.isEmpty()) {
if (headers != null && ! headers.isEmpty()) {
for (Map.Entry<String, String> header : headers.entrySet()) {
if (StringUtils.hasText(header.getValue())) {
requestBuilder.addHeader(header.getKey(), header.getValue());
@@ -783,7 +808,7 @@ public class HttpCollectImpl extends AbstractCollect {
}
}
// add accept
if (DispatchConstants.PARSE_DEFAULT.equals(httpProtocol.getParseType())
if (DispatchConstants.PARSE_DEFAULT. equals(httpProtocol.getParseType())
|| DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
} else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) {
@@ -799,10 +824,10 @@ public class HttpCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization. getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + SignConstants.DOUBLE_MARK + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
requestBuilder.addHeader(HttpHeaders. AUTHORIZATION, DispatchConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
}
}
@@ -817,14 +842,14 @@ public class HttpCollectImpl extends AbstractCollect {
if (enableUrlEncoding) {
// if the url contains parameters directly
if (httpProtocol.getUrl().contains("?")) {
String path = httpProtocol.getUrl().substring(0, httpProtocol.getUrl().indexOf("?"));
String path = httpProtocol.getUrl(). substring(0, httpProtocol.getUrl().indexOf("?"));
String query = httpProtocol.getUrl().substring(httpProtocol.getUrl().indexOf("?") + 1);
uri = UriUtils.encodePath(path, "UTF-8") + "?" + UriUtils.encodeQuery(query, "UTF-8");
} else {
uri = UriUtils.encodePath(httpProtocol.getUrl(), "UTF-8");
}
} else {
uri = httpProtocol.getUrl();
uri = httpProtocol. getUrl();
}
// append query params
@@ -834,10 +859,10 @@ public class HttpCollectImpl extends AbstractCollect {
String finalUri;
if (IpDomainUtil.isHasSchema(httpProtocol.getHost())) {
finalUri = httpProtocol.getHost() + ":" + httpProtocol.getPort() + uri;
finalUri = httpProtocol.getHost() + ":" + httpProtocol. getPort() + uri;
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
String ipAddressType = IpDomainUtil. checkIpAddressType(httpProtocol.getHost());
String baseUri = NetworkConstants.IPV6. equals(ipAddressType)
? String.format("[%s]:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri)
: String.format("%s:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(httpProtocol.getSsl());
@@ -855,21 +880,23 @@ public class HttpCollectImpl extends AbstractCollect {
throw e;
}
// custom timeout
// custom timeout - create RequestConfig separately
RequestConfig requestConfig = null;
int timeout = CollectUtil.getTimeout(httpProtocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
// build the request and return with config
return new HttpRequestWithConfig(requestBuilder. build(), requestConfig);
}
private boolean checkSuccessInvoke(Metrics metrics, int statusCode) {
List<String> successCodes = metrics.getHttp().getSuccessCodes();
List<String> successCodes = metrics.getHttp(). getSuccessCodes();
Set<Integer> successCodeSet = successCodes != null ? successCodes.stream().map(code -> {
try {
return Integer.valueOf(code);
@@ -117,11 +117,11 @@ public class SslCertificateCollectImpl extends AbstractCollect {
} else if (NAME_EXPIRED.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(Boolean.toString(expired));
} else if (NAME_START_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toLocaleString());
valueRowBuilder.addColumn(x509Certificate.getNotBefore().toString());
} else if (NAME_START_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotBefore().getTime()));
} else if (NAME_END_TIME.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toLocaleString());
valueRowBuilder.addColumn(x509Certificate.getNotAfter().toString());
} else if (NAME_END_TIMESTAMP.equalsIgnoreCase(alias)) {
valueRowBuilder.addColumn(String.valueOf(x509Certificate.getNotAfter().getTime()));
} else {
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -44,16 +44,17 @@ import org.apache.hertzbeat.common.entity.job.protocol.NginxProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.MediaType;
/**
@@ -62,6 +63,27 @@ import org.springframework.http.MediaType;
@Slf4j
public class NginxCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final String NGINX_STATUS_NAME = "nginx_status";
private static final String REQ_STATUS_NAME = "req_status";
private static final String AVAILABLE = "available";
@@ -93,34 +115,36 @@ public class NginxCollectImpl extends AbstractCollect {
nginxProtocol.setUrl(url == null ? RIGHT_DASH : RIGHT_DASH + url.trim());
}
HttpContext httpContext = createHttpContext(metrics.getNginx());
HttpUriRequest request = createHttpRequest(metrics.getNginx());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)){
// send an HTTP request and get the response data
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getNginx());
HttpContext httpContext = createHttpContext(metrics.getNginx(), requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + statusCode);
return;
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
Long responseTime = System.currentTimeMillis() - startTime;
// call different parsing methods based on the metrics name
if (StringUtils.equalsAny(metrics.getName(), NGINX_STATUS_NAME, AVAILABLE)) {
if (NGINX_STATUS_NAME.equals(metrics.getName()) || AVAILABLE.equals(metrics.getName())) {
parseNginxStatusResponse(builder, resp, metrics, responseTime);
} else if (REQ_STATUS_NAME.equals(metrics.getName())) {
parseReqStatusResponse(builder, resp, metrics, responseTime);
}
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
}
@@ -130,18 +154,21 @@ public class NginxCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NGINX;
}
private HttpContext createHttpContext(NginxProtocol nginxProtocol) {
private HttpContext createHttpContext(NginxProtocol nginxProtocol, RequestConfig requestConfig) {
HttpHost host = new HttpHost(nginxProtocol.getHost(), Integer.parseInt(nginxProtocol.getPort()));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
return httpClientContext;
}
private HttpUriRequest createHttpRequest(NginxProtocol nginxProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(NginxProtocol nginxProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
String portWithUri = nginxProtocol.getPort() + CollectUtil.replaceUriSpecialChar(nginxProtocol.getUrl());
String host = nginxProtocol.getHost();
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + portWithUri);
} else {
@@ -162,16 +189,16 @@ public class NginxCollectImpl extends AbstractCollect {
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(nginxProtocol.getTimeout());
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -335,4 +362,4 @@ public class NginxCollectImpl extends AbstractCollect {
return GET + result;
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -47,24 +47,24 @@ import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.collector.collect.prometheus.parser.OnlineParser;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -74,11 +74,32 @@ import org.springframework.util.StringUtils;
*/
@Slf4j
public class PrometheusAutoCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder,
Metrics metrics) {
@@ -89,11 +110,13 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
builder.setMsg(e.getMessage());
return null;
}
HttpContext httpContext = createHttpContext(metrics.getPrometheus());
HttpUriRequest request = createHttpRequest(metrics.getPrometheus());
try (CloseableHttpResponse response =
CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getPrometheus());
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getPrometheus(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<List<CollectRep.MetricsData>> responseHandler = response -> {
int statusCode = response.getCode();
boolean isSuccessInvoke = defaultSuccessStatusCodes.contains(statusCode);
log.debug("http response status: {}", statusCode);
if (!isSuccessInvoke) {
@@ -102,12 +125,18 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
return null;
}
try {
// Parse directly from the entity stream inside the handler loop while connection is open
return parseResponseByPrometheusExporter(response.getEntity().getContent(), builder);
} catch (Exception e) {
log.info("parse error: {}.", e.getMessage(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("parse response data error:" + e.getMessage());
return null;
}
};
try {
return CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error(errorMsg);
@@ -133,31 +162,27 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
log.error(errorMsg, e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_PROMETHEUS;
}
private void validateParams(Metrics metrics) throws Exception {
if (metrics == null || metrics.getPrometheus() == null) {
throw new Exception("Prometheus collect must has prometheus params");
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (protocol.getPath() == null
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
|| !StringUtils.hasText(protocol.getPath())
|| !protocol.getPath().startsWith(RIGHT_DASH)) {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
private List<CollectRep.MetricsData> parseResponseByPrometheusExporter(InputStream inputStream, CollectRep.MetricsData.Builder builder) throws IOException {
long endTime = System.currentTimeMillis();
builder.setTime(endTime);
@@ -201,40 +226,49 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
return metricsDataList;
}
/**
* create httpContext
*
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol) {
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
AuthCache authCache = new BasicAuthCache();
authCache.put(new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort())), new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
* @param protocol http params
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -260,7 +294,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
}
// add accept
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE);
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -268,7 +302,7 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -280,17 +314,17 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
if (StringUtils.hasLength(protocol.getPayload()) && (HttpMethod.POST.matches(protocol.getMethod()) || HttpMethod.PUT.matches(protocol.getMethod()))) {
requestBuilder.setEntity(new StringEntity(protocol.getPayload(), StandardCharsets.UTF_8));
}
// uri
String uri = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uri)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uri);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -298,18 +332,19 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -319,11 +354,11 @@ public class PrometheusAutoCollectImpl implements PrometheusCollect {
public static PrometheusAutoCollectImpl getInstance() {
return PrometheusAutoCollectImpl.SingleInstance.INSTANCE;
}
/**
* static instance
*/
private static class SingleInstance {
private static final PrometheusAutoCollectImpl INSTANCE = new PrometheusAutoCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,25 +30,25 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.springframework.http.MediaType;
import org.springframework.util.StringUtils;
@@ -71,6 +71,27 @@ import static org.apache.hertzbeat.common.constants.SignConstants.RIGHT_DASH;
@Slf4j
public class PrometheusProxyCollectImpl implements PrometheusCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@@ -80,7 +101,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
@Override
public List<CollectRep.MetricsData> collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
PrometheusProtocol prometheusProtocol = metrics.getPrometheus();
HttpUriRequest request;
try {
validateParams(metrics);
} catch (Exception e) {
@@ -89,17 +110,19 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
return Collections.singletonList(builder.build());
}
HttpContext httpContext = createHttpContext(prometheusProtocol);
request = createHttpRequest(prometheusProtocol);
HttpRequestWithConfig requestWithConfig = createHttpRequest(prometheusProtocol);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(prometheusProtocol, requestWithConfig.getRequestConfig());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
// Use HttpClientResponseHandler to avoid deprecated execute method and manual resource closing
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
log.debug("Prometheus proxy collect, response status: {}", statusCode);
if (!defaultSuccessStatusCodes.contains(statusCode)) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return Collections.singletonList(builder.build());
return null;
}
String rawTextContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
@@ -118,6 +141,11 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
builder.addValueRow(valueRowBuilder.build());
builder.setCode(CollectRep.Code.SUCCESS);
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (ClientProtocolException e1) {
String errorMsg = CommonUtil.getMessageFromThrowable(e1);
log.error("Prometheus proxy collect error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e1);
@@ -143,10 +171,6 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
log.error("Prometheus proxy collect unknown error: {}. Host: {}, Port: {}", errorMsg, prometheusProtocol.getHost(), prometheusProtocol.getPort(), e);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (request != null) {
request.abort();
}
}
return Collections.singletonList(builder.build());
}
@@ -162,7 +186,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
PrometheusProtocol protocol = metrics.getPrometheus();
if (!StringUtils.hasText(protocol.getHost())
|| !StringUtils.hasText(protocol.getPort())) {
|| !StringUtils.hasText(protocol.getPort())) {
throw new Exception("Prometheus collect must has host and port params");
}
if (protocol.getPath() == null
@@ -171,42 +195,51 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
protocol.setPath(protocol.getPath() == null ? RIGHT_DASH : RIGHT_DASH + protocol.getPath().trim());
}
}
/**
* create httpContext
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol prometheus protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(PrometheusProtocol protocol) {
public HttpContext createHttpContext(PrometheusProtocol protocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
PrometheusProtocol.Authorization auth = protocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (StringUtils.hasText(auth.getDigestAuthUsername())
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
&& StringUtils.hasText(auth.getDigestAuthPassword())) {
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials =
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword());
provider.setCredentials(AuthScope.ANY, credentials);
new UsernamePasswordCredentials(auth.getDigestAuthUsername(), auth.getDigestAuthPassword().toCharArray());
// Fix: Use specific AuthScope instead of AuthScope.ANY
AuthScope authScope = new AuthScope(protocol.getHost(), Integer.parseInt(protocol.getPort()));
provider.setCredentials(authScope, credentials);
AuthCache authCache = new BasicAuthCache();
HttpHost targetHost = new HttpHost(protocol.getHost(), Integer.parseInt(protocol.getPort()));
authCache.put(targetHost, new DigestScheme());
clientContext.setCredentialsProvider(provider);
clientContext.setAuthCache(authCache);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
* This method is adapted from PrometheusAutoCollectImpl
* @param protocol http params
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(PrometheusProtocol protocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PrometheusProtocol protocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// params
Map<String, String> params = protocol.getParams();
if (params != null && !params.isEmpty()) {
@@ -231,7 +264,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
if (headers == null || headers.keySet().stream().noneMatch(HttpHeaders.ACCEPT::equalsIgnoreCase)) {
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE + ";version=0.0.4,*/*;q=0.1");
}
if (protocol.getAuthorization() != null) {
PrometheusProtocol.Authorization authorization = protocol.getAuthorization();
if (DispatchConstants.BEARER_TOKEN.equalsIgnoreCase(authorization.getType())) {
@@ -241,7 +274,7 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
}
} else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
if (StringUtils.hasText(authorization.getBasicAuthUsername())
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
&& StringUtils.hasText(authorization.getBasicAuthPassword())) {
String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
String encodedAuth = Base64Util.encode(authStr);
requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -255,15 +288,15 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN_VALUE);
}
}
String uriPath = CollectUtil.replaceUriSpecialChar(protocol.getPath());
if (IpDomainUtil.isHasSchema(protocol.getHost())) {
requestBuilder.setUri(protocol.getHost() + SignConstants.DOUBLE_MARK + protocol.getPort() + uriPath);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(protocol.getHost());
String baseUri = NetworkConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
? String.format("[%s]:%s%s", protocol.getHost(), protocol.getPort(), uriPath)
: String.format("%s:%s%s", protocol.getHost(), protocol.getPort(), uriPath);
boolean ssl = Boolean.parseBoolean(protocol.getSsl());
if (ssl) {
requestBuilder.setUri(NetworkConstants.HTTPS_HEADER + baseUri);
@@ -271,24 +304,22 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
requestBuilder.setUri(NetworkConstants.HTTP_HEADER + baseUri);
}
}
RequestConfig requestConfig;
// custom timeout
int timeout = CollectUtil.getTimeout(protocol.getTimeout());
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
} else {
RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
requestConfig = RequestConfig.custom()
.setRedirectsEnabled(true)
.build();
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -305,4 +336,4 @@ public class PrometheusProxyCollectImpl implements PrometheusCollect {
private static class SingleInstance {
private static final PrometheusProxyCollectImpl INSTANCE = new PrometheusProxyCollectImpl();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -24,6 +24,15 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -38,15 +47,6 @@ import org.apache.hertzbeat.common.entity.push.PushMetricsDto;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.springframework.http.MediaType;
/**
@@ -55,6 +55,27 @@ import org.springframework.http.MediaType;
@Slf4j
public class PushCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
private static final Map<Long, Long> timeMap = new ConcurrentHashMap<>();
// ms
@@ -83,20 +104,25 @@ public class PushCollectImpl extends AbstractCollect {
Long time = timeMap.getOrDefault(monitorId, curTime - FIRST_COLLECT_INTERVAL);
timeMap.put(monitorId, curTime);
HttpContext httpContext = createHttpContext(pushProtocol);
HttpUriRequest request = createHttpRequest(pushProtocol, monitorId, time);
HttpRequestWithConfig requestWithConfig = createHttpRequest(pushProtocol, monitorId, time);
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(pushProtocol, requestWithConfig.getRequestConfig());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
return;
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
parseResponse(builder, resp, metrics);
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
String resp = CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
if (resp != null) {
parseResponse(builder, resp, metrics);
}
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error(errorMsg, e);
@@ -111,16 +137,17 @@ public class PushCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_PUSH;
}
private HttpContext createHttpContext(PushProtocol pushProtocol) {
HttpHost host = new HttpHost(pushProtocol.getHost(), Integer.parseInt(pushProtocol.getPort()));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
private HttpContext createHttpContext(PushProtocol pushProtocol, RequestConfig requestConfig) {
HttpClientContext httpClientContext = HttpClientContext.create();
if (requestConfig != null) {
httpClientContext.setRequestConfig(requestConfig);
}
return httpClientContext;
}
private HttpUriRequest createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(PushProtocol pushProtocol, Long monitorId, Long startTime) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// uri
String uri = CollectUtil.replaceUriSpecialChar(pushProtocol.getUri());
@@ -142,19 +169,17 @@ public class PushCollectImpl extends AbstractCollect {
requestBuilder.addParameter("time", String.valueOf(startTime));
requestBuilder.addHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
//requestBuilder.setUri(pushProtocol.getUri());
RequestConfig requestConfig = null;
if (DEFAULT_TIMEOUT > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(DEFAULT_TIMEOUT)
.setSocketTimeout(DEFAULT_TIMEOUT)
// Using deprecated setConnectTimeout for request-level override
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
.setResponseTimeout(Timeout.ofMilliseconds(DEFAULT_TIMEOUT))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
private void parseResponse(CollectRep.MetricsData.Builder builder, String resp, Metrics metric) {
@@ -180,4 +205,4 @@ public class PushCollectImpl extends AbstractCollect {
}
builder.setTime(System.currentTimeMillis());
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,20 +18,20 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.job.protocol.RedfishProtocol;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.springframework.http.MediaType;
/**
@@ -58,11 +58,21 @@ public class RedfishClient {
redfishProtocol.getUsername(), redfishProtocol.getPassword(), Integer.parseInt(redfishProtocol.getTimeout()));
}
@SuppressWarnings("deprecation")
public ConnectSession connect() throws Exception {
HttpHost host = new HttpHost(this.host, this.port);
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(host);
RequestBuilder requestBuilder = RequestBuilder.post();
HttpClientContext httpClientContext = HttpClientContext.create();
// Configure RequestConfig if timeout is set
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(this.timeout))
.setResponseTimeout(Timeout.ofMilliseconds(this.timeout))
.setRedirectsEnabled(true)
.build();
httpClientContext.setRequestConfig(requestConfig);
}
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.post();
String uri = REDFISH_SESSION_SERVICE;
if (IpDomainUtil.isHasSchema(this.host)) {
@@ -79,37 +89,31 @@ public class RedfishClient {
requestBuilder.addHeader(HttpHeaders.CONNECTION, NetworkConstants.KEEP_ALIVE);
requestBuilder.addHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE);
requestBuilder.addHeader(HttpHeaders.USER_AGENT, NetworkConstants.USER_AGENT);
requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
// Content-Encoding header is usually for compression, setting charset is done in Content-Type usually,
// but keeping original logic of setting it explicitly if that was the intent, though standard is usually empty or gzip.
// original: requestBuilder.addHeader(HttpHeaders.CONTENT_ENCODING, StandardCharsets.UTF_8 + "");
final String json = "{\"UserName\": \"" + this.username + "\", \"Password\": \"" + this.password + "\"}";
StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8);
requestBuilder.setEntity(entity);
if (this.timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(this.timeout)
.setSocketTimeout(this.timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
ClassicHttpRequest request = requestBuilder.build();
HttpUriRequest request = requestBuilder.build();
Session session;
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpClientContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Session> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_CREATED) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
String location = response.getFirstHeader(NetworkConstants.LOCATION).getValue();
String auth = response.getFirstHeader(NetworkConstants.X_AUTH_TOKEN).getValue();
session = new Session(auth, location, this.host, this.port);
return new Session(auth, location, this.host, this.port);
};
try {
Session session = CommonHttpClient.getHttpClient().execute(request, httpClientContext, responseHandler);
return new RedfishConnectSession(session);
} catch (Exception e) {
throw new Exception("Redfish session create error: " + e.getMessage());
} finally {
request.abort();
throw new Exception("Redfish session create error: " + e.getMessage(), e);
}
return new RedfishConnectSession(session);
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,15 +18,15 @@
package org.apache.hertzbeat.collector.collect.redfish;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
/**
* Redfish connect session
@@ -54,15 +54,20 @@ public class RedfishConnectSession implements ConnectSession {
HttpDelete httpDelete = new HttpDelete(url);
httpDelete.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpDelete.setHeader(NetworkConstants.LOCATION, session.location());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpDelete)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
EntityUtils.consume(response.getEntity());
return null;
};
try {
CommonHttpClient.getHttpClient().execute(httpDelete, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session close error:" + e.getMessage());
} finally {
httpDelete.abort();
throw new Exception("Redfish session close error:" + e.getMessage(), e);
}
}
@@ -84,16 +89,19 @@ public class RedfishConnectSession implements ConnectSession {
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader(NetworkConstants.X_AUTH_TOKEN, session.token());
httpGet.setHeader(NetworkConstants.LOCATION, session.location());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(httpGet)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != HttpStatus.SC_OK) {
throw new Exception(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
throw new org.apache.hc.client5.http.ClientProtocolException(NetworkConstants.STATUS_CODE + SignConstants.BLANK + statusCode);
}
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
return CommonHttpClient.getHttpClient().execute(httpGet, responseHandler);
} catch (Exception e) {
throw new Exception("Redfish session get resource error:" + e.getMessage());
} finally {
httpGet.abort();
throw new Exception("Redfish session get resource error:" + e.getMessage(), e);
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,6 +22,10 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
@@ -32,10 +36,6 @@ import org.apache.hertzbeat.common.entity.sd.ConnectionConfig;
import org.apache.hertzbeat.common.entity.sd.EurekaDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.XmlUtil;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.util.EntityUtils;
import org.springframework.util.CollectionUtils;
/**
@@ -57,38 +57,55 @@ public class EurekaSdCollectImpl extends AbstractCollect {
@Override
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
HttpUriRequest request = RequestBuilder.get().setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH).build();
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request)) {
int statusCode = response.getStatusLine().getStatusCode();
ClassicHttpRequest request = ClassicRequestBuilder.get()
.setUri(metrics.getEureka_sd().getUrl() + APP_LIST_PATH)
.build();
// Use ResponseHandler to avoid manual resource management and deprecated execute methods
HttpClientResponseHandler<String> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != 200) {
log.warn("Failed to fetch eureka sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return;
return null;
}
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
};
try {
String responseBody = CommonHttpClient.getHttpClient().execute(request, responseHandler);
if (responseBody != null) {
EurekaDiscoveryResponseEntity responseEntity =
XmlUtil.fromXml(responseBody, EurekaDiscoveryResponseEntity.class);
if (responseEntity == null || CollectionUtils.isEmpty(responseEntity.getApplications())) {
return;
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
}
responseEntity.getApplications()
.stream()
.filter(application -> !CollectionUtils.isEmpty(application.getInstances()))
.forEach(application -> convertTarget(configList, application));
configList.forEach(config -> {
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
valueRowBuilder.addColumn(config.getHost());
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch eureka sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -108,4 +125,4 @@ public class EurekaSdCollectImpl extends AbstractCollect {
public String supportProtocol() {
return DispatchConstants.PROTOCOL_EUREKA_SD;
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -41,18 +41,18 @@ import org.apache.hertzbeat.common.entity.sd.ServiceDiscoveryResponseEntity;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import org.springframework.web.util.UriUtils;
@@ -63,6 +63,28 @@ import org.springframework.web.util.UriUtils;
*/
@Slf4j
public class HttpSdCollectImpl extends AbstractCollect {
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
}
@@ -71,22 +93,30 @@ public class HttpSdCollectImpl extends AbstractCollect {
public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
List<ConnectionConfig> configList = Lists.newArrayList();
HttpUriRequest request = createHttpRequest(metrics.getHttp_sd());
HttpContext httpContext = createHttpContext(metrics.getHttp_sd());
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
HttpRequestWithConfig requestWithConfig = createHttpRequest(metrics.getHttp_sd());
if (requestWithConfig == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Failed to create HTTP request");
return;
}
ClassicHttpRequest request = requestWithConfig.getRequest();
HttpContext httpContext = createHttpContext(metrics.getHttp_sd(), requestWithConfig.getRequestConfig());
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != 200) {
log.warn("Failed to fetch sd...");
builder.setMsg("StatusCode " + statusCode);
builder.setCode(CollectRep.Code.FAIL);
return;
return null;
}
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
TypeReference<List<ServiceDiscoveryResponseEntity>> typeReference = new TypeReference<>() {};
final List<ServiceDiscoveryResponseEntity> responseEntityList = JsonUtil.fromJson(responseBody, typeReference);
if (CollectionUtils.isEmpty(responseEntityList)) {
return;
return null;
}
responseEntityList.stream()
@@ -100,11 +130,21 @@ public class HttpSdCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(config.getPort());
builder.addValueRow(valueRowBuilder.build());
});
return null;
};
try {
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.warn("Failed to fetch sd... {}", errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
@@ -137,17 +177,23 @@ public class HttpSdCollectImpl extends AbstractCollect {
* create httpContext
*
* @param httpSdProtocol http sd protocol
* @param requestConfig request config
* @return context
*/
public HttpContext createHttpContext(HttpProtocol httpSdProtocol) {
public HttpContext createHttpContext(HttpProtocol httpSdProtocol, RequestConfig requestConfig) {
HttpClientContext clientContext = HttpClientContext.create();
if (requestConfig != null) {
clientContext.setRequestConfig(requestConfig);
}
HttpProtocol.Authorization auth = httpSdProtocol.getAuthorization();
if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
HttpClientContext clientContext = new HttpClientContext();
if (org.springframework.util.StringUtils.hasText(auth.getDigestAuthUsername())
&& org.springframework.util.StringUtils.hasText(auth.getDigestAuthPassword())) {
CredentialsProvider provider = new BasicCredentialsProvider();
BasicCredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
auth.getDigestAuthPassword());
auth.getDigestAuthPassword().toCharArray());
URL url;
try {
url = new URL(httpSdProtocol.getUrl());
@@ -159,20 +205,20 @@ public class HttpSdCollectImpl extends AbstractCollect {
provider.setCredentials(authScope, credentials);
clientContext.setCredentialsProvider(provider);
return clientContext;
}
}
return null;
return clientContext;
}
/**
* create http request
*
* @param httpSdProtocol http request set
* @return http uri request
* @return http uri request wrapper
*/
public HttpUriRequest createHttpRequest(HttpProtocol httpSdProtocol) {
RequestBuilder requestBuilder = RequestBuilder.get();
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(HttpProtocol httpSdProtocol) {
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// The default request header can be overridden if customized
// keep-alive
@@ -227,16 +273,17 @@ public class HttpSdCollectImpl extends AbstractCollect {
throw e;
}
RequestConfig requestConfig = null;
// custom timeout
int timeout = CollectUtil.getTimeout(httpSdProtocol.getTimeout(), 0);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
}
}
@@ -29,7 +29,7 @@ import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingClient;
import org.junit.jupiter.api.BeforeEach;
@@ -113,7 +113,7 @@ class CollectServerTest {
RemotingClient remotingClient = mock(RemotingClient.class);
ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient);
ClusterMsg.Message message = mock(ClusterMsg.Message.class);
ClusterMessage message = mock(ClusterMessage.class);
collectServer.sendMsg(message);
@@ -135,7 +135,7 @@ class CollectServerTest {
collectNettyEventListener.onChannelActive(channel);
verify(timerDispatch, times(1)).goOnline();
verify(remotingClient, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(remotingClient, times(1)).sendMsg(any(ClusterMessage.class));
ScheduledExecutorService scheduledExecutor =
(ScheduledExecutorService) ReflectionTestUtils.getField(collectServer, "scheduledExecutor");
@@ -34,7 +34,7 @@ import org.apache.hertzbeat.collector.dispatch.WorkerPool;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -116,7 +116,7 @@ class CollectJobServiceTest {
collectJobService.collectSyncOneTimeJobData(job);
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
}
@Test
@@ -137,7 +137,7 @@ class CollectJobServiceTest {
.build();
collectJobService.sendAsyncCollectData(metricsData);
verify(collectServer, times(1)).sendMsg(any(ClusterMsg.Message.class));
verify(collectServer, times(1)).sendMsg(any(ClusterMessage.class));
}
@Test
@@ -100,6 +100,35 @@
</build>
<profiles>
<profile>
<id>dev-driver</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>inner</id>
<build>
@@ -28,8 +28,6 @@ spring:
jackson:
default-property-inclusion: ALWAYS
# need to disable spring boot mongodb auto config, or default mongodb connection tried and failed...
autoconfigure:
exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
management:
endpoints:
web:
@@ -1,50 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true">
<springProperty scope="context" name="application_name" source="spring.application.name" defaultValue="collector"/>
<!-- Output logs to ConsoleAppender -->
<appender name="ConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
<appender name="CONSOLE_RAW" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<!--<pattern>%d %p (%file:%line\)- %m%n</pattern>-->
<!-- Format output: %d for date, %thread for thread name, %-5level for level with 5 character width, %msg for log message, %n for new line -->
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="ConsoleAppender" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>0</discardingThreshold>
<queueSize>512</queueSize>
<includeCallerData>true</includeCallerData>
<appender-ref ref="CONSOLE_RAW"/>
</appender>
<appender name="SystemOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- Rolling policy for loggers, by date and by size -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- Archive log file path. %d{yyyy-MM-dd} specifies date format, %i specifies index -->
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}.%i.log.zip</fileNamePattern>
<!-- Log retention duration -->
<maxFileSize>50MB</maxFileSize>
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<!-- Besides logging by day, log files cannot exceed 200M, if exceeded, log files will start from index 0 -->
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
@@ -52,32 +32,23 @@
</appender>
<appender name="ErrOutFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>logs/${application_name}-%d{yyyy-MM-dd}-error.%i.log.zip</fileNamePattern>
<!-- Log retention duration -->
<maxFileSize>50MB</maxFileSize>
<maxHistory>7</maxHistory>
<!-- Maximum size of log retention -->
<totalSizeCap>5GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>50MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<!-- Append logs -->
<append>true</append>
<!-- Log file format -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!-- This log file records error and above levels -->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
</appender>
<!-- Settings for this logger: for example, all output logs under the org.springframework package must be at level info or above to be output! -->
<!-- This can avoid outputting many common debug information of the spring framework! -->
<logger name="org.springframework" level="info" />
<logger name="org.json" level="error"/>
<logger name="io.netty" level="info"/>
@@ -91,9 +62,9 @@
<logger name="org.mongodb" level="warn"/>
<logger name="io.greptime" level="warn"/>
<!-- Production environment configuration -->
<springProfile name="prod">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
@@ -101,12 +72,12 @@
<springProfile name="cluster">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
<appender-ref ref="SystemOutFileAppender"/>
<appender-ref ref="ErrOutFileAppender"/>
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="dev">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -115,7 +86,6 @@
</root>
</springProfile>
<!-- Development environment configuration -->
<springProfile name="mysql">
<root level="INFO">
<appender-ref ref="ConsoleAppender"/>
@@ -124,4 +94,4 @@
</root>
</springProfile>
</configuration>
</configuration>
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,33 +17,30 @@
package org.apache.hertzbeat.collector.collect.common.http;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.X509Certificate;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContexts;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.reactor.ssl.SSLBufferMode;
import org.apache.hc.core5.ssl.SSLContexts;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
/**
* common http client
* common http client for HttpComponents Client 5.x
*/
@Slf4j
public class CommonHttpClient {
@@ -84,11 +81,13 @@ public class CommonHttpClient {
/**
* ssl supported version
* Note: SSLv3 is often disabled in modern JDKs
*/
private static final String[] SUPPORTED_SSL = {"TLSv1", "TLSv1.1", "TLSv1.2", "SSLv3"};
static {
try {
// 1. SSL Context Configuration
SSLContext sslContext = SSLContexts.createDefault();
X509TrustManager x509TrustManager = new X509TrustManager() {
@Override
@@ -102,7 +101,7 @@ public class CommonHttpClient {
for (X509Certificate certificate : x509Certificates) {
Date deadline = certificate.getNotAfter();
if (deadline != null && now.after(deadline)) {
throw new CertificateExpiredException();
throw new CertificateExpiredException("Server certificate expired at " + deadline);
}
}
}
@@ -112,57 +111,63 @@ public class CommonHttpClient {
public X509Certificate[] getAcceptedIssuers() { return null; }
};
sslContext.init(null, new TrustManager[]{x509TrustManager}, null);
SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, SUPPORTED_SSL, null, new NoopHostnameVerifier());
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", sslFactory)
// 2. TlsStrategy Configuration (Replaces deprecated SSLConnectionSocketFactory/Registry)
DefaultClientTlsStrategy tlsStrategy = new DefaultClientTlsStrategy(
sslContext,
SUPPORTED_SSL,
null, // Supported cipher suites (null = default)
SSLBufferMode.STATIC,
NoopHostnameVerifier.INSTANCE
);
// 3. Connection Manager Configuration (Using Builder)
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(CONNECT_TIMEOUT))
.setSocketTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT))
.setValidateAfterInactivity(TimeValue.ofMilliseconds(INACTIVITY_VALIDATED_TIME))
.build();
connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(tlsStrategy)
.setMaxConnTotal(MAX_TOTAL_CONNECTIONS)
.setMaxConnPerRoute(MAX_PER_ROUTE_CONNECTIONS)
.setDefaultConnectionConfig(connectionConfig)
.build();
// 4. Request Config (Connection Request Timeout & Redirects)
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(REQUIRE_CONNECT_TIMEOUT)
.setConnectTimeout(CONNECT_TIMEOUT)
.setSocketTimeout(SOCKET_TIMEOUT)
// auto redirect when 301 302 response status
.setConnectionRequestTimeout(Timeout.ofMilliseconds(REQUIRE_CONNECT_TIMEOUT))
.setRedirectsEnabled(true)
.build();
// connection pool
connectionManager = new PoolingHttpClientConnectionManager(registry);
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS);
connectionManager.setValidateAfterInactivity(INACTIVITY_VALIDATED_TIME);
// 5. Build HttpClient
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
// clean up unavailable expired connections
.evictExpiredConnections()
// clean up available but idle connections
.evictIdleConnections(100, TimeUnit.SECONDS)
.evictIdleConnections(TimeValue.ofSeconds(100))
.build();
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setNameFormat("http-connection-pool-cleaner-%d")
.setDaemon(true)
.build();
ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory);
scheduledExecutor.scheduleWithFixedDelay(() -> {
connectionManager.closeExpiredConnections();
connectionManager.closeIdleConnections(40, TimeUnit.SECONDS);
}, 40L, 40L, TimeUnit.SECONDS);
// shutdown hook
// Shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close));
} catch (Exception ignored) {}
} catch (Exception e) {
log.error("Initialize CommonHttpClient error", e);
}
}
public static CloseableHttpClient getHttpClient() {
return httpClient;
}
public static void close() {
try {
httpClient.close();
if (httpClient != null) {
httpClient.close();
}
} catch (Exception e) {
log.error("close http client error", e);
}
}
}
}
@@ -17,65 +17,48 @@
package org.apache.hertzbeat.collector.dispatch;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Collection task worker thread pool
* Collection task worker thread pool with Virtual Threads
*/
@Component
@Slf4j
public class WorkerPool implements DisposableBean {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public WorkerPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
// thread factory
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable);
log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("collect-worker-%d")
.build();
int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors());
int maxSize = Runtime.getRuntime().availableProcessors() * 16;
workerExecutor = new ThreadPoolExecutor(coreSize,
maxSize,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory virtualThreadFactory = Thread.ofVirtual()
.name("collect-vt-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(virtualThreadFactory);
log.info("WorkerPool initialized with JDK 25 Virtual Threads successfully.");
}
/**
* Run the collection task thread
*
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
* Run the collection task
* @param runnable Task
*/
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable) {
workerExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
public void destroy() {
if (workerExecutor != null) {
workerExecutor.shutdownNow();
workerExecutor.close();
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,7 +18,6 @@
package org.apache.hertzbeat.collector.dispatch.entrance;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
import io.netty.channel.Channel;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties;
@@ -33,7 +32,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProces
import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.RemotingClient;
@@ -65,7 +64,7 @@ public class CollectServer implements CommandLineRunner {
private final TimerDispatch timerDispatch;
private final CollectorInfoProperties infoProperties;
private RemotingClient remotingClient;
private ScheduledExecutorService scheduledExecutor;
@@ -97,13 +96,13 @@ public class CollectServer implements CommandLineRunner {
nettyClientConfig.setServerPort(nettyProperties.getManagerPort());
this.remotingClient = new NettyRemotingClient(nettyClientConfig, new CollectNettyEventListener(), threadPool);
this.remotingClient.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, new HeartbeatProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this));
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_OFFLINE, new GoOfflineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_ONLINE, new GoOnlineProcessor());
this.remotingClient.registerProcessor(ClusterMessage.MessageType.GO_CLOSE, new GoCloseProcessor(this));
}
public void shutdown() {
@@ -116,7 +115,7 @@ public class CollectServer implements CommandLineRunner {
return collectJobService;
}
public void sendMsg(final ClusterMsg.Message message) {
public void sendMsg(final ClusterMessage message) {
this.remotingClient.sendMsg(message);
}
@@ -143,10 +142,10 @@ public class CollectServer implements CommandLineRunner {
.build();
timerDispatch.goOnline();
// send online message
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(collectorInfo)))
ClusterMessage message = ClusterMessage.builder()
.identity(identity)
.type(ClusterMessage.MessageType.GO_ONLINE)
.msg(JsonUtil.toJsonBytes(collectorInfo))
.build();
CollectServer.this.sendMsg(message);
@@ -163,13 +162,13 @@ public class CollectServer implements CommandLineRunner {
// schedule send heartbeat message
scheduledExecutor.scheduleAtFixedRate(() -> {
try {
ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder()
.setIdentity(identity)
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
ClusterMessage heartbeat = ClusterMessage.builder()
.identity(identity)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
.build();
CollectServer.this.sendMsg(heartbeat);
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis());
} catch (Exception e) {
log.error("schedule send heartbeat to server error.{}", e.getMessage());
}
@@ -182,4 +181,4 @@ public class CollectServer implements CommandLineRunner {
log.info("handle idle event triggered. collector is going offline.");
}
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.collector.dispatch.entrance.internal;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.DispatchProperties;
import org.apache.hertzbeat.collector.dispatch.WorkerPool;
@@ -25,7 +24,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.ArrowUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
@@ -117,10 +116,10 @@ public class CollectJobService {
workerPool.executeJob(() -> {
List<CollectRep.MetricsData> metricsDataList = this.collectSyncJobData(oneTimeJob);
byte[] msg = ArrowUtil.serializeMetricsData(metricsDataList);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
ClusterMessage message = ClusterMessage.builder()
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_ONE_TIME_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
});
@@ -153,22 +152,22 @@ public class CollectJobService {
*/
public void sendAsyncCollectData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_DATA)
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_DATA)
.build();
this.collectServer.sendMsg(message);
}
public void sendAsyncServiceDiscoveryData(CollectRep.MetricsData metricsData) {
byte[] msg = ArrowUtil.serializeMetricsData(List.of(metricsData));
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setIdentity(collectorIdentity)
.setMsg(ByteString.copyFrom(msg))
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
ClusterMessage message = ClusterMessage.builder()
.identity(collectorIdentity)
.msg(msg)
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA)
.build();
this.collectServer.sendMsg(message);
}
@@ -184,4 +183,4 @@ public class CollectJobService {
public void setCollectServer(CollectServer collectServer) {
this.collectServer = collectServer;
}
}
}
@@ -21,7 +21,7 @@ import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -37,8 +37,8 @@ public class CollectCyclicDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job job = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job job = JsonUtil.fromJson(message.getMsg(), Job.class);
if (job == null) {
log.error("collector receive cyclic task job is null");
return null;
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -35,8 +35,8 @@ public class CollectOneTimeDataProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
Job oneTimeJob = JsonUtil.fromJson(message.getMsg(), Job.class);
collectServer.getCollectJobService().collectSyncOneTimeJobData(oneTimeJob);
return null;
}
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -39,9 +39,9 @@ public class DeleteCyclicTaskProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
TypeReference<List<Long>> typeReference = new TypeReference<>() {};
List<Long> jobIds = JsonUtil.fromJson(message.getMsg().toStringUtf8(), typeReference);
List<Long> jobIds = JsonUtil.fromJson(message.getMsgString(), typeReference);
if (jobIds == null || jobIds.isEmpty()) {
log.error("collector receive delete cyclic task job ids is null");
return null;
@@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.dispatch.entrance.CollectServer;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
import org.springframework.boot.SpringApplication;
@@ -41,12 +41,12 @@ public class GoCloseProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg());
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsgString());
}
this.timerDispatch.goOffline();
this.collectServer.shutdown();
@@ -17,12 +17,12 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import java.nio.charset.StandardCharsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -36,20 +36,21 @@ public class GoOfflineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
timerDispatch.goOffline();
log.info("receive offline message and handle success");
if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsg());
if (message.getMsgString().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) {
log.error("[Auth Failed]receive client auth failed message and go offline. {}", message.getMsgString());
return null;
}
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
return ClusterMessage.builder()
.identity(message.getIdentity())
.direction(ClusterMessage.Direction.RESPONSE)
.type(ClusterMessage.MessageType.GO_OFFLINE)
.msg(String.valueOf(CommonConstants.SUCCESS_CODE).getBytes(StandardCharsets.UTF_8))
.build();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,13 +17,11 @@
package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.timer.TimerDispatch;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
@@ -35,18 +33,20 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
*/
@Slf4j
public class GoOnlineProcessor implements NettyRemotingProcessor {
private TimerDispatch timerDispatch;
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
if (this.timerDispatch == null) {
this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class);
}
if (message.getMsg().isEmpty()) {
String msgString = message.getMsgString();
if (message.getMsg() == null || msgString == null || msgString.isEmpty()) {
log.warn("The message that server response to collector is empty, please upgrade server");
} else {
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg().toStringUtf8(), ServerInfo.class);
// Use the new JsonUtil.fromJson(byte[], Class) method
ServerInfo serverInfo = JsonUtil.fromJson(message.getMsg(), ServerInfo.class);
if (serverInfo == null || serverInfo.getAesSecret() == null) {
log.warn("The message that server response to collector has not secret empty, please check");
} else {
@@ -55,10 +55,8 @@ public class GoOnlineProcessor implements NettyRemotingProcessor {
}
timerDispatch.goOnline();
log.info("receive online message and handle success");
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(String.valueOf(CommonConstants.SUCCESS_CODE)))
.build();
// Return null to stop the ping-pong loop.
// The collector should not reply to the server's confirmation response.
return null;
}
}
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.collector.dispatch.entrance.processor;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
/**
@@ -28,7 +28,7 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@Slf4j
public class HeartbeatProcessor implements NettyRemotingProcessor {
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
log.info("collector receive manager server response heartbeat, time: {}. ", System.currentTimeMillis());
return null;
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.collector.collect.nebulagraph;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
@@ -25,7 +24,17 @@ import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -37,18 +46,9 @@ import org.apache.hertzbeat.common.entity.job.protocol.NebulaGraphProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/**
* NebulaGraph collect
* NebulaGraph collect
*/
@Slf4j
public class NebulaGraphCollectImpl extends AbstractCollect {
@@ -66,6 +66,27 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
private static final String STORAGE_API = "/rocksdb_stats";
/**
* Inner class to wrap HTTP request with its configuration
*/
private static class HttpRequestWithConfig {
private final ClassicHttpRequest request;
private final RequestConfig requestConfig;
public HttpRequestWithConfig(ClassicHttpRequest request, RequestConfig requestConfig) {
this.request = request;
this.requestConfig = requestConfig;
}
public ClassicHttpRequest getRequest() {
return request;
}
public RequestConfig getRequestConfig() {
return requestConfig;
}
}
@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
if (metrics == null || metrics.getNebulaGraph() == null) {
@@ -91,37 +112,50 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return;
}
String resp;
long responseTime;
HashMap<String, String> resultMap = new HashMap<>(64);
HttpContext httpContext = createHttpContext(nebulaGraph.getHost(), nebulaGraph.getPort());
HttpUriRequest request = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
// Create Request and Config
HttpRequestWithConfig requestWithConfig = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
nebulaGraph.getUrl(), nebulaGraph.getTimeout());
// Create Context and set Config
HttpContext httpContext = createHttpContext(requestWithConfig.getRequestConfig());
ClassicHttpRequest request = requestWithConfig.getRequest();
// Send an HTTP request to obtain response data
try (CloseableHttpResponse response = CommonHttpClient.getHttpClient().execute(request, httpContext)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return;
}
resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
responseTime = System.currentTimeMillis() - startTime;
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
} catch (IOException e) {
try {
// Use ResponseHandler to avoid manual resource management and CloseableHttpResponse deprecation issues
HttpClientResponseHandler<Void> responseHandler = response -> {
int statusCode = response.getCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return null;
}
String resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
long responseTime = System.currentTimeMillis() - startTime;
HashMap<String, String> resultMap = new HashMap<>(64);
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// Parse the response differently depending on the API
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumn(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValueRow(valueRowBuilder.build());
return null;
};
CommonHttpClient.getHttpClient().execute(request, httpContext, responseHandler);
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
@@ -134,43 +168,47 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_NEBULAGRAPH;
}
private HttpContext createHttpContext(String host, String port) {
HttpHost httpHost = new HttpHost(host, Integer.parseInt(port));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(httpHost);
return httpClientContext;
private HttpContext createHttpContext(RequestConfig requestConfig) {
HttpClientContext context = HttpClientContext.create();
if (requestConfig != null) {
context.setRequestConfig(requestConfig);
}
return context;
}
private HttpUriRequest createHttpRequest(String host, String port, String url, String timeoutStr) {
RequestBuilder requestBuilder = RequestBuilder.get();
// uri
@SuppressWarnings("deprecation")
private HttpRequestWithConfig createHttpRequest(String host, String port, String url, String timeoutStr) {
// HttpClient 5 uses ClassicRequestBuilder
ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get();
// uri construction
String uri = CollectUtil.replaceUriSpecialChar(url);
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + port + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(host);
String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s", host, port + uri)
: String.format("%s:%s", host, port + uri);
? String.format("[%s]:%s%s", host, port, uri)
: String.format("%s:%s%s", host, port, uri);
requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri);
}
requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain");
RequestConfig requestConfig = null;
int timeout = Integer.parseInt(timeoutStr);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
// Use setConnectTimeout despite deprecation to allow per-request connection timeout override on a shared client
requestConfig = RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(timeout))
.setResponseTimeout(Timeout.ofMilliseconds(timeout))
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
return new HttpRequestWithConfig(requestBuilder.build(), requestConfig);
}
/**
@@ -178,6 +216,7 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
*
* @param responseBody response body
* @param timePeriod time period
* @param resultMap result map
*/
private void parseStatsResponse(String responseBody, String timePeriod, HashMap<String, String> resultMap) {
// Set up regular expression matching
@@ -188,22 +227,26 @@ public class NebulaGraphCollectImpl extends AbstractCollect {
Matcher matcher = pattern.matcher(str);
if (matcher.find()) {
String[] split = str.split(timeRegex);
resultMap.put(split[0], split[1]);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
}
}
}
/**
* Parse the Storage response and filter by time period
*
* @param responseBody response body
* @param resultMap result map
*/
private void parseStorageResponse(String responseBody, HashMap<String, String> resultMap) {
String[] strArray = responseBody.split(STR_SPLIT);
for (String str : strArray) {
String[] split = str.split(STORAGE_SPLIT_KEY_VALUE);
resultMap.put(split[0], split[1]);
if (split.length > 1) {
resultMap.put(split[0], split[1]);
}
}
}
}
}
+1 -1
View File
@@ -28,7 +28,7 @@
<name>${project.artifactId}</name>
<packaging>pom</packaging>
<properties>
<java.version>17</java.version>
<java.version>21</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
+19 -16
View File
@@ -46,20 +46,9 @@
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<!-- jpa -->
<dependency>
<groupId>org.eclipse.persistence</groupId>
<artifactId>org.eclipse.persistence.jpa</artifactId>
<version>${eclipse-jpa.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<exclusions>
<exclusion>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- validation -->
<dependency>
@@ -76,6 +65,10 @@
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-jackson2</artifactId>
</dependency>
<!-- swagger -->
<dependency>
<groupId>org.springdoc</groupId>
@@ -95,11 +88,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
<!-- Use apache.http.conn.util instead of sun.net.util -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<!-- caffeine-->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
@@ -183,6 +171,21 @@
<version>${javaparser.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-core</artifactId>
<version>${fory.version}</version>
</dependency>
<!-- row/arrow format support -->
<dependency>
<groupId>org.apache.fory</groupId>
<artifactId>fory-format</artifactId>
<version>${fory.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
</dependencies>
</project>
@@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.persistence.config.SessionCustomizer;
import org.eclipse.persistence.descriptors.ClassDescriptor;
import org.eclipse.persistence.mappings.DatabaseMapping;
import org.eclipse.persistence.sessions.Session;
/**
* change column name that from upper-case to lower-case-with-underscore
*/
@Slf4j
public class EclipseLinkCustomizer implements SessionCustomizer {
@Override
public void customize(Session session) throws Exception {
for (ClassDescriptor descriptor : session.getDescriptors().values()) {
for (DatabaseMapping mapping : descriptor.getMappings()) {
if (mapping.isDirectToFieldMapping()) {
// update the column name to lower case with underscore
if (!mapping.getField().getName().equalsIgnoreCase(mapping.getAttributeName())) {
// already custom define the column name, ignore
continue;
}
// todo here i try to change column name that from upper-case to lower-case-with-underscore
// but failed, when update name, something relate exception happen more
mapping.getField().setName(convertCamelCase(mapping.getAttributeName()));
}
}
}
}
private String convertCamelCase(String camelCase) {
StringBuilder result = new StringBuilder();
if (camelCase != null && !camelCase.isEmpty()) {
result.append(Character.toLowerCase(camelCase.charAt(0)));
for (int i = 1; i < camelCase.length(); i++) {
char currentChar = camelCase.charAt(i);
if (Character.isUpperCase(currentChar)) {
result.append('_').append(Character.toLowerCase(currentChar));
} else {
result.append(currentChar);
}
}
}
return result.toString();
}
}
@@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.common.config;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.eclipse.persistence.config.PersistenceUnitProperties;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.orm.jpa.JpaBaseConfiguration;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.orm.jpa.vendor.AbstractJpaVendorAdapter;
import org.springframework.orm.jpa.vendor.EclipseLinkJpaVendorAdapter;
import org.springframework.transaction.jta.JtaTransactionManager;
/**
* jpa eclipselink impl config
*/
@Configuration
@ConditionalOnProperty(prefix = "spring.datasource", name = "url")
public class EclipseLinkJpaConfiguration extends JpaBaseConfiguration {
protected EclipseLinkJpaConfiguration(DataSource dataSource, JpaProperties properties,
ObjectProvider<JtaTransactionManager> jtaTransactionManager) {
super(dataSource, properties, jtaTransactionManager);
}
@Override
protected AbstractJpaVendorAdapter createJpaVendorAdapter() {
return new EclipseLinkJpaVendorAdapter();
}
@Override
protected Map<String, Object> getVendorProperties() {
HashMap<String, Object> map = new HashMap<>(8);
map.put(PersistenceUnitProperties.DDL_GENERATION, "create-or-extend-tables");
map.put(PersistenceUnitProperties.SESSION_CUSTOMIZER, "org.apache.hertzbeat.common.config.EclipseLinkCustomizer");
map.put(PersistenceUnitProperties.ALLOW_NATIVE_SQL_QUERIES, "true");
map.put(PersistenceUnitProperties.WEAVING, "true");
return map;
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -55,7 +55,7 @@ public class ChatConversation {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation title")
private String title;
@@ -79,6 +79,6 @@ public class ChatConversation {
* List of messages in this conversation (one-to-many relationship)
*/
@OneToMany
@JoinColumn(name = "conversation_id")
@JoinColumn(name = "conversation_id", insertable = false, updatable = false)
private List<ChatMessage> messages;
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.ai;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EntityListeners;
import jakarta.persistence.GeneratedValue;
@@ -48,7 +49,7 @@ import lombok.NoArgsConstructor;
@Entity
@EntityListeners(AuditingEntityListener.class)
@Table(name = "hzb_ai_message", indexes = {
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
@Index(name = "idx_message_conversation_id", columnList = "conversation_id")
})
@AllArgsConstructor
@NoArgsConstructor
@@ -57,8 +58,9 @@ public class ChatMessage {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Schema(title = "conversation id")
@Column(name = "conversation_id")
private Long conversationId;
@Schema(title = "message content")
@@ -84,5 +86,5 @@ public class ChatMessage {
@Schema(title = "Record modify time", example = "1612198444000", accessMode = READ_ONLY)
@LastModifiedDate
private LocalDateTime gmtUpdate;
}
}
@@ -101,6 +101,7 @@ public class AlertDefine {
private String datasource;
@Schema(title = "Is Enabled", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "The creator of this record", example = "tom", accessMode = READ_ONLY)
@@ -69,9 +69,11 @@ public class AlertSilence {
private String name;
@Schema(title = "Whether to enable this policy", example = "true")
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to match all", example = "true")
@Builder.Default
private boolean matchAll = true;
@Schema(title = "Silence type 0: once, 1:cyc", example = "1")
@@ -103,11 +103,13 @@ public class NoticeRule {
@Schema(title = "Whether to enable this policy",
description = "Whether to enable this policy",
example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean enable = true;
@Schema(title = "Whether to forward all",
description = "Whether to forward all",
example = "false", accessMode = READ_WRITE)
@Builder.Default
private boolean filterAll = true;
@Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}")
@@ -85,6 +85,7 @@ public class NoticeTemplate {
description = "Is it a preset template: true- preset template false- custom template.",
accessMode = READ_WRITE)
@Column(columnDefinition = "boolean default false")
@Builder.Default
private boolean preset = false;
@Schema(title = "Template content",
@@ -43,5 +43,6 @@ public class CollectorInfo {
private String version;
@NotNull
@Builder.Default
private String mode = CommonConstants.MODE_PUBLIC;
}
@@ -23,7 +23,7 @@ import lombok.EqualsAndHashCode;
import org.apache.hertzbeat.common.constants.ImportTaskStatusEnum;
import org.apache.hertzbeat.common.constants.ManagerEventTypeEnum;
import org.apache.hertzbeat.common.constants.NotifyLevelEnum;
import org.springframework.lang.Nullable;
import org.jspecify.annotations.Nullable;
/**
* Import task message
@@ -50,5 +50,6 @@ public class Configmap implements Serializable {
* 0: number 1: string 2: encrypted string 3: json string mapped by map
* number,string,secret
*/
@Builder.Default
private byte type = 1;
}
@@ -57,6 +57,7 @@ public class Job {
/**
* Tenant id
*/
@Builder.Default
private long tenantId = 0;
/**
* Monitoring Task ID
@@ -78,6 +79,7 @@ public class Job {
/**
* Is hide this app in main menus layout, only for app type, default true.
*/
@Builder.Default
private boolean hide = true;
/**
* Large categories of monitoring
@@ -111,6 +113,7 @@ public class Job {
/**
* Default task collection time interval (unit: second) eg: 30,60,600
*/
@Builder.Default
private long defaultInterval = 600L;
/**
* Refresh time list for one cycle of the job
@@ -119,6 +122,7 @@ public class Job {
/**
* Whether it is a recurring periodic task true is yes, false is no
*/
@Builder.Default
private boolean isCyclic = false;
/**
* monitor input need params
@@ -136,21 +140,25 @@ public class Job {
/**
* Whether it is a service discovery job, true is yes, false is no
*/
@Builder.Default
private boolean isSd = false;
/**
* Whether to use the Prometheus proxy
*/
@Builder.Default
private boolean prometheusProxyMode = false;
/**
* Scheduling type: interval or cron
*/
@Builder.Default
private String scheduleType = "interval";
/**
* Cron expression for scheduling, used when scheduleType is "cron"
*/
@Builder.Default
private String cronExpression = null;
/**
@@ -115,6 +115,7 @@ public class Metrics {
* Is it visible true or false
* if false, web ui will not see this metrics.
*/
@Builder.Default
private boolean visible = true;
/**
* Public attribute - collection and monitoring final result attribute set eg: speed | times | size
@@ -393,10 +394,12 @@ public class Metrics {
/**
* Metric type 0-number: number 1-string: string
*/
@Builder.Default
private byte type = 1;
/**
* Whether this field is the label
*/
@Builder.Default
private boolean label = false;
/**
* Metric unit
@@ -36,6 +36,7 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* enable ssh tunnel
*/
@Builder.Default
private String enable = "false";
/**
@@ -46,11 +47,13 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* Peer host port
*/
@Builder.Default
private String port = "22";
/**
* TIME OUT PERIOD
*/
@Builder.Default
private String timeout = "6000";
/**
@@ -76,5 +79,6 @@ public class SshTunnel implements CommonRequestProtocol, Protocol {
/**
* share connection session
*/
@Builder.Default
private String shareConnection = "true";
}
@@ -64,5 +64,6 @@ public class FtpProtocol implements CommonRequestProtocol, Protocol {
* Whether ftp uses link encryption ssl/tls, i.e. ftp or sftp
*
*/
@Builder.Default
private String ssl = "false";
}
@@ -51,6 +51,7 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether http uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
* http request methods: get, post, put, delete, patch
@@ -99,6 +100,7 @@ public class HttpProtocol implements CommonRequestProtocol, Protocol {
* Whether to enable URL encoding for the path. Default is true.
* When true, the URL path will be encoded. When false, the URL path will not be encoded.
*/
@Builder.Default
private String enableUrlEncoding = "true";
/**
@@ -48,6 +48,7 @@ public class ImapProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -74,6 +74,7 @@ public class JdbcProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
* ssh tunnel
@@ -43,6 +43,7 @@ public class JmxProtocol implements CommonRequestProtocol, Protocol {
/**
* enable ssl?
*/
@Builder.Default
private String ssl = "false";
/**
@@ -54,5 +54,6 @@ public class KafkaProtocol implements CommonRequestProtocol, Protocol {
/**
* Monitor internal topic
*/
@Builder.Default
private String monitorInternalTopic = "false";
}
@@ -54,6 +54,7 @@ public class NginxProtocol implements CommonRequestProtocol, Protocol {
/**
* Whether nginx uses link encryption ssl/tls, i.e. http or https
*/
@Builder.Default
private String ssl = "false";
/**
@@ -49,6 +49,7 @@ public class Pop3Protocol implements CommonRequestProtocol, Protocol {
/**
* Whether to enable SSL encryption [Email Transmission]
*/
@Builder.Default
private String ssl = "false";
/**
@@ -50,6 +50,7 @@ public class PrometheusProtocol implements CommonRequestProtocol, Protocol {
/**
* http Whether to use link-encrypted ssl/tls, that is, http or https
*/
@Builder.Default
private String ssl = "false";
/**
* HTTP REQUEST METHOD: get, post, put, delete, patch
@@ -34,6 +34,7 @@ import org.apache.hertzbeat.common.entity.dto.Field;
public class PushProtocol implements CommonRequestProtocol, Protocol {
private String host;
private String port;
@Builder.Default
private String uri = "/api/push";
private List<Field> fields;
}
@@ -69,6 +69,7 @@ public class SnmpProtocol implements CommonRequestProtocol, Protocol {
/**
* operation: get, walk
*/
@Builder.Default
private String operation = "get";
/**
* oid map
@@ -69,6 +69,7 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* reuse connection session
*/
@Builder.Default
private String reuseConnection = "true";
/**
@@ -104,6 +105,7 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
/**
* flag of use proxy
*/
@Builder.Default
private String useProxy = "false";
/**
@@ -96,6 +96,7 @@ public class ParamDefine {
* Is it mandatory true-required false-optional
*/
@Schema(title = "Is it mandatory true-required false-optional", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean required = false;
/**
@@ -156,6 +157,7 @@ public class ParamDefine {
* Is it an advanced hidden parameter true-yes false-no
*/
@Schema(title = "Is it an advanced hidden parameter true-yes false-no", example = "true", accessMode = READ_WRITE)
@Builder.Default
private boolean hide = false;
/**
@@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.entity.message;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.nio.charset.StandardCharsets;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
/**
* cluster message entity for fury serialization
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ClusterMessage implements Serializable {
/**
* collector identity
*/
@Builder.Default
private String identity = "";
/**
* message direction
*/
@Builder.Default
private Direction direction = Direction.REQUEST;
/**
* message type
*/
private MessageType type;
/**
* message content
* Use byte[] to ensure data integrity for both JSON strings (UTF-8) and Binary data (Arrow).
* Avoids String encoding/decoding issues (Mojibake) across different JVMs or Languages.
*/
private byte[] msg;
@JsonIgnore
public String getMsgString() {
if (this.msg == null) {
return null;
}
return new String(this.msg, StandardCharsets.UTF_8);
}
@JsonIgnore
public void setMsgString(String jsonString) {
if (jsonString != null) {
this.msg = jsonString.getBytes(StandardCharsets.UTF_8);
} else {
this.msg = null;
}
}
/**
* Message Type Enum
*/
public enum MessageType {
// heartbeat message
HEARTBEAT,
// collector go online to master message
GO_ONLINE,
// collector go offline to master message
GO_OFFLINE,
// collector go close to master
GO_CLOSE,
// issue cyclic collect task
ISSUE_CYCLIC_TASK,
// delete cyclic collect task
DELETE_CYCLIC_TASK,
// issue one-time collect task
ISSUE_ONE_TIME_TASK,
// response one-time collect data
RESPONSE_ONE_TIME_TASK_DATA,
// response cyclic collect data
RESPONSE_CYCLIC_TASK_DATA,
// response cyclic service discovery data
RESPONSE_CYCLIC_TASK_SD_DATA
}
/**
* Direction Enum
*/
public enum Direction {
// request message
REQUEST,
// request response
RESPONSE
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.common.entity.sd;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
@@ -17,60 +17,49 @@
package org.apache.hertzbeat.common.support;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* common task worker thread pool
* common task worker thread pool with Virtual Threads
*/
@Component
@Slf4j
public class CommonThreadPool implements DisposableBean {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public CommonThreadPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("common executor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("common-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(1,
Integer.MAX_VALUE,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("common-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
log.info("CommonThreadPool initialized with Virtual Threads.");
}
/**
* Run the task thread
* @param runnable Task
* @throws RejectedExecutionException when thread pool full
* @param runnable Task
*/
public void execute(Runnable runnable) throws RejectedExecutionException {
public void execute(Runnable runnable) {
workerExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
if (workerExecutor != null) {
workerExecutor.shutdownNow();
workerExecutor.close();
}
}
}
}
@@ -21,7 +21,7 @@ import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
import org.springframework.stereotype.Component;
/**
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -37,7 +37,7 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
*/
@Slf4j
public final class ArrowUtil {
private ArrowUtil() {
}
@@ -53,7 +53,7 @@ public final class ArrowUtil {
public static byte[] serializeMultipleRoots(List<VectorSchemaRoot> roots) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(out)) {
dataOut.writeInt(roots.size());
for (VectorSchemaRoot root : roots) {
ArrowStreamWriter writer = new ArrowStreamWriter(
@@ -87,10 +87,10 @@ public final class ArrowUtil {
List<VectorSchemaRoot> roots = new ArrayList<>();
try (ByteArrayInputStream in = new ByteArrayInputStream(data);
DataInputStream dataIn = new DataInputStream(in)) {
int rootCount = dataIn.readInt();
RootAllocator allocator = new RootAllocator();
for (int i = 0; i < rootCount; i++) {
ArrowStreamReader reader = new ArrowStreamReader(
Channels.newChannel(in),
@@ -153,5 +153,4 @@ public final class ArrowUtil {
}
return serializeMultipleRoots(roots);
}
}
}
@@ -24,8 +24,8 @@ import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.net.InetAddressUtils;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.http.conn.util.InetAddressUtils;
import org.springframework.util.StringUtils;
/**
@@ -60,10 +60,10 @@ public final class IpDomainUtil {
if (LOCALHOST.equalsIgnoreCase(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv4Address(ipDomain)) {
if (InetAddressUtils.isIPv4(ipDomain)) {
return true;
}
if (InetAddressUtils.isIPv6Address(ipDomain)) {
if (InetAddressUtils.isIPv6(ipDomain)) {
return true;
}
return DOMAIN_PATTERN.matcher(ipDomain).matches();
@@ -75,7 +75,7 @@ public final class IpDomainUtil {
* @return true or false
*/
public static boolean isHasSchema(String domainIp) {
if (domainIp == null || !StringUtils.hasText(domainIp)) {
if (!StringUtils.hasText(domainIp)) {
return false;
}
return DOMAIN_SCHEMA.matcher(domainIp).matches();
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -62,6 +62,23 @@ public final class JsonUtil {
}
}
/**
* Object to byte array
* @param source object
* @return byte array
*/
public static byte[] toJsonBytes(Object source) {
if (source == null) {
return null;
}
try {
return OBJECT_MAPPER.writeValueAsBytes(source);
} catch (JsonProcessingException e) {
log.error(e.getMessage(), e);
return null;
}
}
public static <T> T fromJson(String jsonStr, Class<T> clazz) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -74,6 +91,25 @@ public final class JsonUtil {
}
}
/**
* byte array to Object
* @param jsonBytes json byte array
* @param clazz object class
* @param <T> object type
* @return object
*/
public static <T> T fromJson(byte[] jsonBytes, Class<T> clazz) {
if (jsonBytes == null || jsonBytes.length == 0) {
return null;
}
try {
return OBJECT_MAPPER.readValue(jsonBytes, clazz);
} catch (Exception e) {
log.error(e.getMessage(), e);
return null;
}
}
public static <T> T fromJson(String jsonStr, TypeReference<T> type) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -85,7 +121,7 @@ public final class JsonUtil {
return null;
}
}
public static JsonNode fromJson(String jsonStr) {
if (!StringUtils.hasText(jsonStr)) {
return null;
@@ -143,4 +179,4 @@ public final class JsonUtil {
char end = jsonStr.charAt(jsonStr.length() - 1);
return (start == '{' && end == '}') || (start == '[' && end == ']');
}
}
}
+6 -14
View File
@@ -99,7 +99,7 @@
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<artifactId>spring-boot-starter-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -139,16 +139,8 @@
</dependency>
<!-- database migration -->
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-database-postgresql</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-flyway</artifactId>
</dependency>
<!-- mysql -->
<dependency>
@@ -176,10 +168,10 @@
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
<!-- okhttp -->
<!-- httpclient -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
<!-- h2 database-->
<dependency>
@@ -26,14 +26,14 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.template.TemplateAvailabilityProvider;
import org.springframework.boot.autoconfigure.template.TemplateAvailabilityProviders;
import org.springframework.boot.autoconfigure.web.WebProperties;
import org.springframework.boot.autoconfigure.web.servlet.error.ErrorViewResolver;
import org.springframework.boot.webmvc.autoconfigure.error.ErrorViewResolver;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.Ordered;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.servlet.ModelAndView;
@@ -19,8 +19,8 @@ package org.apache.hertzbeat.manager.config;
import org.flywaydb.core.Flyway;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.flyway.FlywayMigrationInitializer;
import org.springframework.boot.autoconfigure.flyway.FlywayProperties;
import org.springframework.boot.flyway.autoconfigure.FlywayMigrationInitializer;
import org.springframework.boot.flyway.autoconfigure.FlywayProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -24,7 +24,7 @@ import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestExecution;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.lang.NonNull;
import org.jspecify.annotations.NonNull;
/**
* Rest Template interceptor adds request header information
@@ -22,10 +22,12 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.text.SimpleDateFormat;
import java.util.TimeZone;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import tools.jackson.databind.JacksonModule;
import tools.jackson.databind.json.JsonMapper;
import tools.jackson.databind.module.SimpleModule;
/**
* jackson config
@@ -35,21 +37,17 @@ import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
public class JacksonConfig {
@Bean
public Jackson2ObjectMapperBuilderCustomizer jacksonCustomizer() {
public JsonMapperBuilderCustomizer jacksonCustomizer() {
return builder -> {
JavaTimeModule javaTimeModule = new JavaTimeModule();
final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
builder.defaultTimeZone(TimeZone.getDefault());
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
simpleDateFormat.setTimeZone(TimeZone.getDefault());
builder.modules(javaTimeModule)
.timeZone(TimeZone.getDefault())
.dateFormat(simpleDateFormat);
builder.defaultDateFormat(simpleDateFormat);
};
}
@Bean
public ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) {
public JsonMapper jsonMapper(JsonMapper.Builder builder) {
return builder.build();
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,13 +19,18 @@ package org.apache.hertzbeat.manager.config;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.OkHttp3ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
/**
@@ -44,18 +49,24 @@ public class RestTemplateConfig {
@Bean
public ClientHttpRequestFactory simpleClientHttpRequestFactory() {
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS);
return new OkHttp3ClientHttpRequestFactory(
new OkHttpClient.Builder()
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
.connectionPool(new ConnectionPool(
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
TimeUnit.SECONDS)
).build()
);
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS))
.setSocketTimeout(Timeout.of(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS))
.build();
connectionManager.setDefaultConnectionConfig(connectionConfig);
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
.evictIdleConnections(TimeValue.of(NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT, TimeUnit.SECONDS))
.disableAutomaticRetries()
.build();
return new HttpComponentsClientHttpRequestFactory(httpClient);
}
}
}
@@ -28,12 +28,12 @@ import org.apache.sshd.common.io.nio2.Nio2ServiceFactoryFactory;
import org.apache.sshd.common.session.SessionListener;
import org.apache.sshd.common.util.security.bouncycastle.BouncyCastleSecurityProviderRegistrar;
import org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar;
import org.jspecify.annotations.NonNull;
import org.springframework.aot.hint.ExecutableMode;
import org.springframework.aot.hint.MemberCategory;
import org.springframework.aot.hint.RuntimeHints;
import org.springframework.aot.hint.RuntimeHintsRegistrar;
import org.springframework.aot.hint.TypeReference;
import org.springframework.lang.NonNull;
import org.springframework.util.ClassUtils;
/**
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,6 @@
package org.apache.hertzbeat.manager.scheduler;
import com.google.protobuf.ByteString;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
@@ -44,7 +43,7 @@ import org.apache.hertzbeat.common.entity.manager.CollectorMonitorBind;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.manager.ParamDefine;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
@@ -155,9 +154,9 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
List<Param> params = paramDao.findParamsByMonitorId(monitor.getId());
List<Configmap> configmaps = params.stream()
.map(param -> Configmap.builder()
.key(param.getField())
.value(param.getParamValue())
.type(param.getType()).build()).collect(Collectors.toList());
.key(param.getField())
.value(param.getParamValue())
.type(param.getType()).build()).collect(Collectors.toList());
List<ParamDefine> paramDefaultValue = appDefine.getParams().stream()
.filter(item -> StringUtils.isNotBlank(item.getDefaultValue()))
.toList();
@@ -214,10 +213,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK)
.msg(JsonUtil.toJsonBytes(job))
.build();
this.manageServer.sendMsg(collectorName, message);
}
@@ -229,10 +228,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
assignJobs.getRemovingJobs().forEach(jobId -> collectJobService.cancelAsyncCollectJob(jobId));
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(assignJobs.getRemovingJobs())))
ClusterMessage message = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.DELETE_CYCLIC_TASK)
.msg(JsonUtil.toJsonBytes(assignJobs.getRemovingJobs()))
.build();
this.manageServer.sendMsg(collectorName, message);
}
@@ -243,13 +242,13 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
@Override
public boolean offlineCollector(String identity) {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_OFFLINE)
.setDirection(ClusterMsg.Direction.REQUEST)
.setIdentity(identity)
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.GO_OFFLINE)
.direction(ClusterMessage.Direction.REQUEST)
.identity(identity)
.build();
ClusterMsg.Message response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsg().toStringUtf8())) {
ClusterMessage response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsgString())) {
return false;
}
log.info("send offline collector message to {} success", identity);
@@ -264,14 +263,14 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
return false;
}
ServerInfo serverInfo = ServerInfo.builder().aesSecret(AesUtil.getDefaultSecretKey()).build();
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.GO_ONLINE)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(serverInfo)))
.setIdentity(identity)
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.GO_ONLINE)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(serverInfo))
.identity(identity)
.build();
ClusterMsg.Message response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsg().toStringUtf8())) {
ClusterMessage response = this.manageServer.sendMsgSync(identity, message);
if (response == null || !String.valueOf(CommonConstants.SUCCESS_CODE).equals(response.getMsgString())) {
return false;
}
log.info("send online collector message to {} success", identity);
@@ -304,10 +303,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
CountDownLatch countDownLatch = new CountDownLatch(1);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
boolean result = this.manageServer.sendMsg(node.getIdentity(), message);
@@ -347,10 +346,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
return collectJobService.collectSyncJobData(job);
}
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_ONE_TIME_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
boolean result = this.manageServer.sendMsg(node.getIdentity(), message);
if (result) {
@@ -399,10 +398,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
ClusterMessage message = ClusterMessage.builder()
.type(ClusterMessage.MessageType.ISSUE_CYCLIC_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(job))
.build();
this.manageServer.sendMsg(node.getIdentity(), message);
}
@@ -443,10 +442,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
collectJobService.cancelAsyncCollectJob(jobId);
} else {
ClusterMsg.Message deleteMessage = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(List.of(jobId))))
ClusterMessage deleteMessage = ClusterMessage.builder()
.type(ClusterMessage.MessageType.DELETE_CYCLIC_TASK)
.direction(ClusterMessage.Direction.REQUEST)
.msg(JsonUtil.toJsonBytes(List.of(jobId)))
.build();
this.manageServer.sendMsg(node.getIdentity(), deleteMessage);
}
@@ -471,4 +470,4 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
public void setManageServer(ManageServer manageServer) {
this.manageServer = manageServer;
}
}
}
@@ -17,46 +17,34 @@
package org.apache.hertzbeat.manager.scheduler;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* manager module thread pool
* manager module thread pool with Virtual Threads
*/
@Slf4j
@Component
public class ManagerWorkerPool {
private ThreadPoolExecutor workerExecutor;
private ExecutorService workerExecutor;
public ManagerWorkerPool() {
initWorkExecutor();
}
private void initWorkExecutor() {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.setDaemon(true)
.setNameFormat("manager-worker-%d")
.build();
workerExecutor = new ThreadPoolExecutor(6,
10,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
ThreadFactory factory = Thread.ofVirtual()
.name("manager-worker-", 0)
.factory();
workerExecutor = Executors.newThreadPerTaskExecutor(factory);
}
public void executeJob(Runnable runnable) throws RejectedExecutionException {
public void executeJob(Runnable runnable) {
workerExecutor.execute(runnable);
}
}
@@ -25,7 +25,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler;
import org.apache.hertzbeat.manager.scheduler.SchedulerProperties;
@@ -83,12 +83,12 @@ public class ManageServer implements CommandLineRunner {
this.remotingServer = new NettyRemotingServer(nettyServerConfig, nettyEventListener, threadPool);
// register processor
this.remotingServer.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, new HeartbeatProcessor(this));
this.remotingServer.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new CollectorOnlineProcessor(this));
this.remotingServer.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new CollectorOfflineProcessor(this));
this.remotingServer.registerProcessor(ClusterMsg.MessageType.RESPONSE_ONE_TIME_TASK_DATA, new CollectOneTimeDataResponseProcessor(this));
this.remotingServer.registerProcessor(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_DATA, new CollectCyclicDataResponseProcessor());
this.remotingServer.registerProcessor(ClusterMsg.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA, new CollectCyclicServiceDiscoveryDataResponseProcessor());
this.remotingServer.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, new HeartbeatProcessor(this));
this.remotingServer.registerProcessor(ClusterMessage.MessageType.GO_ONLINE, new CollectorOnlineProcessor(this));
this.remotingServer.registerProcessor(ClusterMessage.MessageType.GO_OFFLINE, new CollectorOfflineProcessor(this));
this.remotingServer.registerProcessor(ClusterMessage.MessageType.RESPONSE_ONE_TIME_TASK_DATA, new CollectOneTimeDataResponseProcessor(this));
this.remotingServer.registerProcessor(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_DATA, new CollectCyclicDataResponseProcessor());
this.remotingServer.registerProcessor(ClusterMessage.MessageType.RESPONSE_CYCLIC_TASK_SD_DATA, new CollectCyclicServiceDiscoveryDataResponseProcessor());
this.channelSchedule = Executors.newSingleThreadScheduledExecutor();
}
@@ -144,7 +144,7 @@ public class ManageServer implements CommandLineRunner {
Channel channel = this.getChannel(identity);
if (channel != null) {
this.collectorJobScheduler.collectorGoOffline(identity);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder().setType(ClusterMsg.MessageType.GO_CLOSE).build();
ClusterMessage message = ClusterMessage.builder().type(ClusterMessage.MessageType.GO_CLOSE).build();
this.remotingServer.sendMsg(channel, message);
this.clientChannelTable.remove(identity);
log.info("close collect client success, identity: {}", identity);
@@ -156,7 +156,7 @@ public class ManageServer implements CommandLineRunner {
return channel != null && channel.isActive();
}
public boolean sendMsg(final String identityId, final ClusterMsg.Message message) {
public boolean sendMsg(final String identityId, final ClusterMessage message) {
Channel channel = this.getChannel(identityId);
if (channel != null) {
this.remotingServer.sendMsg(channel, message);
@@ -165,7 +165,7 @@ public class ManageServer implements CommandLineRunner {
return false;
}
public ClusterMsg.Message sendMsgSync(final String identityId, final ClusterMsg.Message message) {
public ClusterMessage sendMsgSync(final String identityId, final ClusterMessage message) {
Channel channel = this.getChannel(identityId);
if (channel != null) {
return this.remotingServer.sendMsgSync(channel, message, 3000);
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.manager.scheduler.netty.process;
import io.netty.channel.ChannelHandlerContext;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.SpringContextHolder;
@@ -33,9 +33,9 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@Slf4j
public class CollectCyclicDataResponseProcessor implements NettyRemotingProcessor {
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
CommonDataQueue dataQueue = SpringContextHolder.getBean(CommonDataQueue.class);
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg().toByteArray());
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg());
for (CollectRep.MetricsData metricsData : metricsDataList) {
if (metricsData != null) {
dataQueue.sendMetricsData(metricsData);
@@ -43,4 +43,4 @@ public class CollectCyclicDataResponseProcessor implements NettyRemotingProcesso
}
return null;
}
}
}
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.manager.scheduler.netty.process;
import io.netty.channel.ChannelHandlerContext;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.SpringContextHolder;
@@ -33,9 +33,9 @@ import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@Slf4j
public class CollectCyclicServiceDiscoveryDataResponseProcessor implements NettyRemotingProcessor {
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
CommonDataQueue dataQueue = SpringContextHolder.getBean(CommonDataQueue.class);
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg().toByteArray());
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg());
for (CollectRep.MetricsData metricsData : metricsDataList) {
if (metricsData != null) {
dataQueue.sendServiceDiscoveryData(metricsData);
@@ -20,7 +20,7 @@ package org.apache.hertzbeat.manager.scheduler.netty.process;
import io.netty.channel.ChannelHandlerContext;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.ArrowUtil;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
@@ -39,9 +39,9 @@ public class CollectOneTimeDataResponseProcessor implements NettyRemotingProcess
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg().toByteArray());
List<CollectRep.MetricsData> metricsDataList = ArrowUtil.deserializeMetricsData(message.getMsg());
this.manageServer.getCollectorAndJobScheduler().collectSyncJobResponse(metricsDataList);
return null;
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.manager.scheduler.netty.process;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -36,7 +36,7 @@ public class CollectorOfflineProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
String collector = message.getIdentity();
log.info("the collector {} actively requests to go offline.", collector);
this.manageServer.getCollectorAndJobScheduler().collectorGoOffline(collector);
@@ -17,14 +17,13 @@
package org.apache.hertzbeat.manager.scheduler.netty.process;
import com.google.protobuf.ByteString;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
@@ -42,10 +41,10 @@ public class CollectorOnlineProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
String collector = message.getIdentity();
log.info("the collector {} actively requests to go online.", collector);
String msg = message.getMsg().toStringUtf8();
String msg = message.getMsgString();
CollectorInfo collectorInfo = JsonUtil.fromJson(msg, CollectorInfo.class);
if (collectorInfo != null && StringUtils.isBlank(collectorInfo.getIp())) {
// fetch remote ip address
@@ -56,11 +55,11 @@ public class CollectorOnlineProcessor implements NettyRemotingProcessor {
this.manageServer.addChannel(collector, ctx.channel());
this.manageServer.getCollectorAndJobScheduler().collectorGoOnline(collector, collectorInfo);
ServerInfo serverInfo = ServerInfo.builder().aesSecret(AesUtil.getDefaultSecretKey()).build();
return ClusterMsg.Message.newBuilder()
.setIdentity(message.getIdentity())
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(serverInfo)))
.setType(ClusterMsg.MessageType.GO_ONLINE)
return ClusterMessage.builder()
.identity(message.getIdentity())
.direction(ClusterMessage.Direction.RESPONSE)
.msg(JsonUtil.toJsonBytes(serverInfo))
.type(ClusterMessage.MessageType.GO_ONLINE)
.build();
}
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.manager.scheduler.netty.process;
import io.netty.channel.ChannelHandlerContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -36,7 +36,7 @@ public class HeartbeatProcessor implements NettyRemotingProcessor {
}
@Override
public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) {
public ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message) {
String identity = message.getIdentity();
boolean isChannelActive = this.manageServer.isChannelActive(identity);
if (!isChannelActive) {
@@ -52,8 +52,10 @@ public class HeartbeatProcessor implements NettyRemotingProcessor {
if (log.isDebugEnabled()) {
log.debug("server receive collector {} heartbeat", message.getIdentity());
}
return ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.HEARTBEAT)
return ClusterMessage.builder()
.identity(message.getIdentity())
.type(ClusterMessage.MessageType.HEARTBEAT)
.direction(ClusterMessage.Direction.RESPONSE)
.build();
}
}
@@ -5,7 +5,7 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -43,6 +43,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
/**
* class AbstractImExportServiceImpl
@@ -50,6 +53,13 @@ import java.util.Objects;
@Slf4j
public abstract class AbstractImExportServiceImpl implements ImExportService {
/**
* Limit the number of concurrent import tasks to prevent database connection pool exhaustion.
* Although virtual threads are cheap, database connections are limited.
* This acts as a backpressure mechanism.
*/
private static final int IMPORT_CONCURRENCY_LIMIT = 200;
@Resource
@Lazy
private MonitorService monitorService;
@@ -66,12 +76,36 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
if (!CollectionUtils.isEmpty(formList)) {
int totalElements = formList.size();
int progressInterval = Math.max(1, totalElements / 10);
for (int i = 0; i < totalElements; i++) {
MonitorDto monitorDto = formList.get(i);
monitorService.validate(monitorDto, false);
monitorService.addMonitor(monitorDto.getMonitor(), monitorDto.getParams(), monitorDto.getCollector(), monitorDto.getGrafanaDashboard());
if (totalElements >= ImExportTaskConstant.IMPORT_TASK_PROCESS_THRESHOLD && ((i + 1) % progressInterval == 0) && (i + 1 < totalElements)) {
managerSseManager.broadcastImportTaskInProgress(taskName, (int) ((i + 1) * 100.0 / totalElements));
AtomicInteger counter = new AtomicInteger(0);
// Semaphore to limit concurrency
Semaphore semaphore = new Semaphore(IMPORT_CONCURRENCY_LIMIT);
try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {
for (MonitorDto monitorDto : formList) {
// Acquire permit before submitting/executing
try {
semaphore.acquire();
} catch (InterruptedException e) {
log.error("Import interrupted.", e);
Thread.currentThread().interrupt();
return;
}
executor.submit(() -> {
try {
monitorService.validate(monitorDto, false);
monitorService.addMonitor(monitorDto.getMonitor(), monitorDto.getParams(), monitorDto.getCollector(), monitorDto.getGrafanaDashboard());
} catch (Exception e) {
log.error("Import monitor {} failed.", monitorDto.getMonitor().getName(), e);
} finally {
semaphore.release(); // Release permit
int count = counter.incrementAndGet();
if (totalElements >= ImExportTaskConstant.IMPORT_TASK_PROCESS_THRESHOLD && (count % progressInterval == 0) && (count < totalElements)) {
managerSseManager.broadcastImportTaskInProgress(taskName, (int) (count * 100.0 / totalElements));
}
}
});
}
}
managerSseManager.broadcastImportTaskSuccess(taskName);
@@ -205,4 +239,4 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
@Excel(name = "Value")
private String value;
}
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.manager.service.impl;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -47,6 +48,7 @@ import org.apache.hertzbeat.manager.service.MonitorService;
import org.apache.hertzbeat.manager.service.ObjectStoreService;
import org.apache.hertzbeat.warehouse.service.WarehouseService;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
@@ -56,7 +58,6 @@ import org.springframework.util.Assert;
import org.springframework.util.StreamUtils;
import org.yaml.snakeyaml.Yaml;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -82,7 +82,10 @@ import org.springframework.data.jpa.domain.Specification;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
@@ -98,6 +101,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
/**
@@ -145,6 +149,9 @@ public class MonitorServiceImpl implements MonitorService {
@Autowired
private MetricsFavoriteService metricsFavoriteService;
@Autowired
private PlatformTransactionManager transactionManager;
public MonitorServiceImpl(List<ImExportService> imExportServiceList) {
imExportServiceList.forEach(it -> imExportServiceMap.put(it.type(), it));
}
@@ -160,20 +167,10 @@ public class MonitorServiceImpl implements MonitorService {
}
@Override
@Transactional(rollbackFor = Exception.class)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public void addMonitor(Monitor monitor, List<Param> params, String collector, GrafanaDashboard grafanaDashboard) throws RuntimeException {
// Apply for monitor id
long monitorId = SnowFlakeIdGenerator.generateId();
Map<String, String> labels = monitor.getLabels();
if (labels == null) {
labels = new HashMap<>(8);
monitor.setLabels(labels);
}
List<Label> addLabels = labelService.determineNewLabels(labels.entrySet());
if (!addLabels.isEmpty()) {
labelDao.saveAll(addLabels);
}
// Construct the collection task Job entity
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
@@ -200,28 +197,54 @@ public class MonitorServiceImpl implements MonitorService {
return new Configmap(param.getField(), param.getParamValue(), param.getType());
}).collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
// 1. Dispatch the job (Memory operation, fast)
long jobId = collector == null ? collectJobScheduling.addAsyncCollectJob(appDefine, null) :
collectJobScheduling.addAsyncCollectJob(appDefine, collector);
// 2. Perform detection (Network I/O operation, potentially slow) - OUTSIDE TRANSACTION
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
} catch (Exception e) {
// If detection fails, cancel the job and throw exception
collectJobScheduling.cancelAsyncCollectJob(jobId);
throw e;
}
// 3. Save to Database (DB operation, fast) - INSIDE TRANSACTION
TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
try {
if (collector != null) {
CollectorMonitorBind collectorMonitorBind = CollectorMonitorBind.builder()
.collector(collector)
.monitorId(monitorId)
.build();
collectorMonitorBindDao.save(collectorMonitorBind);
}
monitor.setId(monitorId);
monitor.setJobId(jobId);
// create grafana dashboard
if (monitor.getApp().equals(CommonConstants.PROMETHEUS) && grafanaDashboard != null && grafanaDashboard.isEnabled()) {
dashboardService.createOrUpdateDashboard(grafanaDashboard.getTemplate(), monitorId);
}
monitorDao.save(monitor);
paramDao.saveAll(params);
transactionTemplate.executeWithoutResult(status -> {
try {
Map<String, String> labels = monitor.getLabels();
if (labels == null) {
labels = new HashMap<>(8);
monitor.setLabels(labels);
}
List<Label> addLabels = labelService.determineNewLabels(labels.entrySet());
if (!addLabels.isEmpty()) {
labelDao.saveAll(addLabels);
}
if (collector != null) {
CollectorMonitorBind collectorMonitorBind = CollectorMonitorBind.builder()
.collector(collector)
.monitorId(monitorId)
.build();
collectorMonitorBindDao.save(collectorMonitorBind);
}
monitor.setId(monitorId);
monitor.setJobId(jobId);
// create grafana dashboard
if (monitor.getApp().equals(CommonConstants.PROMETHEUS) && grafanaDashboard != null && grafanaDashboard.isEnabled()) {
dashboardService.createOrUpdateDashboard(grafanaDashboard.getTemplate(), monitorId);
}
monitorDao.save(monitor);
paramDao.saveAll(params);
} catch (Exception e) {
throw new MonitorDatabaseException(e.getMessage());
}
});
} catch (Exception e) {
log.error("Error while adding monitor: {}", e.getMessage(), e);
collectJobScheduling.cancelAsyncCollectJob(jobId);
@@ -250,7 +273,7 @@ public class MonitorServiceImpl implements MonitorService {
.stream()
.map(Monitor::getId)
.collect(Collectors.toList());
// Use the existing export method to export all monitors
export(allMonitorIds, type, res);
}
@@ -288,7 +311,7 @@ public class MonitorServiceImpl implements MonitorService {
param.setParamValue(value);
})
.collect(Collectors.toMap(Param::getField, param -> param));
// Check name uniqueness and can not equal app type
// Check name uniqueness and can not equal app type
if (isModify != null) {
Optional<Job> defineOptional = appService.getAppDefineOption(monitor.getName());
if (defineOptional.isPresent()) {
@@ -756,52 +779,56 @@ public class MonitorServiceImpl implements MonitorService {
return;
}
for (Monitor monitor : unManagedMonitors) {
// Construct the collection task Job entity
List<Param> params = paramDao.findParamsByMonitorId(monitor.getId());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
String app = isStatic ? monitor.getApp() : monitor.getScrape();
Job appDefine = appService.getAppDefine(app);
if (!isStatic) {
appDefine.setSd(true);
}
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitor.getId());
appDefine.setDefaultInterval(monitor.getIntervals());
appDefine.setCyclic(true);
appDefine.setTimestamp(System.currentTimeMillis());
appDefine.setScheduleType(monitor.getScheduleType());
appDefine.setCronExpression(monitor.getCronExpression());
Map<String, String> metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(),
CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost());
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<ParamDefine> paramDefaultValue = appDefine.getParams().stream()
.filter(item -> StringUtils.hasText(item.getDefaultValue()))
.toList();
paramDefaultValue.forEach(defaultVar -> {
if (configmaps.stream().noneMatch(item -> item.getKey().equals(defaultVar.getField()))) {
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(), CommonConstants.TYPE_STRING);
configmaps.add(configmap);
}
});
appDefine.setConfigmap(configmaps);
try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {
for (Monitor monitor : unManagedMonitors) {
executor.submit(() -> {
// Construct the collection task Job entity
List<Param> params = paramDao.findParamsByMonitorId(monitor.getId());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape());
String app = isStatic ? monitor.getApp() : monitor.getScrape();
Job appDefine = appService.getAppDefine(app);
if (!isStatic) {
appDefine.setSd(true);
}
if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) {
appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName());
}
appDefine.setMonitorId(monitor.getId());
appDefine.setDefaultInterval(monitor.getIntervals());
appDefine.setCyclic(true);
appDefine.setTimestamp(System.currentTimeMillis());
appDefine.setScheduleType(monitor.getScheduleType());
appDefine.setCronExpression(monitor.getCronExpression());
Map<String, String> metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(),
CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost());
appDefine.setMetadata(metadata);
appDefine.setLabels(monitor.getLabels());
appDefine.setAnnotations(monitor.getAnnotations());
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
List<ParamDefine> paramDefaultValue = appDefine.getParams().stream()
.filter(item -> StringUtils.hasText(item.getDefaultValue()))
.toList();
paramDefaultValue.forEach(defaultVar -> {
if (configmaps.stream().noneMatch(item -> item.getKey().equals(defaultVar.getField()))) {
Configmap configmap = new Configmap(defaultVar.getField(), defaultVar.getDefaultValue(), CommonConstants.TYPE_STRING);
configmaps.add(configmap);
}
});
appDefine.setConfigmap(configmaps);
// Issue collection tasks
Optional<CollectorMonitorBind> bindOptional =
collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(monitor.getId());
String collector = bindOptional.map(CollectorMonitorBind::getCollector).orElse(null);
long newJobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
applicationContext.publishEvent(new MonitorDeletedEvent(applicationContext, monitor.getId()));
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {
// Issue collection tasks
Optional<CollectorMonitorBind> bindOptional =
collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(monitor.getId());
String collector = bindOptional.map(CollectorMonitorBind::getCollector).orElse(null);
long newJobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
applicationContext.publishEvent(new MonitorDeletedEvent(applicationContext, monitor.getId()));
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {
}
});
}
}
monitorDao.saveAll(unManagedMonitors);
@@ -899,7 +926,7 @@ public class MonitorServiceImpl implements MonitorService {
}
}
}
@Override
public Monitor getMonitor(Long monitorId) {
return monitorDao.findById(monitorId).orElse(null);
@@ -983,9 +1010,9 @@ public class MonitorServiceImpl implements MonitorService {
monitor.setStatus(CommonConstants.MONITOR_DOWN_CODE);
throw new MonitorDetectException("Collect Timeout No Response");
}
if (collectRep.get(0).getCode() != CollectRep.Code.SUCCESS) {
if (collectRep.getFirst().getCode() != CollectRep.Code.SUCCESS) {
monitor.setStatus(CommonConstants.MONITOR_DOWN_CODE);
throw new MonitorDetectException(collectRep.get(0).getMsg());
throw new MonitorDetectException(collectRep.getFirst().getMsg());
}
collectRep.forEach(CollectRep.MetricsData::close);
}
@@ -1027,10 +1054,10 @@ public class MonitorServiceImpl implements MonitorService {
monitor.setStatus(CommonConstants.MONITOR_DOWN_CODE);
throw new MonitorDetectException("Collect Timeout No Response");
}
if (collectRep.get(0).getCode() != CollectRep.Code.SUCCESS) {
if (collectRep.getFirst().getCode() != CollectRep.Code.SUCCESS) {
monitor.setStatus(CommonConstants.MONITOR_DOWN_CODE);
throw new MonitorDetectException(collectRep.get(0).getMsg());
throw new MonitorDetectException(collectRep.getFirst().getMsg());
}
collectRep.forEach(CollectRep.MetricsData::close);
}
}
}
@@ -20,6 +20,7 @@ package org.apache.hertzbeat.manager.service.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.obs.services.ObsClient;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.GeneralConfigTypeEnum;
import org.apache.hertzbeat.base.dao.GeneralConfigDao;
@@ -33,7 +34,6 @@ import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;
import javax.annotation.Resource;
import java.lang.reflect.Type;
import java.net.URL;
@@ -22,7 +22,7 @@ import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.manager.CollectorMonitorBind;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.dao.CollectorDao;
@@ -137,11 +137,11 @@ public class CollectorJobSchedulerTest {
collectorJobScheduler.collectorGoOnline(identity, collectorInfo);
// Capture the parameters of sendMsg
ArgumentCaptor<ClusterMsg.Message> msgCaptor = ArgumentCaptor.forClass(ClusterMsg.Message.class);
ArgumentCaptor<ClusterMessage> msgCaptor = ArgumentCaptor.forClass(ClusterMessage.class);
verify(manageServer, atLeastOnce()).sendMsg(eq("collector-1"), msgCaptor.capture());
ClusterMsg.Message message = msgCaptor.getValue();
ClusterMessage message = msgCaptor.getValue();
Job job = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
Job job = JsonUtil.fromJson(message.getMsgString(), Job.class);
assertNotNull(job);
assertNotNull(job.getMetadata());
assertEquals("test-monitor", job.getMetadata().get(CommonConstants.LABEL_INSTANCE_NAME));
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -19,6 +19,7 @@
package org.apache.hertzbeat.push.service.impl;
import jakarta.annotation.PostConstruct;
import java.io.InputStream;
import java.time.Instant;
import java.util.LinkedList;
@@ -46,19 +47,27 @@ import org.springframework.stereotype.Service;
@Slf4j
@Service
public class PushGatewayServiceImpl implements PushGatewayService {
private final CommonDataQueue commonDataQueue;
private final PushMonitorDao pushMonitorDao;
private final Map<String, Long> jobInstanceMap;
public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao) {
this.commonDataQueue = commonDataQueue;
this.pushMonitorDao = pushMonitorDao;
jobInstanceMap = new ConcurrentHashMap<>();
pushMonitorDao.findMonitorsByType((byte) 1).forEach(monitor ->
jobInstanceMap.put(monitor.getApp() + "_" + monitor.getName(), monitor.getId()));
}
@PostConstruct
public void init() {
try {
pushMonitorDao.findMonitorsByType((byte) 1).forEach(monitor ->
jobInstanceMap.put(monitor.getApp() + "_" + monitor.getName(), monitor.getId()));
} catch (Exception e) {
log.error("Failed to load push monitors into cache during initialization: {}", e.getMessage());
}
}
@Override
@@ -130,4 +139,4 @@ public class PushGatewayServiceImpl implements PushGatewayService {
return false;
}
}
}
}
@@ -17,7 +17,7 @@
package org.apache.hertzbeat.remoting;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
/**
@@ -30,13 +30,13 @@ public interface RemotingClient extends RemotingService {
* @param messageType type
* @param processor remoting processor
*/
void registerProcessor(ClusterMsg.MessageType messageType, NettyRemotingProcessor processor);
void registerProcessor(ClusterMessage.MessageType messageType, NettyRemotingProcessor processor);
/**
* send message to server
* @param request request message
*/
void sendMsg(ClusterMsg.Message request);
void sendMsg(ClusterMessage request);
/**
* send message to server and sync waiting receive server message
@@ -44,5 +44,5 @@ public interface RemotingClient extends RemotingService {
* @param timeoutMillis timeout millis
* @return response message
*/
ClusterMsg.Message sendMsgSync(ClusterMsg.Message request, int timeoutMillis);
ClusterMessage sendMsgSync(ClusterMessage request, int timeoutMillis);
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.remoting;
import io.netty.channel.Channel;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.remoting.netty.NettyHook;
import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor;
@@ -33,14 +33,14 @@ public interface RemotingServer extends RemotingService {
* @param messageType type
* @param processor remoting processor
*/
void registerProcessor(ClusterMsg.MessageType messageType, NettyRemotingProcessor processor);
void registerProcessor(ClusterMessage.MessageType messageType, NettyRemotingProcessor processor);
/**
* send message to client
* @param channel client channel
* @param request request message
*/
void sendMsg(Channel channel, ClusterMsg.Message request);
void sendMsg(Channel channel, ClusterMessage request);
/**
* send message to client and receive client message
@@ -49,7 +49,7 @@ public interface RemotingServer extends RemotingService {
* @param timeoutMillis timeout millis
* @return response message
*/
ClusterMsg.Message sendMsgSync(Channel channel, ClusterMsg.Message request, int timeoutMillis);
ClusterMessage sendMsgSync(Channel channel, ClusterMessage request, int timeoutMillis);
/**
* register hook.
@@ -18,13 +18,13 @@
package org.apache.hertzbeat.remoting.netty;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
/**
* hook interface, handle something before request processor
*/
public interface NettyHook {
void doBeforeRequest(ChannelHandlerContext ctx, ClusterMsg.Message message);
void doBeforeRequest(ChannelHandlerContext ctx, ClusterMessage message);
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,19 +27,19 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.util.NetworkUtil;
import org.apache.hertzbeat.remoting.RemotingService;
import org.apache.hertzbeat.remoting.event.NettyEventListener;
/**
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingAbstract
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingAbstract
* netty remote abstract
* @see <a href="https://github.com/apache/rocketmq/blob/develop/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java">NettyRemotingAbstract</a>
*/
@Slf4j
public abstract class NettyRemotingAbstract implements RemotingService {
protected ConcurrentHashMap<ClusterMsg.MessageType, NettyRemotingProcessor> processorTable = new ConcurrentHashMap<>();
protected ConcurrentHashMap<ClusterMessage.MessageType, NettyRemotingProcessor> processorTable = new ConcurrentHashMap<>();
protected ConcurrentHashMap<String, ResponseFuture> responseTable = new ConcurrentHashMap<>();
@@ -51,33 +51,32 @@ public abstract class NettyRemotingAbstract implements RemotingService {
this.nettyEventListener = nettyEventListener;
}
public void registerProcessor(final ClusterMsg.MessageType messageType, final NettyRemotingProcessor processor) {
public void registerProcessor(final ClusterMessage.MessageType messageType, final NettyRemotingProcessor processor) {
this.processorTable.put(messageType, processor);
}
protected void processReceiveMsg(ChannelHandlerContext ctx, ClusterMsg.Message message) {
if (ClusterMsg.Direction.REQUEST.equals(message.getDirection())) {
protected void processReceiveMsg(ChannelHandlerContext ctx, ClusterMessage message) {
if (ClusterMessage.Direction.REQUEST.equals(message.getDirection())) {
this.processRequestMsg(ctx, message);
} else {
this.processResponseMsg(ctx, message);
}
}
protected void processRequestMsg(ChannelHandlerContext ctx, ClusterMsg.Message request) {
protected void processRequestMsg(ChannelHandlerContext ctx, ClusterMessage request) {
this.doBeforeRequest(ctx, request);
NettyRemotingProcessor processor = this.processorTable.get(request.getType());
if (processor == null) {
log.info("request type {} not supported", request.getType());
return;
}
ClusterMsg.Message response = processor.handle(ctx, request);
ClusterMessage response = processor.handle(ctx, request);
if (response != null) {
ctx.writeAndFlush(response);
}
}
private void doBeforeRequest(ChannelHandlerContext ctx, ClusterMsg.Message request) {
private void doBeforeRequest(ChannelHandlerContext ctx, ClusterMessage request) {
if (CollectionUtils.isEmpty(this.nettyHookList)) {
return;
}
@@ -86,7 +85,7 @@ public abstract class NettyRemotingAbstract implements RemotingService {
}
}
protected void processResponseMsg(ChannelHandlerContext ctx, ClusterMsg.Message response) {
protected void processResponseMsg(ChannelHandlerContext ctx, ClusterMessage response) {
// for sync response
if (this.responseTable.containsKey(response.getIdentity())) {
ResponseFuture responseFuture = this.responseTable.get(response.getIdentity());
@@ -95,15 +94,15 @@ public abstract class NettyRemotingAbstract implements RemotingService {
// async response
NettyRemotingProcessor processor = this.processorTable.get(response.getType());
if (processor != null) {
ClusterMsg.Message repMessage = processor.handle(ctx, response);
ClusterMessage repMessage = processor.handle(ctx, response);
if (repMessage != null) {
ctx.writeAndFlush(repMessage);
}
}
}
}
}
protected void sendMsgImpl(final Channel channel, final ClusterMsg.Message request) {
protected void sendMsgImpl(final Channel channel, final ClusterMessage request) {
channel.writeAndFlush(request).addListener(future -> {
if (!future.isSuccess()) {
log.warn("send request message failed. address: {}, ", channel.remoteAddress(), future.cause());
@@ -111,7 +110,7 @@ public abstract class NettyRemotingAbstract implements RemotingService {
});
}
protected ClusterMsg.Message sendMsgSyncImpl(final Channel channel, final ClusterMsg.Message request, final int timeoutMillis) {
protected ClusterMessage sendMsgSyncImpl(final Channel channel, final ClusterMessage request, final int timeoutMillis) {
final String identity = request.getIdentity();
try {
@@ -123,7 +122,7 @@ public abstract class NettyRemotingAbstract implements RemotingService {
log.warn("send request message failed. request: {}, address: {}, ", request, channel.remoteAddress(), future.cause());
}
});
ClusterMsg.Message response = responseFuture.waitResponse(timeoutMillis);
ClusterMessage response = responseFuture.waitResponse(timeoutMillis);
if (response == null) {
log.warn("get response message failed, message is null");
}
@@ -155,4 +154,4 @@ public abstract class NettyRemotingAbstract implements RemotingService {
&& Epoll.isAvailable();
}
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -29,21 +29,18 @@ import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.compression.ZlibCodecFactory;
import io.netty.handler.codec.compression.ZlibWrapper;
import io.netty.handler.codec.protobuf.ProtobufDecoder;
import io.netty.handler.codec.protobuf.ProtobufEncoder;
import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import java.util.concurrent.ThreadFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingClient;
import org.apache.hertzbeat.remoting.event.NettyEventListener;
import org.apache.hertzbeat.remoting.netty.codec.ForyCodec;
/**
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingClient
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingClient
* netty client
* @see <a href="https://github.com/apache/rocketmq/blob/develop/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java">NettyRemotingClient</a>
*/
@@ -51,7 +48,7 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener;
public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient {
private static final int DEFAULT_WORKER_THREAD_NUM = Math.min(4, Runtime.getRuntime().availableProcessors());
private final NettyClientConfig nettyClientConfig;
private final CommonThreadPool threadPool;
@@ -81,7 +78,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
.setDaemon(true)
.setNameFormat("netty-client-worker-%d")
.build();
String envThreadNum = System.getProperty("hertzbeat.client.worker.thread.num");
String envThreadNum = System.getProperty("hertzbeat.client.worker.thread.num");
int workerThreadNum = envThreadNum != null ? Integer.parseInt(envThreadNum) : DEFAULT_WORKER_THREAD_NUM;
this.workerGroup = new NioEventLoopGroup(workerThreadNum, threadFactory);
this.bootstrap.group(workerGroup)
@@ -96,7 +93,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
this.channel = null;
boolean first = true;
while (!Thread.currentThread().isInterrupted()
while (!Thread.currentThread().isInterrupted()
&& (first || this.channel == null || !this.channel.isActive())) {
first = false;
try {
@@ -121,14 +118,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
private void initChannel(final SocketChannel channel) {
ChannelPipeline pipeline = channel.pipeline();
// zip
pipeline.addLast(ZlibCodecFactory.newZlibEncoder(ZlibWrapper.GZIP));
pipeline.addLast(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP));
// protocol buf encode decode
pipeline.addLast(new ProtobufVarint32FrameDecoder());
pipeline.addLast(new ProtobufDecoder(ClusterMsg.Message.getDefaultInstance()));
pipeline.addLast(new ProtobufVarint32LengthFieldPrepender());
pipeline.addLast(new ProtobufEncoder());
// max frame length 10MB
pipeline.addLast(new LengthFieldBasedFrameDecoder(10 * 1024 * 1024, 0, 4, 0, 4));
pipeline.addLast(new LengthFieldPrepender(4));
// fory codec
pipeline.addLast(new ForyCodec());
pipeline.addLast(new NettyClientHandler());
}
@@ -155,16 +149,16 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}
@Override
public void sendMsg(final ClusterMsg.Message request) {
public void sendMsg(final ClusterMessage request) {
this.sendMsgImpl(this.channel, request);
}
@Override
public ClusterMsg.Message sendMsgSync(ClusterMsg.Message request, int timeoutMillis) {
public ClusterMessage sendMsgSync(ClusterMessage request, int timeoutMillis) {
return this.sendMsgSyncImpl(this.channel, request, timeoutMillis);
}
class NettyClientHandler extends SimpleChannelInboundHandler<ClusterMsg.Message> {
class NettyClientHandler extends SimpleChannelInboundHandler<ClusterMessage> {
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
@@ -172,7 +166,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, ClusterMsg.Message msg) throws Exception {
protected void channelRead0(ChannelHandlerContext ctx, ClusterMessage msg) throws Exception {
NettyRemotingClient.this.processReceiveMsg(ctx, msg);
}
@@ -181,4 +175,4 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
NettyRemotingClient.this.channelIdle(ctx, evt);
}
}
}
}
@@ -18,7 +18,7 @@
package org.apache.hertzbeat.remoting.netty;
import io.netty.channel.ChannelHandlerContext;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
/**
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRequestProcessor
@@ -27,6 +27,6 @@ import org.apache.hertzbeat.common.entity.message.ClusterMsg;
*/
public interface NettyRemotingProcessor {
ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message);
ClusterMessage handle(ChannelHandlerContext ctx, ClusterMessage message);
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -32,25 +32,22 @@ import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.compression.ZlibCodecFactory;
import io.netty.handler.codec.compression.ZlibWrapper;
import io.netty.handler.codec.protobuf.ProtobufDecoder;
import io.netty.handler.codec.protobuf.ProtobufEncoder;
import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.timeout.IdleStateHandler;
import java.util.List;
import java.util.concurrent.ThreadFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.RemotingServer;
import org.apache.hertzbeat.remoting.event.NettyEventListener;
import org.apache.hertzbeat.remoting.netty.codec.ForyCodec;
/**
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingServer
* Derived from Apache Rocketmq org.apache.rocketmq.remoting.netty.NettyRemotingServer
* netty server
* @see <a href="https://github.com/apache/rocketmq/blob/develop/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java">NettyRemotingServer</a>
*/
@@ -137,14 +134,11 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
private void initChannel(final SocketChannel channel) {
ChannelPipeline pipeline = channel.pipeline();
// zip
pipeline.addLast(ZlibCodecFactory.newZlibEncoder(ZlibWrapper.GZIP));
pipeline.addLast(ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP));
// protocol buf encode decode
pipeline.addLast(new ProtobufVarint32FrameDecoder());
pipeline.addLast(new ProtobufDecoder(ClusterMsg.Message.getDefaultInstance()));
pipeline.addLast(new ProtobufVarint32LengthFieldPrepender());
pipeline.addLast(new ProtobufEncoder());
// fory codec
// max frame length 10MB
pipeline.addLast(new LengthFieldBasedFrameDecoder(10 * 1024 * 1024, 0, 4, 0, 4));
pipeline.addLast(new LengthFieldPrepender(4));
pipeline.addLast(new ForyCodec());
// idle state
pipeline.addLast(new IdleStateHandler(0, 0, nettyServerConfig.getIdleStateEventTriggerTime()));
pipeline.addLast(new NettyServerHandler());
@@ -165,12 +159,12 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
@Override
public void sendMsg(final Channel channel, final ClusterMsg.Message request) {
public void sendMsg(final Channel channel, final ClusterMessage request) {
this.sendMsgImpl(channel, request);
}
@Override
public ClusterMsg.Message sendMsgSync(final Channel channel, final ClusterMsg.Message request, final int timeoutMillis) {
public ClusterMessage sendMsgSync(final Channel channel, final ClusterMessage request, final int timeoutMillis) {
return this.sendMsgSyncImpl(channel, request, timeoutMillis);
}
@@ -183,7 +177,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
* netty server handler
*/
@ChannelHandler.Sharable
public class NettyServerHandler extends SimpleChannelInboundHandler<ClusterMsg.Message> {
public class NettyServerHandler extends SimpleChannelInboundHandler<ClusterMessage> {
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
@@ -191,7 +185,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, ClusterMsg.Message msg) throws Exception {
protected void channelRead0(ChannelHandlerContext ctx, ClusterMessage msg) {
NettyRemotingServer.this.processReceiveMsg(ctx, msg);
}
@@ -200,4 +194,4 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
NettyRemotingServer.this.channelIdle(ctx, evt);
}
}
}
}
@@ -19,7 +19,7 @@ package org.apache.hertzbeat.remoting.netty;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
/**
* netty response future
@@ -28,14 +28,14 @@ public class ResponseFuture {
private final CountDownLatch countDownLatch = new CountDownLatch(1);
private ClusterMsg.Message response;
private ClusterMessage response;
public ClusterMsg.Message waitResponse(final long timeoutMillis) throws InterruptedException {
public ClusterMessage waitResponse(final long timeoutMillis) throws InterruptedException {
this.countDownLatch.await(timeoutMillis, TimeUnit.MILLISECONDS);
return this.response;
}
public void putResponse(final ClusterMsg.Message response) {
public void putResponse(final ClusterMessage response) {
this.response = response;
this.countDownLatch.countDown();
}
@@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.remoting.netty.codec;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageCodec;
import org.apache.fory.Fory;
import org.apache.fory.ThreadSafeFory;
import org.apache.fory.config.Language;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import java.util.List;
/**
* Netty codec for ClusterMessage using Apache fory
*/
public class ForyCodec extends ByteToMessageCodec<ClusterMessage> {
private static final ThreadSafeFory fory;
static {
// 1. Initialize Fory in XLANG mode
fory = Fory.builder()
.withLanguage(Language.XLANG)
.requireClassRegistration(false)
.buildThreadSafeFory();
// 2. Register classes with specific names for Cross-Language compatibility (Java <-> Go)
// These names (e.g., "ClusterMessage") must match the registration name in the Go client.
fory.register(ClusterMessage.class, "ClusterMessage");
fory.register(ClusterMessage.MessageType.class, "MessageType");
fory.register(ClusterMessage.Direction.class, "Direction");
}
@Override
protected void encode(ChannelHandlerContext ctx, ClusterMessage msg, ByteBuf out) {
byte[] bytes = fory.serialize(msg);
out.writeBytes(bytes);
}
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {
byte[] bytes = new byte[in.readableBytes()];
in.readBytes(bytes);
Object obj = fory.deserialize(bytes);
out.add(obj);
}
}
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,8 +17,8 @@
package org.apache.hertzbeat.remoting;
import com.google.protobuf.ByteString;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import java.nio.charset.StandardCharsets;
import org.apache.hertzbeat.common.entity.message.ClusterMessage;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.remoting.netty.NettyClientConfig;
import org.apache.hertzbeat.remoting.netty.NettyRemotingClient;
@@ -102,15 +102,15 @@ public class RemotingServiceTest {
public void testSendMsg() {
final String msg = "hello world";
this.remotingServer.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, (ctx, message) -> {
Assertions.assertEquals(msg, message.getMsg().toStringUtf8());
this.remotingServer.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, (ctx, message) -> {
Assertions.assertEquals(msg, message.getMsgString());
return null;
});
ClusterMsg.Message request = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
.setMsg(ByteString.copyFromUtf8(msg))
ClusterMessage request = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
.msg(msg.getBytes(StandardCharsets.UTF_8))
.build();
this.remotingClient.sendMsg(request);
}
@@ -120,40 +120,42 @@ public class RemotingServiceTest {
final String requestMsg = "request";
final String responseMsg = "response";
this.remotingServer.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, (ctx, message) -> {
Assertions.assertEquals(requestMsg, message.getMsg().toStringUtf8());
return ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.RESPONSE)
.setMsg(ByteString.copyFromUtf8(responseMsg))
this.remotingServer.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, (ctx, message) -> {
Assertions.assertEquals(requestMsg, message.getMsgString());
return ClusterMessage.builder()
.direction(ClusterMessage.Direction.RESPONSE)
.type(ClusterMessage.MessageType.HEARTBEAT)
.msg(responseMsg.getBytes(StandardCharsets.UTF_8))
.build();
});
ClusterMsg.Message request = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
.setMsg(ByteString.copyFromUtf8(requestMsg))
ClusterMessage request = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
.msg(requestMsg.getBytes(StandardCharsets.UTF_8))
.build();
ClusterMsg.Message response = this.remotingClient.sendMsgSync(request, 3000);
Assertions.assertEquals(responseMsg, response.getMsg().toStringUtf8());
ClusterMessage response = this.remotingClient.sendMsgSync(request, 3000);
Assertions.assertEquals(responseMsg, response.getMsgString());
}
@Test
public void testNettyHook() {
this.remotingServer.registerHook(Lists.newArrayList(
(ctx, message) -> Assertions.assertEquals("hello world", message.getMsg().toStringUtf8())
(ctx, message) -> Assertions.assertEquals("hello world", message.getMsgString())
));
this.remotingServer.registerProcessor(ClusterMsg.MessageType.HEARTBEAT, (ctx, message) ->
ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.RESPONSE)
this.remotingServer.registerProcessor(ClusterMessage.MessageType.HEARTBEAT, (ctx, message) ->
ClusterMessage.builder()
.direction(ClusterMessage.Direction.RESPONSE)
.type(ClusterMessage.MessageType.HEARTBEAT)
.build());
ClusterMsg.Message request = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.HEARTBEAT)
.setMsg(ByteString.copyFromUtf8("hello world"))
ClusterMessage request = ClusterMessage.builder()
.direction(ClusterMessage.Direction.REQUEST)
.type(ClusterMessage.MessageType.HEARTBEAT)
.msg("hello world".getBytes(StandardCharsets.UTF_8))
.build();
this.remotingClient.sendMsg(request);
}
}
}
+38 -74
View File
@@ -1,20 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -33,120 +17,94 @@
<description>Apache HertzBeat startup module</description>
<dependencies>
<!-- Spring Boot Starter -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<!-- Spring Boot JPA -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!-- Spring Boot Web -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- HertzBeat manager -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-jackson2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-manager</artifactId>
</dependency>
<!-- HertzBeat common -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common</artifactId>
</dependency>
<!-- HertzBeat base -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-base</artifactId>
</dependency>
<!-- HertzBeat warehouse -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-warehouse</artifactId>
</dependency>
<!-- HertzBeat alerter -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-alerter</artifactId>
</dependency>
<!-- HertzBeat remoting -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-remoting</artifactId>
</dependency>
<!-- HertzBeat ai -->
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-ai</artifactId>
</dependency>
<!-- Jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- Commons Lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- Sureness for security -->
<dependency>
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
<!-- Netty -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<!-- Database Migration - Flyway -->
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-database-postgresql</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-flyway</artifactId>
</dependency>
<!-- H2 Database (default) -->
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
@@ -180,15 +138,11 @@
<configuration>
<classesDirectory>target/classes/</classesDirectory>
<archive>
<!--Exclude maven descriptors from generated JAR-->
<addMavenDescriptor>false</addMavenDescriptor>
<manifest>
<!--Main class of the project-->
<mainClass>org.apache.hertzbeat.startup.HertzBeatApplication</mainClass>
<useUniqueVersions>false</useUniqueVersions>
<!--Add third-party JARs to the classpath using maven-dependency-plugin-->
<addClasspath>true</addClasspath>
<!--Location of external dependency JARs-->
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
@@ -201,11 +155,39 @@
</build>
<profiles>
<profile>
<id>dev-driver</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>release</id>
<build>
<plugins>
<!--Critical plugin, assembly plugin provided by maven, should be placed last-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
@@ -213,9 +195,7 @@
<executions>
<execution>
<id>make-zip</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -228,9 +208,7 @@
</execution>
<execution>
<id>make-docker-zip</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -243,9 +221,7 @@
</execution>
<execution>
<id>make-docker-compose-script</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -272,9 +248,7 @@
<executions>
<execution>
<id>make-macos-arm64</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -287,9 +261,7 @@
</execution>
<execution>
<id>make-macos-amd64</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -302,9 +274,7 @@
</execution>
<execution>
<id>make-linux-arm64</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -317,9 +287,7 @@
</execution>
<execution>
<id>make-linux-amd64</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -332,9 +300,7 @@
</execution>
<execution>
<id>make-windows-64</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -347,9 +313,7 @@
</execution>
<execution>
<id>make-docker-compose-script</id>
<!--Bound maven operation-->
<phase>package</phase>
<!--Run once-->
<goals>
<goal>single</goal>
</goals>
@@ -367,4 +331,4 @@
</profile>
</profiles>
</project>
</project>
@@ -0,0 +1,149 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.usthe.sureness.configuration;
import com.usthe.sureness.mgt.SecurityManager;
import com.usthe.sureness.processor.exception.DisabledAccountException;
import com.usthe.sureness.processor.exception.ExcessiveAttemptsException;
import com.usthe.sureness.processor.exception.ExpiredCredentialsException;
import com.usthe.sureness.processor.exception.IncorrectCredentialsException;
import com.usthe.sureness.processor.exception.NeedDigestInfoException;
import com.usthe.sureness.processor.exception.UnauthorizedException;
import com.usthe.sureness.processor.exception.UnknownAccountException;
import com.usthe.sureness.subject.SubjectSum;
import com.usthe.sureness.util.SurenessContextHolder;
import jakarta.servlet.Filter;
import jakarta.servlet.FilterChain;
import jakarta.servlet.FilterConfig;
import jakarta.servlet.ServletException;
import jakarta.servlet.ServletRequest;
import jakarta.servlet.ServletResponse;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
/**
* @author wangtao
* @date 2021/7/8
*/
public class SurenessJakartaServletFilter implements Filter {
private final SecurityManager securityManager;
public SurenessJakartaServletFilter(SecurityManager securityManager) {
this.securityManager = securityManager;
}
/** logger **/
private static final Logger logger = LoggerFactory.getLogger(SurenessJakartaServletFilter.class);
private static final String UPGRADE = "Upgrade";
private static final String WEBSOCKET = "websocket";
@Override
public void init(FilterConfig filterConfig) {
logger.info("servlet surenessFilter initialized");
}
@Override
public void destroy() {
logger.info("servlet surenessFilter destroyed");
}
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse,
FilterChain filterChain)
throws IOException, ServletException {
try {
SubjectSum subject = securityManager.checkIn(servletRequest);
// You can consider using SurenessContextHolder to bind subject in threadLocal
// if bind, please remove it when end
if (subject != null) {
SurenessContextHolder.bindSubject(subject);
}
} catch (IncorrectCredentialsException | UnknownAccountException | ExpiredCredentialsException e1) {
logger.debug("this request account info is illegal, {}", e1.getMessage());
responseWrite(ResponseEntity
.status(HttpStatus.UNAUTHORIZED)
.body("Username or password is incorrect or token expired"), servletResponse);
return;
} catch (DisabledAccountException | ExcessiveAttemptsException e2 ) {
logger.debug("the account is disabled, {}", e2.getMessage());
responseWrite(ResponseEntity
.status(HttpStatus.UNAUTHORIZED).body("Account is disabled"), servletResponse);
return;
} catch (NeedDigestInfoException e3) {
logger.debug("you should try once again with digest auth information");
responseWrite(ResponseEntity
.status(HttpStatus.UNAUTHORIZED)
.header("WWW-Authenticate", e3.getAuthenticate()).build(), servletResponse);
return;
} catch (UnauthorizedException e4) {
logger.debug("this account can not access this resource, {}", e4.getMessage());
responseWrite(ResponseEntity
.status(HttpStatus.FORBIDDEN)
.body("This account has no permission to access this resource"), servletResponse);
return;
} catch (RuntimeException e) {
logger.error("other exception happen: ", e);
responseWrite(ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(),
servletResponse);
return;
}
try {
// if ok, doFilter and add subject in request
filterChain.doFilter(servletRequest, servletResponse);
} finally {
int statusCode = ((HttpServletResponse) servletResponse).getStatus();
String upgrade = ((HttpServletResponse) servletResponse).getHeader(UPGRADE);
if (statusCode != HttpStatus.SWITCHING_PROTOCOLS.value() || !WEBSOCKET.equals(upgrade)) {
SurenessContextHolder.clear();
}
}
}
/**
* write response json data
* @param content content
* @param response response
*/
private static void responseWrite(ResponseEntity content, ServletResponse response) {
response.setCharacterEncoding("UTF-8");
((HttpServletResponse)response).setStatus(content.getStatusCode().value());
content.getHeaders().forEach((key, value) ->
((HttpServletResponse) response).addHeader(key, value.get(0)));
try (PrintWriter printWriter = response.getWriter()) {
if (content.getBody() != null) {
printWriter.write(content.getBody().toString());
} else {
printWriter.flush();
}
} catch (IOException e) {
logger.error("responseWrite response error: ", e);
}
}
}
@@ -17,12 +17,13 @@
package org.apache.hertzbeat.startup;
import java.util.Collections;
import javax.annotation.PostConstruct;
import org.apache.hertzbeat.manager.nativex.HertzbeatRuntimeHintsRegistrar;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.boot.persistence.autoconfigure.EntityScan;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.ImportRuntimeHints;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
@@ -46,7 +47,9 @@ import org.springframework.scheduling.annotation.EnableScheduling;
public class HertzBeatApplication {
public static void main(String[] args) {
SpringApplication.run(HertzBeatApplication.class, args);
SpringApplication app = new SpringApplication(HertzBeatApplication.class);
app.setDefaultProperties(Collections.singletonMap("spring.threads.virtual.enabled", "true"));
app.run(args);
}
@PostConstruct

Some files were not shown because too many files have changed in this diff Show More