diff --git a/e2e/data/monitor-ftp.json b/e2e/data/monitor-ftp.json index e6ed517b7a..f8301e4975 100644 --- a/e2e/data/monitor-ftp.json +++ b/e2e/data/monitor-ftp.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "ftp", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorFTP}}" }, "params": [ diff --git a/e2e/data/monitor-http.json b/e2e/data/monitor-http.json index c8280dd1b5..f1983ea6c2 100644 --- a/e2e/data/monitor-http.json +++ b/e2e/data/monitor-http.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "api", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorHTTP}}" }, "params": [ diff --git a/e2e/data/monitor-ping.json b/e2e/data/monitor-ping.json index 4bc332a22e..6df9e15be0 100644 --- a/e2e/data/monitor-ping.json +++ b/e2e/data/monitor-ping.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "ping", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorPing}}" }, "params": [ diff --git a/e2e/data/monitor-port.json b/e2e/data/monitor-port.json index 4c2afc621a..bd0ef2b319 100644 --- a/e2e/data/monitor-port.json +++ b/e2e/data/monitor-port.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "port", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorPort}}" }, "params": [ diff --git a/e2e/data/monitor-sitemap.json b/e2e/data/monitor-sitemap.json index 2be2d1bf61..5146c3570b 100644 --- a/e2e/data/monitor-sitemap.json +++ b/e2e/data/monitor-sitemap.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "fullsite", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorSiteMap}}" }, "params": [ diff --git a/e2e/data/monitor-ssl.json b/e2e/data/monitor-ssl.json index 6ebbc7a15c..af7459d351 100644 --- a/e2e/data/monitor-ssl.json +++ b/e2e/data/monitor-ssl.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "ssl_cert", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorSSL}}" }, "params": [ diff --git a/e2e/data/monitor-udp.json b/e2e/data/monitor-udp.json index 26c41c706e..636db4dc6c 100644 --- a/e2e/data/monitor-udp.json +++ b/e2e/data/monitor-udp.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "udp_port", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorUDP}}" }, "params": [ diff --git a/e2e/data/monitor-website.json b/e2e/data/monitor-website.json index 804d837ff4..cf0ca20491 100644 --- a/e2e/data/monitor-website.json +++ b/e2e/data/monitor-website.json @@ -5,7 +5,7 @@ "intervals": 60, "tags": [], "app": "website", - "host": "127.0.0.1", + "instance": "127.0.0.1", "name": "{{.param.monitorWebsite}}" }, "params": [ diff --git a/e2e/k8s/data/create_k8s_monitor_data.json b/e2e/k8s/data/create_k8s_monitor_data.json index 43eaa8fc0f..c7f0648ed1 100644 --- a/e2e/k8s/data/create_k8s_monitor_data.json +++ b/e2e/k8s/data/create_k8s_monitor_data.json @@ -4,7 +4,7 @@ "intervals": 10, "tags": [], "app": "kubernetes", - "host": "172.29.0.11", + "instance": "172.29.0.11", "name": "Brave_Stingray_55yR" }, "collector": "", diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/MetricsTools.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/MetricsTools.java index dcf7dafc51..1e61579c11 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/MetricsTools.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/MetricsTools.java @@ -33,16 +33,17 @@ public interface MetricsTools { /** * Get historical metrics data for a monitor - * @param monitorId Monitor ID - * @param app Monitor type (e.g., "linux", "mysql", "http") - * @param metrics Metrics name (e.g., "system", "cpu", "memory") - * @param metric Specific metric field (e.g., "usage", "used", "available") - * @param label Label filter for specific instances - * @param history Time range (e.g., "1h", "6h", "24h", "7d") + * + * @param instance Instance identifier (e.g., "ip:port", "ip", or "domain") + * @param app Monitor type (e.g., "linux", "mysql", "http") + * @param metrics Metrics name (e.g., "system", "cpu", "memory") + * @param metric Specific metric field (e.g., "usage", "used", "available") + * @param label Label filter for specific instances + * @param history Time range (e.g., "1h", "6h", "24h", "7d") * @param interval Whether to aggregate data with intervals * @return Historical metrics data formatted for display */ - String getHistoricalMetrics(Long monitorId, String app, String metrics, String metric, String label, String history, Boolean interval); + String getHistoricalMetrics(String instance, String app, String metrics, String metric, String label, String history, Boolean interval); /** * Check warehouse storage server status diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MetricsToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MetricsToolsImpl.java index e5faa9c2c7..439a85d801 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MetricsToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MetricsToolsImpl.java @@ -152,7 +152,7 @@ public class MetricsToolsImpl implements MetricsTools { Ask user to provide the filters for labels, history and interval aggregation """) public String getHistoricalMetrics( - @ToolParam(description = "Monitor ID", required = true) Long monitorId, + @ToolParam(description = "Instance identifier (e.g., 'ip:port', 'ip', or 'domain')") String instance, @ToolParam(description = "Monitor type (e.g., 'linux', 'mysql', 'http')", required = true) String app, @ToolParam(description = "Metrics name (e.g., 'target', 'cpu', 'memory')", required = true) String metrics, @ToolParam(description = "Field Parameter (e.g., 'usage', 'used', 'available')", required = false) String fieldParameter, @@ -161,7 +161,7 @@ public class MetricsToolsImpl implements MetricsTools { @ToolParam(description = "Whether to aggregate data with intervals", required = false) Boolean interval) { try { - log.info("Getting historical metrics for monitor {} and metrics {}", monitorId, metrics); + log.info("Getting historical metrics for monitor instance {} and metrics {}", instance, metrics); if (history == null || history.trim().isEmpty()) { history = "24h"; @@ -170,15 +170,15 @@ public class MetricsToolsImpl implements MetricsTools { interval = true; } - MetricsHistoryData historyData = metricsDataService.getMetricHistoryData( - monitorId, app, metrics, fieldParameter, label, history, interval); + MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(instance, + app, metrics, fieldParameter, history, interval); if (historyData == null) { - return String.format("No historical metrics data found for monitor ID %d and metrics '%s'", monitorId, metrics); + return String.format("No historical metrics data found for monitor %s and metrics '%s'", instance, metrics); } StringBuilder response = new StringBuilder(); - response.append("HISTORICAL METRICS: ").append(metrics).append(" (Monitor ID: ").append(monitorId).append(")\n"); + response.append("HISTORICAL METRICS: ").append(metrics).append(" (Monitor ID: ").append(instance).append(")\n"); response.append("Time Range: ").append(history).append(" | Interval Aggregation: ").append(interval).append("\n"); response.append("=".repeat(60)).append("\n\n"); diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java index f0d2a1b6f7..5da5e53269 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java @@ -36,6 +36,7 @@ import org.apache.hertzbeat.common.entity.manager.ParamDefine; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Implementation of Monitoring Tools functionality @@ -171,7 +172,7 @@ public class MonitorToolsImpl implements MonitorTools { response.append("ID: ").append(monitor.getId()) .append(" | Name: ").append(monitor.getName()) .append(" | Type: ").append(monitor.getApp()) - .append(" | Host: ").append(monitor.getHost()) + .append(" | Instance: ").append(monitor.getInstance()) .append(" | Status: ").append(UtilityClass.getStatusText(monitor.getStatus())); // Add creation date for better context @@ -257,12 +258,14 @@ public class MonitorToolsImpl implements MonitorTools { if (intervals == null || intervals < 10) { intervals = 600; } - + + String instance = Objects.nonNull(port) ? host.trim() + ":" + port : host.trim(); + // Create Monitor entity Monitor monitor = Monitor.builder() .name(name.trim()) .app(app.toLowerCase().trim()) - .host(host.trim()) + .instance(instance) .intervals(intervals) .status((byte) 1) .type((byte) 0) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java index 4597760502..e338126eb0 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculator.java @@ -156,7 +156,7 @@ public class MetricsRealTimeAlertCalculator { long currentTimeMilli = System.currentTimeMillis(); String instance = String.valueOf(metricsData.getId()); String instanceName = metricsData.getInstanceName(); - String instanceHost = metricsData.getInstanceHost(); + String instanceHost = metricsData.getInstance(); String app = metricsData.getApp(); String metrics = metricsData.getMetrics(); if ((CommonConstants.PROMETHEUS_APP_PREFIX + instanceName).equals(metricsData.getApp())) { @@ -210,7 +210,7 @@ public class MetricsRealTimeAlertCalculator { commonFingerPrints.put(CommonConstants.LABEL_DEFINE_ID, String.valueOf(define.getId())); commonFingerPrints.put(CommonConstants.LABEL_ALERT_NAME, define.getName()); commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_NAME, instanceName); - commonFingerPrints.put(CommonConstants.LABEL_INSTANCE_HOST, instanceHost); + commonFingerPrints.put(CommonConstants.LABEL_INSTANCE, instanceHost); commonFingerPrints.putAll(define.getLabels()); if (labels != null) { commonFingerPrints.putAll(labels); diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java index 9a003eca38..f7f62ab8b7 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/service/impl/NoticeConfigServiceImpl.java @@ -303,9 +303,8 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine @Override public boolean sendTestMsg(NoticeReceiver noticeReceiver) { Map labels = new HashMap<>(8); - labels.put(CommonConstants.LABEL_INSTANCE, "1000000"); + labels.put(CommonConstants.LABEL_INSTANCE, "127.0.0.1"); labels.put(CommonConstants.LABEL_ALERT_NAME, "CPU Usage Alert"); - labels.put(CommonConstants.LABEL_INSTANCE_HOST, "127.0.0.1"); Map annotations = new HashMap<>(8); annotations.put("suggest", "Please check the CPU usage of the server"); SingleAlert singleAlert1 = SingleAlert.builder() diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java index e5e4b38a2a..6844bb1bcb 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/MetricsRealTimeAlertCalculatorMatchTest.java @@ -125,7 +125,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest { Map meta = new HashMap<>(); meta.put(MetricDataConstants.INSTANCE_NAME, "Cool_Stingray_34Nj_copy"); - meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1"); + meta.put(MetricDataConstants.INSTANCE, "127.0.0.1"); builder.addMetadataAll(meta); builder.addAllFields(Lists.newArrayList(destination, mode, metricValue)); @@ -175,7 +175,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest { Map meta = new HashMap<>(); meta.put(MetricDataConstants.INSTANCE_NAME, "Cool_Stingray_34Nj"); - meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1"); + meta.put(MetricDataConstants.INSTANCE, "127.0.0.1"); builder.addMetadataAll(meta); builder.addAllFields(Lists.newArrayList(destination, mode, metricValue)); @@ -225,7 +225,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest { Map meta = new HashMap<>(); meta.put(MetricDataConstants.INSTANCE_NAME, "Vibrant_Gazelle_83vJ"); - meta.put(MetricDataConstants.INSTANCE_HOST, "127.0.0.1"); + meta.put(MetricDataConstants.INSTANCE, "127.0.0.1"); builder.addMetadataAll(meta); builder.addAllFields(Lists.newArrayList(responseTime)); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/controller/AlertDefineControllerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/controller/AlertDefineControllerTest.java index f3333096ed..14471e0fef 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/controller/AlertDefineControllerTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/controller/AlertDefineControllerTest.java @@ -88,7 +88,7 @@ class AlertDefineControllerTest { Monitor.builder() .id(1L) .app("app") - .host("localhost") + .instance("localhost") .name("monitor") .build() ) diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java index d86f0ea9c6..c03d872479 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/CommonConstants.java @@ -97,11 +97,6 @@ public interface CommonConstants { */ String LABEL_ALERT_NAME = "alertname"; - /** - * label key: instance host - */ - String LABEL_INSTANCE_HOST = "instancehost"; - /** * label key: instance name */ diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/MetricDataConstants.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/MetricDataConstants.java index f408121114..b73d3ad629 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/MetricDataConstants.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/MetricDataConstants.java @@ -38,5 +38,5 @@ public interface MetricDataConstants { String CODE = "code"; String MSG = "msg"; String INSTANCE_NAME = "instancename"; - String INSTANCE_HOST = "instancehost"; + String INSTANCE = "instance"; } diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/dto/MetricsHistoryData.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/dto/MetricsHistoryData.java index a16867e1bf..498fa7bfba 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/dto/MetricsHistoryData.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/dto/MetricsHistoryData.java @@ -18,13 +18,14 @@ package org.apache.hertzbeat.common.entity.dto; import io.swagger.v3.oas.annotations.media.Schema; -import java.util.List; -import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; +import java.util.Map; + /** * Metric History Range Query Data */ @@ -35,8 +36,8 @@ import lombok.NoArgsConstructor; @Schema(description = "Metric History Range Query Data") public class MetricsHistoryData { - @Schema(title = "Monitoring Task ID") - private Long id; + @Schema(title = "Monitor Instance (e.g., ip:port or domain)") + private String instance; @Schema(title = "Monitoring Type") private String app; diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java index a549c44c8b..585a7018cf 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/manager/Monitor.java @@ -52,7 +52,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; @Entity @Table(name = "hzb_monitor", indexes = { @Index(name = "monitor_query_index", columnList = "app"), - @Index(name = "monitor_query_index", columnList = "host"), + @Index(name = "monitor_query_index", columnList = "instance"), @Index(name = "monitor_query_index", columnList = "name") }) @Data @@ -82,10 +82,10 @@ public class Monitor { @Size(max = 100) private String scrape; - @Schema(title = "peer host: ipv4, ipv6, domain name", example = "192.167.25.11", accessMode = READ_WRITE) + @Schema(title = "the monitor target: ip/domain+port or ip/domain", example = "192.167.25.11:8081", accessMode = READ_WRITE) @Size(max = 100) @HostValid - private String host; + private String instance; @Schema(title = "Monitoring of the acquisition interval time in seconds", example = "600", accessMode = READ_WRITE) @Min(10) diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/message/CollectRep.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/message/CollectRep.java index 438b207ff7..d327879bef 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/message/CollectRep.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/message/CollectRep.java @@ -225,9 +225,9 @@ public final class CollectRep { return metadata.getOrDefault(MetricDataConstants.INSTANCE_NAME, null); } - public String getInstanceHost() { + public String getInstance() { Map metadata = getMetadata(); - return metadata.getOrDefault(MetricDataConstants.INSTANCE_HOST, null); + return metadata.getOrDefault(MetricDataConstants.INSTANCE, null); } public Map getLabels() { @@ -501,8 +501,8 @@ public final class CollectRep { return metadata.getOrDefault(MetricDataConstants.INSTANCE_NAME, null); } - public String getInstanceHost() { - return metadata.getOrDefault(MetricDataConstants.INSTANCE_HOST, null); + public String getInstance() { + return metadata.getOrDefault(MetricDataConstants.INSTANCE, null); } public Map getLabels() { diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/warehouse/History.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/warehouse/History.java index bcbf1f774b..24729b16e8 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/warehouse/History.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/entity/warehouse/History.java @@ -37,7 +37,7 @@ import lombok.NoArgsConstructor; */ @Entity @Table(name = "hzb_history", indexes = { - @Index(name = "history_query_index", columnList = "monitorId"), + @Index(name = "history_query_index", columnList = "instance"), @Index(name = "history_query_index", columnList = "app"), @Index(name = "history_query_index", columnList = "metrics"), @Index(name = "history_query_index", columnList = "metric") @@ -54,8 +54,8 @@ public class History { @Schema(description = "Metric data history entity primary key index ID", example = "87584674384", accessMode = READ_ONLY) private Long id; - @Schema(title = "Monitoring Id", example = "87432674336", accessMode = READ_WRITE) - private Long monitorId; + @Schema(title = "Monitoring instance", example = "127.0.0.1:8080", accessMode = READ_WRITE) + private String instance; @Schema(title = "Monitoring Type mysql oracle db2") private String app; @@ -67,7 +67,7 @@ public class History { private String metric; @Column(length = 5000) - private String instance; + private String metricLabels; @Schema(title = "Metric Type 0: Number 1：String") private Byte metricType; diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/support/valid/HostParamValidator.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/support/valid/HostParamValidator.java index 7462d8c85d..5a5a93fb2c 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/support/valid/HostParamValidator.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/support/valid/HostParamValidator.java @@ -44,7 +44,21 @@ public class HostParamValidator implements ConstraintValidator value.indexOf(":") && value.contains("[")) { + int portIndex = value.lastIndexOf(":"); + hostPart = value.substring(0, portIndex); + } else if (value.split(":").length == 2) { + // it is IPv4 or domain with port + String[] parts = value.split(":"); + hostPart = parts[0]; + } + } + + return IpDomainUtil.validateIpDomain(hostPart); } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java index 10aa00021b..3cbf7e2269 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java @@ -123,13 +123,14 @@ public class ServiceDiscoveryWorker implements InitializingBean { .filter(p -> !p.isEmpty()) .orElse(defaultPort); final String keyStr = host + ":" + port; + final String instance = port.isEmpty() ? host : host + ":" + port; if (subMonitorBindMap.containsKey(keyStr)) { subMonitorBindMap.remove(keyStr); continue; } Monitor newMonitor = mainMonitor.clone(); newMonitor.setId(null); - newMonitor.setHost(host); + newMonitor.setInstance(instance); newMonitor.setName(newMonitor.getName() + "-" + host + ":" + port); newMonitor.setScrape(CommonConstants.SCRAPE_STATIC); newMonitor.setGmtCreate(LocalDateTime.now()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/CollectorJobScheduler.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/CollectorJobScheduler.java index c939200e76..bcc53e30e3 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/CollectorJobScheduler.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/CollectorJobScheduler.java @@ -150,7 +150,7 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch appDefine.setCyclic(true); appDefine.setTimestamp(System.currentTimeMillis()); Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, monitor.getInstance()); appDefine.setMetadata(metadata); List params = paramDao.findParamsByMonitorId(monitor.getId()); List configmaps = params.stream() diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java index 868ac259b6..b4c660099a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/SchedulerInit.java @@ -50,7 +50,7 @@ import org.springframework.util.StringUtils; @Order(value = Ordered.LOWEST_PRECEDENCE - 1) @Slf4j public class SchedulerInit implements CommandLineRunner { - + @Autowired private CollectorScheduling collectorScheduling; @@ -59,6 +59,7 @@ public class SchedulerInit implements CommandLineRunner { private static final String MAIN_COLLECTOR_NODE_IP = "127.0.0.1"; private static final String DEFAULT_COLLECTOR_VERSION = "DEBUG"; + public static final String PARAM_FIELD_PORT = "port"; @Autowired private AppService appService; @@ -117,8 +118,11 @@ public class SchedulerInit implements CommandLineRunner { appDefine.setDefaultInterval(monitor.getIntervals()); appDefine.setCyclic(true); appDefine.setTimestamp(System.currentTimeMillis()); + + String instance = monitor.getInstance(); + Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, instance); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/BulletinServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/BulletinServiceImpl.java index 0376ea1b45..e5cc79dd94 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/BulletinServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/BulletinServiceImpl.java @@ -123,10 +123,11 @@ public class BulletinServiceImpl implements BulletinService { if (null == monitor) { continue; } + BulletinMetricsData.Data.DataBuilder dataBuilder = BulletinMetricsData.Data.builder() .monitorId(monitorId) .monitorName(monitor.getName()) - .host(monitor.getHost()); + .host(monitor.getInstance()); List metrics = new ArrayList<>(); Map> fieldMap = bulletin.getFields(); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java index bde0de1c54..43c9e8da5b 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/MonitorServiceImpl.java @@ -111,6 +111,7 @@ public class MonitorServiceImpl implements MonitorService { public static final String PATTERN_HTTPS = "(?i)https://"; private static final Long MONITOR_ID_TMP = 1000000000L; private static final byte ALL_MONITOR_STATUS = 9; + public static final String PARAM_FIELD_PORT = "port"; private static final String CONTENT_VALUE = MediaType.APPLICATION_OCTET_STREAM_VALUE + SignConstants.SINGLE_MARK + "charset=" + StandardCharsets.UTF_8; private final Map imExportServiceMap = new HashMap<>(); @@ -181,7 +182,7 @@ public class MonitorServiceImpl implements MonitorService { Job appDefine = appService.getAppDefine(app); if (!isStatic) { appDefine.setSd(true); - monitor.setHost("unknow"); + monitor.setInstance("unknow"); } if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); @@ -190,8 +191,21 @@ public class MonitorServiceImpl implements MonitorService { appDefine.setDefaultInterval(monitor.getIntervals()); appDefine.setCyclic(true); appDefine.setTimestamp(System.currentTimeMillis()); + + String instance = monitor.getInstance(); + // The port field may be null + Param portParam = params.stream() + .filter(param -> PARAM_FIELD_PORT.equals(param.getField())) + .findFirst() + .orElse(null); + String portWithMark = Objects.isNull(portParam) ? "" : SignConstants.DOUBLE_MARK + portParam.getParamValue(); + if (Objects.nonNull(instance)) { + instance = instance + portWithMark; + } + monitor.setInstance(instance); + Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, instance); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); @@ -278,7 +292,7 @@ public class MonitorServiceImpl implements MonitorService { // The request monitoring parameter matches the monitoring parameter definition mapping check Monitor monitor = monitorDto.getMonitor(); // The Service Discovery host field may be null - monitor.setHost(StringUtils.hasText(monitor.getHost()) ? monitor.getHost().trim() : null); + monitor.setInstance(StringUtils.hasText(monitor.getInstance()) ? monitor.getInstance().trim() : null); monitor.setName(monitor.getName().trim()); Map paramMap = monitorDto.getParams() .stream() @@ -507,6 +521,18 @@ public class MonitorServiceImpl implements MonitorService { labelDao.saveAll(addLabels); } + String instance = monitor.getInstance(); + // The port field may be null + Param portParam = params.stream() + .filter(param -> PARAM_FIELD_PORT.equals(param.getField())) + .findFirst() + .orElse(null); + String portWithMark = Objects.isNull(portParam) ? "" : SignConstants.DOUBLE_MARK + portParam.getParamValue(); + if (Objects.nonNull(instance)) { + instance = instance + portWithMark; + } + monitor.setInstance(instance); + boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape()) || !StringUtils.hasText(monitor.getScrape()); if (preMonitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE) { // Construct the collection task Job entity @@ -526,7 +552,7 @@ public class MonitorServiceImpl implements MonitorService { appDefine.setScheduleType(monitor.getScheduleType()); appDefine.setCronExpression(monitor.getCronExpression()); Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, monitor.getInstance()); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); @@ -775,7 +801,7 @@ public class MonitorServiceImpl implements MonitorService { appDefine.setScheduleType(monitor.getScheduleType()); appDefine.setCronExpression(monitor.getCronExpression()); Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, monitor.getInstance()); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); @@ -871,7 +897,7 @@ public class MonitorServiceImpl implements MonitorService { appDefine.setCyclic(true); appDefine.setTimestamp(System.currentTimeMillis()); Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, monitor.getInstance()); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); @@ -1003,7 +1029,7 @@ public class MonitorServiceImpl implements MonitorService { appDefine.setCyclic(false); appDefine.setTimestamp(System.currentTimeMillis()); Map metadata = Map.of(CommonConstants.LABEL_INSTANCE_NAME, monitor.getName(), - CommonConstants.LABEL_INSTANCE_HOST, monitor.getHost()); + CommonConstants.LABEL_INSTANCE, monitor.getInstance()); appDefine.setMetadata(metadata); appDefine.setLabels(monitor.getLabels()); appDefine.setAnnotations(monitor.getAnnotations()); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorControllerTest.java index af99394749..b30c392c12 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorControllerTest.java @@ -60,7 +60,7 @@ class MonitorControllerTest { monitor.setJobId(43243543543L); monitor.setName("Api-TanCloud.cn"); monitor.setName("TanCloud"); - monitor.setHost("192.167.25.11"); + monitor.setInstance("192.167.25.11:8989"); monitor.setIntervals(600); monitor.setDescription("对SAAS网站TanCloud的可用性监控"); monitor.setCreator("tom"); @@ -115,7 +115,7 @@ class MonitorControllerTest { monitor.setJobId(43243543543L); monitor.setName("Api-TanCloud.cn"); monitor.setName("TanCloud"); - monitor.setHost("192.167.25.11"); + monitor.setInstance("192.167.25.11:8989"); monitor.setIntervals(600); monitor.setDescription("对SAAS网站TanCloud的可用性监控"); monitor.setCreator("tom"); @@ -142,7 +142,7 @@ class MonitorControllerTest { monitor.setJobId(43243543543L); monitor.setName("Api-TanCloud.cn"); monitor.setName("TanCloud"); - monitor.setHost("192.167.25.11"); + monitor.setInstance("192.167.25.11:8989"); monitor.setIntervals(600); monitor.setDescription("对SAAS网站TanCloud的可用性监控"); monitor.setCreator("tom"); diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/CollectorJobSchedulerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/CollectorJobSchedulerTest.java index e068c7b7fb..e4df4231ca 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/CollectorJobSchedulerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/CollectorJobSchedulerTest.java @@ -116,7 +116,7 @@ public class CollectorJobSchedulerTest { when(collectorMonitorBindDao.findCollectorMonitorBindsByCollector(identity)).thenReturn(List.of(bind)); // mock monitor - Monitor monitor = Monitor.builder().id(1L).name("test-monitor").host("127.0.0.1").app("test-app").intervals(60).status((byte) 1).build(); + Monitor monitor = Monitor.builder().id(1L).name("test-monitor").instance("127.0.0.1:8080").app("test-app").intervals(60).status((byte) 1).build(); when(monitorDao.findMonitorsByIdIn(any())).thenReturn(List.of(monitor)); // mock Params @@ -145,7 +145,7 @@ public class CollectorJobSchedulerTest { assertNotNull(job); assertNotNull(job.getMetadata()); assertEquals("test-monitor", job.getMetadata().get(CommonConstants.LABEL_INSTANCE_NAME)); - assertEquals("127.0.0.1", job.getMetadata().get(CommonConstants.LABEL_INSTANCE_HOST)); + assertEquals("127.0.0.1:8080", job.getMetadata().get(CommonConstants.LABEL_INSTANCE)); } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java index 0441d45292..f465af47b9 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/MonitorServiceTest.java @@ -142,7 +142,7 @@ class MonitorServiceTest { .intervals(1) .name("memory") .app("demoApp") - .host("localhost") + .instance("localhost") .build(); Job job = new Job(); job.setMetrics(new ArrayList<>()); @@ -165,7 +165,7 @@ class MonitorServiceTest { .intervals(1) .name("memory") .app("demoApp") - .host("localhost") + .instance("localhost") .build(); Job job = new Job(); job.setMetrics(new ArrayList<>()); @@ -188,7 +188,7 @@ class MonitorServiceTest { .intervals(1) .name("memory") .app("demoApp") - .host("localhost") + .instance("localhost") .build(); Job job = new Job(); when(appService.getAppDefine(monitor.getApp())).thenReturn(job); @@ -204,7 +204,7 @@ class MonitorServiceTest { Monitor monitor = Monitor.builder() .intervals(1) .name("memory") - .host("localhost") + .instance("localhost") .app("demoApp") .build(); Job job = new Job(); @@ -223,10 +223,10 @@ class MonitorServiceTest { MonitorDto dto = new MonitorDto(); List params = new ArrayList<>(); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(2L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(2L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); try { monitorService.validate(dto, isModify); @@ -249,10 +249,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); ParamDefine pd = ParamDefine.builder() @@ -282,10 +282,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); ParamDefine paramDefine = ParamDefine.builder() @@ -318,10 +318,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); ParamDefine paramDefine = ParamDefine.builder() @@ -354,10 +354,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); Short limit = 3; @@ -398,10 +398,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); Short limit = 3; @@ -443,10 +443,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); Short limit = 3; @@ -490,10 +490,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); Short limit = 3; @@ -541,10 +541,10 @@ class MonitorServiceTest { .build(); params.add(param); dto.setParams(params); - Monitor monitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor monitor = Monitor.builder().name("memory").instance("host").id(1L).build(); dto.setMonitor(monitor); Boolean isModify = true; - Monitor existMonitor = Monitor.builder().name("memory").host("host").id(1L).build(); + Monitor existMonitor = Monitor.builder().name("memory").instance("host").id(1L).build(); when(monitorDao.findMonitorByNameEquals(monitor.getName())).thenReturn(Optional.of(existMonitor)); List paramDefines = new ArrayList<>(); Short limit = 3; @@ -584,7 +584,7 @@ class MonitorServiceTest { params.add(param); dto.setParams(params); long monitorId = 1L; - Monitor monitor = Monitor.builder().jobId(1L).intervals(1).app("app").name("memory").host("host").id(monitorId).build(); + Monitor monitor = Monitor.builder().jobId(1L).intervals(1).app("app").name("memory").instance("host").id(monitorId).build(); dto.setMonitor(monitor); when(monitorDao.findById(monitorId)).thenReturn(Optional.empty()); try { @@ -596,7 +596,7 @@ class MonitorServiceTest { /* The [monitoring type] of monitor cannot be modified. */ - Monitor existErrorMonitor = Monitor.builder().app("app2").name("memory").host("host").id(monitorId).build(); + Monitor existErrorMonitor = Monitor.builder().app("app2").name("memory").instance("host").id(monitorId).build(); when(monitorDao.findById(monitorId)).thenReturn(Optional.of(existErrorMonitor)); try { monitorService.modifyMonitor(dto.getMonitor(), dto.getParams(), null, null); @@ -604,7 +604,7 @@ class MonitorServiceTest { assertEquals("Can not modify monitor's app type", e.getMessage()); } reset(); - Monitor existOkMonitor = Monitor.builder().jobId(1L).intervals(1).app("app").name("memory").host("host").id(monitorId).build(); + Monitor existOkMonitor = Monitor.builder().jobId(1L).intervals(1).app("app").name("memory").instance("host").id(monitorId).build(); when(monitorDao.findById(monitorId)).thenReturn(Optional.of(existOkMonitor)); when(monitorDao.save(monitor)).thenThrow(RuntimeException.class); @@ -617,7 +617,7 @@ class MonitorServiceTest { ids.add(1L); List monitors = new ArrayList<>(); for (Long id : ids) { - Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").host("host").id(id).build(); + Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").instance("host").id(id).build(); monitors.add(monitor); } when(monitorDao.findMonitorsByIdIn(ids)).thenReturn(monitors); @@ -633,7 +633,7 @@ class MonitorServiceTest { List monitors = new ArrayList<>(); for (Long id : ids) { - Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").host("host").id(id).build(); + Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").instance("host").id(id).build(); monitors.add(monitor); } when(monitorDao.findMonitorsByIdIn(ids)).thenReturn(monitors); @@ -643,7 +643,7 @@ class MonitorServiceTest { @Test void getMonitorDto() { long id = 1L; - Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").host("host").id(id).build(); + Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").instance("host").id(id).build(); when(monitorDao.findById(id)).thenReturn(Optional.of(monitor)); List params = Collections.singletonList(new Param()); when(paramDao.findParamsByMonitorId(id)).thenReturn(params); @@ -677,7 +677,7 @@ class MonitorServiceTest { List monitors = new ArrayList<>(); for (Long id : ids) { - Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").host("host").id(id).build(); + Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").instance("host").id(id).build(); monitors.add(monitor); } when(monitorDao.findMonitorsByIdIn(ids)).thenReturn(monitors); @@ -692,7 +692,7 @@ class MonitorServiceTest { List monitors = new ArrayList<>(); for (Long id : ids) { - Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").host("host").id(id).build(); + Monitor monitor = Monitor.builder().jobId(id).intervals(1).app("app").name("memory").instance("host").id(id).build(); monitor.setStatus(CommonConstants.MONITOR_PAUSED_CODE); monitors.add(monitor); } @@ -752,7 +752,7 @@ class MonitorServiceTest { .intervals(1) .name("memory") .app("demoApp") - .host("localhost") + .instance("localhost") .build(); Job job = new Job(); when(appService.getAppDefine(monitor.getApp())).thenReturn(job); @@ -802,7 +802,7 @@ class MonitorServiceTest { Job job = new Job(); job.setApp("testJob"); job.setMetrics(metrics); - Monitor monitor = Monitor.builder().jobId(1L).intervals(1).app(job.getApp()).name(job.getApp()).host("host").build(); + Monitor monitor = Monitor.builder().jobId(1L).intervals(1).app(job.getApp()).name(job.getApp()).instance("host").build(); List params = new ArrayList<>(); diff --git a/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java b/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java index bc0eb95295..1be1a9203f 100644 --- a/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java +++ b/hertzbeat-push/src/main/java/org/apache/hertzbeat/push/service/impl/PushGatewayServiceImpl.java @@ -81,7 +81,7 @@ public class PushGatewayServiceImpl implements PushGatewayService { .id(monitorId) .app(job) .name(instance) - .host(instance) + .instance(instance) .type((byte) 1) .status(CommonConstants.MONITOR_UP_CODE) .build(); diff --git a/hertzbeat-startup/src/main/resources/db/migration/h2/V180__update_column.sql b/hertzbeat-startup/src/main/resources/db/migration/h2/V180__update_column.sql index 70e80cb000..5e67e2fdbc 100644 --- a/hertzbeat-startup/src/main/resources/db/migration/h2/V180__update_column.sql +++ b/hertzbeat-startup/src/main/resources/db/migration/h2/V180__update_column.sql @@ -26,3 +26,68 @@ WHERE type = 'realtime'; UPDATE HZB_ALERT_DEFINE SET type = 'periodic_metric' WHERE type = 'periodic'; + +-- Rename host to instance +CREATE ALIAS RENAME_HOST_TO_INSTANCE AS $$ +void renameHostToInstance(java.sql.Connection conn) throws java.sql.SQLException { + boolean instanceExists = false; + boolean hostExists = false; + try (java.sql.ResultSet rs = conn.getMetaData().getColumns(null, null, "HZB_MONITOR", "INSTANCE")) { + if (rs.next()) instanceExists = true; + } + try (java.sql.ResultSet rs = conn.getMetaData().getColumns(null, null, "HZB_MONITOR", "HOST")) { + if (rs.next()) hostExists = true; + } + try (java.sql.Statement stmt = conn.createStatement()) { + if (instanceExists) { + if (hostExists) { + stmt.execute("UPDATE HZB_MONITOR SET instance = host WHERE instance IS NULL"); + stmt.execute("ALTER TABLE HZB_MONITOR DROP COLUMN host"); + } + } else { + if (hostExists) { + stmt.execute("ALTER TABLE HZB_MONITOR ALTER COLUMN host RENAME TO instance"); + } + } + } +} +$$; +CALL RENAME_HOST_TO_INSTANCE(); +DROP ALIAS RENAME_HOST_TO_INSTANCE; + +-- Update instance with port +UPDATE HZB_MONITOR m +SET instance = CONCAT(instance, ':', (SELECT param_value FROM HZB_PARAM p WHERE p.monitor_id = m.id AND p.field = 'port')) +WHERE EXISTS (SELECT 1 FROM HZB_PARAM p WHERE p.monitor_id = m.id AND p.field = 'port'); + +-- Migrate history table +CREATE ALIAS MIGRATE_HISTORY_TABLE AS $$ +void migrateHistoryTable(java.sql.Connection conn) throws java.sql.SQLException { + boolean monitorIdExists = false; + boolean metricLabelsExists = false; + try (java.sql.ResultSet rs = conn.getMetaData().getColumns(null, null, "HZB_HISTORY", "MONITOR_ID")) { + if (rs.next()) monitorIdExists = true; + } + try (java.sql.ResultSet rs = conn.getMetaData().getColumns(null, null, "HZB_HISTORY", "METRIC_LABELS")) { + if (rs.next()) metricLabelsExists = true; + } + + if (monitorIdExists) { + try (java.sql.Statement stmt = conn.createStatement()) { + if (!metricLabelsExists) { + stmt.execute("ALTER TABLE HZB_HISTORY ALTER COLUMN instance RENAME TO metric_labels"); + stmt.execute("ALTER TABLE HZB_HISTORY ALTER COLUMN metric_labels SET DATA TYPE VARCHAR(5000)"); + stmt.execute("ALTER TABLE HZB_HISTORY ADD COLUMN instance VARCHAR(255)"); + } else { + stmt.execute("UPDATE HZB_HISTORY SET metric_labels = instance WHERE metric_labels IS NULL"); + stmt.execute("UPDATE HZB_HISTORY SET instance = NULL"); + stmt.execute("ALTER TABLE HZB_HISTORY ALTER COLUMN instance SET DATA TYPE VARCHAR(255)"); + } + stmt.execute("UPDATE HZB_HISTORY h SET instance = (SELECT m.instance FROM HZB_MONITOR m WHERE m.id = h.monitor_id) WHERE h.monitor_id IS NOT NULL"); + stmt.execute("ALTER TABLE HZB_HISTORY DROP COLUMN monitor_id"); + } + } +} +$$; +CALL MIGRATE_HISTORY_TABLE(); +DROP ALIAS MIGRATE_HISTORY_TABLE; diff --git a/hertzbeat-startup/src/main/resources/db/migration/mysql/V180__update_column.sql b/hertzbeat-startup/src/main/resources/db/migration/mysql/V180__update_column.sql index dbc21060f9..2c88874cc3 100644 --- a/hertzbeat-startup/src/main/resources/db/migration/mysql/V180__update_column.sql +++ b/hertzbeat-startup/src/main/resources/db/migration/mysql/V180__update_column.sql @@ -41,4 +41,86 @@ DELIMITER ; CALL UpdateAlertDefineColumns(); DROP PROCEDURE IF EXISTS UpdateAlertDefineColumns; + +-- Rename host to instance +DELIMITER // +CREATE PROCEDURE RenameHostToInstance() +BEGIN + DECLARE instance_exists INT; + DECLARE host_exists INT; + SELECT COUNT(*) INTO instance_exists FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'hzb_monitor' AND COLUMN_NAME = 'instance'; + SELECT COUNT(*) INTO host_exists FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'hzb_monitor' AND COLUMN_NAME = 'host'; + IF instance_exists > 0 THEN + IF host_exists > 0 THEN + SET @sql_update = 'UPDATE hzb_monitor SET instance = host WHERE instance IS NULL'; + PREPARE stmt_update FROM @sql_update; + EXECUTE stmt_update; + DEALLOCATE PREPARE stmt_update; + + SET @sql_drop = 'ALTER TABLE hzb_monitor DROP COLUMN host'; + PREPARE stmt_drop FROM @sql_drop; + EXECUTE stmt_drop; + DEALLOCATE PREPARE stmt_drop; + END IF; + ELSE + IF host_exists > 0 THEN + SET @sql_change = 'ALTER TABLE hzb_monitor CHANGE host instance VARCHAR(100)'; + PREPARE stmt_change FROM @sql_change; + EXECUTE stmt_change; + DEALLOCATE PREPARE stmt_change; + END IF; + END IF; +END // +DELIMITER ; +CALL RenameHostToInstance(); +DROP PROCEDURE IF EXISTS RenameHostToInstance; + +-- Update instance with port +UPDATE hzb_monitor m +INNER JOIN hzb_param p ON m.id = p.monitor_id AND p.field = 'port' +SET m.instance = CONCAT(m.instance, ':', p.param_value) +WHERE m.instance IS NOT NULL; + +-- Migrate history table +DELIMITER // +CREATE PROCEDURE MigrateHistoryTable() +BEGIN + DECLARE monitor_id_exists INT; + DECLARE metric_labels_exists INT; + + SELECT COUNT(*) INTO monitor_id_exists FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'hzb_history' AND COLUMN_NAME = 'monitor_id'; + SELECT COUNT(*) INTO metric_labels_exists FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'hzb_history' AND COLUMN_NAME = 'metric_labels'; + + IF monitor_id_exists > 0 THEN + IF metric_labels_exists = 0 THEN + SET @sql_rename = 'ALTER TABLE hzb_history CHANGE instance metric_labels VARCHAR(5000)'; + PREPARE stmt_rename FROM @sql_rename; + EXECUTE stmt_rename; + DEALLOCATE PREPARE stmt_rename; + + SET @sql_add = 'ALTER TABLE hzb_history ADD COLUMN instance VARCHAR(255)'; + PREPARE stmt_add FROM @sql_add; + EXECUTE stmt_add; + DEALLOCATE PREPARE stmt_add; + ELSE + UPDATE hzb_history SET metric_labels = instance WHERE metric_labels IS NULL; + UPDATE hzb_history SET instance = NULL; + SET @sql_resize = 'ALTER TABLE hzb_history MODIFY COLUMN instance VARCHAR(255)'; + PREPARE stmt_resize FROM @sql_resize; + EXECUTE stmt_resize; + DEALLOCATE PREPARE stmt_resize; + END IF; + + UPDATE hzb_history h JOIN hzb_monitor m ON h.monitor_id = m.id SET h.instance = m.instance; + + SET @sql_drop = 'ALTER TABLE hzb_history DROP COLUMN monitor_id'; + PREPARE stmt_drop FROM @sql_drop; + EXECUTE stmt_drop; + DEALLOCATE PREPARE stmt_drop; + END IF; +END // +DELIMITER ; +CALL MigrateHistoryTable(); +DROP PROCEDURE IF EXISTS MigrateHistoryTable; + COMMIT; diff --git a/hertzbeat-startup/src/main/resources/db/migration/postgresql/V180__update_column.sql b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V180__update_column.sql index 4e8b4046be..c37ed10229 100644 --- a/hertzbeat-startup/src/main/resources/db/migration/postgresql/V180__update_column.sql +++ b/hertzbeat-startup/src/main/resources/db/migration/postgresql/V180__update_column.sql @@ -27,4 +27,48 @@ UPDATE HZB_ALERT_DEFINE SET type = 'periodic_metric' WHERE type = 'periodic'; +-- Rename host to instance +DO $$ +BEGIN + IF EXISTS(SELECT * FROM information_schema.columns WHERE table_name = 'hzb_monitor' AND column_name = 'instance') THEN + IF EXISTS(SELECT * FROM information_schema.columns WHERE table_name = 'hzb_monitor' AND column_name = 'host') THEN + EXECUTE 'UPDATE HZB_MONITOR SET instance = host WHERE instance IS NULL'; + EXECUTE 'ALTER TABLE HZB_MONITOR DROP COLUMN host'; + END IF; + ELSE + IF EXISTS(SELECT * FROM information_schema.columns WHERE table_name = 'hzb_monitor' AND column_name = 'host') THEN + EXECUTE 'ALTER TABLE HZB_MONITOR RENAME COLUMN host TO instance'; + END IF; + END IF; +END $$; + +-- Update instance with port +UPDATE HZB_MONITOR m +SET instance = m.instance || ':' || p.param_value +FROM HZB_PARAM p +WHERE m.id = p.monitor_id AND p.field = 'port'; + +-- Migrate history table +DO $$ +BEGIN + IF EXISTS(SELECT * FROM information_schema.columns WHERE table_name = 'hzb_history' AND column_name = 'monitor_id') THEN + IF NOT EXISTS(SELECT * FROM information_schema.columns WHERE table_name = 'hzb_history' AND column_name = 'metric_labels') THEN + ALTER TABLE hzb_history RENAME COLUMN instance TO metric_labels; + ALTER TABLE hzb_history ALTER COLUMN metric_labels TYPE VARCHAR(5000); + ALTER TABLE hzb_history ADD COLUMN instance VARCHAR(255); + ELSE + UPDATE hzb_history SET metric_labels = instance WHERE metric_labels IS NULL; + UPDATE hzb_history SET instance = NULL; + ALTER TABLE hzb_history ALTER COLUMN instance TYPE VARCHAR(255); + END IF; + + UPDATE hzb_history h + SET instance = m.instance + FROM hzb_monitor m + WHERE h.monitor_id = m.id; + + ALTER TABLE hzb_history DROP COLUMN monitor_id; + END IF; +END $$; + commit; diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/dao/MonitorDaoTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/dao/MonitorDaoTest.java index ef9612ea61..a24b50cba8 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/dao/MonitorDaoTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/dao/MonitorDaoTest.java @@ -53,7 +53,7 @@ class MonitorDaoTest extends AbstractSpringIntegrationTest { .jobId(2L) .app("jvm") .name("jvm_test") - .host("192.34.5.43") + .instance("192.34.5.43:8989") .status((byte) 1) .build(); monitor = monitorDao.saveAndFlush(monitor); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/controller/MetricsDataController.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/controller/MetricsDataController.java index d95616fe25..832059cd2d 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/controller/MetricsDataController.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/controller/MetricsDataController.java @@ -75,15 +75,13 @@ public class MetricsDataController { return ResponseEntity.ok(Message.success(metricsData)); } - @GetMapping("/api/monitor/{monitorId}/metric/{metricFull}") + @GetMapping("/api/monitor/{instance}/metric/{metricFull}") @Operation(summary = "Queries historical data for a specified metric for monitoring", description = "Queries historical data for a specified metric under monitoring") public ResponseEntity> getMetricHistoryData( - @Parameter(description = "monitor the task ID", example = "343254354") - @PathVariable Long monitorId, + @Parameter(description = "monitor instance", example = "127.0.0.1:8080") + @PathVariable String instance, @Parameter(description = "monitor metric full path", example = "linux.cpu.usage") @PathVariable() String metricFull, - @Parameter(description = "label filter, empty by default", example = "disk2") - @RequestParam(required = false) String label, @Parameter(description = "query historical time period, default 6h-6 hours: s-seconds, M-minutes, h-hours, d-days, w-weeks", example = "6h") @RequestParam(required = false) String history, @Parameter(description = "aggregate data calc. off by default; 4-hour window, query limit >1 week", example = "false") @@ -99,7 +97,7 @@ public class MetricsDataController { String app = names[0]; String metrics = names[1]; String metric = names[2]; - MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(monitorId, app, metrics, metric, label, history, interval); + MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, interval); return ResponseEntity.ok(Message.success(historyData)); } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/MetricsDataService.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/MetricsDataService.java index c0d7e7788b..6c39b78d58 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/MetricsDataService.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/MetricsDataService.java @@ -41,14 +41,14 @@ public interface MetricsDataService { /** * Queries historical data for a specified metric for monitoring - * @param monitorId Monitor Id - * @param app Monitor Type - * @param metrics Metrics Name - * @param metric Metrics Field Name - * @param label Label Filter - * @param history Query Historical Time Period - * @param interval aggregate data calc - * @return metrics history data + * + * @param instance Instance e.g. ip:port or ip or domain + * @param app Monitor Type + * @param metrics Metrics Name + * @param metric Metrics Field Name + * @param history Query Historical Time Period + * @param interval aggregate data calc + * @return metrics history data */ - MetricsHistoryData getMetricHistoryData(Long monitorId, String app, String metrics, String metric, String label, String history, Boolean interval); + MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval); } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/MetricsDataServiceImpl.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/MetricsDataServiceImpl.java index 7d2935af16..b1595c241d 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/MetricsDataServiceImpl.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/MetricsDataServiceImpl.java @@ -107,22 +107,22 @@ public class MetricsDataServiceImpl implements MetricsDataService { } @Override - public MetricsHistoryData getMetricHistoryData(Long monitorId, String app, String metrics, String metric, String label, String history, Boolean interval) { + public MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval) { if (history == null) { history = "6h"; } Map> instanceValuesMap; if (interval == null || !interval) { - instanceValuesMap = historyDataReader.get().getHistoryMetricData(monitorId, app, metrics, metric, label, history); + instanceValuesMap = historyDataReader.get().getHistoryMetricData(instance, app, metrics, metric, history); } else { - instanceValuesMap = historyDataReader.get().getHistoryIntervalMetricData(monitorId, app, metrics, metric, label, history); + instanceValuesMap = historyDataReader.get().getHistoryIntervalMetricData(instance, app, metrics, metric, history); } if (instanceValuesMap.containsKey("{}")) { instanceValuesMap.put("", instanceValuesMap.get("{}")); instanceValuesMap.remove("{}"); } return MetricsHistoryData.builder() - .id(monitorId).metrics(metrics).values(instanceValuesMap) + .instance(instance).metrics(metrics).values(instanceValuesMap) .field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build()) .build(); } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/HistoryDataReader.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/HistoryDataReader.java index 37baebc42a..89fe0f3adb 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/HistoryDataReader.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/HistoryDataReader.java @@ -34,30 +34,28 @@ public interface HistoryDataReader { /** * query history range metrics data from tsdb - * @param monitorId monitor id - * @param app monitor type - * @param metrics metrics - * @param metric metric - * @param label label - * @param history range + * + * @param instance instance e.g. ip:port or ip or domain + * @param app monitor type + * @param metrics metrics + * @param metric metric + * @param history range * @return metrics data */ - Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, - String label, String history); + Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history); /** * query history range interval metrics data from tsdb * max min mean metrics value - * @param monitorId monitor id - * @param app monitor type - * @param metrics metrics - * @param metric metric - * @param label label - * @param history history range + * + * @param instance instance e.g. ip:port or ip or domain + * @param app monitor type + * @param metrics metrics + * @param metric metric + * @param history history range * @return metrics data */ - Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, - String label, String history); + Map> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history); /** * Query logs with multiple filter conditions diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java index adf411125b..6d6c1d1a2a 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java @@ -139,7 +139,7 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { log.info("[warehouse greptime] flush metrics data {} {}is null, ignore.", metricsData.getId(), metricsData.getMetrics()); return; } - String monitorId = String.valueOf(metricsData.getId()); + String instance = metricsData.getInstance(); String tableName = getTableName(metricsData.getMetrics()); TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(tableName); @@ -160,7 +160,7 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { Table table = Table.from(tableSchemaBuilder.build()); long now = System.currentTimeMillis(); Object[] values = new Object[2 + fields.size()]; - values[0] = monitorId; + values[0] = instance; values[1] = now; RowWrapper rowWrapper = metricsData.readRow(); while (rowWrapper.hasNextRow()) { @@ -206,15 +206,15 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, - String label, String history) { + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, + String history) { Map timeRange = getTimeRange(history); Long start = timeRange.get(LABEL_KEY_START_TIME); Long end = timeRange.get(LABEL_KEY_END_TIME); String step = getTimeStep(start, end); - return getHistoryData(start, end, step, monitorId, app, metrics, metric); + return getHistoryData(start, end, step, instance, app, metrics, metric); } private String getTableName(String metrics) { @@ -222,15 +222,15 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, + String metric, String history) { Map timeRange = getTimeRange(history); Long start = timeRange.get(LABEL_KEY_START_TIME); Long end = timeRange.get(LABEL_KEY_END_TIME); String step = getTimeStep(start, end); - Map> instanceValuesMap = getHistoryData(start, end, step, monitorId, app, metrics, metric); + Map> instanceValuesMap = getHistoryData(start, end, step, instance, app, metrics, metric); // Queries below this point may yield inconsistent results due to exceeding the valid data range. // Therefore, we restrict the valid range by obtaining the post-query timeframe. @@ -241,7 +241,7 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { long effectiveEnd = values.get(values.size() - 1).getTime() / 1000 + Duration.ofHours(4).getSeconds(); String name = getTableName(metrics); - String timeSeriesSelector = name + "{" + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\""; + String timeSeriesSelector = name + "{" + LABEL_KEY_INSTANCE + "=\"" + instance + "\""; if (!CommonConstants.PROMETHEUS.equals(app)) { timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\""; } @@ -317,16 +317,16 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { * @param start start time * @param end end time * @param step step - * @param monitorId monitor id + * @param instance monitor instance * @param app monitor type * @param metrics metrics * @param metric metric * @return history metric data */ - private Map> getHistoryData(long start, long end, String step, Long monitorId, String app, String metrics, String metric) { + private Map> getHistoryData(long start, long end, String step, String instance, String app, String metrics, String metric) { String name = getTableName(metrics); String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + name + "\"" - + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\""; + + "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\""; if (!CommonConstants.PROMETHEUS.equals(app)) { timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\""; } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java index 4805d71e81..f65d6673aa 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/influxdb/InfluxdbDataStorage.java @@ -71,17 +71,14 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { private static final String CREATE_DATABASE = "CREATE DATABASE %s"; - private static final String QUERY_HISTORY_SQL = "SELECT instance, %s FROM %s WHERE time >= now() - %s order by time desc"; - - private static final String QUERY_HISTORY_SQL_WITH_INSTANCE = "SELECT instance, %s FROM %s WHERE instance = '%s' and time >= now() - %s order by time desc"; + private static final String QUERY_HISTORY_SQL = "SELECT metric_labels, \"%s\" FROM \"%s\" WHERE time >= now() - %s order by time desc"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = - "SELECT FIRST(%s), MEAN(%s), MAX(%s), MIN(%s) FROM %s WHERE instance = '%s' and time >= now() - %s GROUP BY time(4h)"; + "SELECT FIRST(\"%s\"), MEAN(\"%s\"), MAX(\"%s\"), MIN(\"%s\") FROM \"%s\" WHERE metric_labels = '%s' and time >= now() - %s GROUP BY time(4h)"; + private static final String QUERY_INSTANCE_SQL = "show tag values from \"%s\" with key = \"metric_labels\""; private static final String CREATE_RETENTION_POLICY = "CREATE RETENTION POLICY \"%s_retention\" ON \"%s\" DURATION %s REPLICATION %d DEFAULT"; - private static final String QUERY_INSTANCE_SQL = "show tag values from %s with key = \"instance\""; - private InfluxDB influxDb; public InfluxdbDataStorage(InfluxdbProperties influxdbProperties) { @@ -153,11 +150,11 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { return; } if (metricsData.getValues().isEmpty()) { - log.info("[warehouse influxdb] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse influxdb] flush metrics data {} is null, ignore.", metricsData.getInstance()); return; } - String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId()); + String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance()); List points = new ArrayList<>(); try { @@ -186,7 +183,7 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { labels.put(cell.getField().getName(), cell.getValue()); } }); - builder.tag("instance", JsonUtil.toJson(labels)); + builder.tag("metric_labels", JsonUtil.toJson(labels)); points.add(builder.build()); } @@ -199,10 +196,9 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String table = this.generateTable(app, metrics, monitorId); - String selectSql = label == null ? String.format(QUERY_HISTORY_SQL, metric, table, history) - : String.format(QUERY_HISTORY_SQL_WITH_INSTANCE, metric, table, label, history); + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) { + String table = this.generateTable(app, metrics, instance); + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); Map> instanceValueMap = new HashMap<>(8); try { QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS); @@ -230,26 +226,21 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String table = this.generateTable(app, metrics, monitorId); + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) { + String table = this.generateTable(app, metrics, instance); Map> instanceValueMap = new HashMap<>(8); Set instances = new HashSet<>(8); - if (label != null) { - instances.add(label); - } - if (instances.isEmpty()) { - // query the instance near 1week - String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); - QueryResult instanceQueryResult = this.influxDb.query(new Query(queryInstanceSql, DATABASE), TimeUnit.MILLISECONDS); - for (QueryResult.Result result : instanceQueryResult.getResults()) { - if (result.getSeries() == null) { - continue; - } - for (QueryResult.Series series : result.getSeries()) { - for (List value : series.getValues()) { - if (value != null && value.get(1) != null) { - instances.add(value.get(1).toString()); - } + // query all metric_labels + String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); + QueryResult instanceQueryResult = this.influxDb.query(new Query(queryInstanceSql, DATABASE), TimeUnit.MILLISECONDS); + for (QueryResult.Result result : instanceQueryResult.getResults()) { + if (result.getSeries() == null) { + continue; + } + for (QueryResult.Series series : result.getSeries()) { + for (List value : series.getValues()) { + if (value != null && value.get(1) != null) { + instances.add(value.get(1).toString()); } } } @@ -309,8 +300,14 @@ public class InfluxdbDataStorage extends AbstractHistoryDataStorage { return instanceValueMap; } - private String generateTable(String app, String metrics, Long monitorId) { - return app + "_" + metrics + "_" + monitorId; + private String generateTable(String app, String metrics, String instance) { + if (instance.contains(".") || instance.contains(":") || instance.contains("[")) { + instance = instance.replace(".", "_") + .replace(":", "_") + .replace("[", "_") + .replace("]", "_"); + } + return app + "_" + metrics + "_" + instance; } private long parseTimeToMillis(Object time) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java index 4d85fba3b1..0b79ce3e9b 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/iotdb/IotDbDataStorage.java @@ -179,7 +179,7 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { return; } if (metricsData.getValues().isEmpty()) { - log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getInstance()); return; } List schemaList = new ArrayList<>(); @@ -216,7 +216,7 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { String label = JsonUtil.toJson(labels); - String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId(), label, false); + String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance(), label, true); if (tabletMap.containsKey(label)) { // Avoid Time repeats now++; @@ -259,8 +259,8 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, - String label, String history) { + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, + String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { log.error(""" @@ -271,27 +271,19 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { """); return instanceValuesMap; } - String deviceId = getDeviceId(app, metrics, monitorId, label, true); + String deviceId = getDeviceId(app, metrics, instance, null, true); String selectSql = ""; try { - if (label != null) { - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); - handleHistorySelect(selectSql, "", instanceValuesMap); - } else { - // First query all the devices below, if there is data for all the devices below, otherwise query the data for the deviceId - List devices = queryAllDevices(deviceId); - if (devices.isEmpty()) { - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); - handleHistorySelect(selectSql, "", instanceValuesMap); - } else { - // todo Transform to a select query: Select Device 1.0. Metric, Device2. Metric from XXX - for (String device : devices) { - String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); - String instanceId = device.substring(prefixDeviceId.length() + 1); - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId + "." + addQuote(instanceId), history); - handleHistorySelect(selectSql, instanceId, instanceValuesMap); - } - } + // query all devices + List devices = queryAllDevices(deviceId); + if (devices.isEmpty()) { + log.warn("no iot device found for deviceId: {}", deviceId); + return instanceValuesMap; + } + for (String device : devices) { + selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), device, history); + String labels = extractLabelsFromDevice(device, deviceId); + handleHistorySelect(selectSql, labels, instanceValuesMap); } } catch (StatementExecutionException | IoTDBConnectionException e) { log.error("select error history sql: {}", selectSql); @@ -323,8 +315,8 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, + String metric, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { log.error(""" @@ -335,26 +327,18 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { """); return instanceValuesMap; } - String deviceId = getDeviceId(app, metrics, monitorId, label, true); + String deviceId = getDeviceId(app, metrics, instance, null, true); String selectSql; - if (label != null) { - selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); - handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); + List devices = queryAllDevices(deviceId); + if (devices.isEmpty()) { + log.warn("no iot device found for deviceId: {}", deviceId); + return instanceValuesMap; } else { - List devices = queryAllDevices(deviceId); - if (devices.isEmpty()) { + for (String device : devices) { selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); - handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); - } else { - for (String device : devices) { - String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); - String instance = device.substring(prefixDeviceId.length() + 1); - selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId + "." + addQuote(instance), history); - handleHistoryIntervalSelect(selectSql, instance, instanceValuesMap); - } + addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), device, history); + String labels = extractLabelsFromDevice(device, deviceId); + handleHistoryIntervalSelect(selectSql, labels, instanceValuesMap); } } return instanceValuesMap; @@ -405,45 +389,67 @@ public class IotDbDataStorage extends AbstractHistoryDataStorage { * @param deviceId deviceId */ private List queryAllDevices(String deviceId) { - String showDevicesSql = String.format(SHOW_DEVICES, deviceId + ".*"); - SessionDataSetWrapper dataSet = null; List devices = new ArrayList<>(); - try { - dataSet = this.sessionPool.executeQueryStatement(showDevicesSql, this.queryTimeoutInMs); - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - devices.add(rowRecord.getFields().get(0).getStringValue()); - } - } catch (StatementExecutionException | IoTDBConnectionException e) { - log.error("query show all devices sql error. sql: {}", showDevicesSql); - log.error(e.getMessage(), e); - } finally { - if (dataSet != null) { - // need to close the result set! ! ! otherwise it will cause server-side heap - this.sessionPool.closeResultSet(dataSet); + List sqls = new ArrayList<>(); + sqls.add(String.format(SHOW_DEVICES, deviceId)); + sqls.add(String.format(SHOW_DEVICES, deviceId + ".*")); + + for (String sql : sqls) { + SessionDataSetWrapper dataSet = null; + try { + dataSet = this.sessionPool.executeQueryStatement(sql, this.queryTimeoutInMs); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + devices.add(rowRecord.getFields().get(0).getStringValue()); + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + log.error("query show devices sql error. sql: {}", sql); + log.error(e.getMessage(), e); + } finally { + if (dataSet != null) { + // need to close the result set! ! ! otherwise it will cause server-side heap + this.sessionPool.closeResultSet(dataSet); + } } } return devices; } /** - * use ${group}.${app}.${metrics}.${monitor}.${labels} to get device id if there is a way to get instanceId + * use ${group}.${app}.${metrics}.${monitor}.${metric_labels} to get device id if labels exist * otherwise use ${group}.${app}.${metrics}.${monitor} * Use ${group}.${app}.${metrics}.${monitor}.* to get all instance data when you tend to query */ - private String getDeviceId(String app, String metrics, Long monitorId, String labels, boolean useQuote) { + private String getDeviceId(String app, String metrics, String instance, String labels, boolean useQuote) { + if (instance.contains(".") || instance.contains(":") || instance.contains("[")) { + instance = instance.replace(".", "_") + .replace(":", "_") + .replace("[", "_") + .replace("]", "_"); + } String deviceId = STORAGE_GROUP + "." + (useQuote ? addQuote(app) : app) + "." + (useQuote ? addQuote(metrics) : metrics) + "." - + addQuote(monitorId.toString()); - if (labels != null && !labels.isEmpty() && !labels.equals(CommonConstants.NULL_VALUE)) { + + addQuote(instance); + if (labels != null && !labels.isEmpty() && !labels.equals(CommonConstants.NULL_VALUE) && !"{}".equals(labels)) { deviceId += "." + addQuote(labels); } return deviceId; } /** - * add quote，prevents keyword errors during queries(eg: nodes) + * Extract labels from device path + */ + private String extractLabelsFromDevice(String device, String baseDeviceId) { + if (device.length() > baseDeviceId.length() + 1) { + String labelsPart = device.substring(baseDeviceId.length() + 1); + return labelsPart.replace("`", ""); + } + return ""; + } + + /** + * add quote,prevents keyword errors during queries(eg: nodes) */ private String addQuote(String text) { if (text == null || text.isEmpty() || (text.startsWith(BACK_QUOTE) && text.endsWith(BACK_QUOTE))) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/jpa/JpaDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/jpa/JpaDatabaseDataStorage.java index 02ce832469..6c31ab9a1c 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/jpa/JpaDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/jpa/JpaDatabaseDataStorage.java @@ -21,23 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.persistence.criteria.Predicate; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.time.Duration; -import java.time.ZonedDateTime; -import java.time.temporal.TemporalAmount; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.MetricDataConstants; @@ -55,8 +39,23 @@ import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + /** - * data storage by mysql/h2 - jpa + * data storage by mysql/h2/pgsql - jpa */ @Component @ConditionalOnProperty(prefix = "warehouse.store.jpa", name = "enabled", havingValue = "true") @@ -126,7 +125,7 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { return; } if (metricsData.getValues().isEmpty()) { - log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getInstance()); return; } String monitorType = metricsData.getApp(); @@ -142,7 +141,7 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { List singleHistoryList = new ArrayList<>(); rowWrapper.cellStream().forEach(cell -> singleHistoryList.add(buildHistory(metricsData, cell, monitorType, metrics, labels))); - singleHistoryList.forEach(history -> history.setInstance(JsonUtil.toJson(labels))); + singleHistoryList.forEach(history -> history.setMetricLabels(JsonUtil.toJson(labels))); allHistoryList.addAll(singleHistoryList); } @@ -155,7 +154,7 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { private History buildHistory(CollectRep.MetricsData metricsData, ArrowCell cell, String monitorType, String metrics, Map labels) { History.HistoryBuilder historyBuilder = History.builder() - .monitorId(metricsData.getId()) + .instance(metricsData.getInstance()) .app(monitorType) .metrics(metrics) .time(metricsData.getTime()) @@ -199,27 +198,22 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) { Map> instanceValuesMap = new HashMap<>(8); Specification specification = (root, query, criteriaBuilder) -> { List andList = new ArrayList<>(); - Predicate predicateMonitorId = criteriaBuilder.equal(root.get("monitorId"), monitorId); + Predicate predicateInstance = criteriaBuilder.equal(root.get("instance"), instance); Predicate predicateMonitorType = criteriaBuilder.equal(root.get("app"), app); if (CommonConstants.PROMETHEUS.equals(app)) { predicateMonitorType = criteriaBuilder.like(root.get("app"), CommonConstants.PROMETHEUS_APP_PREFIX + "%"); } Predicate predicateMonitorMetrics = criteriaBuilder.equal(root.get("metrics"), metrics); Predicate predicateMonitorMetric = criteriaBuilder.equal(root.get("metric"), metric); - andList.add(predicateMonitorId); + andList.add(predicateInstance); andList.add(predicateMonitorType); andList.add(predicateMonitorMetrics); andList.add(predicateMonitorMetric); - if (StringUtils.isNotBlank(label)) { - Predicate predicateMonitorInstance = criteriaBuilder.equal(root.get("instance"), label); - andList.add(predicateMonitorInstance); - } - if (history != null) { try { TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); @@ -247,7 +241,7 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { } else { value = dataItem.getStr(); } - String instanceValue = dataItem.getInstance() == null ? "" : dataItem.getInstance(); + String instanceValue = dataItem.getMetricLabels() == null ? "" : dataItem.getMetricLabels(); List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); valueList.add(new Value(value, dataItem.getTime())); } @@ -269,7 +263,7 @@ public class JpaDatabaseDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) { return new HashMap<>(8); } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java index b6afd7c33a..801d635295 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/questdb/QuestdbDataStorage.java @@ -65,14 +65,15 @@ import org.springframework.stereotype.Component; @Slf4j public class QuestdbDataStorage extends AbstractHistoryDataStorage { - private static final String QUERY_HISTORY_SQL = "SELECT timestamp AS ts, instance, %s AS value FROM \"%s\" WHERE timestamp >= %s ORDER BY timestamp DESC"; + private static final String QUERY_HISTORY_SQL = "SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE timestamp >= %s ORDER BY timestamp DESC"; - private static final String QUERY_HISTORY_SQL_WITH_INSTANCE = "SELECT timestamp AS ts, instance, %s AS value FROM \"%s\" WHERE instance = '%s' AND timestamp >= %s ORDER BY timestamp DESC"; + private static final String QUERY_HISTORY_SQL_WITH_INSTANCE = + "SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s ORDER BY timestamp DESC"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = - "SELECT timestamp AS ts, first(%s) AS origin, avg(%s) AS mean, max(%s) AS max, min(%s) AS min FROM \"%s\" WHERE instance = '%s' AND timestamp >= %s SAMPLE BY 4h"; + "SELECT timestamp AS ts, first(\"%s\") AS origin, avg(\"%s\") AS mean, max(\"%s\") AS max, min(\"%s\") AS min FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s SAMPLE BY 4h"; - private static final String QUERY_INSTANCE_SQL = "SELECT DISTINCT instance FROM \"%s\""; + private static final String QUERY_INSTANCE_SQL = "SELECT DISTINCT metric_labels FROM \"%s\""; private Sender sender; @@ -128,7 +129,7 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS || metricsData.getValues().isEmpty()) { return; } - String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId()); + String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance()); try { RowWrapper rowWrapper = metricsData.readRow(); @@ -146,9 +147,9 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { .filter(cell -> cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) .forEach(cell -> labels.put(cell.getField().getName(), cell.getValue())); if (!labels.isEmpty()) { - sender.symbol("instance", JsonUtil.toJson(labels)); + sender.symbol("metric_labels", JsonUtil.toJson(labels)); } else { - sender.symbol("instance", metricsData.getApp() + sender.symbol("metric_labels", metricsData.getApp() + "_" + metricsData.getMetrics()); } @@ -184,11 +185,10 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String table = this.generateTable(app, metrics, monitorId); + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) { + String table = this.generateTable(app, metrics, instance); String dateAdd = getDateAdd(history); - String selectSql = label == null ? String.format(QUERY_HISTORY_SQL, metric, table, dateAdd) - : String.format(QUERY_HISTORY_SQL_WITH_INSTANCE, metric, table, label.replace("'", "\\'"), dateAdd); + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, dateAdd); Map> instanceValueMap = new HashMap<>(8); try { Map selectResult = executeQuery(selectSql); @@ -203,13 +203,13 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { colMap.put(columns.get(i).get("name"), i); } int tsIdx = colMap.get("ts"); - int instanceIdx = colMap.get("instance"); + int metricLabelsIdx = colMap.get("metric_labels"); int valueIdx = colMap.get("value"); for (List row : dataset) { String tsStr = (String) row.get(tsIdx); long time = Instant.parse(tsStr).toEpochMilli(); - String instanceValue = row.get(instanceIdx) == null ? "" : (String) row.get(instanceIdx); + String instanceValue = row.get(metricLabelsIdx) == null ? "" : (String) row.get(metricLabelsIdx); Object valObj = row.get(valueIdx); String strValue = valObj == null ? null : this.parseDoubleValue(valObj.toString()); if (strValue == null) { @@ -225,24 +225,19 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String table = this.generateTable(app, metrics, monitorId); + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) { + String table = this.generateTable(app, metrics, instance); String dateAdd = getDateAdd(history); Map> instanceValueMap = new HashMap<>(8); Set instances = new HashSet<>(8); - if (label != null) { - instances.add(label); - } - if (instances.isEmpty()) { - // query the instance - String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); - Map instanceQueryResult = executeQuery(queryInstanceSql); - if (instanceQueryResult != null && instanceQueryResult.containsKey("dataset")) { - List> dataset = (List>) instanceQueryResult.get("dataset"); - for (List row : dataset) { - if (!row.isEmpty() && row.get(0) != null) { - instances.add(row.get(0).toString()); - } + // query all metric_labels + String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); + Map instanceQueryResult = executeQuery(queryInstanceSql); + if (instanceQueryResult != null && instanceQueryResult.containsKey("dataset")) { + List> dataset = (List>) instanceQueryResult.get("dataset"); + for (List row : dataset) { + if (!row.isEmpty() && row.get(0) != null) { + instances.add(row.get(0).toString()); } } } @@ -273,32 +268,33 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { for (List row : dataset) { String tsStr = (String) row.get(tsIdx); long time = Instant.parse(tsStr).toEpochMilli(); - Value.ValueBuilder valueBuilder = Value.builder().time(time); + Value.ValueBuilder valueBuilder = Value.builder(); + valueBuilder.time(time); Object originObj = row.get(originIdx); - if (originObj != null) { - valueBuilder.origin(this.parseDoubleValue(originObj.toString())); - } else { + if (originObj == null) { continue; } + valueBuilder.origin(this.parseDoubleValue(originObj.toString())); + Object meanObj = row.get(meanIdx); - if (meanObj != null) { - valueBuilder.mean(this.parseDoubleValue(meanObj.toString())); - } else { + if (meanObj == null) { continue; } + valueBuilder.mean(this.parseDoubleValue(meanObj.toString())); + Object maxObj = row.get(maxIdx); - if (maxObj != null) { - valueBuilder.max(this.parseDoubleValue(maxObj.toString())); - } else { + if (maxObj == null) { continue; } + valueBuilder.max(this.parseDoubleValue(maxObj.toString())); + Object minObj = row.get(minIdx); - if (minObj != null) { - valueBuilder.min(this.parseDoubleValue(minObj.toString())); - } else { + if (minObj == null) { continue; } + valueBuilder.min(this.parseDoubleValue(minObj.toString())); + List valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); valueList.add(valueBuilder.build()); } @@ -359,8 +355,14 @@ public class QuestdbDataStorage extends AbstractHistoryDataStorage { return String.format("dateadd('%s', %d, now())", unit, -count); } - private String generateTable(String app, String metrics, Long monitorId) { - return app + "_" + metrics + "_" + monitorId; + private String generateTable(String app, String metrics, String instance) { + if (instance.contains(".") || instance.contains(":") || instance.contains("[")) { + instance = instance.replace(".", "_") + .replace(":", "_") + .replace("[", "_") + .replace("]", "_"); + } + return app + "_" + metrics + "_" + instance; } private String parseDoubleValue(String value) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java index 22a1acb37e..b81bc5b401 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java @@ -48,6 +48,7 @@ import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; +import org.jetbrains.annotations.NotNull; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -71,13 +72,13 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { private static final String NO_SUPER_TABLE_ERROR = "Table does not exist"; private static final String QUERY_HISTORY_WITH_INSTANCE_SQL = - "SELECT ts, instance, `%s` FROM `%s` WHERE instance = '%s' AND ts >= now - %s order by ts desc"; + "SELECT ts, metric_labels, `%s` FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s order by ts desc"; private static final String QUERY_HISTORY_SQL = - "SELECT ts, instance, `%s` FROM `%s` WHERE ts >= now - %s order by ts desc"; + "SELECT ts, metric_labels, `%s` FROM `%s` WHERE ts >= now - %s order by ts desc"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = - "SELECT first(ts), first(`%s`), avg(`%s`), min(`%s`), max(`%s`) FROM `%s` WHERE instance = '%s' AND ts >= now - %s interval(4h)"; + "SELECT first(ts), first(`%s`), avg(`%s`), min(`%s`), max(`%s`) FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s interval(4h)"; private static final String QUERY_INSTANCE_SQL = - "SELECT DISTINCT instance FROM `%s` WHERE ts >= now - 1w"; + "SELECT DISTINCT metric_labels FROM `%s` WHERE ts >= now - 1w"; private static final String TABLE_NOT_EXIST = "Table does not exist"; @@ -175,15 +176,17 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { if (metricsData.getValues().isEmpty()) { if (log.isInfoEnabled()) { - log.info("[warehouse tdengine] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse tdengine] flush metrics data {} is null, ignore.", metricsData.getInstance()); } return; } - String monitorId = String.valueOf(metricsData.getId()); - String superTable = metricsData.getApp() + "_" + metricsData.getMetrics() + "_super"; - String table = metricsData.getApp() + "_" + metricsData.getMetrics() + "_" + monitorId; + String instance = metricsData.getInstance(); + String app = metricsData.getApp(); + String metrics = metricsData.getMetrics(); + String superTable = getTable(app, metrics, "_super"); + String table = getTable(app, metrics, instance); StringBuilder sqlBuffer = new StringBuilder(); int i = 0; @@ -243,7 +246,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { sqlBuffer.append(" ").append(String.format(sqlRowBuffer.toString(), formatStringValue(JsonUtil.toJson(labels)))); } - String insertDataSql = String.format(INSERT_TABLE_DATA_SQL, table, superTable, monitorId, sqlBuffer); + String insertDataSql = String.format(INSERT_TABLE_DATA_SQL, table, superTable, instance, sqlBuffer); if (log.isDebugEnabled()) { log.debug(insertDataSql); @@ -261,7 +264,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { // stable not exists, create it StringBuilder fieldSqlBuilder = new StringBuilder("("); fieldSqlBuilder.append("ts TIMESTAMP, "); - fieldSqlBuilder.append("instance NCHAR(").append(tableStrColumnDefineMaxLength).append("), "); + fieldSqlBuilder.append("metric_labels NCHAR(").append(tableStrColumnDefineMaxLength).append("), "); for (int index = 0; index < metricsData.getFields().size(); index++) { CollectRep.Field field = metricsData.getFields().get(index); String fieldName = field.getName(); @@ -314,6 +317,17 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } + @NotNull + private static String getTable(String app, String metrics, String instance) { + if (instance.contains(".") || instance.contains(":") || instance.contains("[")) { + instance = instance.replace(".", "_") + .replace(":", "_") + .replace("[", "_") + .replace("]", "_"); + } + return app + "_" + metrics + "_" + instance; + } + private String formatStringValue(String value) { String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0"); // bugfix Argument list too long @@ -331,10 +345,9 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String table = app + "_" + metrics + "_" + monitorId; - String selectSql = label == null ? String.format(QUERY_HISTORY_SQL, metric, table, history) : - String.format(QUERY_HISTORY_WITH_INSTANCE_SQL, metric, table, label, history); + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) { + String table = getTable(app, metrics, instance); + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); if (log.isDebugEnabled()) { log.debug(selectSql); @@ -384,48 +397,43 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, + String metric, String history) { if (!serverAvailable) { INSTANCE_EXCEPTION_PRINT.run(); return Collections.emptyMap(); } - String table = app + "_" + metrics + "_" + monitorId; + String table = getTable(app, metrics, instance); List instances = new LinkedList<>(); - if (label != null) { - instances.add(label); - } - if (instances.isEmpty()) { - // need to confirm that how many instances of current metrics one week ago - String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); - Connection connection = null; - try { - connection = hikariDataSource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(queryInstanceSql); - while (resultSet.next()) { - String instanceValue = resultSet.getString(1); - if (instanceValue == null || StringUtils.isBlank(instanceValue)) { - instances.add("''"); - } else { - instances.add(instanceValue); - } + // query all metric_labels from the table + String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); + Connection connection = null; + try { + connection = hikariDataSource.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(queryInstanceSql); + while (resultSet.next()) { + String instanceValue = resultSet.getString(1); + if (instanceValue == null || StringUtils.isBlank(instanceValue)) { + instances.add("''"); + } else { + instances.add(instanceValue); } + } + } catch (Exception e) { + if (log.isErrorEnabled()) { + log.error(e.getMessage()); + } + } finally { + try { + assert connection != null; + connection.close(); } catch (Exception e) { if (log.isErrorEnabled()) { log.error(e.getMessage()); } - } finally { - try { - assert connection != null; - connection.close(); - } catch (Exception e) { - if (log.isErrorEnabled()) { - log.error(e.getMessage()); - } - } } } Map> instanceValuesMap = new HashMap<>(instances.size()); @@ -441,10 +449,10 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } List values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - Connection connection = null; + Connection conn = null; try { - connection = hikariDataSource.getConnection(); - Statement statement = connection.createStatement(); + conn = hikariDataSource.getConnection(); + Statement statement = conn.createStatement(); ResultSet resultSet = statement.executeQuery(selectSql); while (resultSet.next()) { Timestamp ts = resultSet.getTimestamp(1); @@ -470,8 +478,8 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage { } } finally { try { - assert connection != null; - connection.close(); + assert conn != null; + conn.close(); } catch (Exception e) { if (log.isErrorEnabled()) { log.error(e.getMessage()); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java index 50478e9747..7b25d95472 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsClusterDataStorage.java @@ -98,7 +98,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag private static final String LABEL_KEY_NAME = "__name__"; private static final String LABEL_KEY_JOB = "job"; private static final String LABEL_KEY_INSTANCE = "instance"; - private static final String LABEL_KEY_HOST = "host"; + private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__"; private static final String SPILT = "_"; private static final String MONITOR_METRICS_KEY = "__metrics__"; private static final String MONITOR_METRIC_KEY = "__metric__"; @@ -201,7 +201,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag isPrometheusAuto = false; defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()); } - defaultLabels.put(LABEL_KEY_INSTANCE, String.valueOf(metricsData.getId())); + defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance()); try { @@ -244,7 +244,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag if (!isPrometheusAuto) { labels.put(MONITOR_METRIC_KEY, entry.getKey()); } - labels.put(LABEL_KEY_HOST, metricsData.getInstanceHost()); + labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId())); // add customized labels as identifier var customizedLabels = metricsData.getLabels(); if (!ObjectUtils.isEmpty(customizedLabels)) { @@ -287,15 +287,15 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, - String label, String history) { + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, + String history) { String labelName = metrics + SPILT + metric; if (CommonConstants.PROMETHEUS.equals(app)) { labelName = metrics; } String timeSeriesSelector = Stream.of( LABEL_KEY_NAME + "=\"" + labelName + "\"", - LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"", + LABEL_KEY_INSTANCE + "=\"" + instance + "\"", CommonConstants.PROMETHEUS.equals(app) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\"" ).filter(Objects::nonNull).collect(Collectors.joining(",")); Map> instanceValuesMap = new HashMap<>(8); @@ -331,6 +331,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); @@ -363,8 +364,8 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, + String metric, String history) { if (!serverAvailable) { log.error(""" @@ -396,7 +397,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag } String timeSeriesSelector = Stream.of( LABEL_KEY_NAME + "=\"" + labelName + "\"", - LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"", + LABEL_KEY_INSTANCE + "=\"" + instance + "\"", CommonConstants.PROMETHEUS.equals(app) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\"" ).filter(Objects::nonNull).collect(Collectors.joining(",")); Map> instanceValuesMap = new HashMap<>(8); @@ -432,6 +433,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -470,6 +472,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -508,6 +511,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -546,6 +550,7 @@ public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorag labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java index ab304497be..2219678457 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/vm/VictoriaMetricsDataStorage.java @@ -95,7 +95,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { private static final String LABEL_KEY_NAME = "__name__"; private static final String LABEL_KEY_JOB = "job"; private static final String LABEL_KEY_INSTANCE = "instance"; - private static final String LABEL_KEY_HOST = "host"; + private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__"; private static final String SPILT = "_"; private static final String MONITOR_METRICS_KEY = "__metrics__"; private static final String MONITOR_METRIC_KEY = "__metric__"; @@ -182,7 +182,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { } else { defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()); } - defaultLabels.put(LABEL_KEY_INSTANCE, String.valueOf(metricsData.getId())); + defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance()); List contentList = new LinkedList<>(); @@ -225,7 +225,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { if (!isPrometheusAuto) { labels.put(MONITOR_METRIC_KEY, entry.getKey()); } - labels.put(LABEL_KEY_HOST, metricsData.getInstanceHost()); + labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId())); // add customized labels as identifier var customizedLabels = metricsData.getLabels(); if (!ObjectUtils.isEmpty(customizedLabels)) { @@ -264,13 +264,13 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { + public Map> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) { String labelName = metrics + SPILT + metric; if (CommonConstants.PROMETHEUS.equals(app)) { labelName = metrics; } String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\"" - + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"" + + "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\"" + (CommonConstants.PROMETHEUS.equals(app) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\""); Map> instanceValuesMap = new HashMap<>(8); try { @@ -303,6 +303,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -332,8 +333,8 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { } @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { + public Map> getHistoryIntervalMetricData(String instance, String app, String metrics, + String metric, String history) { if (!serverAvailable) { log.error(""" @@ -364,7 +365,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labelName = metrics; } String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\"" - + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"" + + "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\"" + (CommonConstants.PROMETHEUS.equals(app) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\""); Map> instanceValuesMap = new HashMap<>(8); try { @@ -397,6 +398,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -432,6 +434,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -467,6 +470,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -502,6 +506,7 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage { labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); labels.remove(LABEL_KEY_INSTANCE); + labels.remove(LABEL_KEY_MONITOR_ID); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/controller/MetricsDataControllerTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/controller/MetricsDataControllerTest.java index 28e0655510..674f2b5d33 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/controller/MetricsDataControllerTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/controller/MetricsDataControllerTest.java @@ -116,7 +116,7 @@ class MetricsDataControllerTest { @Test void getMetricHistoryData() throws Exception { - final long monitorId = 343254354; + final String instance = "127.0.0.1:8081"; final String app = "linux"; final String metrics = "cpu"; final String metric = "usage"; @@ -125,11 +125,11 @@ class MetricsDataControllerTest { final String label = "disk2"; final String history = "6h"; final Boolean interval = false; - final String getUrl = "/api/monitor/" + monitorId + "/metric/" + metricFull; - final String getUrlFail = "/api/monitor/" + monitorId + "/metric/" + metricFullFail; + final String getUrl = "/api/monitor/" + instance + "/metric/" + metricFull; + final String getUrlFail = "/api/monitor/" + instance + "/metric/" + metricFullFail; MultiValueMap params = new LinkedMultiValueMap<>(); - params.add("monitorId", String.valueOf(monitorId)); + params.add("instance", instance); params.add("label", label); params.add("history", history); params.add("interval", String.valueOf(interval)); @@ -151,17 +151,17 @@ class MetricsDataControllerTest { assertTrue(exception.getMessage().contains("IllegalArgumentException")); MetricsHistoryData metricsHistoryData = MetricsHistoryData.builder() - .id(monitorId) + .instance(instance) .metrics(metrics) .field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build()) .build(); when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true); - lenient().when(metricsDataService.getMetricHistoryData(eq(monitorId), eq(app), eq(metrics), eq(metric), eq(label), eq(history), eq(interval))) + lenient().when(metricsDataService.getMetricHistoryData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history), eq(interval))) .thenReturn(metricsHistoryData); this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl).params(params)) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) - .andExpect(jsonPath("$.data.id").value(monitorId)) + .andExpect(jsonPath("$.data.instance").value(instance)) .andExpect(jsonPath("$.data.metrics").value(metrics)) .andExpect(jsonPath("$.data.field.name").value(metric)) .andExpect(jsonPath("$.data.field.type").value(String.valueOf(CommonConstants.TYPE_NUMBER))) diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/MetricsDataServiceTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/MetricsDataServiceTest.java index 07c7072bd3..b73646a9e9 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/MetricsDataServiceTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/MetricsDataServiceTest.java @@ -95,7 +95,7 @@ public class MetricsDataServiceTest { @Test public void testGetMetricHistoryData() { - Long monitorId = 1L; + String instance = "127.0.0.1:8080"; String app = "linux"; String metrics = "disk"; String metric = "used"; @@ -104,12 +104,12 @@ public class MetricsDataServiceTest { Boolean intervalFalse = false; Boolean intervalTrue = true; - when(historyDataReader.getHistoryMetricData(eq(monitorId), eq(app), eq(metrics), eq(metric), eq(label), eq(history))).thenReturn(new HashMap<>()); - assertNotNull(metricsDataService.getMetricHistoryData(monitorId, app, metrics, metric, label, history, intervalFalse)); - verify(historyDataReader, times(1)).getHistoryMetricData(eq(monitorId), eq(app), eq(metrics), eq(metric), eq(label), eq(history)); + when(historyDataReader.getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>()); + assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalFalse)); + verify(historyDataReader, times(1)).getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history)); - when(historyDataReader.getHistoryIntervalMetricData(eq(monitorId), eq(app), eq(metrics), eq(metric), eq(label), eq(history))).thenReturn(new HashMap<>()); - assertNotNull(metricsDataService.getMetricHistoryData(monitorId, app, metrics, metric, label, history, intervalTrue)); - verify(historyDataReader, times(1)).getHistoryIntervalMetricData(eq(monitorId), eq(app), eq(metrics), eq(metric), eq(label), eq(history)); + when(historyDataReader.getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>()); + assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalTrue)); + verify(historyDataReader, times(1)).getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history)); } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java index 0a3570b1b9..45a1234c11 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java @@ -160,9 +160,7 @@ class GreptimeDbDataStorageTest { when(restTemplate.exchange(any(), eq(HttpMethod.GET), any(HttpEntity.class), eq(PromQlQueryContent.class))) .thenReturn(responseEntity); - Map> result = greptimeDbDataStorage.getHistoryMetricData( - 1L, "test_app", "test_metrics", "test_metric", "test_label", "6h" - ); + Map> result = greptimeDbDataStorage.getHistoryMetricData("127.0.0.1:8080", "test_app", "test_metrics", "test_metric", "6h"); assertNotNull(result); assertFalse(result.isEmpty()); diff --git a/web-app/src/app/pojo/Monitor.ts b/web-app/src/app/pojo/Monitor.ts index 03af662e68..c2a89db44b 100644 --- a/web-app/src/app/pojo/Monitor.ts +++ b/web-app/src/app/pojo/Monitor.ts @@ -22,7 +22,7 @@ export class Monitor { name!: string; app!: string; scrape!: string; - host!: string; + instance!: string; intervals: number = 60; // Schedule type: interval | cron scheduleType: string = 'interval'; diff --git a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.ts b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.ts index 24269865ee..776320e040 100644 --- a/web-app/src/app/routes/alert/alert-setting/alert-setting.component.ts +++ b/web-app/src/app/routes/alert/alert-setting/alert-setting.component.ts @@ -1524,7 +1524,7 @@ export class AlertSettingComponent implements OnInit { this.transferData = monitors.map(item => ({ key: item.id, title: item.name, - description: item.host, + description: item.instance, direction: this.selectedMonitorIds.has(item.id) ? 'right' : 'left', labels: Object.entries(item.labels).map(([key, value]) => `${key}:${value}`) })); diff --git a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts index 8aeb67637b..0dae9c7802 100644 --- a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts +++ b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts @@ -44,6 +44,8 @@ export class MonitorDataChartComponent implements OnInit, OnDestroy { private _monitorId!: number; @Input() + instance!: string; + @Input() app!: string; @Input() metrics!: string; @@ -256,7 +258,7 @@ export class MonitorDataChartComponent implements OnInit, OnDestroy { // load historical metrics data this.loading = `${this.i18nSvc.fanyi('monitor.detail.chart.data-loading')}`; let metricData$ = this.monitorSvc - .getMonitorMetricHistoryData(this.monitorId, this.app, this.metrics, this.metric, this.timePeriod, isInterval) + .getMonitorMetricHistoryData(this.instance, this.app, this.metrics, this.metric, this.timePeriod, isInterval) .pipe( finalize(() => { if (!this.worker$) { diff --git a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html index 7aae9843da..ee25a56259 100644 --- a/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html +++ b/web-app/src/app/routes/monitor/monitor-data-table/monitor-data-table.component.html @@ -51,14 +51,13 @@
{{ monitorId }} - {{ monitor.host }} - {{ port }} {{ monitor.cronExpression }} {{ monitor.intervals }}s + {{ monitor.instance }}
diff --git a/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.html b/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.html index a97ad24c9f..9daf780eeb 100755 --- a/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.html +++ b/web-app/src/app/routes/monitor/monitor-detail/monitor-detail.component.html @@ -92,6 +92,7 @@ [metric]="item.metric" [unit]="item.unit" [monitorId]="monitorId" + [instance]="monitor.instance" >
@@ -159,6 +160,7 @@ [metric]="metric.metric" [unit]="metric.unit" [monitorId]="monitorId" + [instance]="monitor.instance" >
- + {{ hostName ? hostName : ('monitor.host' | i18n) }} @@ -128,7 +128,7 @@ - + {{ paramDefine.name }} diff --git a/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.ts b/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.ts index 19843b24ef..715e664fbe 100644 --- a/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.ts +++ b/web-app/src/app/routes/monitor/monitor-form/monitor-form.component.ts @@ -60,6 +60,7 @@ export class MonitorFormComponent implements OnChanges { @Output() readonly collectorChange = new EventEmitter(); hasAdvancedParams: boolean = false; + hostParam: Param | undefined; constructor(private notifySvc: NzNotificationService, @Inject(ALAIN_I18N_TOKEN) private i18nSvc: I18NService) {} @@ -70,6 +71,10 @@ export class MonitorFormComponent implements OnChanges { this.monitor.cronExpression = ''; } + if (changes.params && this.params) { + this.hostParam = this.params.find(param => param.field === 'host'); + } + if (changes.advancedParams && changes.advancedParams.currentValue !== changes.advancedParams.previousValue) { for (const advancedParam of changes.advancedParams.currentValue) { if (advancedParam.display !== false) { @@ -99,13 +104,8 @@ export class MonitorFormComponent implements OnChanges { }); return; } - this.monitor.host = this.monitor.host ? this.monitor.host.trim() : ''; this.monitor.name = this.monitor.name.trim(); - // todo Set the host property value separately for now this.params.forEach(param => { - if (param.field === 'host') { - param.paramValue = this.monitor.host; - } if (param.paramValue != null && typeof param.paramValue == 'string') { param.paramValue = (param.paramValue as string).trim(); } @@ -120,6 +120,13 @@ export class MonitorFormComponent implements OnChanges { param.paramValue = (param.paramValue as string).trim(); } }); + + // Set monitor.instance to host param value, let backend handle the port concatenation + const hostParam = this.params.find(param => param.field === 'host'); + if (hostParam) { + this.monitor.instance = hostParam.paramValue; + } + this.formDetect.emit({ monitor: this.monitor, sdParams: this.sdParams, @@ -147,13 +154,8 @@ export class MonitorFormComponent implements OnChanges { return; } } - this.monitor.host = this.monitor.host?.trim(); this.monitor.name = this.monitor.name?.trim(); - // todo Set the host property value separately for now this.params.forEach(param => { - if (param.field === 'host') { - param.paramValue = this.monitor.host; - } if (param.paramValue != null && typeof param.paramValue == 'string') { param.paramValue = (param.paramValue as string).trim(); } @@ -168,6 +170,13 @@ export class MonitorFormComponent implements OnChanges { param.paramValue = (param.paramValue as string).trim(); } }); + + // Set monitor.instance to host param value, let backend handle the port concatenation + const hostParam = this.params.find(param => param.field === 'host'); + if (hostParam) { + this.monitor.instance = hostParam.paramValue; + } + this.formSubmit.emit({ monitor: this.monitor, sdParams: this.sdParams, diff --git a/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.html b/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.html index da7afe0f28..0dbd76053d 100644 --- a/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.html +++ b/web-app/src/app/routes/monitor/monitor-list/monitor-list.component.html @@ -202,13 +202,13 @@ *ngIf="data.scrape == 'static' || !data.scrape" nz-button nzType="text" - [cdkCopyToClipboard]="data.host" + [cdkCopyToClipboard]="data.instance" nz-tooltip [nzTooltipTitle]="'common.button.copy.tip' | i18n" (click)="notifyCopySuccess()" > - {{ data.host }} + {{ data.instance }}