From 8ab71e672f436cbdf0e9328afff6d5f1d4ab04f0 Mon Sep 17 00:00:00 2001 From: Jast Date: Sat, 4 Jan 2025 15:58:22 +0800 Subject: [PATCH] [Improve]Improve e2e code (#2945) --- .../collector/dispatch/CommonDispatcher.java | 39 ++----- .../collector/dispatch/MetricsCollect.java | 40 +++---- .../hertzbeat/collector/util/CollectUtil.java | 109 ++++++++++++------ .../basic/http/DockerMonitorE2eTest.java | 38 +++--- .../http/docker/containers_result.txt | 2 +- .../http/docker/containers_stats.txt | 1 + .../resources/http/docker/system_result.txt | 2 +- .../collect/AbstractCollectE2eTest.java | 48 ++++++-- 8 files changed, 167 insertions(+), 112 deletions(-) create mode 100644 hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_stats.txt diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java index 2248b9f0da..9c36e831a0 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java @@ -19,7 +19,6 @@ package org.apache.hertzbeat.collector.dispatch; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.Gson; -import com.google.gson.JsonElement; import lombok.AllArgsConstructor; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -37,7 +36,6 @@ import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.springframework.stereotype.Component; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -89,7 +87,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc private final List unitConvertList; private final WorkerPool workerPool; - + private final String collectorIdentity; public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, @@ -146,7 +144,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc log.error("Common Dispatcher error: {}.", e.getMessage(), e); } } - + private void monitorCollectTaskTimeout() { try { // Detect whether the collection unit of each metrics has timed out for 4 minutes, @@ -194,7 +192,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc new MetricsTime(System.currentTimeMillis(), metrics, timeout)); } else { metricsTimeoutMonitorMap.put(job.getId() + "-" + metrics.getName(), - new MetricsTime(System.currentTimeMillis(), metrics, timeout)); + new MetricsTime(System.currentTimeMillis(), metrics, timeout)); } }); } @@ -243,7 +241,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc } else if (!metricsSet.isEmpty()) { // The execution of the current level metrics is completed, and the execution of the next level metrics starts // use pre collect metrics data to replace next metrics config params - List> configmapList = getConfigmapFromPreCollectData(metricsData); + List> configmapList = CollectUtil.getConfigmapFromPreCollectData(metricsData); if (configmapList.size() == ENV_CONFIG_SIZE) { job.addEnvConfigmaps(configmapList.get(0)); } @@ -267,9 +265,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc Map preConfigMap = configmapList.get(index); configmap.putAll(preConfigMap); } - JsonElement metricJson = GSON.toJsonTree(metricItem); - CollectUtil.replaceCryPlaceholder(metricJson, configmap); - Metrics metric = GSON.fromJson(metricJson, Metrics.class); + Metrics metric = CollectUtil.replaceCryPlaceholderToMetrics(metricItem, configmap); metric.setSubTaskNum(subTaskNumAtomic); metric.setSubTaskId(index); metric.setSubTaskDataRef(metricsDataReference); @@ -338,7 +334,7 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc interval = interval <= 0 ? 0 : interval; // Reset Construction Execution Metrics Task View job.constructPriorMetrics(); - timerDispatch.cyclicJob(timerJob, interval, TimeUnit.SECONDS); + timerDispatch.cyclicJob(timerJob, interval, TimeUnit.SECONDS); } // it is an asynchronous periodic cyclic task, directly response the collected data metricsDataList.forEach(commonDataQueue::sendMetricsData); @@ -347,30 +343,9 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc // and the result listener is notified of the combination of all metrics data timerDispatch.responseSyncJobData(job.getId(), metricsDataList); } - + } - private List> getConfigmapFromPreCollectData(CollectRep.MetricsData metricsData) { - if (metricsData.getValuesCount() <= 0 || metricsData.getFieldsCount() <= 0) { - return new LinkedList<>(); - } - List> mapList = new LinkedList<>(); - for (CollectRep.ValueRow valueRow : metricsData.getValues()) { - if (valueRow.getColumnsCount() != metricsData.getFieldsCount()) { - continue; - } - Map configmapMap = new HashMap<>(valueRow.getColumnsCount()); - int index = 0; - for (CollectRep.Field field : metricsData.getFields()) { - String value = valueRow.getColumns(index); - index++; - Configmap configmap = new Configmap(field.getName(), value, Integer.valueOf(field.getType()).byteValue()); - configmapMap.put(field.getName(), configmap); - } - mapList.add(configmapMap); - } - return mapList; - } /** * Metrics times. diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java index 9aae8cd6b1..18121db0e8 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java @@ -20,9 +20,9 @@ package org.apache.hertzbeat.collector.dispatch; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.jexl3.JexlExpression; -import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.collect.prometheus.PrometheusAutoCollectImpl; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; import org.apache.hertzbeat.collector.dispatch.timer.Timeout; import org.apache.hertzbeat.collector.dispatch.timer.WheelTimerTask; import org.apache.hertzbeat.collector.dispatch.unit.UnitConvert; @@ -34,6 +34,7 @@ import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.CommonUtil; import org.apache.hertzbeat.common.util.JexlExpressionRunner; import org.apache.hertzbeat.common.util.Pair; +import org.springframework.util.CollectionUtils; import java.util.Collections; import java.util.HashMap; @@ -43,7 +44,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; /** * metrics collection @@ -142,7 +142,7 @@ public class MetricsCollect implements Runnable, Comparable { // for prometheus auto if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(metrics.getProtocol())) { List metricsData = PrometheusAutoCollectImpl - .getInstance().collect(response, metrics); + .getInstance().collect(response, metrics); validateResponse(metricsData.stream().findFirst().orElse(null)); collectDataDispatch.dispatchCollectData(timeout, metrics, metricsData); return; @@ -153,10 +153,10 @@ public class MetricsCollect implements Runnable, Comparable { AbstractCollect abstractCollect = CollectStrategyFactory.invoke(metrics.getProtocol()); if (abstractCollect == null) { log.error("[Dispatcher] - not support this: app: {}, metrics: {}, protocol: {}.", - app, metrics.getName(), metrics.getProtocol()); + app, metrics.getName(), metrics.getProtocol()); response.setCode(CollectRep.Code.FAIL); response.setMsg("not support " + app + ", " - + metrics.getName() + ", " + metrics.getProtocol()); + + metrics.getName() + ", " + metrics.getProtocol()); } else { try { abstractCollect.preCheck(metrics); @@ -192,7 +192,7 @@ public class MetricsCollect implements Runnable, Comparable { * @param metrics Metrics configuration * @param collectData Data collection */ - private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder collectData) { + public void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder collectData) { collectData.setPriority(metrics.getPriority()); List fieldList = new LinkedList<>(); for (Metrics.Field field : metrics.getFields()) { @@ -216,22 +216,22 @@ public class MetricsCollect implements Runnable, Comparable { // eg: database_pages=Database pages unconventional mapping Map fieldAliasMap = new HashMap<>(8); Map fieldExpressionMap = metrics.getCalculates() - .stream() - .map(cal -> transformCal(cal, fieldAliasMap)) - .filter(Objects::nonNull) - .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue)); + .stream() + .map(cal -> transformCal(cal, fieldAliasMap)) + .filter(Objects::nonNull) + .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue)); if (metrics.getUnits() == null) { metrics.setUnits(Collections.emptyList()); } Map> fieldUnitMap = metrics.getUnits() - .stream() - .map(this::transformUnit) - .filter(Objects::nonNull) - .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair) arr[1], (oldValue, newValue) -> newValue)); + .stream() + .map(this::transformUnit) + .filter(Objects::nonNull) + .collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair) arr[1], (oldValue, newValue) -> newValue)); List fields = metrics.getFields(); - List aliasFields = metrics.getAliasFields(); + List aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList); Map aliasFieldValueMap = new HashMap<>(8); Map fieldValueMap = new HashMap<>(8); Map stringTypefieldValueMap = new HashMap<>(8); @@ -245,7 +245,7 @@ public class MetricsCollect implements Runnable, Comparable { aliasFieldValueMap.put(aliasField, aliasFieldValue); // whether the alias field is a number CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil - .extractDoubleAndUnitFromStr(aliasFieldValue); + .extractDoubleAndUnitFromStr(aliasFieldValue); if (doubleAndUnit != null && doubleAndUnit.getValue() != null) { fieldValueMap.put(aliasField, doubleAndUnit.getValue()); if (doubleAndUnit.getUnit() != null) { @@ -290,8 +290,8 @@ public class MetricsCollect implements Runnable, Comparable { } catch (Exception e) { log.info("[calculates execute warning] {}.", e.getMessage()); value = Optional.ofNullable(fieldValueMap.get(expression.getSourceText())) - .map(String::valueOf) - .orElse(null); + .map(String::valueOf) + .orElse(null); } } else { // does not exist then map the alias value @@ -306,7 +306,7 @@ public class MetricsCollect implements Runnable, Comparable { final byte fieldType = field.getType(); if (fieldType == CommonConstants.TYPE_NUMBER) { CollectUtil.DoubleAndUnit doubleAndUnit = CollectUtil - .extractDoubleAndUnitFromStr(value); + .extractDoubleAndUnitFromStr(value); final Double tempValue = doubleAndUnit == null ? null : doubleAndUnit.getValue(); value = tempValue == null ? null : String.valueOf(tempValue); aliasFieldUnit = doubleAndUnit == null ? null : doubleAndUnit.getUnit(); @@ -453,7 +453,7 @@ public class MetricsCollect implements Runnable, Comparable { private void setNewThreadName(long monitorId, String app, long startTime, Metrics metrics) { String builder = monitorId + "-" + app + "-" + metrics.getName() - + "-" + String.valueOf(startTime).substring(9); + + "-" + String.valueOf(startTime).substring(9); Thread.currentThread().setName(builder); } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/CollectUtil.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/CollectUtil.java index ff65344f76..2674a4b7be 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/CollectUtil.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/util/CollectUtil.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.collector.util; import com.beetstra.jutf7.CharsetProvider; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonNull; @@ -29,11 +30,14 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Configmap; import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -57,6 +61,11 @@ public final class CollectUtil { private static final String CRYING_PLACEHOLDER_REGEX = "(\\^o\\^)(\\w|-|$|\\.)+(\\^o\\^)"; private static final Pattern CRYING_PLACEHOLDER_REGEX_PATTERN = Pattern.compile(CRYING_PLACEHOLDER_REGEX); private static final List UNIT_SYMBOLS = Arrays.asList("%", "G", "g", "M", "m", "K", "k", "B", "b", "Ki", "Mi", "Gi"); + private static final Gson GSON = new Gson(); + /** + * Regularly verifying whether a string is a combination of numbers and units + */ + private static final String DOUBLE_AND_UNIT_CHECK_REGEX = "^[.\\d+" + String.join("", UNIT_SYMBOLS) + "]+$"; /** * private constructor, not allow to create instance. @@ -64,13 +73,9 @@ public final class CollectUtil { private CollectUtil() { } - /** - * Regularly verifying whether a string is a combination of numbers and units - */ - private static final String DOUBLE_AND_UNIT_CHECK_REGEX = "^[.\\d+" + String.join("", UNIT_SYMBOLS) + "]+$"; - /** * count match keyword number + * * @param content content * @param keyword keyword * @return match num @@ -134,31 +139,6 @@ public final class CollectUtil { return null; } - /** - * double and unit - */ - public static final class DoubleAndUnit { - - private Double value; - private String unit; - - public Double getValue() { - return value; - } - - public void setValue(Double value) { - this.value = value; - } - - public String getUnit() { - return unit; - } - - public void setUnit(String unit) { - this.unit = unit; - } - } - /** * get timeout integer * @@ -194,7 +174,6 @@ public final class CollectUtil { return CommonConstants.PROM_TIME.equals(aliasField) || CommonConstants.PROM_VALUE.equals(aliasField); } - /** * is contains cryPlaceholder ^o^xxx^o^ * @@ -212,6 +191,7 @@ public final class CollectUtil { /** * match existed cry placeholder fields ^o^field^o^ + * * @param jsonElement json element * @return match field str */ @@ -222,6 +202,19 @@ public final class CollectUtil { .collect(Collectors.toSet()); } + /** + * replace cry placeholder to metrics + * + * @param metricItem metric item + * @param configmap configmap + * @return metrics + */ + public static Metrics replaceCryPlaceholderToMetrics(Metrics metricItem, Map configmap) { + JsonElement metricJson = GSON.toJsonTree(metricItem); + CollectUtil.replaceCryPlaceholder(metricJson, configmap); + return GSON.fromJson(metricJson, Metrics.class); + } + /** * json parameter replacement * @@ -431,6 +424,27 @@ public final class CollectUtil { return uri; } + public static List> getConfigmapFromPreCollectData(CollectRep.MetricsData metricsData) { + if (metricsData.getValuesCount() <= 0 || metricsData.getFieldsCount() <= 0) { + return new LinkedList<>(); + } + List> mapList = new LinkedList<>(); + for (CollectRep.ValueRow valueRow : metricsData.getValues()) { + if (valueRow.getColumnsCount() != metricsData.getFieldsCount()) { + continue; + } + Map configmapMap = new HashMap<>(valueRow.getColumnsCount()); + int index = 0; + for (CollectRep.Field field : metricsData.getFields()) { + String value = valueRow.getColumns(index); + index++; + Configmap configmap = new Configmap(field.getName(), value, Integer.valueOf(field.getType()).byteValue()); + configmapMap.put(field.getName(), configmap); + } + mapList.add(configmapMap); + } + return mapList; + } public static void replaceFieldsForPushStyleMonitor(Metrics metrics, Map configmap) { @@ -461,21 +475,48 @@ public final class CollectUtil { /** * convert original string to UTF-7 String + * * @param original original text - * @param charset encode charset + * @param charset encode charset * @return String */ - public static String stringEncodeUtf7String(String original, String charset) { + public static String stringEncodeUtf7String(String original, String charset) { return new String(original.getBytes(new CharsetProvider().charsetForName(charset)), StandardCharsets.US_ASCII); } /** * convert UTF-7 string to original String + * * @param encoded encoded String * @param charset encode charset * @return String */ - public static String utf7StringDecodeString(String encoded, String charset) { + public static String utf7StringDecodeString(String encoded, String charset) { return new String(encoded.getBytes(StandardCharsets.US_ASCII), new CharsetProvider().charsetForName(charset)); } + + /** + * double and unit + */ + public static final class DoubleAndUnit { + + private Double value; + private String unit; + + public Double getValue() { + return value; + } + + public void setValue(Double value) { + this.value = value; + } + + public String getUnit() { + return unit; + } + + public void setUnit(String unit) { + this.unit = unit; + } + } } diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/DockerMonitorE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/DockerMonitorE2eTest.java index af68686c5c..12cb606d25 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/DockerMonitorE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/http/DockerMonitorE2eTest.java @@ -22,6 +22,8 @@ import com.sun.net.httpserver.HttpServer; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol; @@ -39,6 +41,10 @@ import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.file.Files; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; /** * Integration test for Docker monitoring functionality @@ -52,9 +58,18 @@ public class DockerMonitorE2eTest extends AbstractCollectE2eTest { private static final String LOCALHOST = "127.0.0.1"; private static HttpServer mockServer; + @AfterAll + public static void tearDown() { + if (mockServer != null) { + mockServer.stop(0); + } + } + @BeforeEach public void setUp() throws Exception { + super.setUp(); + // Setup collect instance collect = new HttpCollectImpl(); // Setup mock server and endpoints @@ -65,6 +80,7 @@ public class DockerMonitorE2eTest extends AbstractCollectE2eTest { // Setup Docker API endpoints String containerResponse = loadResponseFromFile("classpath:http/docker/containers_result.txt"); String infoResponse = loadResponseFromFile("classpath:http/docker/system_result.txt"); + String containerStatsResponse = loadResponseFromFile("classpath:http/docker/containers_stats.txt"); mockServer.createContext("/containers/json", exchange -> { String query = exchange.getRequestURI().getQuery(); @@ -77,6 +93,8 @@ public class DockerMonitorE2eTest extends AbstractCollectE2eTest { }); mockServer.createContext("/info", exchange -> sendJsonResponse(exchange, infoResponse)); + mockServer.createContext("/containers/34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64/stats", exchange -> sendJsonResponse(exchange, containerStatsResponse)); + } private String loadResponseFromFile(String resourcePath) throws Exception { @@ -94,13 +112,12 @@ public class DockerMonitorE2eTest extends AbstractCollectE2eTest { @Test public void testDockerMonitor() { Job dockerJob = appService.getAppDefine("docker"); - dockerJob.getMetrics().forEach(metricsDef -> { - // Skip metrics containing "^o^" as parameter substitution is not supported in e2e tests - if (metricsDef.getHttp().getUrl().contains("^o^")) { - return; - } - validateMetricsCollection(metricsDef, metricsDef.getName()); - }); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : dockerJob.getMetrics()) { + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>()); + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName()); + configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData); + } } @Override @@ -122,11 +139,4 @@ public class DockerMonitorE2eTest extends AbstractCollectE2eTest { metrics.setHttp(protocol); return collectMetricsData(metrics, metricsDef); } - - @AfterAll - public static void tearDown() { - if (mockServer != null) { - mockServer.stop(0); - } - } } \ No newline at end of file diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_result.txt b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_result.txt index c26a69a018..0a45cccfe8 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_result.txt +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_result.txt @@ -1 +1 @@ -{"ID":"8483c578-3364-4c9e-914f-bd171d8d9a8e","Containers":3,"ContainersRunning":3,"ContainersPaused":0,"ContainersStopped":0,"Images":13,"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"],["Supports d_type","true"],["Using metacopy","false"],["Native Overlay Diff","true"],["userxattr","false"]],"Plugins":{"Volume":["local"],"Network":["bridge","host","ipvlan","macvlan","null","overlay"],"Authorization":null,"Log":["awslogs","fluentd","gcplogs","gelf","journald","json-file","local","splunk","syslog"]},"MemoryLimit":true,"SwapLimit":true,"KernelMemoryTCP":true,"CpuCfsPeriod":true,"CpuCfsQuota":true,"CPUShares":true,"CPUSet":true,"PidsLimit":true,"IPv4Forwarding":true,"BridgeNfIptables":true,"BridgeNfIp6tables":true,"Debug":false,"NFd":47,"OomKillDisable":true,"NGoroutines":63,"SystemTime":"2024-12-29T14:23:11.284383336+08:00","LoggingDriver":"json-file","CgroupDriver":"cgroupfs","CgroupVersion":"1","NEventsListener":0,"KernelVersion":"3.10.0-1160.71.1.el7.x86_64","OperatingSystem":"CentOS Linux 7 (Core)","OSVersion":"7","OSType":"linux","Architecture":"x86_64","IndexServerAddress":"https://index.docker.io/v1/","RegistryConfig":{"AllowNondistributableArtifactsCIDRs":null,"AllowNondistributableArtifactsHostnames":null,"InsecureRegistryCIDRs":["127.0.0.0/8"],"IndexConfigs":{"docker.io":{"Name":"docker.io","Mirrors":["https://exinp.mirror.aliyuncs.com/"],"Secure":true,"Official":true}},"Mirrors":["https://exinp.mirror.aliyuncs.com/"]},"NCPU":8,"MemTotal":33566269440,"GenericResources":null,"DockerRootDir":"/var/lib/docker","HttpProxy":"","HttpsProxy":"","NoProxy":"","Name":"bigdata-new-25-214","Labels":[],"ExperimentalBuild":false,"ServerVersion":"26.1.4","Runtimes":{"io.containerd.runc.v2":{"path":"runc","status":{"org.opencontainers.runtime-spec.features":"{\"ociVersionMin\":\"1.0.0\",\"ociVersionMax\":\"1.0.2-dev\",\"hooks\":[\"prestart\",\"createRuntime\",\"createContainer\",\"startContainer\",\"poststart\",\"poststop\"],\"mountOptions\":[\"acl\",\"async\",\"atime\",\"bind\",\"defaults\",\"dev\",\"diratime\",\"dirsync\",\"exec\",\"iversion\",\"lazytime\",\"loud\",\"mand\",\"noacl\",\"noatime\",\"nodev\",\"nodiratime\",\"noexec\",\"noiversion\",\"nolazytime\",\"nomand\",\"norelatime\",\"nostrictatime\",\"nosuid\",\"nosymfollow\",\"private\",\"ratime\",\"rbind\",\"rdev\",\"rdiratime\",\"relatime\",\"remount\",\"rexec\",\"rnoatime\",\"rnodev\",\"rnodiratime\",\"rnoexec\",\"rnorelatime\",\"rnostrictatime\",\"rnosuid\",\"rnosymfollow\",\"ro\",\"rprivate\",\"rrelatime\",\"rro\",\"rrw\",\"rshared\",\"rslave\",\"rstrictatime\",\"rsuid\",\"rsymfollow\",\"runbindable\",\"rw\",\"shared\",\"silent\",\"slave\",\"strictatime\",\"suid\",\"symfollow\",\"sync\",\"tmpcopyup\",\"unbindable\"],\"linux\":{\"namespaces\":[\"cgroup\",\"ipc\",\"mount\",\"network\",\"pid\",\"user\",\"uts\"],\"capabilities\":[\"CAP_CHOWN\",\"CAP_DAC_OVERRIDE\",\"CAP_DAC_READ_SEARCH\",\"CAP_FOWNER\",\"CAP_FSETID\",\"CAP_KILL\",\"CAP_SETGID\",\"CAP_SETUID\",\"CAP_SETPCAP\",\"CAP_LINUX_IMMUTABLE\",\"CAP_NET_BIND_SERVICE\",\"CAP_NET_BROADCAST\",\"CAP_NET_ADMIN\",\"CAP_NET_RAW\",\"CAP_IPC_LOCK\",\"CAP_IPC_OWNER\",\"CAP_SYS_MODULE\",\"CAP_SYS_RAWIO\",\"CAP_SYS_CHROOT\",\"CAP_SYS_PTRACE\",\"CAP_SYS_PACCT\",\"CAP_SYS_ADMIN\",\"CAP_SYS_BOOT\",\"CAP_SYS_NICE\",\"CAP_SYS_RESOURCE\",\"CAP_SYS_TIME\",\"CAP_SYS_TTY_CONFIG\",\"CAP_MKNOD\",\"CAP_LEASE\",\"CAP_AUDIT_WRITE\",\"CAP_AUDIT_CONTROL\",\"CAP_SETFCAP\",\"CAP_MAC_OVERRIDE\",\"CAP_MAC_ADMIN\",\"CAP_SYSLOG\",\"CAP_WAKE_ALARM\",\"CAP_BLOCK_SUSPEND\",\"CAP_AUDIT_READ\",\"CAP_PERFMON\",\"CAP_BPF\",\"CAP_CHECKPOINT_RESTORE\"],\"cgroup\":{\"v1\":true,\"v2\":true,\"systemd\":true,\"systemdUser\":true},\"seccomp\":{\"enabled\":true,\"actions\":[\"SCMP_ACT_ALLOW\",\"SCMP_ACT_ERRNO\",\"SCMP_ACT_KILL\",\"SCMP_ACT_KILL_PROCESS\",\"SCMP_ACT_KILL_THREAD\",\"SCMP_ACT_LOG\",\"SCMP_ACT_NOTIFY\",\"SCMP_ACT_TRACE\",\"SCMP_ACT_TRAP\"],\"operators\":[\"SCMP_CMP_EQ\",\"SCMP_CMP_GE\",\"SCMP_CMP_GT\",\"SCMP_CMP_LE\",\"SCMP_CMP_LT\",\"SCMP_CMP_MASKED_EQ\",\"SCMP_CMP_NE\"],\"archs\":[\"SCMP_ARCH_AARCH64\",\"SCMP_ARCH_ARM\",\"SCMP_ARCH_MIPS\",\"SCMP_ARCH_MIPS64\",\"SCMP_ARCH_MIPS64N32\",\"SCMP_ARCH_MIPSEL\",\"SCMP_ARCH_MIPSEL64\",\"SCMP_ARCH_MIPSEL64N32\",\"SCMP_ARCH_PPC\",\"SCMP_ARCH_PPC64\",\"SCMP_ARCH_PPC64LE\",\"SCMP_ARCH_RISCV64\",\"SCMP_ARCH_S390\",\"SCMP_ARCH_S390X\",\"SCMP_ARCH_X32\",\"SCMP_ARCH_X86\",\"SCMP_ARCH_X86_64\"]},\"apparmor\":{\"enabled\":true},\"selinux\":{\"enabled\":true}},\"annotations\":{\"io.github.seccomp.libseccomp.version\":\"2.3.1\",\"org.opencontainers.runc.checkpoint.enabled\":\"true\",\"org.opencontainers.runc.commit\":\"v1.1.12-0-g51d5e94\",\"org.opencontainers.runc.version\":\"1.1.12\"}}"}},"runc":{"path":"runc","status":{"org.opencontainers.runtime-spec.features":"{\"ociVersionMin\":\"1.0.0\",\"ociVersionMax\":\"1.0.2-dev\",\"hooks\":[\"prestart\",\"createRuntime\",\"createContainer\",\"startContainer\",\"poststart\",\"poststop\"],\"mountOptions\":[\"acl\",\"async\",\"atime\",\"bind\",\"defaults\",\"dev\",\"diratime\",\"dirsync\",\"exec\",\"iversion\",\"lazytime\",\"loud\",\"mand\",\"noacl\",\"noatime\",\"nodev\",\"nodiratime\",\"noexec\",\"noiversion\",\"nolazytime\",\"nomand\",\"norelatime\",\"nostrictatime\",\"nosuid\",\"nosymfollow\",\"private\",\"ratime\",\"rbind\",\"rdev\",\"rdiratime\",\"relatime\",\"remount\",\"rexec\",\"rnoatime\",\"rnodev\",\"rnodiratime\",\"rnoexec\",\"rnorelatime\",\"rnostrictatime\",\"rnosuid\",\"rnosymfollow\",\"ro\",\"rprivate\",\"rrelatime\",\"rro\",\"rrw\",\"rshared\",\"rslave\",\"rstrictatime\",\"rsuid\",\"rsymfollow\",\"runbindable\",\"rw\",\"shared\",\"silent\",\"slave\",\"strictatime\",\"suid\",\"symfollow\",\"sync\",\"tmpcopyup\",\"unbindable\"],\"linux\":{\"namespaces\":[\"cgroup\",\"ipc\",\"mount\",\"network\",\"pid\",\"user\",\"uts\"],\"capabilities\":[\"CAP_CHOWN\",\"CAP_DAC_OVERRIDE\",\"CAP_DAC_READ_SEARCH\",\"CAP_FOWNER\",\"CAP_FSETID\",\"CAP_KILL\",\"CAP_SETGID\",\"CAP_SETUID\",\"CAP_SETPCAP\",\"CAP_LINUX_IMMUTABLE\",\"CAP_NET_BIND_SERVICE\",\"CAP_NET_BROADCAST\",\"CAP_NET_ADMIN\",\"CAP_NET_RAW\",\"CAP_IPC_LOCK\",\"CAP_IPC_OWNER\",\"CAP_SYS_MODULE\",\"CAP_SYS_RAWIO\",\"CAP_SYS_CHROOT\",\"CAP_SYS_PTRACE\",\"CAP_SYS_PACCT\",\"CAP_SYS_ADMIN\",\"CAP_SYS_BOOT\",\"CAP_SYS_NICE\",\"CAP_SYS_RESOURCE\",\"CAP_SYS_TIME\",\"CAP_SYS_TTY_CONFIG\",\"CAP_MKNOD\",\"CAP_LEASE\",\"CAP_AUDIT_WRITE\",\"CAP_AUDIT_CONTROL\",\"CAP_SETFCAP\",\"CAP_MAC_OVERRIDE\",\"CAP_MAC_ADMIN\",\"CAP_SYSLOG\",\"CAP_WAKE_ALARM\",\"CAP_BLOCK_SUSPEND\",\"CAP_AUDIT_READ\",\"CAP_PERFMON\",\"CAP_BPF\",\"CAP_CHECKPOINT_RESTORE\"],\"cgroup\":{\"v1\":true,\"v2\":true,\"systemd\":true,\"systemdUser\":true},\"seccomp\":{\"enabled\":true,\"actions\":[\"SCMP_ACT_ALLOW\",\"SCMP_ACT_ERRNO\",\"SCMP_ACT_KILL\",\"SCMP_ACT_KILL_PROCESS\",\"SCMP_ACT_KILL_THREAD\",\"SCMP_ACT_LOG\",\"SCMP_ACT_NOTIFY\",\"SCMP_ACT_TRACE\",\"SCMP_ACT_TRAP\"],\"operators\":[\"SCMP_CMP_EQ\",\"SCMP_CMP_GE\",\"SCMP_CMP_GT\",\"SCMP_CMP_LE\",\"SCMP_CMP_LT\",\"SCMP_CMP_MASKED_EQ\",\"SCMP_CMP_NE\"],\"archs\":[\"SCMP_ARCH_AARCH64\",\"SCMP_ARCH_ARM\",\"SCMP_ARCH_MIPS\",\"SCMP_ARCH_MIPS64\",\"SCMP_ARCH_MIPS64N32\",\"SCMP_ARCH_MIPSEL\",\"SCMP_ARCH_MIPSEL64\",\"SCMP_ARCH_MIPSEL64N32\",\"SCMP_ARCH_PPC\",\"SCMP_ARCH_PPC64\",\"SCMP_ARCH_PPC64LE\",\"SCMP_ARCH_RISCV64\",\"SCMP_ARCH_S390\",\"SCMP_ARCH_S390X\",\"SCMP_ARCH_X32\",\"SCMP_ARCH_X86\",\"SCMP_ARCH_X86_64\"]},\"apparmor\":{\"enabled\":true},\"selinux\":{\"enabled\":true}},\"annotations\":{\"io.github.seccomp.libseccomp.version\":\"2.3.1\",\"org.opencontainers.runc.checkpoint.enabled\":\"true\",\"org.opencontainers.runc.commit\":\"v1.1.12-0-g51d5e94\",\"org.opencontainers.runc.version\":\"1.1.12\"}}"}}},"DefaultRuntime":"runc","Swarm":{"NodeID":"","NodeAddr":"","LocalNodeState":"inactive","ControlAvailable":false,"Error":"","RemoteManagers":null},"LiveRestoreEnabled":false,"Isolation":"","InitBinary":"docker-init","ContainerdCommit":{"ID":"d2d58213f83a351ca8f528a95fbd145f5654e957","Expected":"d2d58213f83a351ca8f528a95fbd145f5654e957"},"RuncCommit":{"ID":"v1.1.12-0-g51d5e94","Expected":"v1.1.12-0-g51d5e94"},"InitCommit":{"ID":"de40ad0","Expected":"de40ad0"},"SecurityOptions":["name=seccomp,profile=builtin"],"CDISpecDirs":[],"Warnings":["[DEPRECATION NOTICE]: API is accessible on http://0.0.0.0:2375 without encryption.\n Access to the remote API is equivalent to root access on the host. Refer\n to the 'Docker daemon attack surface' section in the documentation for\n more information: https://docs.docker.com/go/attack-surface/\nIn future versions this will be a hard failure preventing the daemon from starting! Learn more at: https://docs.docker.com/go/api-security/"]} \ No newline at end of file +[{"Id":"34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64","Names":["/mysql"],"Image":"mysql:8.0","ImageID":"sha256:6c55ddbef96911f9f36d1330ffe3f7557c019d49434e738cafabd1a3dd6b4bac","Command":"docker-entrypoint.sh mysqld","Created":1735286602,"Ports":[{"IP":"0.0.0.0","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"IP":"::","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"PrivatePort":33060,"Type":"tcp"}],"Labels":{},"State":"running","Status":"Up 42 hours","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:04","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"6925c9ece99d84ed57c4b08a41c38d009cd8becc9cc2e941eb2b572478a22747","Gateway":"172.17.0.1","IPAddress":"172.17.0.4","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"bind","Source":"/docker/mysql/conf/my.cnf","Destination":"/etc/mysql/my.cnf","Mode":"","RW":true,"Propagation":"rprivate"},{"Type":"bind","Source":"/docker/mysql/data","Destination":"/var/lib/mysql","Mode":"","RW":true,"Propagation":"rprivate"}]},{"Id":"d1f070438d28de7059113df7e79423f5ba1fdc4968cee4914b661cec222ddb06","Names":["/dbgate-instance"],"Image":"dbgate:latest","ImageID":"sha256:107efa35665b98c97cb9698cae2daf3f80aa88854ac90d81bba24b8de02c830d","Command":"/home/dbgate-docker/entrypoint.sh","Created":1734751223,"Ports":[{"IP":"0.0.0.0","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"},{"IP":"::","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"}],"Labels":{"org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"22.04"},"State":"running","Status":"Up 7 days","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:03","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"a8f31d5d2ec0100137dced7550b3d662b0804bf8f767a457cbaebda863fb4504","Gateway":"172.17.0.1","IPAddress":"172.17.0.3","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"volume","Name":"d0d9b8e8edce3dc092b2b50a1f9c6bb754fd556cffe95be8ae00e016daaaa3fd","Source":"","Destination":"/root/.dbgate","Driver":"local","Mode":"","RW":true,"Propagation":""}]}] \ No newline at end of file diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_stats.txt b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_stats.txt new file mode 100644 index 0000000000..ce75d98163 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/containers_stats.txt @@ -0,0 +1 @@ +{"read":"2000-01-01T00:00:15.864368408Z","preread":"2000-01-01T00:00:14.86209639Z","pids_stats":{"current":41},"blkio_stats":{"io_service_bytes_recursive":[{"major":8,"minor":0,"op":"Read","value":118784},{"major":8,"minor":0,"op":"Write","value":267395072},{"major":8,"minor":0,"op":"Sync","value":267395072},{"major":8,"minor":0,"op":"Async","value":118784},{"major":8,"minor":0,"op":"Total","value":267513856},{"major":253,"minor":0,"op":"Read","value":118784},{"major":253,"minor":0,"op":"Write","value":267395072},{"major":253,"minor":0,"op":"Sync","value":267395072},{"major":253,"minor":0,"op":"Async","value":118784},{"major":253,"minor":0,"op":"Total","value":267513856}],"io_serviced_recursive":[{"major":8,"minor":0,"op":"Read","value":8},{"major":8,"minor":0,"op":"Write","value":7892},{"major":8,"minor":0,"op":"Sync","value":7892},{"major":8,"minor":0,"op":"Async","value":8},{"major":8,"minor":0,"op":"Total","value":7900},{"major":253,"minor":0,"op":"Read","value":8},{"major":253,"minor":0,"op":"Write","value":7892},{"major":253,"minor":0,"op":"Sync","value":7892},{"major":253,"minor":0,"op":"Async","value":8},{"major":253,"minor":0,"op":"Total","value":7900}],"io_queue_recursive":[],"io_service_time_recursive":[],"io_wait_time_recursive":[],"io_merged_recursive":[],"io_time_recursive":[],"sectors_recursive":[]},"num_procs":0,"storage_stats":{},"cpu_stats":{"cpu_usage":{"total_usage":4810174080874,"percpu_usage":[733251885463,588352569755,649761861064,692942680516,618939038384,513206148155,431772846470,581947051067],"usage_in_kernelmode":1191390000000,"usage_in_usermode":1463160000000},"system_cpu_usage":21196078922904483,"online_cpus":8,"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"precpu_stats":{"cpu_usage":{"total_usage":4810165603627,"percpu_usage":[733250739914,588352569755,649761643572,692942567027,618937059553,513205711609,431769258762,581946053435],"usage_in_kernelmode":1191390000000,"usage_in_usermode":1463160000000},"system_cpu_usage":21196070922904483,"online_cpus":8,"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"memory_stats":{"usage":505090048,"max_usage":519450624,"stats":{"active_anon":396529664,"active_file":52400128,"cache":108560384,"dirty":0,"hierarchical_memory_limit":9223372036854771712,"hierarchical_memsw_limit":9223372036854771712,"inactive_anon":0,"inactive_file":56160256,"mapped_file":40960,"pgfault":409148,"pgmajfault":0,"pgpgin":343733,"pgpgout":220420,"rss":396529664,"rss_huge":0,"total_active_anon":396529664,"total_active_file":52400128,"total_cache":108560384,"total_dirty":0,"total_inactive_anon":0,"total_inactive_file":56160256,"total_mapped_file":40960,"total_pgfault":0,"total_pgmajfault":0,"total_pgpgin":0,"total_pgpgout":0,"total_rss":396529664,"total_rss_huge":0,"total_unevictable":0,"total_writeback":0,"unevictable":0,"writeback":0},"limit":33566269440},"name":"/mysql","id":"34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64","networks":{"eth0":{"rx_bytes":68182,"rx_packets":533,"rx_errors":0,"rx_dropped":0,"tx_bytes":211850,"tx_packets":361,"tx_errors":0,"tx_dropped":0}}} \ No newline at end of file diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/system_result.txt b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/system_result.txt index 0a45cccfe8..c26a69a018 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/system_result.txt +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/resources/http/docker/system_result.txt @@ -1 +1 @@ -[{"Id":"34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64","Names":["/mysql"],"Image":"mysql:8.0","ImageID":"sha256:6c55ddbef96911f9f36d1330ffe3f7557c019d49434e738cafabd1a3dd6b4bac","Command":"docker-entrypoint.sh mysqld","Created":1735286602,"Ports":[{"IP":"0.0.0.0","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"IP":"::","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"PrivatePort":33060,"Type":"tcp"}],"Labels":{},"State":"running","Status":"Up 42 hours","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:04","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"6925c9ece99d84ed57c4b08a41c38d009cd8becc9cc2e941eb2b572478a22747","Gateway":"172.17.0.1","IPAddress":"172.17.0.4","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"bind","Source":"/docker/mysql/conf/my.cnf","Destination":"/etc/mysql/my.cnf","Mode":"","RW":true,"Propagation":"rprivate"},{"Type":"bind","Source":"/docker/mysql/data","Destination":"/var/lib/mysql","Mode":"","RW":true,"Propagation":"rprivate"}]},{"Id":"d1f070438d28de7059113df7e79423f5ba1fdc4968cee4914b661cec222ddb06","Names":["/dbgate-instance"],"Image":"dbgate:latest","ImageID":"sha256:107efa35665b98c97cb9698cae2daf3f80aa88854ac90d81bba24b8de02c830d","Command":"/home/dbgate-docker/entrypoint.sh","Created":1734751223,"Ports":[{"IP":"0.0.0.0","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"},{"IP":"::","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"}],"Labels":{"org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"22.04"},"State":"running","Status":"Up 7 days","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:03","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"a8f31d5d2ec0100137dced7550b3d662b0804bf8f767a457cbaebda863fb4504","Gateway":"172.17.0.1","IPAddress":"172.17.0.3","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"volume","Name":"d0d9b8e8edce3dc092b2b50a1f9c6bb754fd556cffe95be8ae00e016daaaa3fd","Source":"","Destination":"/root/.dbgate","Driver":"local","Mode":"","RW":true,"Propagation":""}]}] \ No newline at end of file +{"ID":"8483c578-3364-4c9e-914f-bd171d8d9a8e","Containers":3,"ContainersRunning":3,"ContainersPaused":0,"ContainersStopped":0,"Images":13,"Driver":"overlay2","DriverStatus":[["Backing Filesystem","extfs"],["Supports d_type","true"],["Using metacopy","false"],["Native Overlay Diff","true"],["userxattr","false"]],"Plugins":{"Volume":["local"],"Network":["bridge","host","ipvlan","macvlan","null","overlay"],"Authorization":null,"Log":["awslogs","fluentd","gcplogs","gelf","journald","json-file","local","splunk","syslog"]},"MemoryLimit":true,"SwapLimit":true,"KernelMemoryTCP":true,"CpuCfsPeriod":true,"CpuCfsQuota":true,"CPUShares":true,"CPUSet":true,"PidsLimit":true,"IPv4Forwarding":true,"BridgeNfIptables":true,"BridgeNfIp6tables":true,"Debug":false,"NFd":47,"OomKillDisable":true,"NGoroutines":63,"SystemTime":"2024-12-29T14:23:11.284383336+08:00","LoggingDriver":"json-file","CgroupDriver":"cgroupfs","CgroupVersion":"1","NEventsListener":0,"KernelVersion":"3.10.0-1160.71.1.el7.x86_64","OperatingSystem":"CentOS Linux 7 (Core)","OSVersion":"7","OSType":"linux","Architecture":"x86_64","IndexServerAddress":"https://index.docker.io/v1/","RegistryConfig":{"AllowNondistributableArtifactsCIDRs":null,"AllowNondistributableArtifactsHostnames":null,"InsecureRegistryCIDRs":["127.0.0.0/8"],"IndexConfigs":{"docker.io":{"Name":"docker.io","Mirrors":["https://exinp.mirror.aliyuncs.com/"],"Secure":true,"Official":true}},"Mirrors":["https://exinp.mirror.aliyuncs.com/"]},"NCPU":8,"MemTotal":33566269440,"GenericResources":null,"DockerRootDir":"/var/lib/docker","HttpProxy":"","HttpsProxy":"","NoProxy":"","Name":"bigdata-new-25-214","Labels":[],"ExperimentalBuild":false,"ServerVersion":"26.1.4","Runtimes":{"io.containerd.runc.v2":{"path":"runc","status":{"org.opencontainers.runtime-spec.features":"{\"ociVersionMin\":\"1.0.0\",\"ociVersionMax\":\"1.0.2-dev\",\"hooks\":[\"prestart\",\"createRuntime\",\"createContainer\",\"startContainer\",\"poststart\",\"poststop\"],\"mountOptions\":[\"acl\",\"async\",\"atime\",\"bind\",\"defaults\",\"dev\",\"diratime\",\"dirsync\",\"exec\",\"iversion\",\"lazytime\",\"loud\",\"mand\",\"noacl\",\"noatime\",\"nodev\",\"nodiratime\",\"noexec\",\"noiversion\",\"nolazytime\",\"nomand\",\"norelatime\",\"nostrictatime\",\"nosuid\",\"nosymfollow\",\"private\",\"ratime\",\"rbind\",\"rdev\",\"rdiratime\",\"relatime\",\"remount\",\"rexec\",\"rnoatime\",\"rnodev\",\"rnodiratime\",\"rnoexec\",\"rnorelatime\",\"rnostrictatime\",\"rnosuid\",\"rnosymfollow\",\"ro\",\"rprivate\",\"rrelatime\",\"rro\",\"rrw\",\"rshared\",\"rslave\",\"rstrictatime\",\"rsuid\",\"rsymfollow\",\"runbindable\",\"rw\",\"shared\",\"silent\",\"slave\",\"strictatime\",\"suid\",\"symfollow\",\"sync\",\"tmpcopyup\",\"unbindable\"],\"linux\":{\"namespaces\":[\"cgroup\",\"ipc\",\"mount\",\"network\",\"pid\",\"user\",\"uts\"],\"capabilities\":[\"CAP_CHOWN\",\"CAP_DAC_OVERRIDE\",\"CAP_DAC_READ_SEARCH\",\"CAP_FOWNER\",\"CAP_FSETID\",\"CAP_KILL\",\"CAP_SETGID\",\"CAP_SETUID\",\"CAP_SETPCAP\",\"CAP_LINUX_IMMUTABLE\",\"CAP_NET_BIND_SERVICE\",\"CAP_NET_BROADCAST\",\"CAP_NET_ADMIN\",\"CAP_NET_RAW\",\"CAP_IPC_LOCK\",\"CAP_IPC_OWNER\",\"CAP_SYS_MODULE\",\"CAP_SYS_RAWIO\",\"CAP_SYS_CHROOT\",\"CAP_SYS_PTRACE\",\"CAP_SYS_PACCT\",\"CAP_SYS_ADMIN\",\"CAP_SYS_BOOT\",\"CAP_SYS_NICE\",\"CAP_SYS_RESOURCE\",\"CAP_SYS_TIME\",\"CAP_SYS_TTY_CONFIG\",\"CAP_MKNOD\",\"CAP_LEASE\",\"CAP_AUDIT_WRITE\",\"CAP_AUDIT_CONTROL\",\"CAP_SETFCAP\",\"CAP_MAC_OVERRIDE\",\"CAP_MAC_ADMIN\",\"CAP_SYSLOG\",\"CAP_WAKE_ALARM\",\"CAP_BLOCK_SUSPEND\",\"CAP_AUDIT_READ\",\"CAP_PERFMON\",\"CAP_BPF\",\"CAP_CHECKPOINT_RESTORE\"],\"cgroup\":{\"v1\":true,\"v2\":true,\"systemd\":true,\"systemdUser\":true},\"seccomp\":{\"enabled\":true,\"actions\":[\"SCMP_ACT_ALLOW\",\"SCMP_ACT_ERRNO\",\"SCMP_ACT_KILL\",\"SCMP_ACT_KILL_PROCESS\",\"SCMP_ACT_KILL_THREAD\",\"SCMP_ACT_LOG\",\"SCMP_ACT_NOTIFY\",\"SCMP_ACT_TRACE\",\"SCMP_ACT_TRAP\"],\"operators\":[\"SCMP_CMP_EQ\",\"SCMP_CMP_GE\",\"SCMP_CMP_GT\",\"SCMP_CMP_LE\",\"SCMP_CMP_LT\",\"SCMP_CMP_MASKED_EQ\",\"SCMP_CMP_NE\"],\"archs\":[\"SCMP_ARCH_AARCH64\",\"SCMP_ARCH_ARM\",\"SCMP_ARCH_MIPS\",\"SCMP_ARCH_MIPS64\",\"SCMP_ARCH_MIPS64N32\",\"SCMP_ARCH_MIPSEL\",\"SCMP_ARCH_MIPSEL64\",\"SCMP_ARCH_MIPSEL64N32\",\"SCMP_ARCH_PPC\",\"SCMP_ARCH_PPC64\",\"SCMP_ARCH_PPC64LE\",\"SCMP_ARCH_RISCV64\",\"SCMP_ARCH_S390\",\"SCMP_ARCH_S390X\",\"SCMP_ARCH_X32\",\"SCMP_ARCH_X86\",\"SCMP_ARCH_X86_64\"]},\"apparmor\":{\"enabled\":true},\"selinux\":{\"enabled\":true}},\"annotations\":{\"io.github.seccomp.libseccomp.version\":\"2.3.1\",\"org.opencontainers.runc.checkpoint.enabled\":\"true\",\"org.opencontainers.runc.commit\":\"v1.1.12-0-g51d5e94\",\"org.opencontainers.runc.version\":\"1.1.12\"}}"}},"runc":{"path":"runc","status":{"org.opencontainers.runtime-spec.features":"{\"ociVersionMin\":\"1.0.0\",\"ociVersionMax\":\"1.0.2-dev\",\"hooks\":[\"prestart\",\"createRuntime\",\"createContainer\",\"startContainer\",\"poststart\",\"poststop\"],\"mountOptions\":[\"acl\",\"async\",\"atime\",\"bind\",\"defaults\",\"dev\",\"diratime\",\"dirsync\",\"exec\",\"iversion\",\"lazytime\",\"loud\",\"mand\",\"noacl\",\"noatime\",\"nodev\",\"nodiratime\",\"noexec\",\"noiversion\",\"nolazytime\",\"nomand\",\"norelatime\",\"nostrictatime\",\"nosuid\",\"nosymfollow\",\"private\",\"ratime\",\"rbind\",\"rdev\",\"rdiratime\",\"relatime\",\"remount\",\"rexec\",\"rnoatime\",\"rnodev\",\"rnodiratime\",\"rnoexec\",\"rnorelatime\",\"rnostrictatime\",\"rnosuid\",\"rnosymfollow\",\"ro\",\"rprivate\",\"rrelatime\",\"rro\",\"rrw\",\"rshared\",\"rslave\",\"rstrictatime\",\"rsuid\",\"rsymfollow\",\"runbindable\",\"rw\",\"shared\",\"silent\",\"slave\",\"strictatime\",\"suid\",\"symfollow\",\"sync\",\"tmpcopyup\",\"unbindable\"],\"linux\":{\"namespaces\":[\"cgroup\",\"ipc\",\"mount\",\"network\",\"pid\",\"user\",\"uts\"],\"capabilities\":[\"CAP_CHOWN\",\"CAP_DAC_OVERRIDE\",\"CAP_DAC_READ_SEARCH\",\"CAP_FOWNER\",\"CAP_FSETID\",\"CAP_KILL\",\"CAP_SETGID\",\"CAP_SETUID\",\"CAP_SETPCAP\",\"CAP_LINUX_IMMUTABLE\",\"CAP_NET_BIND_SERVICE\",\"CAP_NET_BROADCAST\",\"CAP_NET_ADMIN\",\"CAP_NET_RAW\",\"CAP_IPC_LOCK\",\"CAP_IPC_OWNER\",\"CAP_SYS_MODULE\",\"CAP_SYS_RAWIO\",\"CAP_SYS_CHROOT\",\"CAP_SYS_PTRACE\",\"CAP_SYS_PACCT\",\"CAP_SYS_ADMIN\",\"CAP_SYS_BOOT\",\"CAP_SYS_NICE\",\"CAP_SYS_RESOURCE\",\"CAP_SYS_TIME\",\"CAP_SYS_TTY_CONFIG\",\"CAP_MKNOD\",\"CAP_LEASE\",\"CAP_AUDIT_WRITE\",\"CAP_AUDIT_CONTROL\",\"CAP_SETFCAP\",\"CAP_MAC_OVERRIDE\",\"CAP_MAC_ADMIN\",\"CAP_SYSLOG\",\"CAP_WAKE_ALARM\",\"CAP_BLOCK_SUSPEND\",\"CAP_AUDIT_READ\",\"CAP_PERFMON\",\"CAP_BPF\",\"CAP_CHECKPOINT_RESTORE\"],\"cgroup\":{\"v1\":true,\"v2\":true,\"systemd\":true,\"systemdUser\":true},\"seccomp\":{\"enabled\":true,\"actions\":[\"SCMP_ACT_ALLOW\",\"SCMP_ACT_ERRNO\",\"SCMP_ACT_KILL\",\"SCMP_ACT_KILL_PROCESS\",\"SCMP_ACT_KILL_THREAD\",\"SCMP_ACT_LOG\",\"SCMP_ACT_NOTIFY\",\"SCMP_ACT_TRACE\",\"SCMP_ACT_TRAP\"],\"operators\":[\"SCMP_CMP_EQ\",\"SCMP_CMP_GE\",\"SCMP_CMP_GT\",\"SCMP_CMP_LE\",\"SCMP_CMP_LT\",\"SCMP_CMP_MASKED_EQ\",\"SCMP_CMP_NE\"],\"archs\":[\"SCMP_ARCH_AARCH64\",\"SCMP_ARCH_ARM\",\"SCMP_ARCH_MIPS\",\"SCMP_ARCH_MIPS64\",\"SCMP_ARCH_MIPS64N32\",\"SCMP_ARCH_MIPSEL\",\"SCMP_ARCH_MIPSEL64\",\"SCMP_ARCH_MIPSEL64N32\",\"SCMP_ARCH_PPC\",\"SCMP_ARCH_PPC64\",\"SCMP_ARCH_PPC64LE\",\"SCMP_ARCH_RISCV64\",\"SCMP_ARCH_S390\",\"SCMP_ARCH_S390X\",\"SCMP_ARCH_X32\",\"SCMP_ARCH_X86\",\"SCMP_ARCH_X86_64\"]},\"apparmor\":{\"enabled\":true},\"selinux\":{\"enabled\":true}},\"annotations\":{\"io.github.seccomp.libseccomp.version\":\"2.3.1\",\"org.opencontainers.runc.checkpoint.enabled\":\"true\",\"org.opencontainers.runc.commit\":\"v1.1.12-0-g51d5e94\",\"org.opencontainers.runc.version\":\"1.1.12\"}}"}}},"DefaultRuntime":"runc","Swarm":{"NodeID":"","NodeAddr":"","LocalNodeState":"inactive","ControlAvailable":false,"Error":"","RemoteManagers":null},"LiveRestoreEnabled":false,"Isolation":"","InitBinary":"docker-init","ContainerdCommit":{"ID":"d2d58213f83a351ca8f528a95fbd145f5654e957","Expected":"d2d58213f83a351ca8f528a95fbd145f5654e957"},"RuncCommit":{"ID":"v1.1.12-0-g51d5e94","Expected":"v1.1.12-0-g51d5e94"},"InitCommit":{"ID":"de40ad0","Expected":"de40ad0"},"SecurityOptions":["name=seccomp,profile=builtin"],"CDISpecDirs":[],"Warnings":["[DEPRECATION NOTICE]: API is accessible on http://0.0.0.0:2375 without encryption.\n Access to the remote API is equivalent to root access on the host. Refer\n to the 'Docker daemon attack surface' section in the documentation for\n more information: https://docs.docker.com/go/attack-surface/\nIn future versions this will be a hard failure preventing the daemon from starting! Learn more at: https://docs.docker.com/go/api-security/"]} \ No newline at end of file diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java index c7c982e74b..3cd21cb973 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java @@ -18,6 +18,11 @@ package org.apache.hertzbeat.collector.collect; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.dispatch.timer.Timeout; +import org.apache.hertzbeat.collector.dispatch.timer.WheelTimerTask; +import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.Protocol; import org.apache.hertzbeat.common.entity.message.CollectRep; @@ -26,9 +31,14 @@ import org.apache.hertzbeat.manager.service.impl.ObjectStoreConfigServiceImpl; import org.junit.jupiter.api.Assertions; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import java.util.List; import java.util.stream.Collectors; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + /** * AbstractCollectE2eTest */ @@ -40,12 +50,26 @@ public abstract class AbstractCollectE2eTest { protected AbstractCollect collect; + protected MetricsCollect metricsCollect; + protected Metrics metrics; @Mock protected ObjectStoreConfigServiceImpl objectStoreConfigService; + @Mock + private WheelTimerTask timerJob; + @Mock + private Timeout timeout; + @Mock + private Job job; public void setUp() throws Exception { + // Initialize mocks + MockitoAnnotations.openMocks(this); + when(timeout.task()).thenReturn(timerJob); + when(timerJob.getJob()).thenReturn(job); + metricsCollect = new MetricsCollect(mock(Metrics.class), timeout, mock(CollectDataDispatch.class), null, List.of()); + // Initialize services and components appService.run(); metrics = new Metrics(); @@ -55,32 +79,36 @@ public abstract class AbstractCollectE2eTest { * Validate metrics collection, check if the metrics values are not empty
* We believe that all monitoring metrics should have data */ - protected void validateMetricsCollection(Metrics metricsDef, String metricName) { + protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, String metricName) { CollectRep.MetricsData.Builder metricsData = collectMetrics(metricsDef); - + + metricsCollect.calculateFields(metricsDef, metricsData); + Assertions.assertTrue(metricsData.getValuesList().size() > 0, String.format("%s metrics values should not be empty", metricName)); - CollectRep.ValueRow firstRow = metricsData.getValuesList().get(0); - for (int i = 0; i < firstRow.getColumnsCount(); i++) { - Assertions.assertFalse(firstRow.getColumns(i).isEmpty(), - String.format("%s metric column %d should not be empty", metricName, i)); + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + for (int i = 0; i < valueRow.getColumnsCount(); i++) { + Assertions.assertFalse(valueRow.getColumns(i).isEmpty(), + String.format("%s metric column %d should not be empty", metricName, i)); + } } log.info("{} metrics validation passed", metricName); + return metricsData.build(); } - + protected void setMetricsAliasFields(Metrics metrics, Metrics metricsDef) { metrics.setAliasFields(metricsDef.getAliasFields() == null ? metricsDef.getFields().stream() - .map(Metrics.Field::getField) - .collect(Collectors.toList()) : + .map(Metrics.Field::getField) + .collect(Collectors.toList()) : metricsDef.getAliasFields()); } protected abstract CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef); - protected CollectRep.MetricsData.Builder collectMetricsData(Metrics metrics, Metrics metricsDef){ + protected CollectRep.MetricsData.Builder collectMetricsData(Metrics metrics, Metrics metricsDef) { setMetricsAliasFields(metrics, metricsDef); // Collect metrics