update parser to parse from prometheus txt metrics data (#1421)

Signed-off-by: tomsun28 <tomsun28@outlook.com>
2026-09-17 09:40:58 +00:00 · 2023-12-15 11:58:12 +08:00
parent 405b8825a0
commit 3ff13aa6ea
4 changed files with 341 additions and 32 deletions
@@ -389,13 +389,13 @@ public class HttpCollectImpl extends AbstractCollect {
                for (String aliasField : aliasFields) {
                    if ("value".equals(aliasField)) {
                        if (metric.getCounter() != null) {
-                            valueRowBuilder.addColumns(metric.getCounter().getValue() + "");
+                            valueRowBuilder.addColumns(String.valueOf(metric.getCounter().getValue()));
                        } else if (metric.getGauge() != null) {
-                            valueRowBuilder.addColumns(metric.getGauge().getValue() + "");
+                            valueRowBuilder.addColumns(String.valueOf(metric.getGauge().getValue()));
                        } else if (metric.getUntyped() != null) {
-                            valueRowBuilder.addColumns(metric.getUntyped().getValue() + "");
+                            valueRowBuilder.addColumns(String.valueOf(metric.getUntyped().getValue()));
                        } else if (metric.getInfo() != null) {
-                            valueRowBuilder.addColumns(metric.getInfo().getValue() + "");
+                            valueRowBuilder.addColumns(String.valueOf(metric.getInfo().getValue()));
                        }
                    } else {
                        valueRowBuilder.addColumns(labelMap.get(aliasField));
@@ -39,9 +39,8 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.util.EntityUtils;
 import org.dromara.hertzbeat.collector.collect.common.http.CommonHttpClient;
-import org.dromara.hertzbeat.collector.collect.http.promethus.exporter.ExporterParser;
-import org.dromara.hertzbeat.collector.collect.http.promethus.exporter.MetricFamily;
-import org.dromara.hertzbeat.collector.collect.http.promethus.exporter.MetricType;
+import org.dromara.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
+import org.dromara.hertzbeat.collector.collect.prometheus.parser.TextParser;
 import org.dromara.hertzbeat.collector.dispatch.DispatchConstants;
 import org.dromara.hertzbeat.collector.util.CollectUtil;
 import org.dromara.hertzbeat.common.constants.CollectorConstants;
@@ -60,7 +59,6 @@ import java.net.ConnectException;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

@@ -170,15 +168,9 @@ public class PrometheusAutoCollectImpl {
        }
    }
    
-    private static final Map<Long, ExporterParser> EXPORTER_PARSER_TABLE = new ConcurrentHashMap<>();
-    
    private List<CollectRep.MetricsData> parseResponseByPrometheusExporter(String resp, List<String> aliasFields,
                                                                           CollectRep.MetricsData.Builder builder) {
-        if (!EXPORTER_PARSER_TABLE.containsKey(builder.getId())) {
-            EXPORTER_PARSER_TABLE.put(builder.getId(), new ExporterParser());
-        }
-        ExporterParser parser = EXPORTER_PARSER_TABLE.get(builder.getId());
-        Map<String, MetricFamily> metricFamilyMap = parser.textToMetric(resp);
+        Map<String, MetricFamily> metricFamilyMap = TextParser.textToMetricFamilies(resp);
        List<CollectRep.MetricsData> metricsDataList = new LinkedList<>();
        for (Map.Entry<String, MetricFamily> entry : metricFamilyMap.entrySet()) {
            builder.clearMetrics();
@@ -187,16 +179,12 @@ public class PrometheusAutoCollectImpl {
            String metricsName = entry.getKey();
            builder.setMetrics(metricsName);
            MetricFamily metricFamily = entry.getValue();
-            if (metricFamily.getMetricType() == MetricType.HISTOGRAM || metricFamily.getMetricType() == MetricType.SUMMARY) {
-                // todo HISTOGRAM SUMMARY
-                continue;
-            }
            if (!metricFamily.getMetricList().isEmpty()) {
                List<String> metricsFields = new LinkedList<>();
                for (int index = 0; index < metricFamily.getMetricList().size(); index++) {
                    MetricFamily.Metric metric = metricFamily.getMetricList().get(index);
                    if (index == 0) {
-                        metric.getLabelPair().forEach(label -> {
+                        metric.getLabels().forEach(label -> {
                            metricsFields.add(label.getName());
                            builder.addFields(CollectRep.Field.newBuilder().setName(label.getName())
                                    .setType(CommonConstants.TYPE_STRING).setLabel(true).build());
@@ -204,7 +192,7 @@ public class PrometheusAutoCollectImpl {
                        builder.addFields(CollectRep.Field.newBuilder().setName("value")
                                .setType(CommonConstants.TYPE_NUMBER).setLabel(false).build());
                    }
-                    Map<String, String> labelMap = metric.getLabelPair()
+                    Map<String, String> labelMap = metric.getLabels()
                            .stream()
                            .collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
                    CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
@@ -212,17 +200,7 @@ public class PrometheusAutoCollectImpl {
                        String fieldValue = labelMap.get(field);
                        valueRowBuilder.addColumns(fieldValue == null ? CommonConstants.NULL_VALUE : fieldValue);
                    }
-                    if (metric.getCounter() != null) {
-                        valueRowBuilder.addColumns(String.valueOf(metric.getCounter().getValue()));
-                    } else if (metric.getGauge() != null) {
-                        valueRowBuilder.addColumns(String.valueOf(metric.getGauge().getValue()));
-                    } else if (metric.getUntyped() != null) {
-                        valueRowBuilder.addColumns(String.valueOf(metric.getUntyped().getValue()));
-                    } else if (metric.getInfo() != null) {
-                        valueRowBuilder.addColumns(String.valueOf(metric.getInfo().getValue()));
-                    } else {
-                        valueRowBuilder.addColumns(CommonConstants.NULL_VALUE);
-                    }
+                    valueRowBuilder.addColumns(String.valueOf(metric.getValue()));
                    builder.addValues(valueRowBuilder.build());
                }
                metricsDataList.add(builder.build());
@@ -0,0 +1,43 @@
+package org.dromara.hertzbeat.collector.collect.prometheus.parser;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+/**
+ * metric family
+ *
+ */
+@Data
+@ToString
+public class MetricFamily {
+    
+    /**
+     * metric name
+     */
+    private String name;
+
+    /**
+     * metrics
+     */
+    private List<Metric> metricList;
+
+    @Data
+    public static class Metric {
+        
+        private List<Label> labels;
+
+        private double value;
+
+        private long timestamp;
+    }
+
+    @Data
+    public static class Label {
+
+        private String name;
+
+        private String value;
+    }
+}
@@ -0,0 +1,288 @@
+package org.dromara.hertzbeat.collector.collect.prometheus.parser;
+
+import lombok.extern.slf4j.Slf4j;
+import org.dromara.hertzbeat.collector.collect.http.promethus.ParseException;
+import org.dromara.hertzbeat.common.util.StrBuffer;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ *
+ *
+ * 解析prometheus的exporter接口传递的数据 http:xxx/metrics
+ * 参考: prometheus的text_parse.go的代码, 入口: TextToMetricFamilies
+ */
+@Slf4j
+public class TextParser {
+    private static final String NAME_LABEL = "__name__";
+    private static final char LEFT_CURLY_BRACKET = '{';
+    private static final char RIGHT_CURLY_BRACKET = '}';
+    private static final char EQUALS = '=';
+    private static final char QUOTES = '"';
+    private static final char ENTER = '\n';
+    private static final char SPACE = ' ';
+    private static final char COMMA = ',';
+
+
+    /**
+     * parser prometheus exporter text metrics data
+     * todo use inputStream bytebuffer instead of resp string
+     * @param resp txt data
+     * @return metrics family
+     */
+    public static Map<String, MetricFamily> textToMetricFamilies(String resp) {
+        // key: metric name, value: metric family
+        Map<String, MetricFamily> metricMap = new ConcurrentHashMap<>(10);
+        try {
+            String[] lines = resp.split("\n");
+            for (String line : lines) {
+                parseLine(metricMap, new StrBuffer(line));
+            }
+            return metricMap;
+        } catch (Exception e) {
+            log.error("parse prometheus exporter data error, msg: {}", e.getMessage(), e);
+        }
+        return metricMap;
+    }
+
+    private static void parseLine(Map<String, MetricFamily> metricMap, StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        if (buffer.isEmpty()) return;
+        switch (buffer.charAt(0)) {
+            case '#':
+            case ENTER:
+                break;
+            default:
+                parseMetric(metricMap, buffer);
+        }
+    }
+
+    private static void parseMetric(Map<String, MetricFamily> metricMap, StrBuffer buffer) {
+        String metricName = readTokenAsMetricName(buffer);
+        if (metricName.isEmpty()) {
+            log.error("error parse metric, metric name is null, line: {}", buffer.toStr());
+            return;
+        }
+        MetricFamily currentMetricFamily = metricMap.computeIfAbsent(metricName, key -> new MetricFamily());
+        List<MetricFamily.Metric> metricList = currentMetricFamily.getMetricList();
+        if (metricList == null) {
+            metricList = new ArrayList<>();
+            currentMetricFamily.setMetricList(metricList);
+        }
+        MetricFamily.Metric metric = new MetricFamily.Metric();
+        metricList.add(metric);
+        readLabels(metric, buffer);
+    }
+
+    private static void readLabels(MetricFamily.Metric metric, StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        if (buffer.isEmpty()) return;
+        metric.setLabels(new LinkedList<>());
+        if (buffer.charAt(0) == LEFT_CURLY_BRACKET) {
+            buffer.read();
+            startReadLabelName(metric, buffer);
+        } else {
+            readLabelValue(metric, buffer);
+        }
+    }
+
+    private static void startReadLabelName(MetricFamily.Metric metric, StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        if (buffer.isEmpty()) return;
+        if (buffer.charAt(0) == RIGHT_CURLY_BRACKET) {
+            buffer.read();
+            buffer.skipBlankTabs();
+            if (buffer.isEmpty()) return;
+            readLabelValue(metric, buffer);
+            return;
+        }
+        String labelName = readTokenAsLabelName(buffer);
+        if (labelName.isEmpty() || NAME_LABEL.equals(labelName)) {
+            throw new ParseException("invalid label name" + labelName + ", label name size = 0 or label name equals " + NAME_LABEL);
+        }
+        MetricFamily.Label label = new MetricFamily.Label();
+        label.setName(labelName);
+        if (buffer.read() != EQUALS) {
+            throw new ParseException("parse error, not match the format of labelName=labelValue");
+        }
+        startReadLabelValue(metric, label, buffer);
+    }
+
+    private static void startReadLabelValue(MetricFamily.Metric metric, MetricFamily.Label label, StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        if (buffer.isEmpty()) return;
+        char c = buffer.read();
+        if (c != QUOTES) {
+            throw new ParseException("expected '\"' at start of label value, line: " + buffer.toStr());
+        }
+        String labelValue = readTokenAsLabelValue(buffer);
+        label.setValue(labelValue);
+        if (!isValidLabelValue(labelValue)) {
+            throw new ParseException("no valid label value: " + labelValue);
+        }
+        metric.getLabels().add(label);
+        if (buffer.isEmpty()) return;
+        c = buffer.read();
+        switch (c) {
+            case COMMA:
+                startReadLabelName(metric, buffer);
+                break;
+            case RIGHT_CURLY_BRACKET:
+                readLabelValue(metric, buffer);
+                break;
+            default:
+                throw new ParseException("expected '}' or ',' at end of label value, line: " + buffer.toStr());
+        }
+    }
+
+    private static void readLabelValue(MetricFamily.Metric metric, StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        if (buffer.isEmpty()) return;
+        metric.setValue(buffer.toDouble());
+    }
+
+    /**
+     * 获取指标的名称
+     *
+     * @param buffer 行数据对象
+     * @return token name
+     */
+    private static String readTokenAsMetricName(StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        StringBuilder builder = new StringBuilder();
+        if (isValidMetricNameStart(buffer.charAt(0))) {
+            while (!buffer.isEmpty()) {
+                char c = buffer.read();
+                if (!isValidMetricNameContinuation(c)) {
+                    buffer.rollback();
+                    break;
+                }
+                builder.append(c);
+            }
+            return builder.toString();
+        }
+        throw new ParseException("parse metric name error");
+    }
+
+    /**
+     * 获取label的名称
+     *
+     * @param buffer 行数据对象
+     * @return label name
+     */
+    private static String readTokenAsLabelName(StrBuffer buffer) {
+        buffer.skipBlankTabs();
+        StringBuilder builder = new StringBuilder();
+        char c = buffer.read();
+        if (isValidLabelNameStart(c)) {
+            builder.append(c);
+            while (!buffer.isEmpty()) {
+                c = buffer.read();
+                if (!isValidLabelNameContinuation(c)) {
+                    buffer.rollback();
+                    break;
+                }
+                builder.append(c);
+            }
+            return builder.toString();
+        }
+        throw new ParseException("parse label name error");
+    }
+
+    /**
+     * 获取Label的值
+     *
+     * @param buffer 行数据对象
+     * @return label value
+     */
+    private static String readTokenAsLabelValue(StrBuffer buffer) {
+        StringBuilder builder = new StringBuilder();
+        boolean escaped = false;
+        while (!buffer.isEmpty()) {
+            char c = buffer.read();
+            // 处理 '\\' 转义
+            if (escaped) {
+                switch (c) {
+                    case QUOTES:
+                    case '\\':
+                        builder.append(c);
+                        break;
+                    case 'n':
+                        builder.append('\n');
+                        break;
+                    default:
+                        throw new ParseException("parse label value error");
+                }
+                escaped = false;
+            } else {
+                switch (c) {
+                    case QUOTES:
+                        return builder.toString();
+                    case ENTER:
+                        throw new ParseException("parse label value error, next line");
+                    case '\\':
+                        escaped = true;
+                        break;
+                    default:
+                        builder.append(c);
+                }
+            }
+        }
+        return builder.toString();
+    }
+
+    /**
+     * 是否符合metric name首字符规则
+     *
+     * @param c metric字符
+     * @return true/false
+     */
+    private static boolean isValidMetricNameStart(char c) {
+        return isValidLabelNameStart(c) || c == ':';
+    }
+
+    /**
+     * 是否符合metric name除首字符其他字符规则
+     *
+     * @param c metric字符
+     * @return true/false
+     */
+    private static boolean isValidMetricNameContinuation(char c) {
+        return isValidLabelNameContinuation(c) || c == ':';
+    }
+
+    /**
+     * 是否符合label name首字符规则
+     *
+     * @param c metric字符
+     * @return true/false
+     */
+    private static boolean isValidLabelNameStart(char c) {
+        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
+    }
+
+    /**
+     * 是否符合label name除首字符其他字符规则
+     *
+     * @param c metric字符
+     * @return true/false
+     */
+    private static boolean isValidLabelNameContinuation(char c) {
+        return isValidLabelNameStart(c) || (c >= '0' && c <= '9');
+    }
+
+    /**
+     * 检测是否是有效的utf8编码的字符串
+     *
+     * @param s label value
+     * @return true/false
+     */
+    private static boolean isValidLabelValue(String s) {
+        return s != null && s.equals(new String(s.getBytes(StandardCharsets.UTF_8)));
+    }
+}