Consolidate entity-free observability for 1.9 (#4302)

This commit is contained in:
Logic
2026-08-22 06:39:42 +08:00
committed by GitHub
parent b8148240f8
commit 0c4136cade
141 changed files with 4162 additions and 1529 deletions
+1 -1
View File
@@ -76,7 +76,7 @@ jobs:
- name: Build backend Maven E2E modules - name: Build backend Maven E2E modules
run: | run: |
mvnd clean -B package \ mvnd clean -B package \
-pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-log-e2e \ -pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-observability-e2e \
-am \ -am \
-Dmaven.test.skip=false \ -Dmaven.test.skip=false \
--file pom.xml --file pom.xml
@@ -177,6 +177,11 @@ public class CommonHttpClient {
static void setBeforeCleanupHookForTest(Runnable hook) { static void setBeforeCleanupHookForTest(Runnable hook) {
beforeCleanupHook = hook; beforeCleanupHook = hook;
} }
static boolean awaitConnectionPoolCleanupIdleForTest(long timeout, TimeUnit unit) throws InterruptedException {
ScheduledDispatchTask currentCleanupTask = cleanupTask;
return currentCleanupTask == null || currentCleanupTask.awaitIdle(timeout, unit);
}
public static void close() { public static void close() {
try { try {
@@ -268,10 +273,24 @@ public class CommonHttpClient {
shouldSchedule = pendingRuns > 0; shouldSchedule = pendingRuns > 0;
if (!shouldSchedule) { if (!shouldSchedule) {
running = false; running = false;
notifyAll();
return; return;
} }
} }
scheduleRun(); scheduleRun();
} }
private synchronized boolean awaitIdle(long timeout, TimeUnit unit) throws InterruptedException {
long deadline = System.nanoTime() + unit.toNanos(timeout);
long remainingNanos = deadline - System.nanoTime();
while (running || pendingRuns > 0) {
if (remainingNanos <= 0) {
return false;
}
TimeUnit.NANOSECONDS.timedWait(this, remainingNanos);
remainingNanos = deadline - System.nanoTime();
}
return true;
}
} }
} }
@@ -96,9 +96,11 @@ class CommonHttpClientVirtualThreadTest {
CommonHttpClient.dispatchConnectionPoolCleanup(); CommonHttpClient.dispatchConnectionPoolCleanup();
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
assertFalse(CommonHttpClient.awaitConnectionPoolCleanupIdleForTest(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown(); releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertTrue(CommonHttpClient.awaitConnectionPoolCleanupIdleForTest(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get()); assertEquals(1, maxConcurrent.get());
} }
@@ -69,6 +69,8 @@ public interface ConfigConstants {
String GRAFANA = "grafana"; String GRAFANA = "grafana";
String LOG = "log"; String LOG = "log";
String OBSERVABILITY = "observability";
} }
} }
@@ -61,7 +61,7 @@ public interface NetworkConstants {
Duration READ_TIMEOUT = Duration.ofSeconds(6); Duration READ_TIMEOUT = Duration.ofSeconds(6);
Duration WRITE_TIMEOUT = Duration.ofSeconds(6); Duration WRITE_TIMEOUT = Duration.ofSeconds(6);
Duration CONNECT_TIMEOUT = Duration.ofSeconds(6); Duration CONNECT_TIMEOUT = Duration.ofSeconds(6);
Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(5); Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(15);
Duration GREPTIME_QUERY_CONNECT_TIMEOUT = Duration.ofSeconds(2); Duration GREPTIME_QUERY_CONNECT_TIMEOUT = Duration.ofSeconds(2);
Duration GREPTIME_WRITE_READ_TIMEOUT = Duration.ofSeconds(3); Duration GREPTIME_WRITE_READ_TIMEOUT = Duration.ofSeconds(3);
Duration GREPTIME_WRITE_CONNECT_TIMEOUT = Duration.ofSeconds(2); Duration GREPTIME_WRITE_CONNECT_TIMEOUT = Duration.ofSeconds(2);
@@ -25,7 +25,7 @@
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>hertzbeat-log-e2e</artifactId> <artifactId>hertzbeat-observability-e2e</artifactId>
<properties> <properties>
<maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.source>${java.version}</maven.compiler.source>
@@ -52,7 +52,7 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hertzbeat</groupId> <groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-log</artifactId> <artifactId>hertzbeat-observability</artifactId>
<version>${hertzbeat.version}</version> <version>${hertzbeat.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.alert; package org.apache.hertzbeat.observability.alert;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler; import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler;
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.alert; package org.apache.hertzbeat.observability.alert;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.ingestion; package org.apache.hertzbeat.observability.ingestion;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
@@ -0,0 +1,176 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.storage;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.protobuf.ByteString;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
import io.opentelemetry.proto.common.v1.AnyValue;
import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.logs.v1.LogRecord;
import io.opentelemetry.proto.logs.v1.ResourceLogs;
import io.opentelemetry.proto.logs.v1.ScopeLogs;
import io.opentelemetry.proto.resource.v1.Resource;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeOtlpSignalStorage;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.junit.jupiter.api.Test;
import org.springframework.web.client.RestTemplate;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
/** Proves the warehouse-owned Entity-free OTLP log path against a real GreptimeDB. */
@Testcontainers
class GreptimeEntityFreeSignalStorageE2eTest {
private static final int GREPTIME_HTTP_PORT = 4000;
private static final int GREPTIME_GRPC_PORT = 4001;
private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql";
private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml";
private static final String PIPELINE_NAME = "hertzbeat_otlp_log_v1";
private static final String TRACE_ID = "0123456789abcdef0123456789abcdef";
private static final String SPAN_ID = "0123456789abcdef";
private static final String BODY = "entity-free greptime proof";
private static final long LOG_TIME_NANOS = 1_710_000_000_123_456_789L;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Container
@SuppressWarnings("resource")
private static final GenericContainer<?> GREPTIME = new GenericContainer<>(
DockerImageName.parse("greptime/greptimedb:latest"))
.withExposedPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT)
.withCommand("standalone", "start",
"--http-addr", "0.0.0.0:" + GREPTIME_HTTP_PORT,
"--rpc-bind-addr", "0.0.0.0:" + GREPTIME_GRPC_PORT)
.waitingFor(Wait.forListeningPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT))
.withStartupTimeout(Duration.ofSeconds(120));
private final HttpClient httpClient = HttpClient.newHttpClient();
@Test
void warehouseStorageShouldPersistEntityFreeOtlpLogs() throws Exception {
executeSql(classpathResource(LOG_SCHEMA).strip().replaceFirst(";\\s*$", ""));
uploadPipeline();
GreptimeOtlpSignalStorage storage = new GreptimeOtlpSignalStorage(
new GreptimeProperties(true, GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_GRPC_PORT),
endpoint(), "public", "", ""),
new RestTemplate());
storage.writeProtobuf("logs", request().toByteArray());
await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> {
String sql = "SELECT COUNT(*) AS count FROM hertzbeat_logs WHERE trace_id = '" + TRACE_ID
+ "' AND body = '" + BODY + "'";
assertThat(queryCount(sql)).isEqualTo(1);
});
}
private void uploadPipeline() throws Exception {
String boundary = "----hertzbeat-entity-free-proof";
String body = "--" + boundary + "\r\n"
+ "Content-Disposition: form-data; name=\"file\"; filename=\"pipeline.yaml\"\r\n"
+ "Content-Type: application/x-yaml\r\n\r\n"
+ classpathResource(LOG_PIPELINE) + "\r\n"
+ "--" + boundary + "--\r\n";
HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
.uri(URI.create(endpoint() + "/v1/pipelines/" + PIPELINE_NAME))
.header("Content-Type", "multipart/form-data; boundary=" + boundary)
.POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
.build(), HttpResponse.BodyHandlers.ofString());
assertThat(response.statusCode()).as(response.body()).isBetween(200, 299);
}
private ExportLogsServiceRequest request() {
LogRecord record = LogRecord.newBuilder()
.setTimeUnixNano(LOG_TIME_NANOS)
.setObservedTimeUnixNano(LOG_TIME_NANOS)
.setSeverityNumberValue(9)
.setSeverityText("INFO")
.setBody(AnyValue.newBuilder().setStringValue(BODY).build())
.setTraceId(ByteString.copyFrom(hexToBytes(TRACE_ID)))
.setSpanId(ByteString.copyFrom(hexToBytes(SPAN_ID)))
.build();
return ExportLogsServiceRequest.newBuilder()
.addResourceLogs(ResourceLogs.newBuilder()
.setResource(Resource.newBuilder()
.addAttributes(stringAttribute("service.name", "checkout"))
.addAttributes(stringAttribute("deployment.environment.name", "test"))
.build())
.addScopeLogs(ScopeLogs.newBuilder().addLogRecords(record).build())
.build())
.build();
}
private int queryCount(String sql) throws Exception {
JsonNode rows = OBJECT_MAPPER.readTree(executeSql(sql).body())
.path("output").path(0).path("records").path("rows");
assertThat(rows.isArray()).isTrue();
assertThat(rows).isNotEmpty();
return rows.get(0).get(0).asInt();
}
private HttpResponse<String> executeSql(String sql) throws Exception {
HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
.uri(URI.create(endpoint() + "/v1/sql?db=public"))
.header("Content-Type", "application/x-www-form-urlencoded")
.POST(HttpRequest.BodyPublishers.ofString(
"sql=" + URLEncoder.encode(sql, StandardCharsets.UTF_8), StandardCharsets.UTF_8))
.build(), HttpResponse.BodyHandlers.ofString());
assertThat(response.statusCode()).as(response.body()).isBetween(200, 299);
return response;
}
private String classpathResource(String path) throws Exception {
try (InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(path)) {
assertThat(input).as(path).isNotNull();
return new String(input.readAllBytes(), StandardCharsets.UTF_8);
}
}
private static KeyValue stringAttribute(String key, String value) {
return KeyValue.newBuilder().setKey(key)
.setValue(AnyValue.newBuilder().setStringValue(value).build()).build();
}
private static byte[] hexToBytes(String value) {
byte[] bytes = new byte[value.length() / 2];
for (int index = 0; index < value.length(); index += 2) {
bytes[index / 2] = (byte) Integer.parseInt(value.substring(index, index + 2), 16);
}
return bytes;
}
private static String endpoint() {
return "http://" + GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_HTTP_PORT);
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.storage; package org.apache.hertzbeat.observability.storage;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
@@ -80,12 +80,12 @@ resourceRole:
- /api/ai/**===post===[admin] - /api/ai/**===post===[admin]
- /api/ai/**===put===[admin] - /api/ai/**===put===[admin]
- /api/ai/**===delete===[admin] - /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest] - /api/otlp/v1/**===post===[admin,user]
# deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0
- /api/logs/otlp/**===post===[admin,user]
- /api/logs/ingest/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user]
- /api/otlp/**===post===[admin,user] - /api/observability/logs===delete===[admin]
- /api/ingestion/otlp/**===get===[admin,user,guest] - /api/observability/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is # The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous # scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin] - /v3/api-docs/**===get===[admin]
@@ -93,19 +93,20 @@ resourceRole:
- /v3/api-docs.yaml/**===get===[admin] - /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin] - /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin] - /swagger-resources/**===get===[admin]
# the alert stream carries full alert payloads and the manager stream carries import
# progress; both are scoped like the log stream above rather than left anonymous
- /api/alert/sse/**===get===[admin,user,guest]
- /api/manager/sse/**===get===[admin,user,guest]
# config the resource restful api that need bypass auth protection # config the resource restful api that need bypass auth protection
# rule: api===method # rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource: excludedResource:
- /api/alert/sse/**===*
- /api/account/auth/**===* - /api/account/auth/**===*
- /api/i18n/**===get - /api/i18n/**===get
- /api/apps/hierarchy===get - /api/apps/hierarchy===get
- /api/observability/capability===get
- /api/push/**===* - /api/push/**===*
- /api/status/page/public/**===* - /api/status/page/public/**===*
- /api/manager/sse/**===*
# web ui resource # web ui resource
- /===get - /===get
- /assets/**===get - /assets/**===get
@@ -102,7 +102,7 @@ sinks:
type: opentelemetry type: opentelemetry
protocol: protocol:
type: http type: http
uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/logs/ingest/otlp" uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/otlp/v1/logs"
method: post method: post
encoding: encoding:
codec: json codec: json
+1 -1
View File
@@ -32,7 +32,7 @@
<module>hertzbeat-collector-kafka-e2e</module> <module>hertzbeat-collector-kafka-e2e</module>
<module>hertzbeat-collector-basic-e2e</module> <module>hertzbeat-collector-basic-e2e</module>
<module>hertzbeat-collector-mysql-r2dbc-e2e</module> <module>hertzbeat-collector-mysql-r2dbc-e2e</module>
<module>hertzbeat-log-e2e</module> <module>hertzbeat-observability-e2e</module>
</modules> </modules>
<properties> <properties>
@@ -1,82 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.log.service.LogProtocolAdapter;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* Generic Log Ingestion Controller
* Provides a fallback endpoint for log protocols that don't have dedicated controllers.
* For OTLP protocol, use OtlpLogController instead.
*/
@Tag(name = "Log Ingestion Controller")
@RestController
@RequestMapping(path = "/api/logs", produces = MediaType.APPLICATION_JSON_VALUE)
@Slf4j
public class LogIngestionController {
private final List<LogProtocolAdapter> protocolAdapters;
public LogIngestionController(List<LogProtocolAdapter> protocolAdapters) {
this.protocolAdapters = protocolAdapters;
}
/**
* Receive log payload pushed from external system specifying the log protocol.
*
* @param protocol log protocol identifier (e.g., "vector", "loki")
* @param content raw request body
*/
@Operation(summary = "Ingest logs by protocol name")
@PostMapping(value = "/ingest/{protocol}", consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Message<Void>> ingestLog(@PathVariable("protocol") String protocol,
@RequestBody String content) {
log.debug("Receive log from protocol: {}, content length: {}", protocol, content == null ? 0 : content.length());
for (LogProtocolAdapter adapter : protocolAdapters) {
if (adapter.supportProtocol().equalsIgnoreCase(protocol)) {
try {
adapter.ingest(content);
return ResponseEntity.ok(Message.success("Add extern log success"));
} catch (Exception e) {
log.error("Add log failed: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Add extern log failed: " + e.getMessage()));
}
}
}
log.warn("Not support extern log from protocol: {}", protocol);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(Message.fail(CommonConstants.FAIL_CODE, "Not support the " + protocol + " protocol log"));
}
}
@@ -1,160 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import com.google.rpc.Status;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* OTLP Log Ingestion Controller
* Implements OTLP/HTTP specification for log ingestion.
* Supports both binary-encoded Protobuf (application/x-protobuf) and JSON-encoded Protobuf (application/json).
*
* @see <a href="https://opentelemetry.io/docs/specs/otlp/#otlphttp">OTLP/HTTP Specification</a>
*/
@Tag(name = "OTLP Log Controller")
@RestController
@RequestMapping(path = "/api/logs/otlp")
@Slf4j
public class OtlpLogController {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
private static final ExportLogsServiceResponse EMPTY_RESPONSE = ExportLogsServiceResponse.newBuilder().build();
private final OtlpLogProtocolAdapter otlpLogProtocolAdapter;
public OtlpLogController(OtlpLogProtocolAdapter otlpLogProtocolAdapter) {
this.otlpLogProtocolAdapter = otlpLogProtocolAdapter;
}
/**
* OTLP/HTTP standard endpoint for logs with JSON-encoded Protobuf payload.
* Content-Type: application/json
*
* Response follows OTLP specification:
* - Success: HTTP 200 with ExportLogsServiceResponse (JSON encoded)
* - Failure: HTTP 400 with google.rpc.Status (JSON encoded)
*
* @param content JSON-encoded ExportLogsServiceRequest
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (JSON format)")
@PostMapping(value = "/v1/logs", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> ingestJsonLogs(@RequestBody String content) {
log.debug("Receive OTLP JSON logs, content length: {}", content == null ? 0 : content.length());
try {
otlpLogProtocolAdapter.ingest(content);
return ResponseEntity.ok(toJsonResponse(EMPTY_RESPONSE));
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(toJsonErrorResponse(e.getMessage()));
} catch (Exception e) {
// Server-side errors - unexpected failure
log.error("Unexpected error ingesting OTLP JSON logs: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(toJsonErrorResponse(e.getMessage()));
}
}
/**
* OTLP/HTTP standard endpoint for logs with binary-encoded Protobuf payload.
* Content-Type: application/x-protobuf
*
* Response follows OTLP specification:
* - Success: HTTP 200 with ExportLogsServiceResponse (binary encoded)
* - Failure: HTTP 400 with google.rpc.Status (binary encoded)
*
* @param content binary-encoded ExportLogsServiceRequest
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (binary Protobuf format)")
@PostMapping(value = "/v1/logs", consumes = CONTENT_TYPE_PROTOBUF, produces = CONTENT_TYPE_PROTOBUF)
public ResponseEntity<byte[]> ingestBinaryLogs(@RequestBody byte[] content) {
log.debug("Receive OTLP binary logs, content length: {}", content == null ? 0 : content.length);
try {
otlpLogProtocolAdapter.ingestBinary(content);
return ResponseEntity.ok(EMPTY_RESPONSE.toByteArray());
} catch (IllegalArgumentException e) {
// Client-side validation errors - malformed request
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(createBinaryErrorResponse(e.getMessage()));
} catch (Exception e) {
// Server-side errors - unexpected failure
log.error("Unexpected error ingesting OTLP binary logs: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(createBinaryErrorResponse(e.getMessage()));
}
}
private String toJsonResponse(ExportLogsServiceResponse response) {
try {
return JsonFormat.printer().print(response);
} catch (InvalidProtocolBufferException e) {
log.error("Failed to convert ExportLogsServiceResponse to JSON: {}", e.getMessage(), e);
return "{}";
}
}
private String toJsonErrorResponse(String message) {
Status status = Status.newBuilder()
.setMessage(message != null ? message : "Unknown error")
.build();
try {
return JsonFormat.printer().print(status);
} catch (InvalidProtocolBufferException e) {
return "{\"message\":\"" + escapeJson(message) + "\"}";
}
}
/**
* Escapes a string value for safe inclusion in JSON.
*
* @param message the string to escape
* @return the escaped string, or empty string if message is null
*/
private String escapeJson(String message) {
if (message == null) {
return "";
}
char[] escaped = JsonStringEncoder.getInstance().quoteAsString(message);
return new String(escaped);
}
private byte[] createBinaryErrorResponse(String message) {
return Status.newBuilder()
.setMessage(message != null ? message : "Unknown error")
.build()
.toByteArray();
}
}
@@ -1,117 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import java.util.Arrays;
import java.util.List;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.log.service.LogProtocolAdapter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
/**
* Unit test for {@link LogIngestionController}
*/
@ExtendWith(MockitoExtension.class)
class LogIngestionControllerTest {
private MockMvc mockMvc;
@Mock
private LogProtocolAdapter vectorAdapter;
private LogIngestionController logIngestionController;
@BeforeEach
void setUp() {
List<LogProtocolAdapter> adapters = Arrays.asList(vectorAdapter);
this.logIngestionController = new LogIngestionController(adapters);
this.mockMvc = MockMvcBuilders.standaloneSetup(logIngestionController).build();
}
@Test
void testIngestLogWithKnownProtocol() throws Exception {
String logContent = "{\"message\":\"Test log message\"}";
when(vectorAdapter.supportProtocol()).thenReturn("vector");
doNothing().when(vectorAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest/vector")
.contentType(MediaType.APPLICATION_JSON)
.content(logContent)
)
.andDo(print())
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.msg").value("Add extern log success"))
.andReturn();
}
@Test
void testIngestLogWithUnsupportedProtocol() throws Exception {
String logContent = "{\"message\":\"Unsupported protocol log\"}";
when(vectorAdapter.supportProtocol()).thenReturn("vector");
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest/unsupported")
.contentType(MediaType.APPLICATION_JSON)
.content(logContent)
)
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
.andExpect(jsonPath("$.msg").value("Not support the unsupported protocol log"));
}
@Test
void testIngestLogWithAdapterException() throws Exception {
String logContent = "{\"message\":\"Log message that will cause exception\"}";
when(vectorAdapter.supportProtocol()).thenReturn("vector");
doThrow(new IllegalArgumentException("Invalid log format")).when(vectorAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/ingest/vector")
.contentType(MediaType.APPLICATION_JSON)
.content(logContent)
)
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
.andExpect(jsonPath("$.msg").value("Add extern log failed: Invalid log format"));
}
}
@@ -1,131 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.controller;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
/**
* Unit test for {@link OtlpLogController}
*/
@ExtendWith(MockitoExtension.class)
class OtlpLogControllerTest {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
private MockMvc mockMvc;
@Mock
private OtlpLogProtocolAdapter otlpLogProtocolAdapter;
private OtlpLogController otlpLogController;
@BeforeEach
void setUp() {
this.otlpLogController = new OtlpLogController(otlpLogProtocolAdapter);
this.mockMvc = MockMvcBuilders.standaloneSetup(otlpLogController).build();
}
@Test
void testIngestJsonLogsSuccess() throws Exception {
String jsonContent = "{\"resourceLogs\":[]}";
doNothing().when(otlpLogProtocolAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content(jsonContent)
)
.andDo(print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
}
@Test
void testIngestJsonLogsFailure() throws Exception {
String jsonContent = "{\"invalid\":\"content\"}";
doThrow(new IllegalArgumentException("Invalid OTLP JSON log content"))
.when(otlpLogProtocolAdapter).ingest(anyString());
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content(jsonContent)
)
.andExpect(status().isBadRequest())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
}
@Test
void testIngestBinaryLogsSuccess() throws Exception {
byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c};
doNothing().when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class));
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(CONTENT_TYPE_PROTOBUF)
.content(binaryContent)
)
.andDo(print())
.andExpect(status().isOk())
.andExpect(content().contentType(CONTENT_TYPE_PROTOBUF))
.andReturn();
}
@Test
void testIngestBinaryLogsFailure() throws Exception {
byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c};
doThrow(new IllegalArgumentException("Invalid OTLP binary log content"))
.when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class));
mockMvc.perform(
MockMvcRequestBuilders
.post("/api/logs/otlp/v1/logs")
.contentType(CONTENT_TYPE_PROTOBUF)
.content(binaryContent)
)
.andExpect(status().isBadRequest())
.andExpect(content().contentType(CONTENT_TYPE_PROTOBUF))
.andReturn();
}
}
+2 -2
View File
@@ -91,10 +91,10 @@
<groupId>org.apache.hertzbeat</groupId> <groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-otel</artifactId> <artifactId>hertzbeat-otel</artifactId>
</dependency> </dependency>
<!-- log --> <!-- observability -->
<dependency> <dependency>
<groupId>org.apache.hertzbeat</groupId> <groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-log</artifactId> <artifactId>hertzbeat-observability</artifactId>
</dependency> </dependency>
<!-- spring --> <!-- spring -->
<dependency> <dependency>
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import java.net.http.HttpClient;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.junit.jupiter.api.Test;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.test.util.ReflectionTestUtils;
class RestTemplateConfigTest {
@Test
void greptimeQueryRequestFactoryUsesDedicatedTimeouts() {
ClientHttpRequestFactory factory = new RestTemplateConfig().greptimeQueryClientHttpRequestFactory();
JdkClientHttpRequestFactory jdkFactory = assertInstanceOf(JdkClientHttpRequestFactory.class, factory);
assertEquals(NetworkConstants.HttpClientConstants.GREPTIME_QUERY_READ_TIMEOUT,
ReflectionTestUtils.getField(jdkFactory, "readTimeout"));
HttpClient httpClient = (HttpClient) ReflectionTestUtils.getField(jdkFactory, "httpClient");
assertEquals(NetworkConstants.HttpClientConstants.GREPTIME_QUERY_CONNECT_TIMEOUT,
httpClient.connectTimeout().orElseThrow());
}
}
@@ -80,12 +80,12 @@ resourceRole:
- /api/ai/**===post===[admin] - /api/ai/**===post===[admin]
- /api/ai/**===put===[admin] - /api/ai/**===put===[admin]
- /api/ai/**===delete===[admin] - /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest] - /api/otlp/v1/**===post===[admin,user]
# deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0
- /api/logs/otlp/**===post===[admin,user]
- /api/logs/ingest/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user]
- /api/otlp/**===post===[admin,user] - /api/observability/logs===delete===[admin]
- /api/ingestion/otlp/**===get===[admin,user,guest] - /api/observability/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is # The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous # scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin] - /v3/api-docs/**===get===[admin]
@@ -93,19 +93,20 @@ resourceRole:
- /v3/api-docs.yaml/**===get===[admin] - /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin] - /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin] - /swagger-resources/**===get===[admin]
# the alert stream carries full alert payloads and the manager stream carries import
# progress; both are scoped like the log stream above rather than left anonymous
- /api/alert/sse/**===get===[admin,user,guest]
- /api/manager/sse/**===get===[admin,user,guest]
# config the resource restful api that need bypass auth protection # config the resource restful api that need bypass auth protection
# rule: api===method # rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource: excludedResource:
- /api/alert/sse/**===*
- /api/account/auth/**===* - /api/account/auth/**===*
- /api/i18n/**===get - /api/i18n/**===get
- /api/apps/hierarchy===get - /api/apps/hierarchy===get
- /api/observability/capability===get
- /api/push/**===* - /api/push/**===*
- /api/status/page/public/**===* - /api/status/page/public/**===*
- /api/manager/sse/**===*
# web ui resource # web ui resource
- /===get - /===get
- /assets/**===get - /assets/**===get
@@ -25,7 +25,7 @@
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>hertzbeat-log</artifactId> <artifactId>hertzbeat-observability</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>
<properties> <properties>
@@ -15,21 +15,19 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.otel.config; package org.apache.hertzbeat.observability.config;
import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants; import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
/** /**
* Log auto configuration. * Entity-free observability module auto configuration.
*/ */
@AutoConfiguration
@ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG
+ SignConstants.DOT + SignConstants.DOT
+ ConfigConstants.FunctionModuleConstants.LOG + ConfigConstants.FunctionModuleConstants.OBSERVABILITY)
) public class ObservabilityAutoConfiguration {
@EnableConfigurationProperties(GreptimeProperties.class)
public class LogAutoConfiguration {
} }
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.config; package org.apache.hertzbeat.observability.config;
import io.grpc.ForwardingServerCallListener; import io.grpc.ForwardingServerCallListener;
import io.grpc.Metadata; import io.grpc.Metadata;
@@ -41,10 +41,11 @@ import java.net.InetSocketAddress;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator;
import org.apache.hertzbeat.log.service.OtlpSignalForwarder; import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.log.service.SignalQueryRejectedException; import org.apache.hertzbeat.observability.service.OtlpSignalForwarder;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -62,14 +63,16 @@ public class OtlpGrpcServerConfig {
matchIfMissing = true) matchIfMissing = true)
public OtlpGrpcServerRunner otlpGrpcServerRunner( public OtlpGrpcServerRunner otlpGrpcServerRunner(
@Value("${hertzbeat.otlp.grpc.host:0.0.0.0}") String host, @Value("${hertzbeat.otlp.grpc.host:0.0.0.0}") String host,
@Value("${hertzbeat.otlp.grpc.port:4317}") int port, @Value("${hertzbeat.otlp.grpc.port:14317}") int port,
OtlpSignalForwarder signalForwarder, OtlpSignalForwarder signalForwarder,
OtlpLogIngestionService logIngestionService,
SignalWorkloadGuard workloadGuard, SignalWorkloadGuard workloadGuard,
ObjectProvider<OtlpAccessTokenValidator> tokenValidatorProvider) { ObjectProvider<OtlpAccessTokenValidator> tokenValidatorProvider) {
OtlpAccessTokenValidator validator = tokenValidatorProvider.getIfAvailable( OtlpAccessTokenValidator validator = tokenValidatorProvider.getIfAvailable(
() -> token -> "OTLP token validation is unavailable"); () -> token -> "OTLP token validation is unavailable");
ServerInterceptor interceptor = new BearerTokenInterceptor(validator); ServerInterceptor interceptor = new BearerTokenInterceptor(validator);
return new OtlpGrpcServerRunner(host, port, signalForwarder, workloadGuard, interceptor); return new OtlpGrpcServerRunner(host, port, signalForwarder, logIngestionService,
workloadGuard, interceptor);
} }
@Slf4j @Slf4j
@@ -78,20 +81,42 @@ public class OtlpGrpcServerConfig {
private final String host; private final String host;
private final int port; private final int port;
private final OtlpSignalForwarder signalForwarder; private final OtlpSignalForwarder signalForwarder;
private final OtlpLogIngestionService logIngestionService;
private final SignalWorkloadGuard workloadGuard; private final SignalWorkloadGuard workloadGuard;
private final ServerInterceptor authInterceptor; private final ServerInterceptor authInterceptor;
private Server server; private Server server;
public void start() throws IOException { /**
server = NettyServerBuilder.forAddress(new InetSocketAddress(host, port)) * Binds the OTLP/gRPC listener, or gives it up for this run.
.addService(ServerInterceptors.intercept(new MetricsService(signalForwarder, workloadGuard), *
authInterceptor)) * <p>The default 14317 keeps clear of 4317, the OpenTelemetry standard port an OTel Collector
.addService(ServerInterceptors.intercept(new LogsService(signalForwarder, workloadGuard), * on the same host would already hold - and that host is exactly the one most likely to want
authInterceptor)) * this listener. A bind can still fail for other reasons, and letting it escape would abort
.addService(ServerInterceptors.intercept(new TracesService(signalForwarder, workloadGuard), * the whole context: an optional side channel would then keep the monitoring system itself
authInterceptor)) * from starting, while HTTP ingestion on the main port stays perfectly able to serve the same
.build().start(); * signals. Degrade to "unavailable" and say how to fix it.
log.info("OTLP gRPC listener started on {}:{}", host, port); */
public void start() {
try {
server = NettyServerBuilder.forAddress(new InetSocketAddress(host, port))
.addService(ServerInterceptors.intercept(new MetricsService(signalForwarder, workloadGuard),
authInterceptor))
.addService(ServerInterceptors.intercept(new LogsService(logIngestionService, workloadGuard),
authInterceptor))
.addService(ServerInterceptors.intercept(new TracesService(signalForwarder, workloadGuard),
authInterceptor))
.build().start();
log.info("OTLP gRPC listener started on {}:{}", host, port);
} catch (IOException exception) {
server = null;
log.error("OTLP gRPC listener could not bind {}:{}, so gRPC ingestion is unavailable for this run. "
+ "OTLP/HTTP ingestion on /api/otlp/v1 is unaffected. Set hertzbeat.otlp.grpc.port to a free "
+ "port, or hertzbeat.otlp.grpc.enabled=false to stop starting it.", host, port, exception);
}
}
boolean isRunning() {
return server != null;
} }
public void stop() { public void stop() {
@@ -123,14 +148,14 @@ public class OtlpGrpcServerConfig {
@RequiredArgsConstructor @RequiredArgsConstructor
static final class LogsService extends LogsServiceGrpc.LogsServiceImplBase { static final class LogsService extends LogsServiceGrpc.LogsServiceImplBase {
private final OtlpSignalForwarder signalForwarder; private final OtlpLogIngestionService logIngestionService;
private final SignalWorkloadGuard workloadGuard; private final SignalWorkloadGuard workloadGuard;
@Override @Override
public void export(ExportLogsServiceRequest request, StreamObserver<ExportLogsServiceResponse> observer) { public void export(ExportLogsServiceRequest request, StreamObserver<ExportLogsServiceResponse> observer) {
try { try {
byte[] response = workloadGuard.execute(Workload.OTLP_WRITE, byte[] response = workloadGuard.execute(Workload.OTLP_WRITE,
() -> signalForwarder.forwardProtobuf("logs", request.toByteArray())); () -> logIngestionService.ingestProtobuf(request.toByteArray()));
observer.onNext(response.length == 0 ? ExportLogsServiceResponse.getDefaultInstance() observer.onNext(response.length == 0 ? ExportLogsServiceResponse.getDefaultInstance()
: ExportLogsServiceResponse.parseFrom(response)); : ExportLogsServiceResponse.parseFrom(response));
observer.onCompleted(); observer.onCompleted();
@@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RestController;
/**
* Deprecated 1.8.x OTLP log ingestion routes kept as aliases of {@code POST /api/otlp/v1/logs}.
*
* <p>HertzBeat 1.8.0 documented {@code POST /api/logs/otlp/v1/logs} and also exposed
* {@code POST /api/logs/ingest/{protocol}}. Collectors and SDKs configured against those paths keep
* working on 1.9.x through this alias, which forwards to the same log fan-out as the canonical
* route and stamps a {@code Deprecation} / {@code Link} header on every response.
*
* <p>These aliases are scheduled for removal in HertzBeat 2.0. New integrations must use
* {@code /api/otlp/v1/logs}.
*
* @deprecated since 1.9.0, use {@code POST /api/otlp/v1/logs}
*/
@Deprecated(since = "1.9.0", forRemoval = true)
@Slf4j
@RestController
@Tag(name = "OTLP Log Controller (deprecated 1.8 aliases)")
public class LegacyOtlpLogRouteController {
/**
* Canonical route the aliases forward to.
*/
public static final String CANONICAL_LOGS_ROUTE = "/api/otlp/v1/logs";
/**
* 1.8.0 documented OTLP/HTTP log route.
*/
public static final String LEGACY_OTLP_LOGS_ROUTE = "/api/logs/otlp/v1/logs";
/**
* 1.8.0 protocol-named log route; only the {@code otlp} protocol ever had an adapter.
*/
public static final String LEGACY_INGEST_ROUTE = "/api/logs/ingest/{protocol}";
private static final String OTLP_PROTOCOL = "otlp";
private static final String UNSUPPORTED_PROTOCOL_KEY = "unsupported-protocol";
private static final String DEPRECATION_HEADER = "Deprecation";
private static final String LINK_HEADER = "Link";
private static final String LINK_VALUE = "<" + CANONICAL_LOGS_ROUTE + ">; rel=\"successor-version\"";
/**
* Keys already reported at warn level.
*
* <p>A collector that has not been migrated yet pushes continuously, so warning per request buries
* every other line in the log - including the ones the upgrade notes ask operators to watch for.
* Only constant keys are ever added, never a caller supplied value, so this cannot grow unbounded.
*/
private final Set<String> reportedDeprecations = ConcurrentHashMap.newKeySet();
private final OtlpLogIngestionService logIngestionService;
private final SignalWorkloadGuard workloadGuard;
public LegacyOtlpLogRouteController(OtlpLogIngestionService logIngestionService,
SignalWorkloadGuard workloadGuard) {
this.logIngestionService = logIngestionService;
this.workloadGuard = workloadGuard;
}
@PostMapping(LEGACY_OTLP_LOGS_ROUTE)
@Operation(summary = "Deprecated alias of POST /api/otlp/v1/logs", deprecated = true)
public ResponseEntity<byte[]> legacyOtlpLogs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) {
return forward(LEGACY_OTLP_LOGS_ROUTE, content, headers);
}
@PostMapping(LEGACY_INGEST_ROUTE)
@Operation(summary = "Deprecated alias of POST /api/otlp/v1/logs (otlp protocol only)", deprecated = true)
public ResponseEntity<byte[]> legacyIngest(@PathVariable("protocol") String protocol,
@RequestBody byte[] content,
@RequestHeader HttpHeaders headers) {
if (!OTLP_PROTOCOL.equalsIgnoreCase(protocol)) {
reportOnce(UNSUPPORTED_PROTOCOL_KEY,
"Deprecated route /api/logs/ingest/{} rejected: only the otlp protocol is supported", protocol);
// Thrown rather than answered inline so OtlpHttpExceptionHandler encodes it as a
// google.rpc.Status in the format the caller used, with the deprecation headers kept.
throw new IllegalArgumentException(
"Only the otlp protocol is supported, use " + CANONICAL_LOGS_ROUTE);
}
return forward("/api/logs/ingest/" + OTLP_PROTOCOL, content, headers);
}
private ResponseEntity<byte[]> forward(String legacyRoute, byte[] content, HttpHeaders headers) {
reportOnce(legacyRoute,
"Deprecated OTLP log route {} was called; migrate the exporter to {} before HertzBeat 2.0",
legacyRoute, CANONICAL_LOGS_ROUTE);
ResponseEntity<byte[]> canonical = workloadGuard.execute(Workload.OTLP_WRITE,
() -> logIngestionService.ingestHttp(content, headers));
return deprecated(ResponseEntity.status(canonical.getStatusCode()).headers(canonical.getHeaders()))
.body(canonical.getBody());
}
/**
* Warns the first time a key is seen and drops to debug afterwards, so a still unmigrated
* exporter leaves one actionable line instead of one line per request.
*/
private void reportOnce(String key, String message, Object... arguments) {
if (reportedDeprecations.add(key)) {
log.warn(message, arguments);
} else {
log.debug(message, arguments);
}
}
static ResponseEntity.BodyBuilder deprecated(ResponseEntity.BodyBuilder builder) {
return builder
.header(DEPRECATION_HEADER, "true")
.header(LINK_HEADER, LINK_VALUE);
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
@@ -37,7 +37,7 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE;
* Controller for managing log entries in HertzBeat. * Controller for managing log entries in HertzBeat.
*/ */
@RestController @RestController
@RequestMapping(path = "/api/logs", produces = "application/json") @RequestMapping(path = "/api/observability/logs", produces = "application/json")
@Tag(name = "Log Management Controller") @Tag(name = "Log Management Controller")
@Slf4j @Slf4j
public class LogManagerController { public class LogManagerController {
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
@@ -37,8 +37,8 @@ import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter; import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
@@ -49,7 +49,7 @@ import org.springframework.web.bind.annotation.RestController;
* Log query and statistics APIs for UI consumption * Log query and statistics APIs for UI consumption
*/ */
@RestController @RestController
@RequestMapping(path = "/api/logs", produces = "application/json") @RequestMapping(path = "/api/observability/logs", produces = "application/json")
@Tag(name = "Log Query Controller") @Tag(name = "Log Query Controller")
@Slf4j @Slf4j
public class LogQueryController { public class LogQueryController {
@@ -67,7 +67,7 @@ public class LogQueryController {
this.workloadGuard = workloadGuard; this.workloadGuard = workloadGuard;
} }
@GetMapping("/list") @GetMapping
@Operation(summary = "Query logs by time range with optional filters", @Operation(summary = "Query logs by time range with optional filters",
description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.") description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.")
public ResponseEntity<Message<Page<LogEntry>>> list( public ResponseEntity<Message<Page<LogEntry>>> list(
@@ -101,7 +101,7 @@ public class LogQueryController {
}); });
} }
@GetMapping("/stats/overview") @GetMapping("/overview")
@Operation(summary = "Log overview statistics", @Operation(summary = "Log overview statistics",
description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.") description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.")
public ResponseEntity<Message<Map<String, Object>>> overviewStats( public ResponseEntity<Message<Map<String, Object>>> overviewStats(
@@ -132,7 +132,7 @@ public class LogQueryController {
return ResponseEntity.ok(Message.success(historyDataReader.queryLogOverviewAggregate(filter))); return ResponseEntity.ok(Message.success(historyDataReader.queryLogOverviewAggregate(filter)));
} }
@GetMapping("/stats/trace-coverage") @GetMapping("/trace-coverage")
@Operation(summary = "Trace coverage statistics", @Operation(summary = "Trace coverage statistics",
description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.") description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.")
public ResponseEntity<Message<Map<String, Object>>> traceCoverageStats( public ResponseEntity<Message<Map<String, Object>>> traceCoverageStats(
@@ -165,7 +165,7 @@ public class LogQueryController {
return ResponseEntity.ok(Message.success(result)); return ResponseEntity.ok(Message.success(result));
} }
@GetMapping("/stats/trend") @GetMapping("/trend")
@Operation(summary = "Log trend over time", @Operation(summary = "Log trend over time",
description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.") description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.")
public ResponseEntity<Message<Map<String, Object>>> trendStats( public ResponseEntity<Message<Map<String, Object>>> trendStats(
@@ -17,12 +17,12 @@
* under the License. * under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.springframework.http.MediaType.TEXT_EVENT_STREAM_VALUE; import static org.springframework.http.MediaType.TEXT_EVENT_STREAM_VALUE;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator; import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria;
import org.apache.hertzbeat.log.notice.LogSseManager; import org.apache.hertzbeat.observability.notice.LogSseManager;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
@@ -34,7 +34,7 @@ import io.swagger.v3.oas.annotations.Operation;
* SSE controller for log streaming with filtering support * SSE controller for log streaming with filtering support
*/ */
@RestController @RestController
@RequestMapping(path = "/api/logs/sse", produces = {TEXT_EVENT_STREAM_VALUE}) @RequestMapping(path = "/api/observability/logs", produces = {TEXT_EVENT_STREAM_VALUE})
public class LogSseController { public class LogSseController {
private final LogSseManager emitterManager; private final LogSseManager emitterManager;
@@ -48,10 +48,10 @@ public class LogSseController {
* @param filterCriteria Filter criteria for log events (all parameters are optional) * @param filterCriteria Filter criteria for log events (all parameters are optional)
* @return SSE emitter for streaming log events * @return SSE emitter for streaming log events
*/ */
@GetMapping(path = "/subscribe") @GetMapping(path = "/stream")
@Operation(summary = "Subscribe to log events with optional filtering", description = "Subscribe to log events with optional filtering") @Operation(summary = "Subscribe to log events with optional filtering", description = "Subscribe to log events with optional filtering")
public SseEmitter subscribe(@ModelAttribute LogSseFilterCriteria filterCriteria) { public SseEmitter subscribe(@ModelAttribute LogSseFilterCriteria filterCriteria) {
Long clientId = SnowFlakeIdGenerator.generateId(); Long clientId = SnowFlakeIdGenerator.generateId();
return emitterManager.createEmitter(clientId, filterCriteria); return emitterManager.createEmitter(clientId, filterCriteria);
} }
} }
@@ -15,13 +15,13 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.dto.observability.ObservabilityCapability; import org.apache.hertzbeat.common.entity.dto.observability.ObservabilityCapability;
import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@@ -0,0 +1,213 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import com.google.rpc.Code;
import com.google.rpc.Status;
import jakarta.servlet.http.HttpServletRequest;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.support.exception.StorageUnavailableException;
import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.InvalidMediaTypeException;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.ErrorResponse;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestClientException;
import org.springframework.web.method.HandlerMethod;
/**
* Maps OTLP/HTTP ingestion failures to the response shape the OTLP specification requires.
*
* <p>Every non-2xx response carries a {@code google.rpc.Status} body encoded in the same format the
* client used for the request (JSON for {@code application/json}, binary protobuf otherwise), so OTLP
* exporters and collectors can surface the rejection reason instead of an opaque status code. Retryable
* failures (overload, storage unavailable) additionally carry {@code Retry-After}. Deprecated 1.8 alias
* routes keep their {@code Deprecation} / {@code Link} headers on error responses too.
*
* <p>This advice is scoped to the OTLP ingestion controllers only and runs ahead of
* {@link SignalWorkloadExceptionHandler}, whose {@code Message} JSON envelope is meant for the HertzBeat
* query API rather than OTLP clients.
*
* @see <a href="https://opentelemetry.io/docs/specs/otlp/#failures-1">OTLP/HTTP failures</a>
*/
@Slf4j
@RestControllerAdvice(assignableTypes = {
OtlpLogController.class,
OtlpSignalController.class,
LegacyOtlpLogRouteController.class
})
@Order(Ordered.HIGHEST_PRECEDENCE)
public class OtlpHttpExceptionHandler {
static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf");
private static final String RETRY_AFTER_SECONDS = "1";
private static final String UNKNOWN_ERROR = "Unknown error";
private static final String STORAGE_UNAVAILABLE = "GreptimeDB storage is unavailable";
private static final String UNREADABLE_BODY = "Malformed or missing OTLP request body";
private static final String UNEXPECTED_FAILURE = "Unexpected OTLP ingestion failure";
private static final String CLIENT_ERROR = "OTLP request rejected";
@ExceptionHandler(IllegalArgumentException.class)
public ResponseEntity<byte[]> handleInvalidPayload(IllegalArgumentException exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
String message = messageOf(exception);
// Covers both malformed payloads and GreptimeDB 4xx rejections; the message is the only place
// the rejection reason survives, so record it server-side as well.
log.warn("OTLP/HTTP {} rejected as invalid: {}", request.getRequestURI(), message);
return respond(HttpStatus.BAD_REQUEST, Code.INVALID_ARGUMENT, message, request, handlerMethod, false);
}
@ExceptionHandler(SignalQueryRejectedException.class)
public ResponseEntity<byte[]> handleOverloaded(SignalQueryRejectedException exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
String message = messageOf(exception);
// Back-pressure is expected under load; keep it below warn to avoid log storms.
log.debug("OTLP/HTTP {} throttled: {}", request.getRequestURI(), message);
return respond(HttpStatus.TOO_MANY_REQUESTS, Code.RESOURCE_EXHAUSTED, message, request, handlerMethod, true);
}
@ExceptionHandler(StorageUnavailableException.class)
public ResponseEntity<byte[]> handleStorageUnavailable(StorageUnavailableException exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
String message = messageOf(exception);
log.error("OTLP/HTTP {} failed, storage unavailable: {}", request.getRequestURI(), message, exception);
return respond(HttpStatus.SERVICE_UNAVAILABLE, Code.UNAVAILABLE, message, request, handlerMethod, true);
}
@ExceptionHandler(RestClientException.class)
public ResponseEntity<byte[]> handleStorageClientFailure(RestClientException exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
log.error("OTLP/HTTP {} failed, GreptimeDB write error: {}", request.getRequestURI(),
messageOf(exception), exception);
// Do not echo transport details (endpoints, raw 5xx bodies) back to the exporter.
return respond(HttpStatus.SERVICE_UNAVAILABLE, Code.UNAVAILABLE, STORAGE_UNAVAILABLE,
request, handlerMethod, true);
}
/**
* An absent or undecodable request body is a client error the exporter must not retry.
*
* <p>Spring signals it with {@code HttpMessageNotReadableException}, which is raised while the
* arguments are resolved rather than inside the controller, and which does not implement
* {@link ErrorResponse}. Without this handler it falls through to {@link #handleUnexpected} and
* comes back as a {@code 500}, which OTLP exporters read as retryable and replay forever.
*/
@ExceptionHandler(HttpMessageNotReadableException.class)
public ResponseEntity<byte[]> handleUnreadableBody(HttpMessageNotReadableException exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
// The framework message embeds the handler signature, so keep it server-side only.
log.warn("OTLP/HTTP {} rejected, unreadable body: {}", request.getRequestURI(), messageOf(exception));
return respond(HttpStatus.BAD_REQUEST, Code.INVALID_ARGUMENT, UNREADABLE_BODY,
request, handlerMethod, false);
}
@ExceptionHandler(Exception.class)
public ResponseEntity<byte[]> handleUnexpected(Exception exception,
HttpServletRequest request,
HandlerMethod handlerMethod) {
if (exception instanceof ErrorResponse errorResponse) {
// Spring MVC's own request-level failures (unsupported method, unsupported media type, ...)
// already carry the right status; keep it instead of degrading a client error into a 500.
HttpStatus status = HttpStatus.valueOf(errorResponse.getStatusCode().value());
log.warn("OTLP/HTTP {} rejected ({}): {}", request.getRequestURI(), status.value(),
messageOf(exception));
return respond(status, status.is4xxClientError() ? Code.INVALID_ARGUMENT : Code.INTERNAL,
clientFacingDetail(errorResponse), request, handlerMethod, false);
}
// Never echo an unhandled exception message: it routinely carries handler signatures, package
// names and internal state that the exporter has no business seeing.
log.error("OTLP/HTTP {} failed unexpectedly: {}", request.getRequestURI(), messageOf(exception),
exception);
return respond(HttpStatus.INTERNAL_SERVER_ERROR, Code.INTERNAL, UNEXPECTED_FAILURE,
request, handlerMethod, false);
}
/**
* Prefers the curated {@code ProblemDetail} text spring builds for the client over the raw
* exception message, which is written for a log line rather than for a response body.
*/
private static String clientFacingDetail(ErrorResponse errorResponse) {
String detail = errorResponse.getBody() == null ? null : errorResponse.getBody().getDetail();
return StringUtils.hasText(detail) ? detail : CLIENT_ERROR;
}
private static ResponseEntity<byte[]> respond(HttpStatus status, Code code, String message,
HttpServletRequest request, HandlerMethod handlerMethod,
boolean retryable) {
Status rpcStatus = Status.newBuilder().setCode(code.getNumber()).setMessage(message).build();
boolean json = isJsonRequest(request);
ResponseEntity.BodyBuilder builder = ResponseEntity.status(status)
.contentType(json ? MediaType.APPLICATION_JSON : PROTOBUF);
if (retryable) {
builder.header(HttpHeaders.RETRY_AFTER, RETRY_AFTER_SECONDS);
}
if (handlerMethod != null
&& LegacyOtlpLogRouteController.class.isAssignableFrom(handlerMethod.getBeanType())) {
LegacyOtlpLogRouteController.deprecated(builder);
}
return builder.body(json ? toJson(rpcStatus) : rpcStatus.toByteArray());
}
private static boolean isJsonRequest(HttpServletRequest request) {
String contentType = request.getContentType();
if (contentType == null) {
return false;
}
try {
return MediaType.APPLICATION_JSON.includes(MediaType.parseMediaType(contentType));
} catch (InvalidMediaTypeException exception) {
return false;
}
}
private static byte[] toJson(Status status) {
String json;
try {
json = JsonFormat.printer().omittingInsignificantWhitespace().print(status);
} catch (InvalidProtocolBufferException exception) {
json = "{\"code\":" + status.getCode() + ",\"message\":\""
+ new String(JsonStringEncoder.getInstance().quoteAsString(status.getMessage())) + "\"}";
}
return json.getBytes(StandardCharsets.UTF_8);
}
private static String messageOf(Throwable throwable) {
return Objects.requireNonNullElse(throwable.getMessage(), UNKNOWN_ERROR);
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/** Canonical OTLP/HTTP log endpoint backed by the HertzBeat log fan-out. */
@RestController
@RequestMapping("/api/otlp/v1")
@Tag(name = "OTLP Log Controller")
public class OtlpLogController {
private final OtlpLogIngestionService logIngestionService;
private final SignalWorkloadGuard workloadGuard;
public OtlpLogController(OtlpLogIngestionService logIngestionService, SignalWorkloadGuard workloadGuard) {
this.logIngestionService = logIngestionService;
this.workloadGuard = workloadGuard;
}
@PostMapping("/logs")
@Operation(summary = "Ingest OTLP logs")
public ResponseEntity<byte[]> logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) {
return workloadGuard.execute(Workload.OTLP_WRITE,
() -> logIngestionService.ingestHttp(content, headers));
}
}
@@ -15,19 +15,16 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import java.nio.charset.StandardCharsets; import org.apache.hertzbeat.observability.service.OtlpSignalForwarder;
import org.apache.hertzbeat.log.service.OtlpSignalForwarder; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestHeader;
@@ -55,12 +52,6 @@ public class OtlpSignalController {
return forward("metrics", content, headers); return forward("metrics", content, headers);
} }
@PostMapping("/logs")
@Operation(summary = "Ingest OTLP logs")
public ResponseEntity<byte[]> logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) {
return forward("logs", content, headers);
}
@PostMapping("/traces") @PostMapping("/traces")
@Operation(summary = "Ingest OTLP traces") @Operation(summary = "Ingest OTLP traces")
public ResponseEntity<byte[]> traces(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { public ResponseEntity<byte[]> traces(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) {
@@ -71,10 +62,4 @@ public class OtlpSignalController {
return workloadGuard.execute(Workload.OTLP_WRITE, return workloadGuard.execute(Workload.OTLP_WRITE,
() -> signalForwarder.forwardHttp(signal, content, headers)); () -> signalForwarder.forwardHttp(signal, content, headers));
} }
@ExceptionHandler(IllegalArgumentException.class)
public ResponseEntity<byte[]> invalidPayload(IllegalArgumentException exception) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(exception.getMessage().getBytes(StandardCharsets.UTF_8));
}
} }
@@ -15,13 +15,13 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE;
import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException;
import org.apache.hertzbeat.log.service.SignalQueryRejectedException; import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@@ -31,9 +31,14 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestClientException;
/** Maps bounded signal workload rejection to a retryable HTTP response. */ /**
@RestControllerAdvice(basePackages = "org.apache.hertzbeat.log.controller") * Maps bounded signal workload rejection to a retryable HTTP response.
@Order(Ordered.HIGHEST_PRECEDENCE) *
* <p>Ordered just behind {@link OtlpHttpExceptionHandler}: the OTLP ingestion controllers need
* {@code google.rpc.Status} bodies, every other observability controller gets the {@code Message} envelope.
*/
@RestControllerAdvice(basePackages = "org.apache.hertzbeat.observability.controller")
@Order(Ordered.HIGHEST_PRECEDENCE + 1)
public class SignalWorkloadExceptionHandler { public class SignalWorkloadExceptionHandler {
@ExceptionHandler(IllegalArgumentException.class) @ExceptionHandler(IllegalArgumentException.class)
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
@@ -26,19 +26,21 @@ import org.apache.hertzbeat.common.entity.dto.observability.SignalPage;
import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail; import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail;
import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem;
import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview;
import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
/** Public entity-free query APIs for OTLP metrics and traces. */ /** Public entity-free query APIs for OTLP metrics and traces. */
@RestController @RestController
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
@RequestMapping(path = "/api/observability", produces = "application/json")
@Tag(name = "Three Signal Query Controller") @Tag(name = "Three Signal Query Controller")
public class ThreeSignalQueryController { public class ThreeSignalQueryController {
@@ -50,7 +52,7 @@ public class ThreeSignalQueryController {
this.workloadGuard = workloadGuard; this.workloadGuard = workloadGuard;
} }
@GetMapping(path = "/api/ingestion/otlp/metrics/console", produces = "application/json") @GetMapping("/metrics/query")
@Operation(summary = "Query OTLP metrics") @Operation(summary = "Query OTLP metrics")
public ResponseEntity<Message<OtlpMetricsConsole>> metrics( public ResponseEntity<Message<OtlpMetricsConsole>> metrics(
@RequestParam(value = "query", required = false) String query, @RequestParam(value = "query", required = false) String query,
@@ -69,7 +71,7 @@ public class ThreeSignalQueryController {
filter, groupBy, aggregation, step, operationName)))); filter, groupBy, aggregation, step, operationName))));
} }
@GetMapping(path = "/api/ingestion/otlp/metrics/inventory", produces = "application/json") @GetMapping("/metrics/inventory")
@Operation(summary = "List OTLP metric names") @Operation(summary = "List OTLP metric names")
public ResponseEntity<Message<OtlpMetricsInventory>> metricInventory( public ResponseEntity<Message<OtlpMetricsInventory>> metricInventory(
@RequestParam(value = "start", required = false) Long start, @RequestParam(value = "start", required = false) Long start,
@@ -82,7 +84,7 @@ public class ThreeSignalQueryController {
queryService.metricInventory(start, end, serviceName, serviceNamespace, environment, limit)))); queryService.metricInventory(start, end, serviceName, serviceNamespace, environment, limit))));
} }
@GetMapping(path = "/api/traces/list", produces = "application/json") @GetMapping("/traces")
@Operation(summary = "Query traces") @Operation(summary = "Query traces")
public ResponseEntity<Message<SignalPage<TraceListItem>>> traces( public ResponseEntity<Message<SignalPage<TraceListItem>>> traces(
@RequestParam(value = "start", required = false) Long start, @RequestParam(value = "start", required = false) Long start,
@@ -102,7 +104,7 @@ public class ThreeSignalQueryController {
environment, operationName, minDurationMs, maxDurationMs, pageIndex, pageSize)))); environment, operationName, minDurationMs, maxDurationMs, pageIndex, pageSize))));
} }
@GetMapping(path = "/api/traces/stats/overview", produces = "application/json") @GetMapping("/traces/overview")
@Operation(summary = "Get trace overview") @Operation(summary = "Get trace overview")
public ResponseEntity<Message<TraceOverview>> traceOverview( public ResponseEntity<Message<TraceOverview>> traceOverview(
@RequestParam(value = "start", required = false) Long start, @RequestParam(value = "start", required = false) Long start,
@@ -120,14 +122,14 @@ public class ThreeSignalQueryController {
environment, operationName, minDurationMs, maxDurationMs)))); environment, operationName, minDurationMs, maxDurationMs))));
} }
@GetMapping(path = "/api/traces/{traceId}", produces = "application/json") @GetMapping("/traces/{traceId}")
@Operation(summary = "Get trace detail") @Operation(summary = "Get trace detail")
public ResponseEntity<Message<TraceDetail>> traceDetail(@PathVariable("traceId") String traceId) { public ResponseEntity<Message<TraceDetail>> traceDetail(@PathVariable("traceId") String traceId) {
return workloadGuard.execute(Workload.TRACES, return workloadGuard.execute(Workload.TRACES,
() -> ResponseEntity.ok(Message.success(queryService.traceDetail(traceId)))); () -> ResponseEntity.ok(Message.success(queryService.traceDetail(traceId))));
} }
@GetMapping(path = "/api/traces/{traceId}/spans", produces = "application/json") @GetMapping("/traces/{traceId}/spans")
@Operation(summary = "Get trace spans") @Operation(summary = "Get trace spans")
public ResponseEntity<Message<?>> traceSpans(@PathVariable("traceId") String traceId) { public ResponseEntity<Message<?>> traceSpans(@PathVariable("traceId") String traceId) {
return workloadGuard.execute(Workload.TRACES, return workloadGuard.execute(Workload.TRACES,
@@ -17,7 +17,7 @@
* under the License. * under the License.
*/ */
package org.apache.hertzbeat.log.notice; package org.apache.hertzbeat.observability.notice;
import lombok.Data; import lombok.Data;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@@ -17,7 +17,7 @@
* under the License. * under the License.
*/ */
package org.apache.hertzbeat.log.notice; package org.apache.hertzbeat.observability.notice;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.service;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
/**
* Canonical OTLP log ingestion boundary shared by the HTTP and gRPC transports.
*/
public interface OtlpLogIngestionService {
/**
* Decode and ingest an OTLP/HTTP log request.
*
* @param content encoded OTLP request body
* @param headers request headers
* @return an OTLP response encoded for the request content type
*/
ResponseEntity<byte[]> ingestHttp(byte[] content, HttpHeaders headers);
/**
* Decode and ingest an OTLP/gRPC protobuf request.
*
* @param content encoded {@code ExportLogsServiceRequest}
* @return encoded {@code ExportLogsServiceResponse}
*/
byte[] ingestProtobuf(byte[] content);
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.observability.service;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.observability.service;
/** Raised when a signal workload exceeds its isolated concurrency budget. */ /** Raised when a signal workload exceeds its isolated concurrency budget. */
public class SignalQueryRejectedException extends RuntimeException { public class SignalQueryRejectedException extends RuntimeException {
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.observability.service;
import java.time.Duration; import java.time.Duration;
import java.util.EnumMap; import java.util.EnumMap;
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.service.impl;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.zip.GZIPInputStream;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
/** Default OTLP log ingestion service backed by the HertzBeat log fan-out. */
@Service
public class DefaultOtlpLogIngestionService implements OtlpLogIngestionService {
private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf");
private static final byte[] JSON_RESPONSE = "{}".getBytes(StandardCharsets.UTF_8);
private static final byte[] PROTOBUF_RESPONSE = ExportLogsServiceResponse.getDefaultInstance().toByteArray();
private final OtlpLogProtocolAdapter protocolAdapter;
public DefaultOtlpLogIngestionService(OtlpLogProtocolAdapter protocolAdapter) {
this.protocolAdapter = protocolAdapter;
}
@Override
public ResponseEntity<byte[]> ingestHttp(byte[] content, HttpHeaders headers) {
HttpHeaders safeHeaders = headers == null ? HttpHeaders.EMPTY : headers;
byte[] normalizedContent = maybeDecompress(content, safeHeaders);
MediaType contentType = safeHeaders.getContentType();
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
protocolAdapter.ingest(new String(normalizedContent, StandardCharsets.UTF_8));
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(JSON_RESPONSE);
}
protocolAdapter.ingestBinary(normalizedContent);
return ResponseEntity.ok().contentType(PROTOBUF).body(PROTOBUF_RESPONSE);
}
@Override
public byte[] ingestProtobuf(byte[] content) {
protocolAdapter.ingestBinary(content);
return PROTOBUF_RESPONSE;
}
private byte[] maybeDecompress(byte[] content, HttpHeaders headers) {
byte[] safeContent = content == null ? new byte[0] : content;
List<String> encodings = headers.get(HttpHeaders.CONTENT_ENCODING);
if (encodings == null || encodings.stream().noneMatch(this::containsGzipEncoding)) {
return safeContent;
}
try (GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(safeContent));
ByteArrayOutputStream output = new ByteArrayOutputStream()) {
input.transferTo(output);
return output.toByteArray();
} catch (IOException exception) {
throw new IllegalArgumentException("Malformed gzip OTLP log payload", exception);
}
}
private boolean containsGzipEncoding(String value) {
if (value == null) {
return false;
}
for (String encoding : value.split(",")) {
if ("gzip".equalsIgnoreCase(encoding.trim())) {
return true;
}
}
return false;
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.observability.service.impl;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message; import com.google.protobuf.Message;
@@ -34,20 +34,15 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import org.apache.hertzbeat.log.service.OtlpSignalForwarder; import org.apache.hertzbeat.observability.service.OtlpSignalForwarder;
import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
/** Greptime native OTLP forwarder for metrics, logs, and traces. */ /** Greptime native OTLP forwarder for metrics, logs, and traces. */
@Service @Service
@@ -55,25 +50,11 @@ import org.springframework.web.client.RestTemplate;
public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder {
private static final String PROTOBUF = "application/x-protobuf"; private static final String PROTOBUF = "application/x-protobuf";
private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name";
private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name";
private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name";
private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name";
private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name";
private static final String GREPTIME_PROMOTE_RESOURCE_HEADER =
"X-Greptime-OTLP-Metric-Promote-Resource-Attrs";
private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of(
"service.name", "service.namespace", "service.version", "deployment.environment.name",
"host.name", "k8s.namespace.name", "k8s.pod.name"));
private static final Set<String> SIGNALS = Set.of("metrics", "logs", "traces"); private static final Set<String> SIGNALS = Set.of("metrics", "logs", "traces");
private final GreptimeProperties greptimeProperties; private final OtlpSignalStorage signalStorage;
private final RestTemplate restTemplate;
public GreptimeOtlpSignalForwarder(GreptimeProperties greptimeProperties, public GreptimeOtlpSignalForwarder(OtlpSignalStorage signalStorage) {
@Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE) this.signalStorage = signalStorage;
RestTemplate restTemplate) {
this.greptimeProperties = greptimeProperties;
this.restTemplate = restTemplate;
} }
@Override @Override
@@ -81,10 +62,11 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder {
HttpHeaders safeHeaders = requestHeaders == null ? new HttpHeaders() : requestHeaders; HttpHeaders safeHeaders = requestHeaders == null ? new HttpHeaders() : requestHeaders;
byte[] normalized = maybeDecompress(content, safeHeaders); byte[] normalized = maybeDecompress(content, safeHeaders);
MediaType contentType = safeHeaders.getContentType(); MediaType contentType = safeHeaders.getContentType();
byte[] protobuf = contentType != null && MediaType.APPLICATION_JSON.includes(contentType) boolean json = contentType != null && MediaType.APPLICATION_JSON.includes(contentType);
? jsonToProtobuf(signal, normalized) : validateProtobuf(signal, normalized); // Both branches yield already-validated protobuf, so write directly instead of re-parsing via forwardProtobuf.
byte[] response = forwardProtobuf(signal, protobuf); byte[] protobuf = json ? jsonToProtobuf(signal, normalized) : validateProtobuf(signal, normalized);
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) { byte[] response = signalStorage.writeProtobuf(signal, protobuf);
if (json) {
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON) return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON)
.body("{}".getBytes(StandardCharsets.UTF_8)); .body("{}".getBytes(StandardCharsets.UTF_8));
} }
@@ -93,40 +75,8 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder {
@Override @Override
public byte[] forwardProtobuf(String signal, byte[] content) { public byte[] forwardProtobuf(String signal, byte[] content) {
String normalizedSignal = normalizeSignal(signal); byte[] validContent = validateProtobuf(signal, content);
byte[] validContent = validateProtobuf(normalizedSignal, content); return signalStorage.writeProtobuf(signal, validContent);
HttpHeaders headers = greptimeHeaders(normalizedSignal);
ResponseEntity<byte[]> response = restTemplate.exchange(
endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal),
HttpMethod.POST,
new HttpEntity<>(validContent, headers),
byte[].class);
if (!response.getStatusCode().is2xxSuccessful()) {
throw new IllegalStateException("GreptimeDB rejected OTLP " + normalizedSignal);
}
return response.getBody() == null ? new byte[0] : response.getBody();
}
private HttpHeaders greptimeHeaders(String signal) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType(PROTOBUF));
headers.setAccept(List.of(MediaType.parseMediaType(PROTOBUF)));
headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database())
? greptimeProperties.database() : "public");
if ("metrics".equals(signal)) {
headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES);
} else if ("traces".equals(signal)) {
headers.set(GREPTIME_TRACE_TABLE_HEADER, "hzb_traces");
headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1");
} else {
headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME);
headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1");
}
if (StringUtils.hasText(greptimeProperties.username()) && StringUtils.hasText(greptimeProperties.password())) {
String credentials = greptimeProperties.username() + ":" + greptimeProperties.password();
headers.setBasicAuth(Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)));
}
return headers;
} }
private byte[] jsonToProtobuf(String signal, byte[] content) { private byte[] jsonToProtobuf(String signal, byte[] content) {
@@ -213,7 +163,4 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder {
return normalized; return normalized;
} }
private String endpoint(String base, String path) {
return StringUtils.trimTrailingCharacter(base, '/') + path;
}
} }
@@ -15,8 +15,11 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.observability.service.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
@@ -29,14 +32,16 @@ import io.opentelemetry.proto.logs.v1.ScopeLogs;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.log.notice.LogSseManager; import org.apache.hertzbeat.observability.notice.LogSseManager;
import org.apache.hertzbeat.log.service.LogProtocolAdapter;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap; import java.util.HashMap;
import java.util.HexFormat;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
/** /**
* Adapter for OpenTelemetry OTLP/HTTP log ingestion. * Adapter for OpenTelemetry OTLP/HTTP log ingestion.
@@ -46,9 +51,10 @@ import java.util.Map;
*/ */
@Slf4j @Slf4j
@Service @Service
public class OtlpLogProtocolAdapter implements LogProtocolAdapter { public class OtlpLogProtocolAdapter {
private static final String PROTOCOL_NAME = "otlp"; private static final Set<String> OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId");
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final CommonDataQueue commonDataQueue; private final CommonDataQueue commonDataQueue;
private final LogSseManager logSseManager; private final LogSseManager logSseManager;
@@ -58,7 +64,6 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
this.logSseManager = logSseManager; this.logSseManager = logSseManager;
} }
@Override
public void ingest(String content) { public void ingest(String content) {
if (content == null || content.isEmpty()) { if (content == null || content.isEmpty()) {
log.warn("Received empty OTLP JSON log payload - skip processing."); log.warn("Received empty OTLP JSON log payload - skip processing.");
@@ -66,7 +71,7 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
} }
ExportLogsServiceRequest.Builder builder = ExportLogsServiceRequest.newBuilder(); ExportLogsServiceRequest.Builder builder = ExportLogsServiceRequest.newBuilder();
try { try {
JsonFormat.parser().ignoringUnknownFields().merge(content, builder); JsonFormat.parser().ignoringUnknownFields().merge(normalizeOtlpJson(content), builder);
ExportLogsServiceRequest request = builder.build(); ExportLogsServiceRequest request = builder.build();
processLogsRequest(request, "JSON"); processLogsRequest(request, "JSON");
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
@@ -274,8 +279,58 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
return hexString.toString(); return hexString.toString();
} }
@Override /**
public String supportProtocol() { * Accept the hexadecimal IDs emitted by the HertzBeat onboarding probe while preserving
return PROTOCOL_NAME; * standard base64-encoded OTLP/JSON IDs.
*/
private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException {
try {
JsonNode root = OBJECT_MAPPER.readTree(content);
normalizeOtlpHexEncodedIds(root);
return OBJECT_MAPPER.writeValueAsString(root);
} catch (Exception exception) {
throw new InvalidProtocolBufferException("Failed to normalize OTLP JSON: " + exception.getMessage());
}
} }
}
private void normalizeOtlpHexEncodedIds(JsonNode node) {
if (node == null) {
return;
}
if (node.isObject()) {
ObjectNode objectNode = (ObjectNode) node;
objectNode.fieldNames().forEachRemaining(fieldName -> {
JsonNode child = objectNode.get(fieldName);
if (OTLP_HEX_ID_FIELDS.contains(fieldName) && child != null && child.isTextual()) {
String normalized = tryConvertHexToBase64(child.asText());
if (normalized != null) {
objectNode.put(fieldName, normalized);
}
} else {
normalizeOtlpHexEncodedIds(child);
}
});
return;
}
if (node.isArray()) {
node.forEach(this::normalizeOtlpHexEncodedIds);
}
}
private String tryConvertHexToBase64(String value) {
if (value == null || value.isBlank() || (value.length() & 1) != 0) {
return null;
}
for (int i = 0; i < value.length(); i++) {
if (Character.digit(value.charAt(i), 16) < 0) {
return null;
}
}
try {
return Base64.getEncoder().encodeToString(HexFormat.of().parseHex(value));
} catch (IllegalArgumentException exception) {
return null;
}
}
}
@@ -13,5 +13,4 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ org.apache.hertzbeat.observability.config.ObservabilityAutoConfiguration
org.apache.hertzbeat.otel.config.LogAutoConfiguration
@@ -15,13 +15,14 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.config; package org.apache.hertzbeat.observability.config;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import io.grpc.ManagedChannel; import io.grpc.ManagedChannel;
@@ -33,15 +34,21 @@ import io.grpc.StatusRuntimeException;
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
import io.grpc.stub.MetadataUtils; import io.grpc.stub.MetadataUtils;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Supplier; import java.util.function.Supplier;
import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator;
import org.apache.hertzbeat.log.service.OtlpSignalForwarder; import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.OtlpSignalForwarder;
import org.apache.hertzbeat.log.service.SignalQueryRejectedException; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -50,10 +57,12 @@ class OtlpGrpcServerConfigTest {
private Server server; private Server server;
private ManagedChannel channel; private ManagedChannel channel;
private OtlpLogIngestionService logIngestionService;
@BeforeEach @BeforeEach
void setUp() throws Exception { void setUp() throws Exception {
OtlpSignalForwarder forwarder = mock(OtlpSignalForwarder.class); OtlpSignalForwarder forwarder = mock(OtlpSignalForwarder.class);
logIngestionService = mock(OtlpLogIngestionService.class);
SignalWorkloadGuard guard = mock(SignalWorkloadGuard.class); SignalWorkloadGuard guard = mock(SignalWorkloadGuard.class);
OtlpAccessTokenValidator validator = token -> "probe-token".equals(token) ? null : "Invalid token"; OtlpAccessTokenValidator validator = token -> "probe-token".equals(token) ? null : "Invalid token";
when(guard.execute(eq(SignalWorkloadGuard.Workload.OTLP_WRITE), any())).thenAnswer(invocation -> { when(guard.execute(eq(SignalWorkloadGuard.Workload.OTLP_WRITE), any())).thenAnswer(invocation -> {
@@ -61,10 +70,14 @@ class OtlpGrpcServerConfigTest {
return action.get(); return action.get();
}); });
when(forwarder.forwardProtobuf(eq("metrics"), any())).thenReturn(new byte[0]); when(forwarder.forwardProtobuf(eq("metrics"), any())).thenReturn(new byte[0]);
when(logIngestionService.ingestProtobuf(any())).thenReturn(new byte[0]);
server = NettyServerBuilder.forPort(0) server = NettyServerBuilder.forPort(0)
.addService(ServerInterceptors.intercept( .addService(ServerInterceptors.intercept(
new OtlpGrpcServerConfig.MetricsService(forwarder, guard), new OtlpGrpcServerConfig.MetricsService(forwarder, guard),
new OtlpGrpcServerConfig.BearerTokenInterceptor(validator))) new OtlpGrpcServerConfig.BearerTokenInterceptor(validator)))
.addService(ServerInterceptors.intercept(
new OtlpGrpcServerConfig.LogsService(logIngestionService, guard),
new OtlpGrpcServerConfig.BearerTokenInterceptor(validator)))
.build().start(); .build().start();
channel = NettyChannelBuilder.forAddress("127.0.0.1", server.getPort()).usePlaintext().build(); channel = NettyChannelBuilder.forAddress("127.0.0.1", server.getPort()).usePlaintext().build();
} }
@@ -86,6 +99,18 @@ class OtlpGrpcServerConfigTest {
assertThat(response).isEqualTo(ExportMetricsServiceResponse.getDefaultInstance()); assertThat(response).isEqualTo(ExportMetricsServiceResponse.getDefaultInstance());
} }
@Test
void shouldRouteAuthorizedOtlpGrpcLogsThroughLogIngestionService() {
Metadata headers = new Metadata();
headers.put(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER), "Bearer probe-token");
ExportLogsServiceResponse response = LogsServiceGrpc.newBlockingStub(channel)
.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers))
.export(ExportLogsServiceRequest.getDefaultInstance());
assertThat(response).isEqualTo(ExportLogsServiceResponse.getDefaultInstance());
verify(logIngestionService).ingestProtobuf(any());
}
@Test @Test
void shouldRejectMissingOtlpGrpcBearerToken() { void shouldRejectMissingOtlpGrpcBearerToken() {
assertThatThrownBy(() -> MetricsServiceGrpc.newBlockingStub(channel) assertThatThrownBy(() -> MetricsServiceGrpc.newBlockingStub(channel)
@@ -104,4 +129,46 @@ class OtlpGrpcServerConfigTest {
assertThat(OtlpGrpcServerConfig.toGrpcStatus(new IllegalStateException("storage"))) assertThat(OtlpGrpcServerConfig.toGrpcStatus(new IllegalStateException("storage")))
.isEqualTo(Status.UNAVAILABLE); .isEqualTo(Status.UNAVAILABLE);
} }
/**
* A bind failure must cost the gRPC channel only, never the whole application: HTTP ingestion
* serves the same signals on the main port, so an optional side channel has no business keeping
* the monitoring system from starting.
*/
@Test
void shouldGiveUpTheGrpcListenerInsteadOfTheApplicationWhenThePortIsTaken() throws Exception {
// Bind the same address the runner will use: on bsd a wildcard socket does not stop a
// listener from taking 127.0.0.1 on the same port.
try (ServerSocket occupied = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) {
OtlpGrpcServerConfig.OtlpGrpcServerRunner runner = runnerOn(occupied.getLocalPort());
runner.start();
assertThat(runner.isRunning()).isFalse();
runner.stop();
}
}
@Test
void shouldReportTheGrpcListenerRunningOnceItBinds() throws Exception {
int freePort;
try (ServerSocket probe = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) {
freePort = probe.getLocalPort();
}
OtlpGrpcServerConfig.OtlpGrpcServerRunner runner = runnerOn(freePort);
runner.start();
try {
assertThat(runner.isRunning()).isTrue();
} finally {
runner.stop();
}
}
private OtlpGrpcServerConfig.OtlpGrpcServerRunner runnerOn(int port) {
return new OtlpGrpcServerConfig.OtlpGrpcServerRunner("127.0.0.1", port,
mock(OtlpSignalForwarder.class), mock(OtlpLogIngestionService.class),
mock(SignalWorkloadGuard.class),
new OtlpGrpcServerConfig.BearerTokenInterceptor(token -> null));
}
} }
@@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import com.google.rpc.Code;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
/** Deprecated 1.8.x OTLP log route alias contract tests. */
@ExtendWith(MockitoExtension.class)
class LegacyOtlpLogRouteControllerTest {
@Mock
private OtlpLogIngestionService logIngestionService;
private MockMvc mockMvc;
@BeforeEach
void setUp() {
mockMvc = MockMvcBuilders.standaloneSetup(
new LegacyOtlpLogRouteController(logIngestionService, new SignalWorkloadGuard()))
.setControllerAdvice(new OtlpHttpExceptionHandler())
.build();
}
@Test
void legacyOtlpRouteShouldForwardToTheCanonicalLogFanOutWithDeprecationHeaders() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes()));
mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content("{}"))
.andExpect(status().isOk())
.andExpect(header().string("Deprecation", "true"))
.andExpect(header().string("Link", "</api/otlp/v1/logs>; rel=\"successor-version\""))
.andExpect(content().string("{}"));
verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class));
}
@Test
void legacyIngestOtlpRouteShouldForwardToTheCanonicalLogFanOut() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes()));
mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/ingest/OTLP")
.contentType(MediaType.APPLICATION_JSON)
.content("{}"))
.andExpect(status().isOk())
.andExpect(header().string("Deprecation", "true"));
verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class));
}
@Test
void legacyIngestRouteShouldRejectNonOtlpProtocolsWithoutTouchingTheFanOut() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/ingest/vector")
.contentType(MediaType.APPLICATION_JSON)
.content("{}"))
.andExpect(status().isBadRequest())
.andExpect(header().string("Deprecation", "true"));
verifyNoInteractions(logIngestionService);
}
@Test
void legacyRouteShouldPreserveCanonicalErrorStatus() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenThrow(new IllegalArgumentException("Malformed OTLP logs JSON payload"));
mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content("{"))
.andExpect(status().isBadRequest())
.andExpect(header().string("Deprecation", "true"))
.andExpect(header().string("Link", "</api/otlp/v1/logs>; rel=\"successor-version\""))
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber()))
.andExpect(jsonPath("$.message").value("Malformed OTLP logs JSON payload"));
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -61,7 +61,7 @@ class LogManagerControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.delete("/api/logs") .delete("/api/observability/logs")
.param("timeUnixNanos", "1734005477630000000", "1734005477640000000") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000")
) )
.andDo(print()) .andDo(print())
@@ -77,7 +77,7 @@ class LogManagerControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.delete("/api/logs") .delete("/api/observability/logs")
.param("timeUnixNanos", "1734005477630000000", "1734005477640000000") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
@@ -91,7 +91,7 @@ class LogManagerControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.delete("/api/logs") .delete("/api/observability/logs")
.param("timeUnixNanos", "1734005477630000000") .param("timeUnixNanos", "1734005477630000000")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
@@ -32,7 +32,7 @@ import java.util.Map;
import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -69,8 +69,8 @@ class LogQueryControllerTest {
void shouldReturnFriendlyFailureWhenLogQueryUnsupported() throws Exception { void shouldReturnFriendlyFailureWhenLogQueryUnsupported() throws Exception {
when(historyDataReader.supportsLogQuery()).thenReturn(false); when(historyDataReader.supportsLogQuery()).thenReturn(false);
for (String path : List.of("/api/logs/list", "/api/logs/stats/overview", for (String path : List.of("/api/observability/logs", "/api/observability/logs/overview",
"/api/logs/stats/trace-coverage", "/api/logs/stats/trend")) { "/api/observability/logs/trace-coverage", "/api/observability/logs/trend")) {
mockMvc.perform(MockMvcRequestBuilders.get(path)) mockMvc.perform(MockMvcRequestBuilders.get(path))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
@@ -108,7 +108,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/list") .get("/api/observability/logs")
.param("start", "1734005477000") .param("start", "1734005477000")
.param("end", "1734005478000") .param("end", "1734005478000")
.param("traceId", "trace123") .param("traceId", "trace123")
@@ -144,7 +144,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/list") .get("/api/observability/logs")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -157,7 +157,7 @@ class LogQueryControllerTest {
when(historyDataReader.countObservabilityLogs(any())).thenReturn(0L); when(historyDataReader.countObservabilityLogs(any())).thenReturn(0L);
when(historyDataReader.queryObservabilityLogs(any(), any(), any())).thenReturn(List.of()); when(historyDataReader.queryObservabilityLogs(any(), any(), any())).thenReturn(List.of());
mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs")
.param("pageIndex", "-1") .param("pageIndex", "-1")
.param("pageSize", "1000")) .param("pageSize", "1000"))
.andExpect(status().isOk()) .andExpect(status().isOk())
@@ -172,7 +172,7 @@ class LogQueryControllerTest {
when(historyDataReader.countObservabilityLogs(any())) when(historyDataReader.countObservabilityLogs(any()))
.thenThrow(new IllegalArgumentException("Invalid resource filter expression")); .thenThrow(new IllegalArgumentException("Invalid resource filter expression"));
mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs")
.param("resource", "bad-expression")) .param("resource", "bad-expression"))
.andExpect(status().isBadRequest()) .andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)); .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE));
@@ -186,7 +186,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/stats/overview") .get("/api/observability/logs/overview")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -205,7 +205,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/stats/overview") .get("/api/observability/logs/overview")
.param("start", "1734005477000") .param("start", "1734005477000")
.param("end", "1734005478000") .param("end", "1734005478000")
) )
@@ -221,7 +221,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/stats/trace-coverage") .get("/api/observability/logs/trace-coverage")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -238,7 +238,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/stats/trend") .get("/api/observability/logs/trend")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -251,7 +251,7 @@ class LogQueryControllerTest {
mockMvc.perform( mockMvc.perform(
MockMvcRequestBuilders MockMvcRequestBuilders
.get("/api/logs/stats/trend") .get("/api/observability/logs/trend")
) )
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyLong;
@@ -24,8 +24,8 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria;
import org.apache.hertzbeat.log.notice.LogSseManager; import org.apache.hertzbeat.observability.notice.LogSseManager;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@@ -64,7 +64,7 @@ class LogSseControllerTest {
@Test @Test
void testSubscribeWithoutFilters() throws Exception { void testSubscribeWithoutFilters() throws Exception {
// When: A request is made to the subscribe endpoint without any parameters // When: A request is made to the subscribe endpoint without any parameters
mockMvc.perform(get("/api/logs/sse/subscribe") mockMvc.perform(get("/api/observability/logs/stream")
.accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .accept(MediaType.TEXT_EVENT_STREAM_VALUE))
.andExpect(status().isOk()); .andExpect(status().isOk());
@@ -86,7 +86,7 @@ class LogSseControllerTest {
.standaloneSetup(new LogSseController(realEmitterManager)) .standaloneSetup(new LogSseController(realEmitterManager))
.build(); .build();
MvcResult result = realMockMvc.perform(get("/api/logs/sse/subscribe") MvcResult result = realMockMvc.perform(get("/api/observability/logs/stream")
.accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .accept(MediaType.TEXT_EVENT_STREAM_VALUE))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(request().asyncStarted()) .andExpect(request().asyncStarted())
@@ -109,7 +109,7 @@ class LogSseControllerTest {
String spanId = "abcdef1234567890"; String spanId = "abcdef1234567890";
// When: A request is made with all filter parameters // When: A request is made with all filter parameters
mockMvc.perform(get("/api/logs/sse/subscribe") mockMvc.perform(get("/api/observability/logs/stream")
.param("severityText", severityText) .param("severityText", severityText)
.param("severityNumber", severityNumber) .param("severityNumber", severityNumber)
.param("traceId", traceId) .param("traceId", traceId)
@@ -130,7 +130,7 @@ class LogSseControllerTest {
@Test @Test
void testSubscribeWithInvalidSeverityNumber() throws Exception { void testSubscribeWithInvalidSeverityNumber() throws Exception {
// When: A request is made with a non-integer value for severityNumber // When: A request is made with a non-integer value for severityNumber
mockMvc.perform(get("/api/logs/sse/subscribe") mockMvc.perform(get("/api/observability/logs/stream")
.param("severityNumber", "not-a-number") .param("severityNumber", "not-a-number")
.accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .accept(MediaType.TEXT_EVENT_STREAM_VALUE))
.andExpect(status().is4xxClientError()); .andExpect(status().is4xxClientError());
@@ -15,14 +15,14 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -0,0 +1,205 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import com.google.protobuf.util.JsonFormat;
import com.google.rpc.Code;
import com.google.rpc.Status;
import java.nio.charset.StandardCharsets;
import org.apache.hertzbeat.common.support.exception.StorageUnavailableException;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.junit.jupiter.api.Test;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.http.MockHttpInputMessage;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.HttpRequestMethodNotSupportedException;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.method.HandlerMethod;
/** OTLP/HTTP failure response contract tests. */
class OtlpHttpExceptionHandlerTest {
private final OtlpHttpExceptionHandler handler = new OtlpHttpExceptionHandler();
@Test
void shouldEncodeRpcStatusAsJsonForJsonRequests() throws Exception {
ResponseEntity<byte[]> response = handler.handleInvalidPayload(
new IllegalArgumentException("Malformed OTLP metrics JSON payload"),
request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode());
assertEquals(MediaType.APPLICATION_JSON, response.getHeaders().getContentType());
assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
Status status = parseJson(response.getBody());
assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode());
assertEquals("Malformed OTLP metrics JSON payload", status.getMessage());
}
@Test
void shouldEncodeRpcStatusAsProtobufForProtobufRequests() throws Exception {
ResponseEntity<byte[]> response = handler.handleInvalidPayload(
new IllegalArgumentException("Malformed OTLP metrics protobuf payload"),
request("application/x-protobuf"), null);
assertEquals(OtlpHttpExceptionHandler.PROTOBUF, response.getHeaders().getContentType());
Status status = Status.parseFrom(response.getBody());
assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode());
assertEquals("Malformed OTLP metrics protobuf payload", status.getMessage());
}
@Test
void shouldFallBackToProtobufWhenContentTypeIsMissingOrUnparseable() throws Exception {
for (String contentType : new String[] {null, "not a media type"}) {
ResponseEntity<byte[]> response = handler.handleInvalidPayload(
new IllegalArgumentException("bad"), request(contentType), null);
assertEquals(OtlpHttpExceptionHandler.PROTOBUF, response.getHeaders().getContentType());
assertEquals("bad", Status.parseFrom(response.getBody()).getMessage());
}
}
@Test
void shouldNotFailWhenExceptionMessageIsNull() throws Exception {
ResponseEntity<byte[]> response = handler.handleInvalidPayload(
new IllegalArgumentException(), request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode());
assertEquals("Unknown error", parseJson(response.getBody()).getMessage());
}
@Test
void shouldReturnRetryableTooManyRequestsWhenOverloaded() throws Exception {
ResponseEntity<byte[]> response = handler.handleOverloaded(
new SignalQueryRejectedException("OTLP_WRITE"), request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.TOO_MANY_REQUESTS, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
assertEquals(Code.RESOURCE_EXHAUSTED.getNumber(), parseJson(response.getBody()).getCode());
}
@Test
void shouldReturnRetryableServiceUnavailableWhenStorageIsDown() throws Exception {
ResponseEntity<byte[]> response = handler.handleStorageUnavailable(
new StorageUnavailableException("GreptimeDB log storage is unavailable", new RuntimeException()),
request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
Status status = parseJson(response.getBody());
assertEquals(Code.UNAVAILABLE.getNumber(), status.getCode());
assertEquals("GreptimeDB log storage is unavailable", status.getMessage());
}
@Test
void shouldHideTransportDetailsForGreptimeClientFailures() throws Exception {
ResponseEntity<byte[]> response = handler.handleStorageClientFailure(
new HttpServerErrorException(HttpStatus.BAD_GATEWAY, "upstream http://greptime:4000 exploded"),
request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
assertEquals("GreptimeDB storage is unavailable", parseJson(response.getBody()).getMessage());
}
@Test
void shouldReturnInternalErrorStatusForUnexpectedFailures() throws Exception {
ResponseEntity<byte[]> response = handler.handleUnexpected(
new NullPointerException("boom"), request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, response.getStatusCode());
assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
Status status = parseJson(response.getBody());
assertEquals(Code.INTERNAL.getNumber(), status.getCode());
// The exporter gets a fixed phrase; the real cause only reaches the server log.
assertEquals("Unexpected OTLP ingestion failure", status.getMessage());
assertFalse(status.getMessage().contains("boom"));
}
/**
* A missing body is raised while spring resolves the arguments, so it never reaches the
* controller and is not an {@code ErrorResponse}. It must still answer 400 rather than 500:
* OTLP exporters replay 5xx, so a permanently malformed client would retry forever.
*/
@Test
void shouldRejectUnreadableBodyAsNonRetryableClientError() throws Exception {
HttpMessageNotReadableException exception = new HttpMessageNotReadableException(
"Required request body is missing: public org.springframework.http.ResponseEntity<byte[]> "
+ "org.apache.hertzbeat.observability.controller.OtlpLogController.logs(byte[])",
new MockHttpInputMessage(new byte[0]));
ResponseEntity<byte[]> response = handler.handleUnreadableBody(
exception, request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode());
assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER));
Status status = parseJson(response.getBody());
assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode());
assertEquals("Malformed or missing OTLP request body", status.getMessage());
// The framework message names the handler method; that must not travel back to the caller.
assertFalse(status.getMessage().contains("org.apache.hertzbeat"));
}
@Test
void shouldKeepSpringMvcClientErrorStatusInsteadOfInternalError() throws Exception {
ResponseEntity<byte[]> response = handler.handleUnexpected(
new HttpRequestMethodNotSupportedException("GET"), request(MediaType.APPLICATION_JSON_VALUE), null);
assertEquals(HttpStatus.METHOD_NOT_ALLOWED, response.getStatusCode());
assertEquals(Code.INVALID_ARGUMENT.getNumber(), parseJson(response.getBody()).getCode());
}
@Test
void shouldKeepDeprecationHeadersOnLegacyRouteErrors() throws Exception {
LegacyOtlpLogRouteController legacy = new LegacyOtlpLogRouteController(
mock(OtlpLogIngestionService.class), new SignalWorkloadGuard());
HandlerMethod legacyHandler = new HandlerMethod(legacy,
LegacyOtlpLogRouteController.class.getMethod("legacyOtlpLogs", byte[].class, HttpHeaders.class));
ResponseEntity<byte[]> response = handler.handleInvalidPayload(
new IllegalArgumentException("bad"), request(MediaType.APPLICATION_JSON_VALUE), legacyHandler);
assertEquals("true", response.getHeaders().getFirst("Deprecation"));
assertTrue(response.getHeaders().getFirst("Link").contains(LegacyOtlpLogRouteController.CANONICAL_LOGS_ROUTE));
}
private static MockHttpServletRequest request(String contentType) {
MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/otlp/v1/metrics");
if (contentType != null) {
request.setContentType(contentType);
}
return request;
}
private static Status parseJson(byte[] body) throws Exception {
Status.Builder builder = Status.newBuilder();
JsonFormat.parser().merge(new String(body, StandardCharsets.UTF_8), builder);
return builder.build();
}
}
@@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.not;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import com.google.rpc.Code;
import com.google.rpc.Status;
import org.apache.hertzbeat.observability.service.OtlpLogIngestionService;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
/** OTLP/HTTP log route contract tests. */
@ExtendWith(MockitoExtension.class)
class OtlpLogControllerTest {
@Mock
private OtlpLogIngestionService logIngestionService;
private MockMvc mockMvc;
@BeforeEach
void setUp() {
mockMvc = MockMvcBuilders.standaloneSetup(
new OtlpLogController(logIngestionService, new SignalWorkloadGuard()))
.setControllerAdvice(new OtlpHttpExceptionHandler())
.build();
}
@Test
void shouldRouteLogsToTheLogFanOut() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes()));
mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content("{}"))
.andExpect(status().isOk());
verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class));
}
@Test
void shouldReturnBadRequestForMalformedOtlpPayload() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenThrow(new IllegalArgumentException("Malformed OTLP logs JSON payload"));
mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON)
.content("{"))
.andExpect(status().isBadRequest())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber()))
.andExpect(jsonPath("$.message").value("Malformed OTLP logs JSON payload"));
}
@Test
void shouldReturnBinaryRpcStatusForMalformedProtobufPayload() throws Exception {
when(logIngestionService.ingestHttp(any(), any()))
.thenThrow(new IllegalArgumentException("Malformed OTLP logs protobuf payload"));
byte[] body = mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs")
.contentType(OtlpHttpExceptionHandler.PROTOBUF)
.content(new byte[] {1, 2, 3}))
.andExpect(status().isBadRequest())
.andExpect(content().contentType(OtlpHttpExceptionHandler.PROTOBUF))
.andReturn().getResponse().getContentAsByteArray();
Status status = Status.parseFrom(body);
assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode());
assertEquals("Malformed OTLP logs protobuf payload", status.getMessage());
}
/**
* End to end guard for the missing body path: it is resolved before the controller runs, so only a
* request that actually goes through the dispatcher proves the advice turns it into a 400 with a
* google.rpc.Status body that does not name the handler method.
*/
@Test
void shouldRejectMissingBodyWithClientErrorInsteadOfServerError() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs")
.contentType(MediaType.APPLICATION_JSON))
.andExpect(status().isBadRequest())
.andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber()))
.andExpect(jsonPath("$.message").value("Malformed or missing OTLP request body"))
.andExpect(content().string(not(containsString("org.apache.hertzbeat"))));
verifyNoInteractions(logIngestionService);
}
}
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.Test;
class OtlpLogIngestionOwnershipTest {
@Test
void httpAndGrpcLogsShouldUseTheLogFanOutInsteadOfTheRawGreptimeForwarder() throws Exception {
String httpController = Files.readString(Path.of(
"src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java"));
String grpcConfig = Files.readString(Path.of(
"src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java"));
assertTrue(httpController.contains("OtlpLogIngestionService"));
assertTrue(httpController.contains("logIngestionService.ingestHttp(content, headers)"));
assertFalse(httpController.contains("forward(\"logs\""));
assertTrue(grpcConfig.contains("OtlpLogIngestionService"));
assertTrue(grpcConfig.contains("logIngestionService.ingestProtobuf(request.toByteArray())"));
assertFalse(grpcConfig.contains("forwardProtobuf(\"logs\""));
}
}
@@ -15,17 +15,23 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.apache.hertzbeat.log.service.OtlpSignalForwarder; import com.google.rpc.Code;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import com.google.rpc.Status;
import org.apache.hertzbeat.observability.service.OtlpSignalForwarder;
import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@@ -37,6 +43,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.client.ResourceAccessException;
/** OTLP/HTTP route contract tests. */ /** OTLP/HTTP route contract tests. */
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
@@ -50,21 +57,25 @@ class OtlpSignalControllerTest {
@BeforeEach @BeforeEach
void setUp() { void setUp() {
mockMvc = MockMvcBuilders.standaloneSetup( mockMvc = MockMvcBuilders.standaloneSetup(
new OtlpSignalController(signalForwarder, new SignalWorkloadGuard())).build(); new OtlpSignalController(signalForwarder, new SignalWorkloadGuard()))
.setControllerAdvice(new OtlpHttpExceptionHandler())
.build();
} }
@Test @Test
void shouldRouteAllThreeSignals() throws Exception { void shouldRouteGreptimeSignals() throws Exception {
when(signalForwarder.forwardHttp(any(), any(), any())) when(signalForwarder.forwardHttp(any(), any(), any()))
.thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes()));
for (String signal : new String[] {"metrics", "logs", "traces"}) { for (String signal : new String[] {"metrics", "traces"}) {
mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/" + signal) mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/" + signal)
.contentType(MediaType.APPLICATION_JSON) .contentType(MediaType.APPLICATION_JSON)
.content("{}")) .content("{}"))
.andExpect(status().isOk()); .andExpect(status().isOk());
verify(signalForwarder).forwardHttp(eq(signal), any(), any(HttpHeaders.class)); verify(signalForwarder).forwardHttp(eq(signal), any(), any(HttpHeaders.class));
} }
verifyNoMoreInteractions(signalForwarder);
} }
@Test @Test
@@ -76,6 +87,26 @@ class OtlpSignalControllerTest {
.contentType(MediaType.APPLICATION_JSON) .contentType(MediaType.APPLICATION_JSON)
.content("{")) .content("{"))
.andExpect(status().isBadRequest()) .andExpect(status().isBadRequest())
.andExpect(content().string("Malformed OTLP metrics JSON payload")); .andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber()))
.andExpect(jsonPath("$.message").value("Malformed OTLP metrics JSON payload"));
}
@Test
void shouldReturnRetryableServiceUnavailableWhenGreptimeWriteFails() throws Exception {
when(signalForwarder.forwardHttp(eq("traces"), any(), any()))
.thenThrow(new ResourceAccessException("connect refused: http://greptime:4000"));
byte[] body = mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/traces")
.contentType(OtlpHttpExceptionHandler.PROTOBUF)
.content(new byte[] {1, 2, 3}))
.andExpect(status().isServiceUnavailable())
.andExpect(header().string(HttpHeaders.RETRY_AFTER, "1"))
.andExpect(content().contentType(OtlpHttpExceptionHandler.PROTOBUF))
.andReturn().getResponse().getContentAsByteArray();
Status status = Status.parseFrom(body);
assertEquals(Code.UNAVAILABLE.getNumber(), status.getCode());
assertEquals("GreptimeDB storage is unavailable", status.getMessage());
} }
} }
@@ -15,13 +15,14 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.log.service.SignalQueryRejectedException; import org.apache.hertzbeat.observability.service.SignalQueryRejectedException;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
@@ -54,6 +55,16 @@ class SignalWorkloadExceptionHandlerTest {
Order order = SignalWorkloadExceptionHandler.class.getAnnotation(Order.class); Order order = SignalWorkloadExceptionHandler.class.getAnnotation(Order.class);
assertNotNull(order); assertNotNull(order);
assertEquals(Ordered.HIGHEST_PRECEDENCE, order.value()); assertTrue(order.value() < Ordered.LOWEST_PRECEDENCE);
}
@Test
void shouldYieldToTheOtlpHttpAdvice() {
Order order = SignalWorkloadExceptionHandler.class.getAnnotation(Order.class);
Order otlpOrder = OtlpHttpExceptionHandler.class.getAnnotation(Order.class);
assertNotNull(order);
assertNotNull(otlpOrder);
assertTrue(otlpOrder.value() < order.value());
} }
} }
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.controller; package org.apache.hertzbeat.observability.controller;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
@@ -30,8 +30,8 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail;
import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem;
import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview;
import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode;
import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard; import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@@ -62,7 +62,7 @@ class ThreeSignalQueryControllerTest {
"prod", null, null, null, 60, null)) "prod", null, null, null, 60, null))
.thenReturn(new OtlpMetricsConsole("http_server_duration", 1000L, 2000L, 60, List.of())); .thenReturn(new OtlpMetricsConsole("http_server_duration", 1000L, 2000L, 60, List.of()));
mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/console") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/query")
.param("query", "http_server_duration") .param("query", "http_server_duration")
.param("start", "1000") .param("start", "1000")
.param("end", "2000") .param("end", "2000")
@@ -80,7 +80,7 @@ class ThreeSignalQueryControllerTest {
when(queryService.metricInventory(1000L, 2000L, "checkout", "payments", "prod", 50)) when(queryService.metricInventory(1000L, 2000L, "checkout", "payments", "prod", 50))
.thenReturn(new OtlpMetricsInventory(List.of("http_server_duration"))); .thenReturn(new OtlpMetricsInventory(List.of("http_server_duration")));
mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/inventory") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/inventory")
.param("start", "1000") .param("start", "1000")
.param("end", "2000") .param("end", "2000")
.param("serviceName", "checkout") .param("serviceName", "checkout")
@@ -102,7 +102,7 @@ class ThreeSignalQueryControllerTest {
"SERVER", "failed", 10_000_000L, 1000L, Map.of(), Map.of(), List.of()); "SERVER", "failed", 10_000_000L, 1000L, Map.of(), Map.of(), List.of());
when(queryService.traceDetail("trace-1")).thenReturn(new TraceDetail(item, List.of(span))); when(queryService.traceDetail("trace-1")).thenReturn(new TraceDetail(item, List.of(span)));
mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/list") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces")
.param("start", "1000") .param("start", "1000")
.param("end", "2000") .param("end", "2000")
.param("traceId", "trace-1") .param("traceId", "trace-1")
@@ -116,7 +116,7 @@ class ThreeSignalQueryControllerTest {
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.data.content[0].traceId").value("trace-1")); .andExpect(jsonPath("$.data.content[0].traceId").value("trace-1"));
mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/trace-1")) mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/trace-1"))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(jsonPath("$.data.spans[0].spanId").value("span-1")); .andExpect(jsonPath("$.data.spans[0].spanId").value("span-1"));
} }
@@ -127,7 +127,7 @@ class ThreeSignalQueryControllerTest {
"GET /cart", 1L, 100L)) "GET /cart", 1L, 100L))
.thenReturn(new TraceOverview(4, 1, 0.25, 12.5, 24.0)); .thenReturn(new TraceOverview(4, 1, 0.25, 12.5, 24.0));
mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/stats/overview") mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/overview")
.param("start", "1000") .param("start", "1000")
.param("end", "2000") .param("end", "2000")
.param("traceId", "trace-1") .param("traceId", "trace-1")
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.notice; package org.apache.hertzbeat.observability.notice;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@@ -17,7 +17,7 @@
* under the License. * under the License.
*/ */
package org.apache.hertzbeat.log.notice; package org.apache.hertzbeat.observability.notice;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.observability.service;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -25,8 +25,8 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Limits; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Limits;
import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
/** Tests bounded signal workload isolation and recovery. */ /** Tests bounded signal workload isolation and recovery. */
@@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.observability.service.impl;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.verify;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.zip.GZIPOutputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@ExtendWith(MockitoExtension.class)
class DefaultOtlpLogIngestionServiceTest {
@Mock
private OtlpLogProtocolAdapter protocolAdapter;
private DefaultOtlpLogIngestionService service;
@BeforeEach
void setUp() {
service = new DefaultOtlpLogIngestionService(protocolAdapter);
}
@Test
void shouldIngestJsonAndReturnJsonOtlpResponse() {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
var response = service.ingestHttp("{}".getBytes(StandardCharsets.UTF_8), headers);
verify(protocolAdapter).ingest("{}");
assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON);
assertThat(response.getBody()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8));
}
@Test
void shouldIngestProtobufAndReturnProtobufOtlpResponse() {
byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType("application/x-protobuf"));
var response = service.ingestHttp(request, headers);
verify(protocolAdapter).ingestBinary(request);
assertThat(response.getHeaders().getContentType())
.isEqualTo(MediaType.parseMediaType("application/x-protobuf"));
assertThat(response.getBody()).isEmpty();
}
@Test
void shouldDecompressCommaSeparatedGzipContentEncoding() throws Exception {
byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType("application/x-protobuf"));
headers.add(HttpHeaders.CONTENT_ENCODING, "identity, gzip");
service.ingestHttp(gzip(request), headers);
verify(protocolAdapter).ingestBinary(request);
}
@Test
void shouldRejectMalformedGzipBeforeFanOut() {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType("application/x-protobuf"));
headers.set(HttpHeaders.CONTENT_ENCODING, "gzip");
assertThatThrownBy(() -> service.ingestHttp(new byte[] {1, 2, 3}, headers))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Malformed gzip OTLP log payload");
}
private byte[] gzip(byte[] content) throws Exception {
try (ByteArrayOutputStream output = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(output)) {
gzip.write(content);
gzip.finish();
return output.toByteArray();
}
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.observability.service.impl;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -28,33 +28,27 @@ import static org.mockito.Mockito.when;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
/** Greptime native OTLP forwarding tests. */ /** Greptime native OTLP forwarding tests. */
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
class GreptimeOtlpSignalForwarderTest { class GreptimeOtlpSignalForwarderTest {
@Mock @Mock
private RestTemplate restTemplate; private OtlpSignalStorage signalStorage;
@Test @Test
void shouldConvertJsonAndForwardMetricsWithResourcePromotion() { void shouldConvertJsonBeforeWritingMetrics() {
when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), when(signalStorage.writeProtobuf(eq("metrics"), any())).thenReturn(new byte[0]);
any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage);
GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(
new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000",
"public", "greptime", "secret"), restTemplate);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON); headers.setContentType(MediaType.APPLICATION_JSON);
@@ -62,21 +56,15 @@ class GreptimeOtlpSignalForwarderTest {
headers); headers);
assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
ArgumentCaptor<HttpEntity<byte[]>> request = ArgumentCaptor.forClass(HttpEntity.class); ArgumentCaptor<byte[]> content = ArgumentCaptor.forClass(byte[].class);
verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), verify(signalStorage).writeProtobuf(eq("metrics"), content.capture());
request.capture(), eq(byte[].class)); assertThat(content.getValue()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray());
assertThat(request.getValue().getHeaders().getFirst("X-Greptime-OTLP-Metric-Promote-Resource-Attrs"))
.contains("service.name", "deployment.environment.name");
assertThat(request.getValue().getBody()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray());
} }
@Test @Test
void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception { void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception {
when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), when(signalStorage.writeProtobuf(eq("traces"), any())).thenReturn(new byte[0]);
any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage);
GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(
new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000",
"public", "greptime", "secret"), restTemplate);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON); headers.setContentType(MediaType.APPLICATION_JSON);
String json = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{" String json = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{"
@@ -86,10 +74,9 @@ class GreptimeOtlpSignalForwarderTest {
forwarder.forwardHttp("traces", json.getBytes(StandardCharsets.UTF_8), headers); forwarder.forwardHttp("traces", json.getBytes(StandardCharsets.UTF_8), headers);
ArgumentCaptor<HttpEntity<byte[]>> request = ArgumentCaptor.forClass(HttpEntity.class); ArgumentCaptor<byte[]> content = ArgumentCaptor.forClass(byte[].class);
verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), verify(signalStorage).writeProtobuf(eq("traces"), content.capture());
request.capture(), eq(byte[].class)); ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(content.getValue());
ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(request.getValue().getBody());
assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getTraceId().toByteArray()) assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getTraceId().toByteArray())
.containsExactly(java.util.HexFormat.of().parseHex("0123456789abcdef0123456789abcdef")); .containsExactly(java.util.HexFormat.of().parseHex("0123456789abcdef0123456789abcdef"));
assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getParentSpanId().toByteArray()) assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getParentSpanId().toByteArray())
@@ -98,29 +85,25 @@ class GreptimeOtlpSignalForwarderTest {
@Test @Test
void shouldRejectMalformedJsonBeforeCallingGreptime() { void shouldRejectMalformedJsonBeforeCallingGreptime() {
GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage);
new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000",
"public", "greptime", "secret"), restTemplate);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON); headers.setContentType(MediaType.APPLICATION_JSON);
assertThatThrownBy(() -> forwarder.forwardHttp("metrics", "{".getBytes(StandardCharsets.UTF_8), headers)) assertThatThrownBy(() -> forwarder.forwardHttp("metrics", "{".getBytes(StandardCharsets.UTF_8), headers))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Malformed OTLP JSON payload"); .hasMessage("Malformed OTLP JSON payload");
verifyNoInteractions(restTemplate); verifyNoInteractions(signalStorage);
} }
@Test @Test
void shouldRejectMalformedProtobufBeforeCallingGreptime() { void shouldRejectMalformedProtobufBeforeCallingGreptime() {
GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage);
new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000",
"public", "greptime", "secret"), restTemplate);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); headers.setContentType(MediaType.parseMediaType("application/x-protobuf"));
assertThatThrownBy(() -> forwarder.forwardHttp("traces", new byte[] {(byte) 0xff}, headers)) assertThatThrownBy(() -> forwarder.forwardHttp("traces", new byte[] {(byte) 0xff}, headers))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Malformed OTLP traces protobuf payload"); .hasMessage("Malformed OTLP traces protobuf payload");
verifyNoInteractions(restTemplate); verifyNoInteractions(signalStorage);
} }
} }
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.observability.service.impl;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat;
@@ -29,7 +29,7 @@ import io.opentelemetry.proto.resource.v1.Resource;
import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.InstrumentationScope;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.queue.CommonDataQueue; import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.log.notice.LogSseManager; import org.apache.hertzbeat.observability.notice.LogSseManager;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@@ -211,6 +211,31 @@ class OtlpLogProtocolAdapterTest {
assertEquals(1, capturedEntry.getTraceFlags()); assertEquals(1, capturedEntry.getTraceFlags());
} }
@Test
void testIngestWithHexEncodedTraceAndSpanIds() {
String otlpPayload = """
{
"resourceLogs": [{
"scopeLogs": [{
"logRecords": [{
"body": {"stringValue": "hex id probe"},
"traceId": "1234567890abcdef1234567890abcdef",
"spanId": "1234567890abcdef"
}]
}]
}]
}
""";
adapter.ingest(otlpPayload);
ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
verify(commonDataQueue).sendLogEntryToStorageBatch(listCaptor.capture());
LogEntry capturedEntry = listCaptor.getValue().getFirst();
assertEquals("1234567890abcdef1234567890abcdef", capturedEntry.getTraceId());
assertEquals("1234567890abcdef", capturedEntry.getSpanId());
}
@Test @Test
void testIngestWithInvalidJsonContent() { void testIngestWithInvalidJsonContent() {
String invalidJson = "{ invalid json content }"; String invalidJson = "{ invalid json content }";
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.otel.config;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
/**
* HertzBeat self-telemetry auto configuration.
*/
@AutoConfiguration
@ComponentScan(basePackageClasses = OpenTelemetryConfig.class)
public class OpenTelemetryAutoConfiguration {
}
@@ -51,8 +51,8 @@ public class OpenTelemetryConfig {
private static final String HERTZBEAT_SERVICE_NAME = "HertzBeat"; private static final String HERTZBEAT_SERVICE_NAME = "HertzBeat";
private static final String DEFAULT_GREPTIME_DB_NAME = "public"; private static final String DEFAULT_GREPTIME_DB_NAME = "public";
private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_logs"; private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_internal_logs";
private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_traces"; private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_internal_traces";
private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name"; private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name";
private static final String GREPTIME_LOG_TABLE_NAME_HEADER = "X-Greptime-Log-Table-Name"; private static final String GREPTIME_LOG_TABLE_NAME_HEADER = "X-Greptime-Log-Table-Name";
private static final String GREPTIME_TRACE_TABLE_NAME_HEADER = "X-Greptime-Trace-Table-Name"; private static final String GREPTIME_TRACE_TABLE_NAME_HEADER = "X-Greptime-Trace-Table-Name";
@@ -144,4 +144,4 @@ public class OpenTelemetryConfig {
return sdkLoggerProviderBuilder.addLogRecordProcessor(batchLogProcessor); return sdkLoggerProviderBuilder.addLogRecordProcessor(batchLogProcessor);
}); });
} }
} }
@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.hertzbeat.otel.config.OpenTelemetryAutoConfiguration
@@ -346,6 +346,17 @@ grafana:
password: admin password: admin
hertzbeat: hertzbeat:
otlp:
grpc:
# OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at
# this port on every deployment - docker publishes it unchanged.
# 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host
# normally holds already. Set 4317 here if you want the standard port and know it is free.
# A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps
# working either way.
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
collector: collector:
mysql: mysql:
# MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics.
@@ -88,8 +88,6 @@ resourceRole:
# route forwards a raw promql expression straight to the time series database # route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin] - /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user] - /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user] - /api/status/page/**===post===[admin,user]
@@ -111,12 +109,12 @@ resourceRole:
- /api/ai/**===post===[admin] - /api/ai/**===post===[admin]
- /api/ai/**===put===[admin] - /api/ai/**===put===[admin]
- /api/ai/**===delete===[admin] - /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest] - /api/otlp/v1/**===post===[admin,user]
# deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0
- /api/logs/otlp/**===post===[admin,user]
- /api/logs/ingest/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user]
- /api/otlp/**===post===[admin,user] - /api/observability/logs===delete===[admin]
- /api/ingestion/otlp/**===get===[admin,user,guest] - /api/observability/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
- /api/account/token===get===[admin] - /api/account/token===get===[admin]
- /api/account/token/**===post===[admin] - /api/account/token/**===post===[admin]
- /api/account/token/**===delete===[admin] - /api/account/token/**===delete===[admin]
@@ -141,7 +139,6 @@ excludedResource:
- /api/account/auth/**===* - /api/account/auth/**===*
- /api/i18n/**===get - /api/i18n/**===get
- /api/apps/hierarchy===get - /api/apps/hierarchy===get
- /api/observability/capability===get
- /api/push/**===* - /api/push/**===*
- /api/status/page/public/**===* - /api/status/page/public/**===*
# web ui resource # web ui resource
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.startup;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.AlerterProperties;
import org.apache.hertzbeat.alert.AlerterWorkerPool; import org.apache.hertzbeat.alert.AlerterWorkerPool;
@@ -43,8 +44,8 @@ import org.apache.hertzbeat.common.config.CommonProperties;
import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue; import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue;
import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.apache.hertzbeat.alert.service.impl.TencentSmsClientImpl; import org.apache.hertzbeat.alert.service.impl.TencentSmsClientImpl;
import org.apache.hertzbeat.log.controller.OtlpSignalController; import org.apache.hertzbeat.observability.controller.OtlpSignalController;
import org.apache.hertzbeat.log.controller.ThreeSignalQueryController; import org.apache.hertzbeat.observability.controller.ThreeSignalQueryController;
import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool;
import org.apache.hertzbeat.warehouse.controller.MetricsDataController; import org.apache.hertzbeat.warehouse.controller.MetricsDataController;
import org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb.IotDbDataStorage; import org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb.IotDbDataStorage;
@@ -54,6 +55,8 @@ import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisDataStorage;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;
/** /**
* Manager Test * Manager Test
@@ -113,4 +116,14 @@ class ContextTest extends AbstractSpringIntegrationTest {
assertThrows(NoSuchBeanDefinitionException.class, () -> ctx.getBean(ThreeSignalQueryController.class)); assertThrows(NoSuchBeanDefinitionException.class, () -> ctx.getBean(ThreeSignalQueryController.class));
} }
@Test
void canonicalOtlpLogRouteShouldBeAvailableWithoutGreptime() {
RequestMappingHandlerMapping handlerMapping = ctx.getBean(
"requestMappingHandlerMapping", RequestMappingHandlerMapping.class);
assertTrue(handlerMapping.getHandlerMethods().keySet().stream()
.anyMatch(mapping -> mapping.getPatternValues().contains("/api/otlp/v1/logs")
&& mapping.getMethodsCondition().getMethods().contains(RequestMethod.POST)));
}
} }
@@ -0,0 +1,271 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.startup;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.yaml.snakeyaml.Yaml;
class EntityFreeObservabilityTransitionContractTest {
private static final Path REPOSITORY_ROOT = repositoryRoot();
@Test
void reactorAndRuntimeShouldOwnOneObservabilityModule() throws IOException {
String rootPom = Files.readString(REPOSITORY_ROOT.resolve("pom.xml"));
String managerPom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-manager/pom.xml"));
assertTrue(rootPom.contains("<module>hertzbeat-observability</module>"));
assertFalse(rootPom.contains("<module>hertzbeat-log</module>"));
assertTrue(managerPom.contains("<artifactId>hertzbeat-observability</artifactId>"));
assertFalse(managerPom.contains("<artifactId>hertzbeat-log</artifactId>"));
assertFalse(Files.exists(REPOSITORY_ROOT.resolve("hertzbeat-log")));
}
@Test
void productionSourcesShouldExposeOnlyTheCanonicalEntityFreeContract() throws IOException {
Path sourceRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main/java");
assertTrue(Files.isDirectory(sourceRoot));
// The deprecated 1.8.x OTLP log ingestion aliases live in exactly one class so the
// 2.0 removal is a single file delete plus the matching Sureness rules.
Path legacyAliasController = sourceRoot.resolve(
"org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java");
assertTrue(Files.isRegularFile(legacyAliasController));
String legacyAliasSource = Files.readString(legacyAliasController);
assertTrue(legacyAliasSource.contains("@Deprecated(since = \"1.9.0\", forRemoval = true)"));
assertTrue(legacyAliasSource.contains("/api/logs/otlp/v1/logs"));
assertTrue(legacyAliasSource.contains("/api/logs/ingest/{protocol}"));
assertTrue(legacyAliasSource.contains("logIngestionService.ingestHttp(content, headers)"));
String productionSources;
try (Stream<Path> sources = Files.walk(sourceRoot)) {
productionSources = sources
.filter(path -> path.toString().endsWith(".java"))
.filter(path -> !path.equals(legacyAliasController))
.map(EntityFreeObservabilityTransitionContractTest::readSource)
.reduce("", (left, right) -> left + '\n' + right);
}
assertTrue(productionSources.contains("/api/otlp/v1"));
assertTrue(productionSources.contains("/api/observability"));
assertFalse(productionSources.contains("/api/logs/otlp"));
assertFalse(productionSources.contains("/api/logs/ingest"));
assertFalse(productionSources.contains("/api/logs"));
assertFalse(productionSources.contains("/api/ingestion/otlp"));
assertFalse(productionSources.contains("/api/traces"));
assertFalse(productionSources.contains("org.apache.hertzbeat.manager.service.entity"));
assertFalse(productionSources.contains("HERTZBEAT_ENTITY_ID"));
assertFalse(productionSources.contains("HERTZBEAT_ENTITY_TYPE"));
assertFalse(productionSources.contains("EntityObservability"));
assertFalse(productionSources.contains("EntityTrace"));
assertFalse(productionSources.contains("OtlpEntity"));
}
@Test
void warehouseShouldOwnGreptimeSignalStorageSchemaAndQueries() throws IOException {
Path warehouseRoot = REPOSITORY_ROOT.resolve("hertzbeat-warehouse/src/main");
Path observabilityRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main");
assertTrue(Files.isRegularFile(warehouseRoot.resolve(
"java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java")));
assertTrue(Files.isRegularFile(warehouseRoot.resolve(
"java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java")));
assertTrue(Files.isRegularFile(warehouseRoot.resolve(
"java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java")));
assertTrue(Files.isRegularFile(warehouseRoot.resolve("resources/greptime/tables/hertzbeat_traces.sql")));
assertTrue(Files.isRegularFile(warehouseRoot.resolve(
"resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml")));
assertFalse(Files.exists(observabilityRoot.resolve(
"java/org/apache/hertzbeat/observability/config/GreptimeSignalInitializer.java")));
assertFalse(Files.exists(observabilityRoot.resolve(
"java/org/apache/hertzbeat/observability/service/impl/GreptimeThreeSignalQueryService.java")));
assertFalse(Files.exists(observabilityRoot.resolve("resources/greptime")));
String warehousePom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-warehouse/pom.xml"));
assertFalse(warehousePom.contains("<artifactId>hertzbeat-observability</artifactId>"));
String observabilityForwarder = Files.readString(observabilityRoot.resolve(
"java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java"));
assertTrue(observabilityForwarder.contains("OtlpSignalStorage"));
assertFalse(observabilityForwarder.contains("RestTemplate"));
}
@Test
void currentSecurityConfigurationsShouldProtectOnlyCanonicalObservabilityRoutes() throws IOException {
// The runtime configuration is the source of truth; pinning its observability surface
// once here keeps all nine copies from drifting together unnoticed.
SurenessRules baseline = observabilityRules("hertzbeat-startup/src/main/resources/sureness.yml");
assertEquals(Set.of(
"/api/otlp/v1/**===post===[admin,user]",
// deprecated 1.8.x ingestion aliases keep the canonical write roles
"/api/logs/otlp/**===post===[admin,user]",
"/api/logs/ingest/**===post===[admin,user]",
"/api/observability/logs===delete===[admin]",
"/api/observability/**===get===[admin,user,guest]",
"/api/alert/sse/**===get===[admin,user,guest]",
"/api/manager/sse/**===get===[admin,user,guest]"),
baseline.resourceRole());
assertEquals(Set.of(), baseline.excludedResource());
// Every deployable and test copy must carry exactly the baseline rules on this surface,
// so a partial edit that skips a file fails on the file that was missed.
for (String relativePath : new String[] {
"hertzbeat-manager/src/test/resources/sureness.yml",
"hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml",
"script/sureness.yml",
"script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml",
"script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml",
"script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml",
"script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml",
"script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml"
}) {
SurenessRules copy = observabilityRules(relativePath);
assertEquals(baseline.resourceRole(), copy.resourceRole(), relativePath);
assertEquals(baseline.excludedResource(), copy.excludedResource(), relativePath);
}
}
private record SurenessRules(Set<String> resourceRole, Set<String> excludedResource) {
}
@SuppressWarnings("unchecked")
private static SurenessRules observabilityRules(String relativePath) throws IOException {
Map<String, Object> document = new Yaml().load(Files.readString(REPOSITORY_ROOT.resolve(relativePath)));
return new SurenessRules(
observabilitySubset((List<String>) document.get("resourceRole")),
observabilitySubset((List<String>) document.get("excludedResource")));
}
private static Set<String> observabilitySubset(List<String> rules) {
// The route prefixes this transition owns, plus the sse streams scoped alongside it.
List<String> observabilityRoutePrefixes = List.of(
"/api/otlp", "/api/observability", "/api/logs", "/api/traces",
"/api/ingestion", "/api/alert/sse", "/api/manager/sse");
return rules.stream()
.filter(rule -> observabilityRoutePrefixes.stream().anyMatch(rule::startsWith))
.collect(Collectors.toSet());
}
@Test
void productAndSelfTelemetryShouldUseSeparateSignalTables() throws IOException {
String traceSchema = Files.readString(REPOSITORY_ROOT.resolve(
"hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql"));
assertTrue(traceSchema.contains("CREATE TABLE IF NOT EXISTS hertzbeat_traces"));
assertFalse(traceSchema.contains("CREATE TABLE IF NOT EXISTS hzb_traces"));
String selfTelemetry = Files.readString(REPOSITORY_ROOT.resolve(
"hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java"));
assertTrue(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hzb_internal_logs\""));
assertTrue(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hzb_internal_traces\""));
assertFalse(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hertzbeat_logs\""));
assertFalse(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hertzbeat_traces\""));
}
@Test
void currentClientsDocsAndProbesShouldUseCanonicalObservabilityRoutes() throws IOException {
for (String relativePath : new String[] {
"web-app/src/app/service/log.service.ts",
"web-app/src/app/service/observability.service.ts",
"web-app/src/app/routes/log/log-stream/log-stream.component.ts",
"hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml"
}) {
String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath));
assertFalse(content.contains("/api/logs"), relativePath);
assertFalse(content.contains("/logs/list"), relativePath);
assertFalse(content.contains("/logs/stats"), relativePath);
assertFalse(content.contains("/ingestion/otlp/metrics"), relativePath);
assertFalse(content.contains("/traces/list"), relativePath);
assertFalse(content.contains("/traces/stats"), relativePath);
}
// Integration docs must lead with the canonical route; the 1.8.x ingestion paths may only
// appear inside the upgrade notice that marks them as deprecated aliases.
for (String relativePath : new String[] {
"web-app/src/assets/doc/log-integration/otlp.en-US.md",
"web-app/src/assets/doc/log-integration/otlp.zh-CN.md",
"home/docs/help/log_integration.md",
"home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md"
}) {
String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath));
assertTrue(content.contains("POST /api/otlp/v1/logs"), relativePath);
assertTrue(content.contains("/api/otlp/v1/logs"), relativePath);
assertTrue(content.contains("Deprecation: true"), relativePath);
assertFalse(content.contains("logs_endpoint: http://{hertzbeat_host}:1157/api/logs/"), relativePath);
assertFalse(content.contains("/logs/list"), relativePath);
assertFalse(content.contains("/logs/stats"), relativePath);
assertFalse(content.contains("/ingestion/otlp/metrics"), relativePath);
assertFalse(content.contains("/traces/list"), relativePath);
assertFalse(content.contains("/traces/stats"), relativePath);
}
// The upgrade guide must carry the 1.8.x -> 1.9.0 route table so operators see the break.
for (String relativePath : new String[] {
"home/docs/start/upgrade.md",
"home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md"
}) {
String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath));
assertTrue(content.contains("`POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs`"), relativePath);
assertTrue(content.contains("`GET /api/logs/list` | `GET /api/observability/logs`"), relativePath);
assertTrue(content.contains("`GET /api/traces/**` | `GET /api/observability/traces/**`"), relativePath);
}
String logService = Files.readString(REPOSITORY_ROOT.resolve("web-app/src/app/service/log.service.ts"));
assertTrue(logService.contains("/observability/logs"));
String observabilityService = Files.readString(
REPOSITORY_ROOT.resolve("web-app/src/app/service/observability.service.ts"));
assertTrue(observabilityService.contains("/observability/metrics/query"));
assertTrue(observabilityService.contains("/observability/traces"));
String streamComponent = Files.readString(REPOSITORY_ROOT.resolve(
"web-app/src/app/routes/log/log-stream/log-stream.component.ts"));
assertTrue(streamComponent.contains("/api/observability/logs/stream"));
String vectorConfig = Files.readString(REPOSITORY_ROOT.resolve(
"hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml"));
assertTrue(vectorConfig.contains("/api/otlp/v1/logs"));
}
private static String readSource(Path path) {
try {
return Files.readString(path);
} catch (IOException exception) {
throw new IllegalStateException("Failed to read " + path, exception);
}
}
private static Path repositoryRoot() {
Path candidate = Path.of("").toAbsolutePath();
while (candidate != null && !Files.isRegularFile(candidate.resolve("mvnw"))) {
candidate = candidate.getParent();
}
if (candidate == null) {
throw new IllegalStateException("Unable to locate repository root");
}
return candidate;
}
}
@@ -39,7 +39,8 @@ import org.yaml.snakeyaml.Yaml;
* `curl -N` stayed subscribed and received every alert the deployment raised - internal * `curl -N` stayed subscribed and received every alert the deployment raised - internal
* hostnames, addresses, metric values and alert content - because * hostnames, addresses, metric values and alert content - because
* `AlertNoticeDispatch` broadcasts each alert to every subscriber with no per subscriber * `AlertNoticeDispatch` broadcasts each alert to every subscriber with no per subscriber
* filtering. `/api/logs/sse/**` was already scoped this way; these now match it. * filtering. The log stream at `/api/observability/logs/stream` was already scoped this
* way; these now match it.
*/ */
class SurenessSseRuleTest { class SurenessSseRuleTest {
@@ -98,6 +99,6 @@ class SurenessSseRuleTest {
@Test @Test
void theLogStreamScopingIsUnchanged() { void theLogStreamScopingIsUnchanged() {
assertEquals("[admin,user,guest]", assertEquals("[admin,user,guest]",
roleTree.searchPathFilterRoles("/api/logs/sse/subscribe" + SEPARATOR + "get")); roleTree.searchPathFilterRoles("/api/observability/logs/stream" + SEPARATOR + "get"));
} }
} }
@@ -78,21 +78,21 @@ class SurenessUnruledEndpointTest {
@Test @Test
void deletingLogsIsRestrictedToAdmin() { void deletingLogsIsRestrictedToAdmin() {
assertEquals("[admin]", rolesFor("/api/logs", "delete")); assertEquals("[admin]", rolesFor("/api/observability/logs", "delete"));
} }
@Test @Test
void readingLogsStaysOpenToEveryRole() { void readingLogsStaysOpenToEveryRole() {
assertEquals("[admin,user,guest]", rolesFor("/api/logs/list", "get")); assertEquals("[admin,user,guest]", rolesFor("/api/observability/logs", "get"));
} }
/** /**
* Matches how the sibling ingestion routes `/api/otlp/**` and `/api/logs/ingest/**` * Authenticated operators may write telemetry to the canonical OTLP boundary;
* are already scoped, so a low privileged account can no longer forge log records. * guests may not.
*/ */
@Test @Test
void ingestingOtlpLogsRequiresAtLeastUser() { void ingestingOtlpLogsRequiresAtLeastUser() {
assertEquals("[admin,user]", rolesFor("/api/logs/otlp/v1/logs", "post")); assertEquals("[admin,user]", rolesFor("/api/otlp/v1/logs", "post"));
} }
/** /**
@@ -19,7 +19,9 @@ package org.apache.hertzbeat.warehouse.config;
import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants; import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
/** /**
@@ -31,5 +33,6 @@ import org.springframework.context.annotation.ComponentScan;
@ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG
+ SignConstants.DOT + SignConstants.DOT
+ ConfigConstants.FunctionModuleConstants.WAREHOUSE) + ConfigConstants.FunctionModuleConstants.WAREHOUSE)
@EnableConfigurationProperties(GreptimeProperties.class)
public class WarehouseAutoConfiguration { public class WarehouseAutoConfiguration {
} }
@@ -72,6 +72,8 @@ public interface WarehouseConstants {
String LOG_TABLE_NAME = "hertzbeat_logs"; String LOG_TABLE_NAME = "hertzbeat_logs";
String TRACE_TABLE_NAME = "hertzbeat_traces";
String GREPTIME_QUERY_REST_TEMPLATE = "greptimeQueryRestTemplate"; String GREPTIME_QUERY_REST_TEMPLATE = "greptimeQueryRestTemplate";
String GREPTIME_WRITE_REST_TEMPLATE = "greptimeWriteRestTemplate"; String GREPTIME_WRITE_REST_TEMPLATE = "greptimeWriteRestTemplate";
@@ -15,27 +15,17 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.warehouse.service;
/** /** Storage boundary for validated OTLP protobuf signals. */
* Adapter interface for ingesting logs pushed via different protocols public interface OtlpSignalStorage {
* (e.g. OTLP, Loki, Filebeat, Vector).
* Implementations should:
* 1. Parse raw HTTP payload of their protocol.
* 2. Convert data to LogEntry.
* 3. Forward / persist it to downstream pipeline.
*/
public interface LogProtocolAdapter {
/** /**
* Ingest log payload pushed from external system. * Persist one validated OTLP export request.
* *
* @param content raw request body string * @param signal metrics, logs, or traces
* @param content encoded OTLP protobuf request
* @return encoded OTLP protobuf response
*/ */
void ingest(String content); byte[] writeProtobuf(String signal, byte[] content);
}
/**
* Identifier of the protocol this adapter supports ("otlp", "vector", etc.)
*/
String supportProtocol();
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service; package org.apache.hertzbeat.warehouse.service;
import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsConsole; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsConsole;
import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsInventory; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsInventory;
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.warehouse.service.impl;
import java.net.URI; import java.net.URI;
import java.time.Instant; import java.time.Instant;
@@ -40,9 +40,9 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanEvent;
import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode;
import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException;
import org.apache.hertzbeat.log.service.ThreeSignalQueryService;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
@@ -63,13 +63,14 @@ import tools.jackson.core.type.TypeReference;
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService { public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService {
private static final String TRACE_TABLE = "hzb_traces"; private static final String TRACE_TABLE = WarehouseConstants.TRACE_TABLE_NAME;
private static final Pattern SAFE_IDENTIFIER = Pattern.compile("[A-Za-z_:][A-Za-z0-9_:.-]*"); private static final Pattern SAFE_IDENTIFIER = Pattern.compile("[A-Za-z_:][A-Za-z0-9_:.-]*");
private static final Pattern SAFE_LABEL = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); private static final Pattern SAFE_LABEL = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*");
private static final Set<String> AGGREGATIONS = Set.of("sum", "avg", "min", "max", "count"); private static final Set<String> AGGREGATIONS = Set.of("sum", "avg", "min", "max", "count");
private static final long DEFAULT_WINDOW_MILLIS = 30 * 60 * 1000L; private static final long DEFAULT_WINDOW_MILLIS = 30 * 60 * 1000L;
private static final int DEFAULT_STEP_SECONDS = 30; private static final int DEFAULT_STEP_SECONDS = 30;
private static final int MAX_PAGE_SIZE = 200; private static final int MAX_PAGE_SIZE = 200;
private static final long MAX_DURATION_MS = Long.MAX_VALUE / 1_000_000L;
private final GreptimeProperties greptimeProperties; private final GreptimeProperties greptimeProperties;
private final GreptimeSqlQueryExecutor sqlQueryExecutor; private final GreptimeSqlQueryExecutor sqlQueryExecutor;
private final RestTemplate restTemplate; private final RestTemplate restTemplate;
@@ -178,7 +179,7 @@ public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService
} }
String sql = "SELECT *, COUNT(*) OVER () AS total_count FROM (" + grouped String sql = "SELECT *, COUNT(*) OVER () AS total_count FROM (" + grouped
+ ") trace_page ORDER BY start_time DESC LIMIT " + effectiveSize + ") trace_page ORDER BY start_time DESC LIMIT " + effectiveSize
+ " OFFSET " + effectivePage * effectiveSize; + " OFFSET " + (long) effectivePage * effectiveSize;
List<Map<String, Object>> rows = sqlQueryExecutor.execute(sql); List<Map<String, Object>> rows = sqlQueryExecutor.execute(sql);
long total = rows.isEmpty() ? 0 : asLong(rows.getFirst().get("total_count")); long total = rows.isEmpty() ? 0 : asLong(rows.getFirst().get("total_count"));
return new SignalPage<>(rows.stream().map(this::toTraceListItem).toList(), effectivePage, effectiveSize, total); return new SignalPage<>(rows.stream().map(this::toTraceListItem).toList(), effectivePage, effectiveSize, total);
@@ -335,14 +336,18 @@ public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService
addFlattenedResourceFilter(filters, "resource_attributes.service.namespace", serviceNamespace); addFlattenedResourceFilter(filters, "resource_attributes.service.namespace", serviceNamespace);
addFlattenedResourceFilter(filters, "resource_attributes.deployment.environment.name", environment); addFlattenedResourceFilter(filters, "resource_attributes.deployment.environment.name", environment);
if (minDurationMs != null) { if (minDurationMs != null) {
filters.add("duration_nano >= " + Math.max(0, minDurationMs) * 1_000_000L); filters.add("duration_nano >= " + clampDurationMs(minDurationMs) * 1_000_000L);
} }
if (maxDurationMs != null) { if (maxDurationMs != null) {
filters.add("duration_nano <= " + Math.max(0, maxDurationMs) * 1_000_000L); filters.add("duration_nano <= " + clampDurationMs(maxDurationMs) * 1_000_000L);
} }
return String.join(" AND ", filters); return String.join(" AND ", filters);
} }
private long clampDurationMs(long durationMs) {
return Math.min(Math.max(0, durationMs), MAX_DURATION_MS);
}
private void addTextFilter(List<String> filters, String column, String value) { private void addTextFilter(List<String> filters, String column, String value) {
if (StringUtils.hasText(value)) { if (StringUtils.hasText(value)) {
filters.add(column + " = '" + escapeSql(value) + "'"); filters.add(column + " = '" + escapeSql(value) + "'");
@@ -101,6 +101,10 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
private static final String LABEL_KEY_END_TIME = "end"; private static final String LABEL_KEY_END_TIME = "end";
private static final String LABEL_KEY_TS = "ts"; private static final String LABEL_KEY_TS = "ts";
private static final int LOG_BATCH_SIZE = 500; private static final int LOG_BATCH_SIZE = 500;
private static final Map<String, String> OTLP_RESOURCE_KEY_ALIASES = Map.of(
"service_name", "service.name",
"service_namespace", "service.namespace",
"deployment_environment_name", "deployment.environment.name");
private GreptimeDB greptimeDb; private GreptimeDB greptimeDb;
@@ -756,7 +760,7 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
addJsonCondition(resourceConditions, "deployment.environment.name", filter.environment()); addJsonCondition(resourceConditions, "deployment.environment.name", filter.environment());
if (StringUtils.hasText(filter.resourceFilter())) { if (StringUtils.hasText(filter.resourceFilter())) {
for (ResourceFilterExpression.Clause clause : ResourceFilterExpression.parse(filter.resourceFilter())) { for (ResourceFilterExpression.Clause clause : ResourceFilterExpression.parse(filter.resourceFilter())) {
String attribute = "json_get_string(resource, '$[\"" + clause.key() + "\"]')"; String attribute = resourceAttributeExpression(clause.key());
switch (clause.operator()) { switch (clause.operator()) {
case EQUALS -> resourceConditions.add(attribute + " = '" + safeString(clause.value()) + "'"); case EQUALS -> resourceConditions.add(attribute + " = '" + safeString(clause.value()) + "'");
case NOT_EQUALS -> resourceConditions.add(attribute + " <> '" + safeString(clause.value()) + "'"); case NOT_EQUALS -> resourceConditions.add(attribute + " <> '" + safeString(clause.value()) + "'");
@@ -774,10 +778,20 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
private void addJsonCondition(List<String> conditions, String key, String value) { private void addJsonCondition(List<String> conditions, String key, String value) {
if (StringUtils.hasText(value)) { if (StringUtils.hasText(value)) {
conditions.add("json_get_string(resource, '$[\"" + key + "\"]') = '" + safeString(value) + "'"); conditions.add(resourceAttributeExpression(key) + " = '" + safeString(value) + "'");
} }
} }
private String resourceAttributeExpression(String key) {
String normalizedKey = key.replace('.', '_');
String canonical = "json_get_string(resource, '$[\"" + key + "\"]')";
if (normalizedKey.equals(key)) {
return canonical;
}
String normalized = "json_get_string(resource, '$[\"" + normalizedKey + "\"]')";
return "COALESCE(" + canonical + ", " + normalized + ")";
}
private static long msToNs(Long ms) { private static long msToNs(Long ms) {
return ms * 1_000_000L; return ms * 1_000_000L;
} }
@@ -857,7 +871,8 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
Object bodyObj = parseJsonMaybe(row.get("body")); Object bodyObj = parseJsonMaybe(row.get("body"));
Map<String, Object> attributes = castToMap(parseJsonMaybe(row.get("attributes"))); Map<String, Object> attributes = castToMap(parseJsonMaybe(row.get("attributes")));
Map<String, Object> resource = castToMap(parseJsonMaybe(row.get("resource"))); Map<String, Object> resource = canonicalizeOtlpResource(
castToMap(parseJsonMaybe(row.get("resource"))));
LogEntry entry = LogEntry.builder() LogEntry entry = LogEntry.builder()
.timeUnixNano(castToLong(row.get("time_unix_nano"))) .timeUnixNano(castToLong(row.get("time_unix_nano")))
@@ -913,6 +928,19 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
return null; return null;
} }
private static Map<String, Object> canonicalizeOtlpResource(Map<String, Object> resource) {
if (resource == null || resource.isEmpty()) {
return resource;
}
Map<String, Object> canonical = new LinkedHashMap<>(resource);
OTLP_RESOURCE_KEY_ALIASES.forEach((normalized, dotted) -> {
if (!canonical.containsKey(dotted) && resource.containsKey(normalized)) {
canonical.put(dotted, resource.get(normalized));
}
});
return canonical;
}
private static Long castToLong(Object obj) { private static Long castToLong(Object obj) {
if (obj == null) return null; if (obj == null) return null;
if (obj instanceof Number n) return n.longValue(); if (obj instanceof Number n) return n.longValue();
@@ -0,0 +1,166 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.http.HttpStatusCode;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
/** GreptimeDB storage implementation for validated OTLP protobuf requests. */
@Slf4j
@Service
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
public class GreptimeOtlpSignalStorage implements OtlpSignalStorage {
private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf");
private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name";
private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name";
private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name";
private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name";
private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name";
private static final String GREPTIME_PROMOTE_RESOURCE_HEADER =
"X-Greptime-OTLP-Metric-Promote-Resource-Attrs";
private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of(
"service.name", "service.namespace", "service.version", "deployment.environment.name",
"host.name", "k8s.namespace.name", "k8s.pod.name"));
private static final Set<String> SIGNALS = Set.of("metrics", "logs", "traces");
/**
* 4xx codes that still mean "come back later" rather than "this payload is wrong", so the
* batch must keep its retryable semantics instead of being dropped by the exporter.
*/
private static final Set<Integer> RETRYABLE_CLIENT_ERRORS = Set.of(408, 425, 429);
/** Upper bound on how much of a GreptimeDB error body travels back to the exporter. */
private static final int MAX_REJECTION_DETAIL_LENGTH = 200;
private final GreptimeProperties greptimeProperties;
private final RestTemplate restTemplate;
public GreptimeOtlpSignalStorage(GreptimeProperties greptimeProperties,
@Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE)
RestTemplate restTemplate) {
this.greptimeProperties = greptimeProperties;
this.restTemplate = restTemplate;
}
@Override
public byte[] writeProtobuf(String signal, byte[] content) {
String normalizedSignal = normalizeSignal(signal);
HttpHeaders headers = greptimeHeaders(normalizedSignal);
ResponseEntity<byte[]> response;
try {
response = restTemplate.exchange(
endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal),
HttpMethod.POST,
new HttpEntity<>(content == null ? new byte[0] : content, headers),
byte[].class);
} catch (HttpClientErrorException exception) {
if (isRetryable(exception.getStatusCode())) {
// Let it stay a RestClientException so the ingestion boundary answers with a retryable
// 503 and the exporter replays the batch instead of discarding it.
throw exception;
}
// Any other 4xx means the payload itself was rejected: surface it as a client error instead of
// a retryable storage failure so exporters stop retrying a request that can never succeed.
// The full body only goes to the log; the exporter gets a bounded excerpt.
log.warn("GreptimeDB rejected OTLP {} with {}: {}", normalizedSignal,
exception.getStatusCode().value(), exception.getResponseBodyAsString(StandardCharsets.UTF_8));
throw new IllegalArgumentException(rejectionMessage(normalizedSignal, exception), exception);
}
if (!response.getStatusCode().is2xxSuccessful()) {
throw new RestClientException("GreptimeDB returned unexpected OTLP " + normalizedSignal
+ " status " + response.getStatusCode().value());
}
// 3xx, 5xx, and transport failures remain RestClientException and keep their retryable semantics.
return response.getBody() == null ? new byte[0] : response.getBody();
}
private static boolean isRetryable(HttpStatusCode statusCode) {
return RETRYABLE_CLIENT_ERRORS.contains(statusCode.value());
}
private static String rejectionMessage(String signal, HttpClientErrorException exception) {
String detail = exception.getResponseBodyAsString(StandardCharsets.UTF_8);
if (!StringUtils.hasText(detail)) {
detail = exception.getStatusText();
}
return "GreptimeDB rejected OTLP " + signal + " (" + exception.getStatusCode().value() + "): "
+ truncate(detail.strip());
}
/**
* GreptimeDB error bodies can carry long internal detail, and this text ends up in a
* {@code google.rpc.Status} message and in a grpc status description, where an oversized value
* can overflow the trailer budget and cost the client the error itself.
*/
private static String truncate(String detail) {
return detail.length() <= MAX_REJECTION_DETAIL_LENGTH
? detail : detail.substring(0, MAX_REJECTION_DETAIL_LENGTH) + "...";
}
private HttpHeaders greptimeHeaders(String signal) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(PROTOBUF);
headers.setAccept(List.of(PROTOBUF));
headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database())
? greptimeProperties.database() : "public");
if ("metrics".equals(signal)) {
headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES);
} else if ("traces".equals(signal)) {
headers.set(GREPTIME_TRACE_TABLE_HEADER, WarehouseConstants.TRACE_TABLE_NAME);
headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1");
} else {
headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME);
headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1");
}
if (StringUtils.hasText(greptimeProperties.username())
&& StringUtils.hasText(greptimeProperties.password())) {
headers.setBasicAuth(greptimeProperties.username(), greptimeProperties.password(),
StandardCharsets.UTF_8);
}
return headers;
}
private String normalizeSignal(String signal) {
String normalized = StringUtils.hasText(signal) ? signal.toLowerCase(Locale.ROOT) : "";
if (!SIGNALS.contains(normalized)) {
throw new IllegalArgumentException("Unsupported OTLP signal");
}
return normalized;
}
private String endpoint(String base, String path) {
return StringUtils.trimTrailingCharacter(base, '/') + path;
}
}
@@ -15,12 +15,11 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.config; package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.event.ApplicationReadyEvent;
@@ -43,7 +42,7 @@ import org.springframework.web.client.RestTemplate;
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
public class GreptimeSignalInitializer { public class GreptimeSignalInitializer {
private static final String TRACE_SCHEMA = "greptime/tables/hzb_traces.sql"; private static final String TRACE_SCHEMA = "greptime/tables/hertzbeat_traces.sql";
private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql"; private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql";
private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml"; private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml";
private final GreptimeProperties greptimeProperties; private final GreptimeProperties greptimeProperties;
@@ -65,9 +64,9 @@ public class GreptimeSignalInitializer {
String traceSchema = new ClassPathResource(TRACE_SCHEMA) String traceSchema = new ClassPathResource(TRACE_SCHEMA)
.getContentAsString(StandardCharsets.UTF_8).strip(); .getContentAsString(StandardCharsets.UTF_8).strip();
sqlQueryExecutor.execute(StringUtils.trimTrailingCharacter(traceSchema, ';')); sqlQueryExecutor.execute(StringUtils.trimTrailingCharacter(traceSchema, ';'));
sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " sqlQueryExecutor.execute("ALTER TABLE " + WarehouseConstants.TRACE_TABLE_NAME + " ADD COLUMN IF NOT EXISTS "
+ "\"resource_attributes.service.namespace\" STRING NULL"); + "\"resource_attributes.service.namespace\" STRING NULL");
sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " sqlQueryExecutor.execute("ALTER TABLE " + WarehouseConstants.TRACE_TABLE_NAME + " ADD COLUMN IF NOT EXISTS "
+ "\"resource_attributes.deployment.environment.name\" STRING NULL"); + "\"resource_attributes.deployment.environment.name\" STRING NULL");
String logSchema = new ClassPathResource(LOG_SCHEMA) String logSchema = new ClassPathResource(LOG_SCHEMA)
.getContentAsString(StandardCharsets.UTF_8).strip(); .getContentAsString(StandardCharsets.UTF_8).strip();
@@ -13,7 +13,7 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
CREATE TABLE IF NOT EXISTS hzb_traces ( CREATE TABLE IF NOT EXISTS hertzbeat_traces (
"timestamp" TIMESTAMP(9) TIME INDEX, "timestamp" TIMESTAMP(9) TIME INDEX,
"timestamp_end" TIMESTAMP(9) NULL, "timestamp_end" TIMESTAMP(9) NULL,
"duration_nano" BIGINT UNSIGNED NULL, "duration_nano" BIGINT UNSIGNED NULL,
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.service.impl; package org.apache.hertzbeat.warehouse.service.impl;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -53,6 +53,7 @@ import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.arrow.ArrowCell; import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter;
import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
@@ -296,6 +297,38 @@ class GreptimeDbDataStorageTest {
} }
} }
@Test
void shouldQueryNormalizedOtlpResourceKeysAndRestoreCanonicalNames() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(
greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
Map<String, Object> row = new HashMap<>();
row.put("time_unix_nano", System.nanoTime());
row.put("severity_text", "INFO");
row.put("body", "OTLP resource proof");
row.put("resource", Map.of(
"service_name", "checkout-api",
"service_namespace", "storefront",
"deployment_environment_name", "preview"));
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(List.of(row));
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
List<LogEntry> result = greptimeDbDataStorage.queryObservabilityLogs(
new LogQueryFilter(null, null, null, null, null, null, null,
"checkout-api", "storefront", "preview", null),
0, 20);
verify(greptimeSqlQueryExecutor).execute(sqlCaptor.capture());
assertTrue(sqlCaptor.getValue().contains("service_name"));
assertTrue(sqlCaptor.getValue().contains("service_namespace"));
assertTrue(sqlCaptor.getValue().contains("deployment_environment_name"));
assertEquals("checkout-api", result.getFirst().getResource().get("service.name"));
assertEquals("storefront", result.getFirst().getResource().get("service.namespace"));
assertEquals("preview", result.getFirst().getResource().get("deployment.environment.name"));
}
}
@Test @Test
void testBatchDeleteLogsWithValidList() { void testBatchDeleteLogsWithValidList() {
@@ -0,0 +1,193 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
@ExtendWith(MockitoExtension.class)
class GreptimeOtlpSignalStorageTest {
@Mock
private RestTemplate restTemplate;
private GreptimeOtlpSignalStorage storage;
@BeforeEach
void setUp() {
storage = new GreptimeOtlpSignalStorage(
new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000/",
"public", "greptime", "secret"),
restTemplate);
}
@Test
void shouldWriteMetricsWithResourcePromotionAndAuthentication() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenReturn(ResponseEntity.ok("response".getBytes(StandardCharsets.UTF_8)));
byte[] response = storage.writeProtobuf("metrics", new byte[] {1, 2});
ArgumentCaptor<HttpEntity<byte[]>> request = ArgumentCaptor.forClass(HttpEntity.class);
verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST),
request.capture(), eq(byte[].class));
assertThat(request.getValue().getHeaders().getFirst(
"X-Greptime-OTLP-Metric-Promote-Resource-Attrs"))
.contains("service.name", "deployment.environment.name");
assertThat(request.getValue().getHeaders().getFirst("Authorization")).startsWith("Basic ");
assertThat(request.getValue().getBody()).containsExactly(1, 2);
assertThat(response).isEqualTo("response".getBytes(StandardCharsets.UTF_8));
}
@Test
void shouldSelectWarehouseOwnedTraceAndLogSchemas() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenReturn(ResponseEntity.ok(new byte[0]));
storage.writeProtobuf("traces", new byte[0]);
storage.writeProtobuf("logs", new byte[0]);
ArgumentCaptor<HttpEntity<byte[]>> requests = ArgumentCaptor.forClass(HttpEntity.class);
verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST),
requests.capture(), eq(byte[].class));
verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/logs"), eq(HttpMethod.POST),
requests.capture(), eq(byte[].class));
assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Trace-Table-Name"))
.isEqualTo("hertzbeat_traces");
assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Pipeline-Name"))
.isEqualTo("greptime_trace_v1");
assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Table-Name"))
.isEqualTo("hertzbeat_logs");
assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Pipeline-Name"))
.isEqualTo("hertzbeat_otlp_log_v1");
}
@Test
void shouldRejectUnsupportedSignalBeforeStorageCall() {
assertThatThrownBy(() -> storage.writeProtobuf("profiles", new byte[0]))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Unsupported OTLP signal");
}
@Test
void shouldSurfaceGreptimeRejectionAsClientErrorWithResponseBody() {
byte[] body = "{\"error\":\"pipeline hertzbeat_otlp_log_v1 not found\"}".getBytes(StandardCharsets.UTF_8);
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpClientErrorException.create(HttpStatus.BAD_REQUEST, "Bad Request",
new HttpHeaders(), body, StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0]))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("GreptimeDB rejected OTLP logs (400): "
+ "{\"error\":\"pipeline hertzbeat_otlp_log_v1 not found\"}")
.hasCauseInstanceOf(HttpClientErrorException.class);
}
@Test
void shouldFallBackToStatusTextWhenGreptimeRejectionHasNoBody() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpClientErrorException.create(HttpStatus.UNAUTHORIZED, "Unauthorized",
new HttpHeaders(), new byte[0], StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("metrics", new byte[0]))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("GreptimeDB rejected OTLP metrics (401): Unauthorized");
}
@Test
void shouldKeepServerErrorsRetryable() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpServerErrorException.create(HttpStatus.SERVICE_UNAVAILABLE, "Service Unavailable",
new HttpHeaders(), new byte[0], StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("traces", new byte[0]))
.isInstanceOf(HttpServerErrorException.class);
}
@Test
void shouldKeepUnexpectedRedirectResponsesRetryable() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenReturn(ResponseEntity.status(HttpStatus.TEMPORARY_REDIRECT).body(new byte[0]));
assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0]))
.isInstanceOf(RestClientException.class)
.hasMessage("GreptimeDB returned unexpected OTLP logs status 307");
}
/**
* 429 and 408 are 4xx codes that still mean "try again", so converting them into a client error
* would make the exporter drop a batch GreptimeDB was willing to take a moment later.
*/
@Test
void shouldKeepGreptimeBackPressureRetryable() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpClientErrorException.create(HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests",
new HttpHeaders(), new byte[0], StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0]))
.isInstanceOf(HttpClientErrorException.class);
}
@Test
void shouldKeepGreptimeRequestTimeoutRetryable() {
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpClientErrorException.create(HttpStatus.REQUEST_TIMEOUT, "Request Timeout",
new HttpHeaders(), new byte[0], StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("metrics", new byte[0]))
.isInstanceOf(HttpClientErrorException.class);
}
/**
* The message travels back to the exporter inside a google.rpc.Status and, on the grpc transport,
* inside a status description that shares the trailer budget, so it must stay bounded.
*/
@Test
void shouldTruncateOversizedGreptimeRejectionDetail() {
byte[] body = ("x".repeat(5000)).getBytes(StandardCharsets.UTF_8);
when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class)))
.thenThrow(HttpClientErrorException.create(HttpStatus.BAD_REQUEST, "Bad Request",
new HttpHeaders(), body, StandardCharsets.UTF_8));
assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0]))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageEndingWith("...")
.satisfies(thrown -> assertThat(thrown.getMessage().length()).isLessThan(300));
}
}
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.config; package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import org.mockito.InOrder; import org.mockito.InOrder;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@@ -71,11 +70,11 @@ class GreptimeSignalInitializerTest {
InOrder sqlOrder = org.mockito.Mockito.inOrder(sqlQueryExecutor); InOrder sqlOrder = org.mockito.Mockito.inOrder(sqlQueryExecutor);
sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains(
"CREATE TABLE IF NOT EXISTS hzb_traces")); "CREATE TABLE IF NOT EXISTS hertzbeat_traces"));
sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith(
"ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\"")); "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\""));
sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith(
"ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\"")); "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\""));
sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains(
"CREATE TABLE IF NOT EXISTS hertzbeat_logs")); "CREATE TABLE IF NOT EXISTS hertzbeat_logs"));
sqlOrder.verify(sqlQueryExecutor).execute("SELECT 1 AS ready"); sqlOrder.verify(sqlQueryExecutor).execute("SELECT 1 AS ready");
@@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hertzbeat.log.config; package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
+42 -2
View File
@@ -11,6 +11,14 @@ keywords: [open source monitoring, log integration, log management, multi-source
The log integration feature is currently in Beta (experimental) stage. There may be potential defects and limitations. The feature is under active development and iteration. The log integration feature is currently in Beta (experimental) stage. There may be potential defects and limitations. The feature is under active development and iteration.
::: :::
:::warning Upgrading from 1.8.x? The ingestion path changed
The 1.8.x endpoint `POST /api/logs/otlp/v1/logs` (and `POST /api/logs/ingest/otlp`) is replaced by `POST /api/otlp/v1/logs`. Update the `logs_endpoint` of every OpenTelemetry Collector / SDK exporter that points at HertzBeat. On 1.9.x the old paths still work as deprecated aliases (the response carries `Deprecation: true` and HertzBeat logs a warning); they are removed in 2.0. The query paths `/api/logs/**`, `/api/traces/**` and `/api/ingestion/otlp/**` moved to `/api/observability/**` with no alias. See the [Version Upgrade Guide](../start/upgrade) for the full old/new path table.
:::
:::info HertzBeat 1.9.0 transition
Metrics, logs, and traces share `/api/otlp/v1/{signal}` for ingestion and `/api/observability/**` for queries. This release intentionally does not create or bind Entity records from telemetry. External OTLP signal tables are also separate from HertzBeat's internal self-telemetry tables.
:::
## Core Capabilities ## Core Capabilities
- **Multi-source Log Integration**: Support receiving log data from mainstream platforms such as OpenTelemetry, Filebeat, Vector, Loki - **Multi-source Log Integration**: Support receiving log data from mainstream platforms such as OpenTelemetry, Filebeat, Vector, Loki
@@ -36,9 +44,41 @@ You can view specific integration methods and configuration examples through Her
HertzBeat provides the following interface for receiving OTLP log data: HertzBeat provides the following interface for receiving OTLP log data:
```text ```text
POST /api/logs/otlp/v1/logs POST /api/otlp/v1/logs
``` ```
### OTLP/gRPC Endpoint
HertzBeat also runs an OTLP/gRPC listener when GreptimeDB storage is enabled, accepting metrics, logs
and traces. It expects the same `Authorization: Bearer {token}` credential as the HTTP endpoint.
```text
{hertzbeat_host}:14317
```
The port is 14317 on every deployment - the docker images publish it unchanged, so there is no
container-versus-host translation to remember.
It is deliberately not the OpenTelemetry standard 4317: an OTel Collector, Jaeger or Tempo on the
same host normally holds that port already, and a clash on a published port stops the container from
starting at all. HertzBeat serves OTLP/HTTP on its own port too, so this is consistent with the rest
of the product rather than an exception.
To use 4317 anyway, or to turn the listener off, set it in `application.yml` or through the matching
environment variables (and update the port mapping in `docker-compose.yaml` to match):
```yaml
hertzbeat:
otlp:
grpc:
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
```
If the port cannot be bound, HertzBeat logs the failure and starts without gRPC ingestion; OTLP/HTTP
on `/api/otlp/v1` keeps working.
### Request Configuration ### Request Configuration
#### Request Headers #### Request Headers
@@ -112,7 +152,7 @@ Add HertzBeat as a log export target in the OpenTelemetry Collector configuratio
```yaml ```yaml
exporters: exporters:
otlphttp: otlphttp:
logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs
compression: none compression: none
encoding: json encoding: json
headers: headers:
+86
View File
@@ -14,6 +14,92 @@ Apache HertzBeat's metadata information is stored in H2 or Mysql, PostgreSQL rel
**You need to save and back up the data files of the database and monitoring templates yml files before upgrading** **You need to save and back up the data files of the database and monitoring templates yml files before upgrading**
## Breaking Changes In 1.9.0
### Observability (OTLP / logs / traces) API paths moved
1.9.0 consolidates the 1.8.x log module into `hertzbeat-observability`. Metrics, logs and traces now share one ingestion prefix (`/api/otlp/v1/{signal}`) and one query prefix (`/api/observability/**`). Any OpenTelemetry Collector, Vector, SDK exporter, script or dashboard that was configured against a 1.8.x path must be updated.
| 1.8.x path | 1.9.0 path | Status in 1.9.x |
|---|---|---|
| `POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs` | **Deprecated alias kept**, still works, responds with `Deprecation: true`; removed in 2.0 |
| `POST /api/logs/ingest/otlp` | `POST /api/otlp/v1/logs` | **Deprecated alias kept**, still works, responds with `Deprecation: true`; removed in 2.0 |
| `POST /api/logs/ingest/{other protocol}` | — | Removed (`400`), only `otlp` ever had an adapter |
| `GET /api/logs/list` | `GET /api/observability/logs` | Removed (`404`) |
| `GET /api/logs/stats/overview` | `GET /api/observability/logs/overview` | Removed (`404`) |
| `GET /api/logs/stats/trace-coverage` | `GET /api/observability/logs/trace-coverage` | Removed (`404`) |
| `GET /api/logs/stats/trend` | `GET /api/observability/logs/trend` | Removed (`404`) |
| `GET /api/logs/sse/subscribe` | `GET /api/observability/logs/stream` | Removed (`404`); the new route requires an authenticated `admin/user/guest` instead of anonymous access |
| `DELETE /api/logs` | `DELETE /api/observability/logs` | Removed (`404`) |
| `GET /api/traces/**` | `GET /api/observability/traces/**` | Removed (`404`) |
| `GET /api/ingestion/otlp/metrics/console` | `GET /api/observability/metrics/query` | Removed (`404`) |
| `GET /api/ingestion/otlp/metrics/inventory` | `GET /api/observability/metrics/inventory` | Removed (`404`) |
Recommended upgrade steps:
- Before upgrading, search your collector / exporter configuration for `/api/logs/` and change it to `/api/otlp/v1/logs`. OTLP HTTP exporters treat a `404` as a permanent error and silently drop the batch, so a stale path shows up only as "logs stopped arriving".
- If you cannot change the exporters in the same maintenance window, the two ingestion aliases above keep accepting data on 1.9.x. Watch the HertzBeat log for `Deprecated OTLP log route ... was called` warnings and migrate before 2.0.
- If you use a customised `sureness.yml`, add `/api/otlp/v1/**===post===[admin,user]` and `/api/observability/**===get===[admin,user,guest]` (see the packaged `sureness.yml`); the old `/api/logs/**`, `/api/traces/**` and `/api/ingestion/otlp/**` rules can be dropped once your exporters are migrated.
### New OTLP/gRPC listener on port 14317
When `warehouse.store.greptime.enabled=true`, 1.9.0 additionally starts an OTLP/gRPC listener on
`0.0.0.0:14317` so exporters can push metrics, logs and traces over gRPC. The packaged Dockerfile and
the docker-compose files publish it unchanged, so the port is the same on every deployment.
- **It is not the OpenTelemetry standard 4317.** An OTel Collector, Jaeger or Tempo on the same host
normally holds 4317 already, and a clash on a published port makes `docker compose up` fail
outright. HertzBeat serves OTLP/HTTP on its own port as well, so 14317 is consistent with the rest
of the product.
- Existing deployments gain one newly bound port. If your firewall or security policy enumerates
listening ports, add 14317.
- A port that cannot be bound does **not** stop HertzBeat: the failure is logged and the process
starts without gRPC ingestion, while OTLP/HTTP on `/api/otlp/v1` keeps working.
- To move the listener to 4317, or disable it, set these in `application.yml` or through the matching
environment variables, and update the docker-compose port mapping to match:
```yaml
hertzbeat:
otlp:
grpc:
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
```
- If you deploy with the Helm chart, note that the chart is maintained in `apache/hertzbeat-helm-chart`;
check that its release exposes 14317 before relying on gRPC ingestion there.
### GreptimeDB signal tables renamed
When `warehouse.store.greptime.enabled=true`, HertzBeat writes two different kinds of telemetry to GreptimeDB: the traces and logs **you** send it over OTLP, and its **own** runtime logs and traces shipped via OpenTelemetry. On 1.8.x both kinds of traces landed in the same `hzb_traces` table. 1.9.0 separates them, which renames one product table and both self-monitoring tables:
| Data | 1.8.x table | 1.9.0 table |
|---|---|---|
| Product OTLP traces (the traces page, trace queries) | `hzb_traces` | `hertzbeat_traces` |
| Product OTLP logs (the logs page, log alerting, SQL editor) | `hertzbeat_logs` | `hertzbeat_logs` (unchanged) |
| HertzBeat internal logs (self-monitoring) | `hzb_logs` | `hzb_internal_logs` |
| HertzBeat internal traces (self-monitoring) | `hzb_traces` | `hzb_internal_traces` |
- **The traces page will be empty for data ingested before the upgrade.** 1.9.0 creates `hertzbeat_traces` and queries only that table, so spans written to `hzb_traces` on 1.8.x are no longer visible in the UI until you copy them over.
- The product log table `hertzbeat_logs` is **not** renamed; historical logs ingested on 1.8.x remain queryable with no action needed.
- No automatic migration is performed. The old `hzb_logs` / `hzb_traces` tables are left untouched but no longer receive new data. Once 1.9.0 has created the new tables you can copy the history manually, for example:
```sql
INSERT INTO hzb_internal_logs SELECT * FROM hzb_logs;
```
Copying traces needs more care, because `hzb_traces` holds your spans and HertzBeat's own spans mixed together. Filter by service so the self-monitoring spans do not end up in the product table:
```sql
-- keep only your own services; HertzBeat's self-telemetry uses service.name = 'HertzBeat'
INSERT INTO hertzbeat_traces SELECT * FROM hzb_traces WHERE service_name <> 'HertzBeat';
INSERT INTO hzb_internal_traces SELECT * FROM hzb_traces WHERE service_name = 'HertzBeat';
```
Otherwise you can `DROP` the old tables when the retention no longer matters.
- If you have dashboards or ad-hoc SQL against `hzb_logs` / `hzb_traces`, point them at the new table names.
## Upgrade For Docker Deploy ## Upgrade For Docker Deploy
1. If using custom monitoring templates 1. If using custom monitoring templates
@@ -11,6 +11,14 @@ keywords: [开源监控, 日志集成, 日志管理, 多源日志]
日志集成功能目前处于 Beta(实验性)阶段,可能存在潜在缺陷和局限性。该功能正在积极开发和迭代中。 日志集成功能目前处于 Beta(实验性)阶段,可能存在潜在缺陷和局限性。该功能正在积极开发和迭代中。
::: :::
:::warning 从 1.8.x 升级?接收路径已变更
1.8.x 的接收端点 `POST /api/logs/otlp/v1/logs`(以及 `POST /api/logs/ingest/otlp`)已由 `POST /api/otlp/v1/logs` 取代,请更新所有指向 HertzBeat 的 OpenTelemetry Collector / SDK exporter 的 `logs_endpoint`。1.9.x 仍保留旧路径作为 deprecated 别名(响应带 `Deprecation: true`,HertzBeat 日志会打印告警),2.0 将移除。查询侧的 `/api/logs/**``/api/traces/**``/api/ingestion/otlp/**` 已迁移到 `/api/observability/**`,没有别名。完整的新旧路径对照表见[版本升级指南](../start/upgrade)。
:::
:::info HertzBeat 1.9.0 过渡版本
指标、日志和链路统一通过 `/api/otlp/v1/{signal}` 接收,并通过 `/api/observability/**` 查询。该版本不会根据遥测数据创建或绑定 Entity;外部 OTLP 三信号表也与 HertzBeat 自身遥测表分开存储。
:::
## 核心能力 ## 核心能力
- **多源日志接入**:支持从 OpenTelemetry、Filebeat、Vector、Loki 等主流平台接收日志数据 - **多源日志接入**:支持从 OpenTelemetry、Filebeat、Vector、Loki 等主流平台接收日志数据
@@ -36,9 +44,34 @@ HertzBeat 当前已支持以下协议进行日志数据接入:
HertzBeat 提供以下接口用于接收 OTLP 日志数据: HertzBeat 提供以下接口用于接收 OTLP 日志数据:
```text ```text
POST /api/logs/otlp/v1/logs POST /api/otlp/v1/logs
``` ```
### OTLP/gRPC 端点
启用 GreptimeDB 存储时,HertzBeat 同时会启动一个 OTLP/gRPC 监听器,接收指标、日志与链路。凭证与 HTTP 端点一致,同样使用 `Authorization: Bearer {token}`
```text
{hertzbeat_host}:14317
```
所有部署方式下都是这一个端口——Docker 镜像原样发布,不存在"容器内一个、宿主上另一个"的换算。
这里刻意没有使用 OpenTelemetry 标准的 4317:同机的 OTel Collector、Jaeger 或 Tempo 通常已经占着该端口,而已发布端口一旦冲突,容器会直接起不来。HertzBeat 的 OTLP/HTTP 同样走自有端口,因此这个选择与产品其余部分是一致的,并非特例。
如果确实想用 4317,或想关闭该监听器,可在 `application.yml` 中配置,或使用对应的环境变量(同时记得把 `docker-compose.yaml` 里的端口映射改成一致):
```yaml
hertzbeat:
otlp:
grpc:
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
```
若端口无法绑定,HertzBeat 会记录错误并在没有 gRPC 接收能力的情况下继续启动,`/api/otlp/v1` 上的 OTLP/HTTP 不受影响。
### 请求配置 ### 请求配置
#### 请求头 #### 请求头
@@ -112,7 +145,7 @@ POST /api/logs/otlp/v1/logs
```yaml ```yaml
exporters: exporters:
otlphttp: otlphttp:
logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs
compression: none compression: none
encoding: json encoding: json
headers: headers:
@@ -14,6 +14,83 @@ HertzBeat 的元数据信息保存在 H2 或 Mysql, PostgreSQL 关系型数据
**升级前您需要保存备份好数据库的数据文件和监控模板文件** **升级前您需要保存备份好数据库的数据文件和监控模板文件**
## 1.9.0 不兼容变更
### 可观测(OTLP / 日志 / 链路)接口路径变更
1.9.0 将 1.8.x 的日志模块合并为 `hertzbeat-observability`,指标、日志、链路统一使用 `/api/otlp/v1/{signal}` 接收、`/api/observability/**` 查询。所有按 1.8.x 路径配置的 OpenTelemetry Collector、Vector、SDK exporter、脚本或看板都需要更新。
| 1.8.x 路径 | 1.9.0 路径 | 1.9.x 状态 |
|---|---|---|
| `POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs` | **保留为 deprecated 别名**,仍可用,响应带 `Deprecation: true`2.0 移除 |
| `POST /api/logs/ingest/otlp` | `POST /api/otlp/v1/logs` | **保留为 deprecated 别名**,仍可用,响应带 `Deprecation: true`2.0 移除 |
| `POST /api/logs/ingest/{其他协议}` | — | 已移除(`400`),历史上只有 `otlp` 有适配器 |
| `GET /api/logs/list` | `GET /api/observability/logs` | 已移除(`404` |
| `GET /api/logs/stats/overview` | `GET /api/observability/logs/overview` | 已移除(`404` |
| `GET /api/logs/stats/trace-coverage` | `GET /api/observability/logs/trace-coverage` | 已移除(`404` |
| `GET /api/logs/stats/trend` | `GET /api/observability/logs/trend` | 已移除(`404` |
| `GET /api/logs/sse/subscribe` | `GET /api/observability/logs/stream` | 已移除(`404`);新路径需要 `admin/user/guest` 登录,不再匿名放行 |
| `DELETE /api/logs` | `DELETE /api/observability/logs` | 已移除(`404` |
| `GET /api/traces/**` | `GET /api/observability/traces/**` | 已移除(`404` |
| `GET /api/ingestion/otlp/metrics/console` | `GET /api/observability/metrics/query` | 已移除(`404` |
| `GET /api/ingestion/otlp/metrics/inventory` | `GET /api/observability/metrics/inventory` | 已移除(`404` |
建议的升级步骤:
- 升级前在 collector / exporter 配置中搜索 `/api/logs/`,改为 `/api/otlp/v1/logs`。OTLP HTTP exporter 会把 `404` 视为永久错误并静默丢弃该批数据,路径过期的表现只是"日志突然没了"。
- 如果无法在同一维护窗口内改完 exporter,上表两条接收别名在 1.9.x 仍然可用;请关注 HertzBeat 日志中的 `Deprecated OTLP log route ... was called` 告警并在 2.0 之前完成迁移。
- 如果使用了自定义 `sureness.yml`,请补充 `/api/otlp/v1/**===post===[admin,user]``/api/observability/**===get===[admin,user,guest]`(参考安装包内的 `sureness.yml`);旧的 `/api/logs/**``/api/traces/**``/api/ingestion/otlp/**` 规则在 exporter 迁移完成后即可删除。
### 新增 OTLP/gRPC 监听端口 14317
`warehouse.store.greptime.enabled=true` 时,1.9.0 会额外启动一个 OTLP/gRPC 监听器,绑定 `0.0.0.0:14317`,供 exporter 通过 gRPC 推送指标、日志与链路。官方 Dockerfile 与 docker-compose 原样发布该端口,因此所有部署方式下端口一致。
- **这里没有使用 OpenTelemetry 标准的 4317。** 同机的 OTel Collector、Jaeger 或 Tempo 通常已经占着 4317,而已发布端口一旦冲突,`docker compose up` 会直接失败。HertzBeat 的 OTLP/HTTP 同样走自有端口,因此 14317 与产品其余部分是一致的。
- 存量部署升级后会多出一个监听端口。如果你的防火墙或安全策略按端口清单管理,请把 14317 加进去。
- 端口绑定失败**不会**导致 HertzBeat 启动失败:失败会被记录到日志,进程在没有 gRPC 接收能力的情况下继续启动,`/api/otlp/v1` 上的 OTLP/HTTP 不受影响。
- 如需把监听器改到 4317 或关闭它,可在 `application.yml` 中配置,或使用对应的环境变量,并同步修改 docker-compose 的端口映射:
```yaml
hertzbeat:
otlp:
grpc:
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
```
- 使用 Helm 部署时请注意:Chart 维护在 `apache/hertzbeat-helm-chart` 仓库,依赖 gRPC 接入前请先确认其发布版本已暴露 14317。
### GreptimeDB 信号表改名
当 `warehouse.store.greptime.enabled=true` 时,GreptimeDB 里同时存着两类遥测数据:**你**通过 OTLP 推送的日志与链路,以及 HertzBeat 通过 OpenTelemetry 写入的**自身**运行日志与链路。1.8.x 中两类链路数据落在同一张 `hzb_traces` 表里,1.9.0 将其拆开,因此一张产品表和两张自监控表都改了名:
| 数据 | 1.8.x 表名 | 1.9.0 表名 |
|---|---|---|
| 产品 OTLP 链路(链路页面、链路查询) | `hzb_traces` | `hertzbeat_traces` |
| 产品 OTLP 日志(日志页面、日志告警、SQL 编辑器) | `hertzbeat_logs` | `hertzbeat_logs`(不变) |
| HertzBeat 自身日志(自监控) | `hzb_logs` | `hzb_internal_logs` |
| HertzBeat 自身链路(自监控) | `hzb_traces` | `hzb_internal_traces` |
- **升级前接入的链路数据在链路页面上会是空的。** 1.9.0 只创建并查询 `hertzbeat_traces`1.8.x 期间写入 `hzb_traces` 的 span 在手动迁移之前不会显示在界面上。
- 产品日志表 `hertzbeat_logs` **没有**改名,1.8.x 期间接入的历史日志升级后无需任何操作即可正常查询。
- 不做自动迁移。旧的 `hzb_logs` / `hzb_traces` 表会原样保留但不再写入新数据。在 1.9.0 建好新表后可手动迁移历史数据,例如:
```sql
INSERT INTO hzb_internal_logs SELECT * FROM hzb_logs;
```
迁移链路数据需要更谨慎:`hzb_traces` 里混着你的 span 和 HertzBeat 自身的 span,需按服务名过滤,避免把自监控数据灌进产品表:
```sql
-- 只保留业务服务;HertzBeat 自监控使用 service.name = 'HertzBeat'
INSERT INTO hertzbeat_traces SELECT * FROM hzb_traces WHERE service_name <> 'HertzBeat';
INSERT INTO hzb_internal_traces SELECT * FROM hzb_traces WHERE service_name = 'HertzBeat';
```
如果不需要历史数据,待保留期过后直接 `DROP` 旧表即可。
- 如果有看板或临时 SQL 直接查询 `hzb_logs` / `hzb_traces`,请改为新表名。
## Docker部署方式的升级 ## Docker部署方式的升级
1. 若使用了自定义监控模板 1. 若使用了自定义监控模板
+2 -2
View File
@@ -8,7 +8,7 @@
```sql ```sql
SELECT timestamp, severity_text, body SELECT timestamp, severity_text, body
FROM hzb_logs FROM hzb_internal_logs
WHERE <> WHERE <>
ORDER BY timestamp DESC ORDER BY timestamp DESC
LIMIT <1-100> LIMIT <1-100>
@@ -36,7 +36,7 @@ LIMIT <1-100>
} }
``` ```
当前 `hzb_logs` 表没有 `monitorId` 字段,因此本接口不提供无效的监控 ID 过滤。如果后续需要该能力,应先在 OpenTelemetry 日志写入链中定义并提取统一的监控 ID 字段。 当前 `hzb_internal_logs` 表没有 `monitorId` 字段,因此本接口不提供无效的监控 ID 过滤。如果后续需要该能力,应先在 OpenTelemetry 日志写入链中定义并提取统一的监控 ID 字段。
## GreptimeDB 账号 ## GreptimeDB 账号
@@ -48,7 +48,7 @@ import java.util.Set;
@Slf4j @Slf4j
public class LogService { public class LogService {
private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_logs"; private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_internal_logs";
private static final String TIMESTAMP_COLUMN = "timestamp"; private static final String TIMESTAMP_COLUMN = "timestamp";
private static final String SEVERITY_TEXT_COLUMN = "severity_text"; private static final String SEVERITY_TEXT_COLUMN = "severity_text";
private static final String BODY_COLUMN = "body"; private static final String BODY_COLUMN = "body";
@@ -45,7 +45,7 @@ import org.springframework.test.web.client.MockRestServiceServer;
*/ */
class LogServiceTest { class LogServiceTest {
private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_logs"; private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_internal_logs";
@Test @Test
void shouldBuildDefaultReadOnlyQuery() { void shouldBuildDefaultReadOnlyQuery() {
@@ -70,13 +70,13 @@ class LogServiceTest {
@Test @Test
void shouldBuildQueryFromValidatedFilters() { void shouldBuildQueryFromValidatedFilters() {
String query = LogService.buildQuery( String query = LogService.buildQuery(
" error ", " x'); DELETE FROM hzb_logs; -- ", 1L, 2L, 10); " error ", " x'); DELETE FROM hzb_internal_logs; -- ", 1L, 2L, 10);
assertThat(query).isEqualTo(BASE_QUERY assertThat(query).isEqualTo(BASE_QUERY
+ " WHERE timestamp >= 1000000" + " WHERE timestamp >= 1000000"
+ " AND timestamp <= 2000000" + " AND timestamp <= 2000000"
+ " AND severity_text = 'ERROR'" + " AND severity_text = 'ERROR'"
+ " AND matches_term(body, 'x''); DELETE FROM hzb_logs; --')" + " AND matches_term(body, 'x''); DELETE FROM hzb_internal_logs; --')"
+ " ORDER BY timestamp DESC LIMIT 10"); + " ORDER BY timestamp DESC LIMIT 10");
} }
+2 -2
View File
@@ -92,7 +92,7 @@
<module>hertzbeat-otel</module> <module>hertzbeat-otel</module>
<module>hertzbeat-e2e</module> <module>hertzbeat-e2e</module>
<module>hertzbeat-base</module> <module>hertzbeat-base</module>
<module>hertzbeat-log</module> <module>hertzbeat-observability</module>
<module>hertzbeat-ai</module> <module>hertzbeat-ai</module>
</modules> </modules>
@@ -269,7 +269,7 @@
<!-- log --> <!-- log -->
<dependency> <dependency>
<groupId>org.apache.hertzbeat</groupId> <groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-log</artifactId> <artifactId>hertzbeat-observability</artifactId>
<version>${hertzbeat.version}</version> <version>${hertzbeat.version}</version>
</dependency> </dependency>
<!-- collector-basic --> <!-- collector-basic -->
+11
View File
@@ -346,6 +346,17 @@ grafana:
password: admin password: admin
hertzbeat: hertzbeat:
otlp:
grpc:
# OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at
# this port on every deployment - docker publishes it unchanged.
# 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host
# normally holds already. Set 4317 here if you want the standard port and know it is free.
# A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps
# working either way.
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
collector: collector:
mysql: mysql:
# MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics.
@@ -246,6 +246,17 @@ grafana:
password: admin password: admin
hertzbeat: hertzbeat:
otlp:
grpc:
# OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at
# this port on every deployment - docker publishes it unchanged.
# 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host
# normally holds already. Set 4317 here if you want the standard port and know it is free.
# A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps
# working either way.
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
collector: collector:
mysql: mysql:
# MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics.
@@ -84,8 +84,6 @@ resourceRole:
# route forwards a raw promql expression straight to the time series database # route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin] - /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user] - /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user] - /api/status/page/**===post===[admin,user]
@@ -107,8 +105,12 @@ resourceRole:
- /api/ai/**===post===[admin] - /api/ai/**===post===[admin]
- /api/ai/**===put===[admin] - /api/ai/**===put===[admin]
- /api/ai/**===delete===[admin] - /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest] - /api/otlp/v1/**===post===[admin,user]
# deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0
- /api/logs/otlp/**===post===[admin,user]
- /api/logs/ingest/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user]
- /api/observability/logs===delete===[admin]
- /api/observability/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is # The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous # scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin] - /v3/api-docs/**===get===[admin]
@@ -130,7 +132,6 @@ excludedResource:
- /api/account/auth/**===* - /api/account/auth/**===*
- /api/i18n/**===get - /api/i18n/**===get
- /api/apps/hierarchy===get - /api/apps/hierarchy===get
- /api/observability/capability===get
- /api/push/**===* - /api/push/**===*
- /api/status/page/public/**===* - /api/status/page/public/**===*
# web ui resource # web ui resource
@@ -84,5 +84,8 @@ services:
ports: ports:
- "1157:1157" - "1157:1157"
- "1158:1158" - "1158:1158"
# OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local
# OTel Collector normally holds - and a clash on a published port stops the container.
- "14317:14317"
networks: networks:
- hertzbeat - hertzbeat
@@ -243,6 +243,17 @@ grafana:
password: admin password: admin
hertzbeat: hertzbeat:
otlp:
grpc:
# OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at
# this port on every deployment - docker publishes it unchanged.
# 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host
# normally holds already. Set 4317 here if you want the standard port and know it is free.
# A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps
# working either way.
enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true}
host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0}
port: ${HERTZBEAT_OTLP_GRPC_PORT:14317}
collector: collector:
mysql: mysql:
# MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics.

Some files were not shown because too many files have changed in this diff Show More