From 0c4136cade8ea527c21b2b639a7c01d591128e8d Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 22 Aug 2026 06:39:42 +0800 Subject: [PATCH] Consolidate entity-free observability for 1.9 (#4302) --- .github/workflows/backend-build-test.yml | 2 +- .../collect/common/http/CommonHttpClient.java | 19 + .../CommonHttpClientVirtualThreadTest.java | 2 + .../common/constants/ConfigConstants.java | 2 + .../common/constants/NetworkConstants.java | 2 +- .../pom.xml | 4 +- .../alert/LogPeriodicAlertE2eTest.java | 2 +- .../alert/LogRealTimeAlertE2eTest.java | 2 +- .../ingestion/LogIngestionE2eTest.java | 2 +- ...reptimeEntityFreeSignalStorageE2eTest.java | 176 ++++++++ .../storage/GreptimeLogStorageE2eTest.java | 2 +- .../src/test/resources/sureness.yml | 17 +- .../src/test/resources/vector.yml | 2 +- hertzbeat-e2e/pom.xml | 2 +- .../controller/LogIngestionController.java | 82 ---- .../log/controller/OtlpLogController.java | 160 -------- .../LogIngestionControllerTest.java | 117 ------ .../log/controller/OtlpLogControllerTest.java | 131 ------ hertzbeat-manager/pom.xml | 4 +- .../config/RestTemplateConfigTest.java | 44 ++ .../src/test/resources/sureness.yml | 17 +- .../pom.xml | 2 +- .../ObservabilityAutoConfiguration.java | 16 +- .../config/OtlpGrpcServerConfig.java | 63 ++- .../LegacyOtlpLogRouteController.java | 145 +++++++ .../controller/LogManagerController.java | 4 +- .../controller/LogQueryController.java | 16 +- .../controller/LogSseController.java | 12 +- .../ObservabilityCapabilityController.java | 4 +- .../controller/OtlpHttpExceptionHandler.java | 213 ++++++++++ .../controller/OtlpLogController.java | 53 +++ .../controller/OtlpSignalController.java | 23 +- .../SignalWorkloadExceptionHandler.java | 15 +- .../ThreeSignalQueryController.java | 22 +- .../notice/LogSseFilterCriteria.java | 2 +- .../observability}/notice/LogSseManager.java | 2 +- .../service/OtlpLogIngestionService.java | 44 ++ .../service/OtlpSignalForwarder.java | 2 +- .../service/SignalQueryRejectedException.java | 2 +- .../service/SignalWorkloadGuard.java | 2 +- .../impl/DefaultOtlpLogIngestionService.java | 92 +++++ .../impl/GreptimeOtlpSignalForwarder.java | 79 +--- .../service/impl/OtlpLogProtocolAdapter.java | 77 +++- ...ot.autoconfigure.AutoConfiguration.imports | 3 +- .../config/OtlpGrpcServerConfigTest.java | 75 +++- .../LegacyOtlpLogRouteControllerTest.java | 117 ++++++ .../controller/LogManagerControllerTest.java | 8 +- .../controller/LogQueryControllerTest.java | 26 +- .../controller/LogSseControllerTest.java | 14 +- ...ObservabilityCapabilityControllerTest.java | 4 +- .../OtlpHttpExceptionHandlerTest.java | 205 ++++++++++ .../controller/OtlpLogControllerTest.java | 124 ++++++ .../OtlpLogIngestionOwnershipTest.java | 44 ++ .../controller/OtlpSignalControllerTest.java | 45 ++- .../SignalWorkloadExceptionHandlerTest.java | 17 +- .../ThreeSignalQueryControllerTest.java | 16 +- .../notice/LogSseFilterCriteriaTest.java | 2 +- .../notice/LogSseManagerTest.java | 2 +- .../service/SignalWorkloadGuardTest.java | 6 +- .../DefaultOtlpLogIngestionServiceTest.java | 106 +++++ .../impl/GreptimeOtlpSignalForwarderTest.java | 53 +-- .../impl/OtlpLogProtocolAdapterTest.java | 29 +- .../OpenTelemetryAutoConfiguration.java | 29 ++ .../otel/config/OpenTelemetryConfig.java | 6 +- ...ot.autoconfigure.AutoConfiguration.imports | 16 + .../src/main/resources/application.yml | 11 + .../src/main/resources/sureness.yml | 13 +- .../apache/hertzbeat/startup/ContextTest.java | 17 +- ...eeObservabilityTransitionContractTest.java | 271 +++++++++++++ .../startup/security/SurenessSseRuleTest.java | 5 +- .../security/SurenessUnruledEndpointTest.java | 10 +- .../config/WarehouseAutoConfiguration.java | 3 + .../constants/WarehouseConstants.java | 2 + .../warehouse/service/OtlpSignalStorage.java | 28 +- .../service/ThreeSignalQueryService.java | 2 +- .../impl/GreptimeThreeSignalQueryService.java | 17 +- .../tsdb/greptime/GreptimeDbDataStorage.java | 34 +- .../greptime/GreptimeOtlpSignalStorage.java | 166 ++++++++ .../greptime}/GreptimeSignalInitializer.java | 9 +- .../pipelines/hertzbeat_otlp_log_v1.yaml | 0 .../greptime/tables/hertzbeat_logs.sql | 0 .../greptime/tables/hertzbeat_traces.sql | 2 +- .../GreptimeThreeSignalQueryServiceTest.java | 2 +- .../greptime/GreptimeDbDataStorageTest.java | 33 ++ .../GreptimeOtlpSignalStorageTest.java | 193 +++++++++ .../GreptimeSignalInitializerTest.java | 9 +- .../greptime}/GreptimeSignalPipelineTest.java | 2 +- home/docs/help/log_integration.md | 44 +- home/docs/start/upgrade.md | 86 ++++ .../current/help/log_integration.md | 37 +- .../current/start/upgrade.md | 77 ++++ mcp-servers/mcp-log-server/README.md | 4 +- .../mcp/log/server/service/LogService.java | 2 +- .../mcp/server/service/LogServiceTest.java | 6 +- pom.xml | 4 +- script/application.yml | 11 + .../conf/application.yml | 11 + .../hertzbeat-mysql-iotdb/conf/sureness.yml | 9 +- .../hertzbeat-mysql-iotdb/docker-compose.yaml | 3 + .../conf/application.yml | 11 + .../conf/sureness.yml | 9 +- .../docker-compose.yaml | 3 + .../conf/application.yml | 11 + .../conf/sureness.yml | 9 +- .../docker-compose.yaml | 3 + .../conf/application.yml | 11 + .../conf/sureness.yml | 13 +- .../docker-compose.yaml | 3 + .../conf/application.yml | 11 + .../conf/sureness.yml | 9 +- .../docker-compose.yaml | 3 + script/docker/server/Dockerfile | 2 +- script/sureness.yml | 13 +- .../log/log-manage/log-manage.component.html | 283 ++++++++----- .../log/log-manage/log-manage.component.less | 123 ++---- .../log/log-manage/log-manage.component.ts | 2 + .../log-stream/log-stream.component.spec.ts | 2 +- .../log/log-stream/log-stream.component.ts | 2 +- .../metrics/metrics-manage.component.html | 193 +++++---- .../metrics/metrics-manage.component.less | 116 +----- .../metrics/metrics-manage.component.ts | 4 +- .../signal-navigation.component.ts | 34 +- .../signal-time-range.component.ts | 42 +- .../signal-workspace.component.html | 39 ++ .../signal-workspace.component.less | 283 +++++++++++++ .../signal-workspace.component.spec.ts | 70 ++++ .../signal-workspace.component.ts | 36 ++ .../routes/trace/trace-manage.component.html | 375 +++++++++++------- .../routes/trace/trace-manage.component.less | 89 +---- .../routes/trace/trace-manage.component.ts | 4 +- web-app/src/app/service/log.service.spec.ts | 44 ++ web-app/src/app/service/log.service.ts | 8 +- .../app/service/observability.service.spec.ts | 21 + .../src/app/service/observability.service.ts | 16 +- .../assets/doc/log-integration/otlp.en-US.md | 16 +- .../assets/doc/log-integration/otlp.zh-CN.md | 12 +- web-app/src/assets/i18n/en-US.json | 3 + web-app/src/assets/i18n/ja-JP.json | 3 + web-app/src/assets/i18n/pt-BR.json | 3 + web-app/src/assets/i18n/zh-CN.json | 3 + web-app/src/assets/i18n/zh-TW.json | 3 + 141 files changed, 4162 insertions(+), 1529 deletions(-) rename hertzbeat-e2e/{hertzbeat-log-e2e => hertzbeat-observability-e2e}/pom.xml (97%) rename hertzbeat-e2e/{hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability}/alert/LogPeriodicAlertE2eTest.java (99%) rename hertzbeat-e2e/{hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability}/alert/LogRealTimeAlertE2eTest.java (99%) rename hertzbeat-e2e/{hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability}/ingestion/LogIngestionE2eTest.java (98%) create mode 100644 hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java rename hertzbeat-e2e/{hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability}/storage/GreptimeLogStorageE2eTest.java (99%) rename hertzbeat-e2e/{hertzbeat-log-e2e => hertzbeat-observability-e2e}/src/test/resources/sureness.yml (91%) rename hertzbeat-e2e/{hertzbeat-log-e2e => hertzbeat-observability-e2e}/src/test/resources/vector.yml (99%) delete mode 100644 hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java delete mode 100644 hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java delete mode 100644 hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java delete mode 100644 hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java rename {hertzbeat-log => hertzbeat-observability}/pom.xml (98%) rename hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java (72%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/config/OtlpGrpcServerConfig.java (75%) create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/LogManagerController.java (95%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/LogQueryController.java (97%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/LogSseController.java (87%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/ObservabilityCapabilityController.java (96%) create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandler.java create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/OtlpSignalController.java (74%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/SignalWorkloadExceptionHandler.java (83%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/controller/ThreeSignalQueryController.java (91%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/notice/LogSseFilterCriteria.java (98%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/notice/LogSseManager.java (99%) create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/service/OtlpSignalForwarder.java (95%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/service/SignalQueryRejectedException.java (95%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/service/SignalWorkloadGuard.java (98%) create mode 100644 hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/service/impl/GreptimeOtlpSignalForwarder.java (62%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability}/service/impl/OtlpLogProtocolAdapter.java (79%) rename hertzbeat-otel/src/main/resources/META-INF/spring.factories => hertzbeat-observability/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports (86%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/config/OtlpGrpcServerConfigTest.java (58%) create mode 100644 hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteControllerTest.java rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/LogManagerControllerTest.java (94%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/LogQueryControllerTest.java (91%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/LogSseControllerTest.java (92%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/ObservabilityCapabilityControllerTest.java (96%) create mode 100644 hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandlerTest.java create mode 100644 hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java create mode 100644 hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/OtlpSignalControllerTest.java (58%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/SignalWorkloadExceptionHandlerTest.java (78%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/controller/ThreeSignalQueryControllerTest.java (91%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/notice/LogSseFilterCriteriaTest.java (99%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/notice/LogSseManagerTest.java (99%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/service/SignalWorkloadGuardTest.java (92%) create mode 100644 hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/service/impl/GreptimeOtlpSignalForwarderTest.java (63%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability}/service/impl/OtlpLogProtocolAdapterTest.java (94%) create mode 100644 hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java create mode 100644 hertzbeat-otel/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java rename hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java => hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java (56%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse}/service/ThreeSignalQueryService.java (98%) rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log => hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse}/service/impl/GreptimeThreeSignalQueryService.java (97%) create mode 100644 hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java rename {hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config => hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime}/GreptimeSignalInitializer.java (94%) rename {hertzbeat-log => hertzbeat-warehouse}/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml (100%) rename {hertzbeat-log => hertzbeat-warehouse}/src/main/resources/greptime/tables/hertzbeat_logs.sql (100%) rename hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql => hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql (97%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log => hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse}/service/impl/GreptimeThreeSignalQueryServiceTest.java (99%) create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config => hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime}/GreptimeSignalInitializerTest.java (90%) rename {hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config => hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime}/GreptimeSignalPipelineTest.java (95%) create mode 100644 web-app/src/app/routes/observability/signal-workspace.component.html create mode 100644 web-app/src/app/routes/observability/signal-workspace.component.less create mode 100644 web-app/src/app/routes/observability/signal-workspace.component.spec.ts create mode 100644 web-app/src/app/routes/observability/signal-workspace.component.ts create mode 100644 web-app/src/app/service/log.service.spec.ts diff --git a/.github/workflows/backend-build-test.yml b/.github/workflows/backend-build-test.yml index abb52e8960..a71c0a0b19 100644 --- a/.github/workflows/backend-build-test.yml +++ b/.github/workflows/backend-build-test.yml @@ -76,7 +76,7 @@ jobs: - name: Build backend Maven E2E modules run: | mvnd clean -B package \ - -pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-log-e2e \ + -pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-observability-e2e \ -am \ -Dmaven.test.skip=false \ --file pom.xml diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java index 54001cdae0..cda6ce8259 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java @@ -177,6 +177,11 @@ public class CommonHttpClient { static void setBeforeCleanupHookForTest(Runnable hook) { beforeCleanupHook = hook; } + + static boolean awaitConnectionPoolCleanupIdleForTest(long timeout, TimeUnit unit) throws InterruptedException { + ScheduledDispatchTask currentCleanupTask = cleanupTask; + return currentCleanupTask == null || currentCleanupTask.awaitIdle(timeout, unit); + } public static void close() { try { @@ -268,10 +273,24 @@ public class CommonHttpClient { shouldSchedule = pendingRuns > 0; if (!shouldSchedule) { running = false; + notifyAll(); return; } } scheduleRun(); } + + private synchronized boolean awaitIdle(long timeout, TimeUnit unit) throws InterruptedException { + long deadline = System.nanoTime() + unit.toNanos(timeout); + long remainingNanos = deadline - System.nanoTime(); + while (running || pendingRuns > 0) { + if (remainingNanos <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, remainingNanos); + remainingNanos = deadline - System.nanoTime(); + } + return true; + } } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java index 1bdace1667..017419772a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java @@ -96,9 +96,11 @@ class CommonHttpClientVirtualThreadTest { CommonHttpClient.dispatchConnectionPoolCleanup(); assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + assertFalse(CommonHttpClient.awaitConnectionPoolCleanupIdleForTest(200, TimeUnit.MILLISECONDS)); releaseFirst.countDown(); assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertTrue(CommonHttpClient.awaitConnectionPoolCleanupIdleForTest(5, TimeUnit.SECONDS)); assertEquals(1, maxConcurrent.get()); } diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java index 47e8bacfe3..e3b1997774 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java @@ -69,6 +69,8 @@ public interface ConfigConstants { String GRAFANA = "grafana"; String LOG = "log"; + + String OBSERVABILITY = "observability"; } } diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java index 7f1f364785..8e1210d43a 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java @@ -61,7 +61,7 @@ public interface NetworkConstants { Duration READ_TIMEOUT = Duration.ofSeconds(6); Duration WRITE_TIMEOUT = Duration.ofSeconds(6); Duration CONNECT_TIMEOUT = Duration.ofSeconds(6); - Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(5); + Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(15); Duration GREPTIME_QUERY_CONNECT_TIMEOUT = Duration.ofSeconds(2); Duration GREPTIME_WRITE_READ_TIMEOUT = Duration.ofSeconds(3); Duration GREPTIME_WRITE_CONNECT_TIMEOUT = Duration.ofSeconds(2); diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml similarity index 97% rename from hertzbeat-e2e/hertzbeat-log-e2e/pom.xml rename to hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml index a5638e23a2..dfffdb62c7 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml @@ -25,7 +25,7 @@ 4.0.0 - hertzbeat-log-e2e + hertzbeat-observability-e2e ${java.version} @@ -52,7 +52,7 @@ org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability ${hertzbeat.version} test diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java index 9957ce417b..cb5abb26f4 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.alert; +package org.apache.hertzbeat.observability.alert; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java index 4ae973b343..1e93a4c77a 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.alert; +package org.apache.hertzbeat.observability.alert; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java similarity index 98% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java index 57feada98e..5fcb3e0668 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.ingestion; +package org.apache.hertzbeat.observability.ingestion; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; diff --git a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java new file mode 100644 index 0000000000..c6ab50e64a --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.storage; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.resource.v1.Resource; +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeOtlpSignalStorage; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.junit.jupiter.api.Test; +import org.springframework.web.client.RestTemplate; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +/** Proves the warehouse-owned Entity-free OTLP log path against a real GreptimeDB. */ +@Testcontainers +class GreptimeEntityFreeSignalStorageE2eTest { + + private static final int GREPTIME_HTTP_PORT = 4000; + private static final int GREPTIME_GRPC_PORT = 4001; + private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql"; + private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml"; + private static final String PIPELINE_NAME = "hertzbeat_otlp_log_v1"; + private static final String TRACE_ID = "0123456789abcdef0123456789abcdef"; + private static final String SPAN_ID = "0123456789abcdef"; + private static final String BODY = "entity-free greptime proof"; + private static final long LOG_TIME_NANOS = 1_710_000_000_123_456_789L; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Container + @SuppressWarnings("resource") + private static final GenericContainer GREPTIME = new GenericContainer<>( + DockerImageName.parse("greptime/greptimedb:latest")) + .withExposedPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT) + .withCommand("standalone", "start", + "--http-addr", "0.0.0.0:" + GREPTIME_HTTP_PORT, + "--rpc-bind-addr", "0.0.0.0:" + GREPTIME_GRPC_PORT) + .waitingFor(Wait.forListeningPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT)) + .withStartupTimeout(Duration.ofSeconds(120)); + + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @Test + void warehouseStorageShouldPersistEntityFreeOtlpLogs() throws Exception { + executeSql(classpathResource(LOG_SCHEMA).strip().replaceFirst(";\\s*$", "")); + uploadPipeline(); + + GreptimeOtlpSignalStorage storage = new GreptimeOtlpSignalStorage( + new GreptimeProperties(true, GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_GRPC_PORT), + endpoint(), "public", "", ""), + new RestTemplate()); + + storage.writeProtobuf("logs", request().toByteArray()); + + await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> { + String sql = "SELECT COUNT(*) AS count FROM hertzbeat_logs WHERE trace_id = '" + TRACE_ID + + "' AND body = '" + BODY + "'"; + assertThat(queryCount(sql)).isEqualTo(1); + }); + } + + private void uploadPipeline() throws Exception { + String boundary = "----hertzbeat-entity-free-proof"; + String body = "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"pipeline.yaml\"\r\n" + + "Content-Type: application/x-yaml\r\n\r\n" + + classpathResource(LOG_PIPELINE) + "\r\n" + + "--" + boundary + "--\r\n"; + HttpResponse response = httpClient.send(HttpRequest.newBuilder() + .uri(URI.create(endpoint() + "/v1/pipelines/" + PIPELINE_NAME)) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).as(response.body()).isBetween(200, 299); + } + + private ExportLogsServiceRequest request() { + LogRecord record = LogRecord.newBuilder() + .setTimeUnixNano(LOG_TIME_NANOS) + .setObservedTimeUnixNano(LOG_TIME_NANOS) + .setSeverityNumberValue(9) + .setSeverityText("INFO") + .setBody(AnyValue.newBuilder().setStringValue(BODY).build()) + .setTraceId(ByteString.copyFrom(hexToBytes(TRACE_ID))) + .setSpanId(ByteString.copyFrom(hexToBytes(SPAN_ID))) + .build(); + return ExportLogsServiceRequest.newBuilder() + .addResourceLogs(ResourceLogs.newBuilder() + .setResource(Resource.newBuilder() + .addAttributes(stringAttribute("service.name", "checkout")) + .addAttributes(stringAttribute("deployment.environment.name", "test")) + .build()) + .addScopeLogs(ScopeLogs.newBuilder().addLogRecords(record).build()) + .build()) + .build(); + } + + private int queryCount(String sql) throws Exception { + JsonNode rows = OBJECT_MAPPER.readTree(executeSql(sql).body()) + .path("output").path(0).path("records").path("rows"); + assertThat(rows.isArray()).isTrue(); + assertThat(rows).isNotEmpty(); + return rows.get(0).get(0).asInt(); + } + + private HttpResponse executeSql(String sql) throws Exception { + HttpResponse response = httpClient.send(HttpRequest.newBuilder() + .uri(URI.create(endpoint() + "/v1/sql?db=public")) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString( + "sql=" + URLEncoder.encode(sql, StandardCharsets.UTF_8), StandardCharsets.UTF_8)) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).as(response.body()).isBetween(200, 299); + return response; + } + + private String classpathResource(String path) throws Exception { + try (InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(path)) { + assertThat(input).as(path).isNotNull(); + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static KeyValue stringAttribute(String key, String value) { + return KeyValue.newBuilder().setKey(key) + .setValue(AnyValue.newBuilder().setStringValue(value).build()).build(); + } + + private static byte[] hexToBytes(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int index = 0; index < value.length(); index += 2) { + bytes[index / 2] = (byte) Integer.parseInt(value.substring(index, index + 2), 16); + } + return bytes; + } + + private static String endpoint() { + return "http://" + GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_HTTP_PORT); + } +} diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java index d56a46ee6c..7c62fb3846 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.storage; +package org.apache.hertzbeat.observability.storage; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml similarity index 91% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml index dff4e65966..f4c1dd2dd3 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml @@ -80,12 +80,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -93,19 +93,20 @@ resourceRole: - /v3/api-docs.yaml/**===get===[admin] - /v2/api-docs/**===get===[admin] - /swagger-resources/**===get===[admin] + # the alert stream carries full alert payloads and the manager stream carries import + # progress; both are scoped like the log stream above rather than left anonymous + - /api/alert/sse/**===get===[admin,user,guest] + - /api/manager/sse/**===get===[admin,user,guest] # config the resource restful api that need bypass auth protection # rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - - /api/alert/sse/**===* - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - - /api/manager/sse/**===* # web ui resource - /===get - /assets/**===get diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml index f86f2b0100..baea29f4c7 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml @@ -102,7 +102,7 @@ sinks: type: opentelemetry protocol: type: http - uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/logs/ingest/otlp" + uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/otlp/v1/logs" method: post encoding: codec: json diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index 6249cf0d44..2f26189752 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -32,7 +32,7 @@ hertzbeat-collector-kafka-e2e hertzbeat-collector-basic-e2e hertzbeat-collector-mysql-r2dbc-e2e - hertzbeat-log-e2e + hertzbeat-observability-e2e diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java deleted file mode 100644 index e32510e4d6..0000000000 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.extern.slf4j.Slf4j; -import java.util.List; - -import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.common.entity.dto.Message; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * Generic Log Ingestion Controller - * Provides a fallback endpoint for log protocols that don't have dedicated controllers. - * For OTLP protocol, use OtlpLogController instead. - */ -@Tag(name = "Log Ingestion Controller") -@RestController -@RequestMapping(path = "/api/logs", produces = MediaType.APPLICATION_JSON_VALUE) -@Slf4j -public class LogIngestionController { - - private final List protocolAdapters; - - public LogIngestionController(List protocolAdapters) { - this.protocolAdapters = protocolAdapters; - } - - /** - * Receive log payload pushed from external system specifying the log protocol. - * - * @param protocol log protocol identifier (e.g., "vector", "loki") - * @param content raw request body - */ - @Operation(summary = "Ingest logs by protocol name") - @PostMapping(value = "/ingest/{protocol}", consumes = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity> ingestLog(@PathVariable("protocol") String protocol, - @RequestBody String content) { - log.debug("Receive log from protocol: {}, content length: {}", protocol, content == null ? 0 : content.length()); - - for (LogProtocolAdapter adapter : protocolAdapters) { - if (adapter.supportProtocol().equalsIgnoreCase(protocol)) { - try { - adapter.ingest(content); - return ResponseEntity.ok(Message.success("Add extern log success")); - } catch (Exception e) { - log.error("Add log failed: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(Message.fail(CommonConstants.FAIL_CODE, "Add extern log failed: " + e.getMessage())); - } - } - } - log.warn("Not support extern log from protocol: {}", protocol); - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(Message.fail(CommonConstants.FAIL_CODE, "Not support the " + protocol + " protocol log")); - } -} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java deleted file mode 100644 index 3d01075081..0000000000 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import com.fasterxml.jackson.core.io.JsonStringEncoder; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.google.rpc.Status; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * OTLP Log Ingestion Controller - * Implements OTLP/HTTP specification for log ingestion. - * Supports both binary-encoded Protobuf (application/x-protobuf) and JSON-encoded Protobuf (application/json). - * - * @see OTLP/HTTP Specification - */ -@Tag(name = "OTLP Log Controller") -@RestController -@RequestMapping(path = "/api/logs/otlp") -@Slf4j -public class OtlpLogController { - - private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf"; - - private static final ExportLogsServiceResponse EMPTY_RESPONSE = ExportLogsServiceResponse.newBuilder().build(); - - private final OtlpLogProtocolAdapter otlpLogProtocolAdapter; - - public OtlpLogController(OtlpLogProtocolAdapter otlpLogProtocolAdapter) { - this.otlpLogProtocolAdapter = otlpLogProtocolAdapter; - } - - /** - * OTLP/HTTP standard endpoint for logs with JSON-encoded Protobuf payload. - * Content-Type: application/json - * - * Response follows OTLP specification: - * - Success: HTTP 200 with ExportLogsServiceResponse (JSON encoded) - * - Failure: HTTP 400 with google.rpc.Status (JSON encoded) - * - * @param content JSON-encoded ExportLogsServiceRequest - * @return ExportLogsServiceResponse on success, Status on failure - */ - @Operation(summary = "Ingest OTLP logs (JSON format)") - @PostMapping(value = "/v1/logs", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity ingestJsonLogs(@RequestBody String content) { - log.debug("Receive OTLP JSON logs, content length: {}", content == null ? 0 : content.length()); - try { - otlpLogProtocolAdapter.ingest(content); - return ResponseEntity.ok(toJsonResponse(EMPTY_RESPONSE)); - } catch (IllegalArgumentException e) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(toJsonErrorResponse(e.getMessage())); - } catch (Exception e) { - // Server-side errors - unexpected failure - log.error("Unexpected error ingesting OTLP JSON logs: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(toJsonErrorResponse(e.getMessage())); - } - } - - /** - * OTLP/HTTP standard endpoint for logs with binary-encoded Protobuf payload. - * Content-Type: application/x-protobuf - * - * Response follows OTLP specification: - * - Success: HTTP 200 with ExportLogsServiceResponse (binary encoded) - * - Failure: HTTP 400 with google.rpc.Status (binary encoded) - * - * @param content binary-encoded ExportLogsServiceRequest - * @return ExportLogsServiceResponse on success, Status on failure - */ - @Operation(summary = "Ingest OTLP logs (binary Protobuf format)") - @PostMapping(value = "/v1/logs", consumes = CONTENT_TYPE_PROTOBUF, produces = CONTENT_TYPE_PROTOBUF) - public ResponseEntity ingestBinaryLogs(@RequestBody byte[] content) { - log.debug("Receive OTLP binary logs, content length: {}", content == null ? 0 : content.length); - try { - otlpLogProtocolAdapter.ingestBinary(content); - return ResponseEntity.ok(EMPTY_RESPONSE.toByteArray()); - } catch (IllegalArgumentException e) { - // Client-side validation errors - malformed request - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(createBinaryErrorResponse(e.getMessage())); - } catch (Exception e) { - // Server-side errors - unexpected failure - log.error("Unexpected error ingesting OTLP binary logs: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(createBinaryErrorResponse(e.getMessage())); - } - } - - private String toJsonResponse(ExportLogsServiceResponse response) { - try { - return JsonFormat.printer().print(response); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to convert ExportLogsServiceResponse to JSON: {}", e.getMessage(), e); - return "{}"; - } - } - - private String toJsonErrorResponse(String message) { - Status status = Status.newBuilder() - .setMessage(message != null ? message : "Unknown error") - .build(); - try { - return JsonFormat.printer().print(status); - } catch (InvalidProtocolBufferException e) { - return "{\"message\":\"" + escapeJson(message) + "\"}"; - } - } - - /** - * Escapes a string value for safe inclusion in JSON. - * - * @param message the string to escape - * @return the escaped string, or empty string if message is null - */ - private String escapeJson(String message) { - if (message == null) { - return ""; - } - - char[] escaped = JsonStringEncoder.getInstance().quoteAsString(message); - return new String(escaped); - } - - private byte[] createBinaryErrorResponse(String message) { - return Status.newBuilder() - .setMessage(message != null ? message : "Unknown error") - .build() - .toByteArray(); - } -} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java deleted file mode 100644 index cb6eb3678f..0000000000 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; - -/** - * Unit test for {@link LogIngestionController} - */ -@ExtendWith(MockitoExtension.class) -class LogIngestionControllerTest { - - private MockMvc mockMvc; - - @Mock - private LogProtocolAdapter vectorAdapter; - - private LogIngestionController logIngestionController; - - @BeforeEach - void setUp() { - List adapters = Arrays.asList(vectorAdapter); - this.logIngestionController = new LogIngestionController(adapters); - this.mockMvc = MockMvcBuilders.standaloneSetup(logIngestionController).build(); - } - - @Test - void testIngestLogWithKnownProtocol() throws Exception { - String logContent = "{\"message\":\"Test log message\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - doNothing().when(vectorAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/vector") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) - .andExpect(jsonPath("$.msg").value("Add extern log success")) - .andReturn(); - } - - @Test - void testIngestLogWithUnsupportedProtocol() throws Exception { - String logContent = "{\"message\":\"Unsupported protocol log\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/unsupported") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) - .andExpect(jsonPath("$.msg").value("Not support the unsupported protocol log")); - } - - @Test - void testIngestLogWithAdapterException() throws Exception { - String logContent = "{\"message\":\"Log message that will cause exception\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - doThrow(new IllegalArgumentException("Invalid log format")).when(vectorAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/vector") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) - .andExpect(jsonPath("$.msg").value("Add extern log failed: Invalid log format")); - } -} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java deleted file mode 100644 index d2e67bcb8e..0000000000 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; - -import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; - -/** - * Unit test for {@link OtlpLogController} - */ -@ExtendWith(MockitoExtension.class) -class OtlpLogControllerTest { - - private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf"; - - private MockMvc mockMvc; - - @Mock - private OtlpLogProtocolAdapter otlpLogProtocolAdapter; - - private OtlpLogController otlpLogController; - - @BeforeEach - void setUp() { - this.otlpLogController = new OtlpLogController(otlpLogProtocolAdapter); - this.mockMvc = MockMvcBuilders.standaloneSetup(otlpLogController).build(); - } - - @Test - void testIngestJsonLogsSuccess() throws Exception { - String jsonContent = "{\"resourceLogs\":[]}"; - - doNothing().when(otlpLogProtocolAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(MediaType.APPLICATION_JSON) - .content(jsonContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - } - - @Test - void testIngestJsonLogsFailure() throws Exception { - String jsonContent = "{\"invalid\":\"content\"}"; - - doThrow(new IllegalArgumentException("Invalid OTLP JSON log content")) - .when(otlpLogProtocolAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(MediaType.APPLICATION_JSON) - .content(jsonContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - } - - @Test - void testIngestBinaryLogsSuccess() throws Exception { - byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c}; - - doNothing().when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class)); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(CONTENT_TYPE_PROTOBUF) - .content(binaryContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(CONTENT_TYPE_PROTOBUF)) - .andReturn(); - } - - @Test - void testIngestBinaryLogsFailure() throws Exception { - byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c}; - - doThrow(new IllegalArgumentException("Invalid OTLP binary log content")) - .when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class)); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(CONTENT_TYPE_PROTOBUF) - .content(binaryContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(content().contentType(CONTENT_TYPE_PROTOBUF)) - .andReturn(); - } -} diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 57994eac2d..f98eb08c52 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -91,10 +91,10 @@ org.apache.hertzbeat hertzbeat-otel - + org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java new file mode 100644 index 0000000000..99c0e84a80 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +import java.net.http.HttpClient; +import org.apache.hertzbeat.common.constants.NetworkConstants; +import org.junit.jupiter.api.Test; +import org.springframework.http.client.ClientHttpRequestFactory; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.test.util.ReflectionTestUtils; + +class RestTemplateConfigTest { + + @Test + void greptimeQueryRequestFactoryUsesDedicatedTimeouts() { + ClientHttpRequestFactory factory = new RestTemplateConfig().greptimeQueryClientHttpRequestFactory(); + + JdkClientHttpRequestFactory jdkFactory = assertInstanceOf(JdkClientHttpRequestFactory.class, factory); + assertEquals(NetworkConstants.HttpClientConstants.GREPTIME_QUERY_READ_TIMEOUT, + ReflectionTestUtils.getField(jdkFactory, "readTimeout")); + + HttpClient httpClient = (HttpClient) ReflectionTestUtils.getField(jdkFactory, "httpClient"); + assertEquals(NetworkConstants.HttpClientConstants.GREPTIME_QUERY_CONNECT_TIMEOUT, + httpClient.connectTimeout().orElseThrow()); + } +} diff --git a/hertzbeat-manager/src/test/resources/sureness.yml b/hertzbeat-manager/src/test/resources/sureness.yml index dff4e65966..f4c1dd2dd3 100644 --- a/hertzbeat-manager/src/test/resources/sureness.yml +++ b/hertzbeat-manager/src/test/resources/sureness.yml @@ -80,12 +80,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -93,19 +93,20 @@ resourceRole: - /v3/api-docs.yaml/**===get===[admin] - /v2/api-docs/**===get===[admin] - /swagger-resources/**===get===[admin] + # the alert stream carries full alert payloads and the manager stream carries import + # progress; both are scoped like the log stream above rather than left anonymous + - /api/alert/sse/**===get===[admin,user,guest] + - /api/manager/sse/**===get===[admin,user,guest] # config the resource restful api that need bypass auth protection # rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. excludedResource: - - /api/alert/sse/**===* - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - - /api/manager/sse/**===* # web ui resource - /===get - /assets/**===get diff --git a/hertzbeat-log/pom.xml b/hertzbeat-observability/pom.xml similarity index 98% rename from hertzbeat-log/pom.xml rename to hertzbeat-observability/pom.xml index c5c432aca5..5d62967651 100644 --- a/hertzbeat-log/pom.xml +++ b/hertzbeat-observability/pom.xml @@ -25,7 +25,7 @@ 4.0.0 - hertzbeat-log + hertzbeat-observability ${project.artifactId} diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java similarity index 72% rename from hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java index f58973b12d..5405a66b49 100644 --- a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java @@ -15,21 +15,19 @@ * limitations under the License. */ -package org.apache.hertzbeat.otel.config; +package org.apache.hertzbeat.observability.config; import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.SignConstants; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.context.annotation.ComponentScan; /** - * Log auto configuration. + * Entity-free observability module auto configuration. */ +@AutoConfiguration @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG - + SignConstants.DOT - + ConfigConstants.FunctionModuleConstants.LOG -) -@EnableConfigurationProperties(GreptimeProperties.class) -public class LogAutoConfiguration { + + SignConstants.DOT + + ConfigConstants.FunctionModuleConstants.OBSERVABILITY) +public class ObservabilityAutoConfiguration { } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java similarity index 75% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java index 9e6d3bc067..fb5d9072d1 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.observability.config; import io.grpc.ForwardingServerCallListener; import io.grpc.Metadata; @@ -41,10 +41,11 @@ import java.net.InetSocketAddress; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -62,14 +63,16 @@ public class OtlpGrpcServerConfig { matchIfMissing = true) public OtlpGrpcServerRunner otlpGrpcServerRunner( @Value("${hertzbeat.otlp.grpc.host:0.0.0.0}") String host, - @Value("${hertzbeat.otlp.grpc.port:4317}") int port, + @Value("${hertzbeat.otlp.grpc.port:14317}") int port, OtlpSignalForwarder signalForwarder, + OtlpLogIngestionService logIngestionService, SignalWorkloadGuard workloadGuard, ObjectProvider tokenValidatorProvider) { OtlpAccessTokenValidator validator = tokenValidatorProvider.getIfAvailable( () -> token -> "OTLP token validation is unavailable"); ServerInterceptor interceptor = new BearerTokenInterceptor(validator); - return new OtlpGrpcServerRunner(host, port, signalForwarder, workloadGuard, interceptor); + return new OtlpGrpcServerRunner(host, port, signalForwarder, logIngestionService, + workloadGuard, interceptor); } @Slf4j @@ -78,20 +81,42 @@ public class OtlpGrpcServerConfig { private final String host; private final int port; private final OtlpSignalForwarder signalForwarder; + private final OtlpLogIngestionService logIngestionService; private final SignalWorkloadGuard workloadGuard; private final ServerInterceptor authInterceptor; private Server server; - public void start() throws IOException { - server = NettyServerBuilder.forAddress(new InetSocketAddress(host, port)) - .addService(ServerInterceptors.intercept(new MetricsService(signalForwarder, workloadGuard), - authInterceptor)) - .addService(ServerInterceptors.intercept(new LogsService(signalForwarder, workloadGuard), - authInterceptor)) - .addService(ServerInterceptors.intercept(new TracesService(signalForwarder, workloadGuard), - authInterceptor)) - .build().start(); - log.info("OTLP gRPC listener started on {}:{}", host, port); + /** + * Binds the OTLP/gRPC listener, or gives it up for this run. + * + *

The default 14317 keeps clear of 4317, the OpenTelemetry standard port an OTel Collector + * on the same host would already hold - and that host is exactly the one most likely to want + * this listener. A bind can still fail for other reasons, and letting it escape would abort + * the whole context: an optional side channel would then keep the monitoring system itself + * from starting, while HTTP ingestion on the main port stays perfectly able to serve the same + * signals. Degrade to "unavailable" and say how to fix it. + */ + public void start() { + try { + server = NettyServerBuilder.forAddress(new InetSocketAddress(host, port)) + .addService(ServerInterceptors.intercept(new MetricsService(signalForwarder, workloadGuard), + authInterceptor)) + .addService(ServerInterceptors.intercept(new LogsService(logIngestionService, workloadGuard), + authInterceptor)) + .addService(ServerInterceptors.intercept(new TracesService(signalForwarder, workloadGuard), + authInterceptor)) + .build().start(); + log.info("OTLP gRPC listener started on {}:{}", host, port); + } catch (IOException exception) { + server = null; + log.error("OTLP gRPC listener could not bind {}:{}, so gRPC ingestion is unavailable for this run. " + + "OTLP/HTTP ingestion on /api/otlp/v1 is unaffected. Set hertzbeat.otlp.grpc.port to a free " + + "port, or hertzbeat.otlp.grpc.enabled=false to stop starting it.", host, port, exception); + } + } + + boolean isRunning() { + return server != null; } public void stop() { @@ -123,14 +148,14 @@ public class OtlpGrpcServerConfig { @RequiredArgsConstructor static final class LogsService extends LogsServiceGrpc.LogsServiceImplBase { - private final OtlpSignalForwarder signalForwarder; + private final OtlpLogIngestionService logIngestionService; private final SignalWorkloadGuard workloadGuard; @Override public void export(ExportLogsServiceRequest request, StreamObserver observer) { try { byte[] response = workloadGuard.execute(Workload.OTLP_WRITE, - () -> signalForwarder.forwardProtobuf("logs", request.toByteArray())); + () -> logIngestionService.ingestProtobuf(request.toByteArray())); observer.onNext(response.length == 0 ? ExportLogsServiceResponse.getDefaultInstance() : ExportLogsServiceResponse.parseFrom(response)); observer.onCompleted(); diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java new file mode 100644 index 0000000000..77123e6293 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RestController; + +/** + * Deprecated 1.8.x OTLP log ingestion routes kept as aliases of {@code POST /api/otlp/v1/logs}. + * + *

HertzBeat 1.8.0 documented {@code POST /api/logs/otlp/v1/logs} and also exposed + * {@code POST /api/logs/ingest/{protocol}}. Collectors and SDKs configured against those paths keep + * working on 1.9.x through this alias, which forwards to the same log fan-out as the canonical + * route and stamps a {@code Deprecation} / {@code Link} header on every response. + * + *

These aliases are scheduled for removal in HertzBeat 2.0. New integrations must use + * {@code /api/otlp/v1/logs}. + * + * @deprecated since 1.9.0, use {@code POST /api/otlp/v1/logs} + */ +@Deprecated(since = "1.9.0", forRemoval = true) +@Slf4j +@RestController +@Tag(name = "OTLP Log Controller (deprecated 1.8 aliases)") +public class LegacyOtlpLogRouteController { + + /** + * Canonical route the aliases forward to. + */ + public static final String CANONICAL_LOGS_ROUTE = "/api/otlp/v1/logs"; + + /** + * 1.8.0 documented OTLP/HTTP log route. + */ + public static final String LEGACY_OTLP_LOGS_ROUTE = "/api/logs/otlp/v1/logs"; + + /** + * 1.8.0 protocol-named log route; only the {@code otlp} protocol ever had an adapter. + */ + public static final String LEGACY_INGEST_ROUTE = "/api/logs/ingest/{protocol}"; + + private static final String OTLP_PROTOCOL = "otlp"; + + private static final String UNSUPPORTED_PROTOCOL_KEY = "unsupported-protocol"; + + private static final String DEPRECATION_HEADER = "Deprecation"; + private static final String LINK_HEADER = "Link"; + private static final String LINK_VALUE = "<" + CANONICAL_LOGS_ROUTE + ">; rel=\"successor-version\""; + + /** + * Keys already reported at warn level. + * + *

A collector that has not been migrated yet pushes continuously, so warning per request buries + * every other line in the log - including the ones the upgrade notes ask operators to watch for. + * Only constant keys are ever added, never a caller supplied value, so this cannot grow unbounded. + */ + private final Set reportedDeprecations = ConcurrentHashMap.newKeySet(); + + private final OtlpLogIngestionService logIngestionService; + private final SignalWorkloadGuard workloadGuard; + + public LegacyOtlpLogRouteController(OtlpLogIngestionService logIngestionService, + SignalWorkloadGuard workloadGuard) { + this.logIngestionService = logIngestionService; + this.workloadGuard = workloadGuard; + } + + @PostMapping(LEGACY_OTLP_LOGS_ROUTE) + @Operation(summary = "Deprecated alias of POST /api/otlp/v1/logs", deprecated = true) + public ResponseEntity legacyOtlpLogs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { + return forward(LEGACY_OTLP_LOGS_ROUTE, content, headers); + } + + @PostMapping(LEGACY_INGEST_ROUTE) + @Operation(summary = "Deprecated alias of POST /api/otlp/v1/logs (otlp protocol only)", deprecated = true) + public ResponseEntity legacyIngest(@PathVariable("protocol") String protocol, + @RequestBody byte[] content, + @RequestHeader HttpHeaders headers) { + if (!OTLP_PROTOCOL.equalsIgnoreCase(protocol)) { + reportOnce(UNSUPPORTED_PROTOCOL_KEY, + "Deprecated route /api/logs/ingest/{} rejected: only the otlp protocol is supported", protocol); + // Thrown rather than answered inline so OtlpHttpExceptionHandler encodes it as a + // google.rpc.Status in the format the caller used, with the deprecation headers kept. + throw new IllegalArgumentException( + "Only the otlp protocol is supported, use " + CANONICAL_LOGS_ROUTE); + } + return forward("/api/logs/ingest/" + OTLP_PROTOCOL, content, headers); + } + + private ResponseEntity forward(String legacyRoute, byte[] content, HttpHeaders headers) { + reportOnce(legacyRoute, + "Deprecated OTLP log route {} was called; migrate the exporter to {} before HertzBeat 2.0", + legacyRoute, CANONICAL_LOGS_ROUTE); + ResponseEntity canonical = workloadGuard.execute(Workload.OTLP_WRITE, + () -> logIngestionService.ingestHttp(content, headers)); + return deprecated(ResponseEntity.status(canonical.getStatusCode()).headers(canonical.getHeaders())) + .body(canonical.getBody()); + } + + /** + * Warns the first time a key is seen and drops to debug afterwards, so a still unmigrated + * exporter leaves one actionable line instead of one line per request. + */ + private void reportOnce(String key, String message, Object... arguments) { + if (reportedDeprecations.add(key)) { + log.warn(message, arguments); + } else { + log.debug(message, arguments); + } + } + + static ResponseEntity.BodyBuilder deprecated(ResponseEntity.BodyBuilder builder) { + return builder + .header(DEPRECATION_HEADER, "true") + .header(LINK_HEADER, LINK_VALUE); + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java index f552e5c20f..8d9fbe506f 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -37,7 +37,7 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; * Controller for managing log entries in HertzBeat. */ @RestController -@RequestMapping(path = "/api/logs", produces = "application/json") +@RequestMapping(path = "/api/observability/logs", produces = "application/json") @Tag(name = "Log Management Controller") @Slf4j public class LogManagerController { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java similarity index 97% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java index 7cde958b43..a5e18c1c8c 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -37,8 +37,8 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter; import org.apache.hertzbeat.common.entity.log.LogEntry; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -49,7 +49,7 @@ import org.springframework.web.bind.annotation.RestController; * Log query and statistics APIs for UI consumption */ @RestController -@RequestMapping(path = "/api/logs", produces = "application/json") +@RequestMapping(path = "/api/observability/logs", produces = "application/json") @Tag(name = "Log Query Controller") @Slf4j public class LogQueryController { @@ -67,7 +67,7 @@ public class LogQueryController { this.workloadGuard = workloadGuard; } - @GetMapping("/list") + @GetMapping @Operation(summary = "Query logs by time range with optional filters", description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.") public ResponseEntity>> list( @@ -101,7 +101,7 @@ public class LogQueryController { }); } - @GetMapping("/stats/overview") + @GetMapping("/overview") @Operation(summary = "Log overview statistics", description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.") public ResponseEntity>> overviewStats( @@ -132,7 +132,7 @@ public class LogQueryController { return ResponseEntity.ok(Message.success(historyDataReader.queryLogOverviewAggregate(filter))); } - @GetMapping("/stats/trace-coverage") + @GetMapping("/trace-coverage") @Operation(summary = "Trace coverage statistics", description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.") public ResponseEntity>> traceCoverageStats( @@ -165,7 +165,7 @@ public class LogQueryController { return ResponseEntity.ok(Message.success(result)); } - @GetMapping("/stats/trend") + @GetMapping("/trend") @Operation(summary = "Log trend over time", description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.") public ResponseEntity>> trendStats( diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java similarity index 87% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java index e519ae06ee..7095616761 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.springframework.http.MediaType.TEXT_EVENT_STREAM_VALUE; import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator; -import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.RequestMapping; @@ -34,7 +34,7 @@ import io.swagger.v3.oas.annotations.Operation; * SSE controller for log streaming with filtering support */ @RestController -@RequestMapping(path = "/api/logs/sse", produces = {TEXT_EVENT_STREAM_VALUE}) +@RequestMapping(path = "/api/observability/logs", produces = {TEXT_EVENT_STREAM_VALUE}) public class LogSseController { private final LogSseManager emitterManager; @@ -48,10 +48,10 @@ public class LogSseController { * @param filterCriteria Filter criteria for log events (all parameters are optional) * @return SSE emitter for streaming log events */ - @GetMapping(path = "/subscribe") + @GetMapping(path = "/stream") @Operation(summary = "Subscribe to log events with optional filtering", description = "Subscribe to log events with optional filtering") public SseEmitter subscribe(@ModelAttribute LogSseFilterCriteria filterCriteria) { Long clientId = SnowFlakeIdGenerator.generateId(); return emitterManager.createEmitter(clientId, filterCriteria); } -} \ No newline at end of file +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java similarity index 96% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java index 055167f403..9943c2f69d 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.observability.ObservabilityCapability; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.springframework.beans.factory.ObjectProvider; import org.springframework.http.ResponseEntity; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandler.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandler.java new file mode 100644 index 0000000000..f3cd7d23d7 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandler.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.google.rpc.Code; +import com.google.rpc.Status; +import jakarta.servlet.http.HttpServletRequest; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.InvalidMediaTypeException; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.ErrorResponse; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestClientException; +import org.springframework.web.method.HandlerMethod; + +/** + * Maps OTLP/HTTP ingestion failures to the response shape the OTLP specification requires. + * + *

Every non-2xx response carries a {@code google.rpc.Status} body encoded in the same format the + * client used for the request (JSON for {@code application/json}, binary protobuf otherwise), so OTLP + * exporters and collectors can surface the rejection reason instead of an opaque status code. Retryable + * failures (overload, storage unavailable) additionally carry {@code Retry-After}. Deprecated 1.8 alias + * routes keep their {@code Deprecation} / {@code Link} headers on error responses too. + * + *

This advice is scoped to the OTLP ingestion controllers only and runs ahead of + * {@link SignalWorkloadExceptionHandler}, whose {@code Message} JSON envelope is meant for the HertzBeat + * query API rather than OTLP clients. + * + * @see OTLP/HTTP failures + */ +@Slf4j +@RestControllerAdvice(assignableTypes = { + OtlpLogController.class, + OtlpSignalController.class, + LegacyOtlpLogRouteController.class +}) +@Order(Ordered.HIGHEST_PRECEDENCE) +public class OtlpHttpExceptionHandler { + + static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf"); + + private static final String RETRY_AFTER_SECONDS = "1"; + private static final String UNKNOWN_ERROR = "Unknown error"; + private static final String STORAGE_UNAVAILABLE = "GreptimeDB storage is unavailable"; + private static final String UNREADABLE_BODY = "Malformed or missing OTLP request body"; + private static final String UNEXPECTED_FAILURE = "Unexpected OTLP ingestion failure"; + private static final String CLIENT_ERROR = "OTLP request rejected"; + + @ExceptionHandler(IllegalArgumentException.class) + public ResponseEntity handleInvalidPayload(IllegalArgumentException exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + String message = messageOf(exception); + // Covers both malformed payloads and GreptimeDB 4xx rejections; the message is the only place + // the rejection reason survives, so record it server-side as well. + log.warn("OTLP/HTTP {} rejected as invalid: {}", request.getRequestURI(), message); + return respond(HttpStatus.BAD_REQUEST, Code.INVALID_ARGUMENT, message, request, handlerMethod, false); + } + + @ExceptionHandler(SignalQueryRejectedException.class) + public ResponseEntity handleOverloaded(SignalQueryRejectedException exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + String message = messageOf(exception); + // Back-pressure is expected under load; keep it below warn to avoid log storms. + log.debug("OTLP/HTTP {} throttled: {}", request.getRequestURI(), message); + return respond(HttpStatus.TOO_MANY_REQUESTS, Code.RESOURCE_EXHAUSTED, message, request, handlerMethod, true); + } + + @ExceptionHandler(StorageUnavailableException.class) + public ResponseEntity handleStorageUnavailable(StorageUnavailableException exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + String message = messageOf(exception); + log.error("OTLP/HTTP {} failed, storage unavailable: {}", request.getRequestURI(), message, exception); + return respond(HttpStatus.SERVICE_UNAVAILABLE, Code.UNAVAILABLE, message, request, handlerMethod, true); + } + + @ExceptionHandler(RestClientException.class) + public ResponseEntity handleStorageClientFailure(RestClientException exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + log.error("OTLP/HTTP {} failed, GreptimeDB write error: {}", request.getRequestURI(), + messageOf(exception), exception); + // Do not echo transport details (endpoints, raw 5xx bodies) back to the exporter. + return respond(HttpStatus.SERVICE_UNAVAILABLE, Code.UNAVAILABLE, STORAGE_UNAVAILABLE, + request, handlerMethod, true); + } + + /** + * An absent or undecodable request body is a client error the exporter must not retry. + * + *

Spring signals it with {@code HttpMessageNotReadableException}, which is raised while the + * arguments are resolved rather than inside the controller, and which does not implement + * {@link ErrorResponse}. Without this handler it falls through to {@link #handleUnexpected} and + * comes back as a {@code 500}, which OTLP exporters read as retryable and replay forever. + */ + @ExceptionHandler(HttpMessageNotReadableException.class) + public ResponseEntity handleUnreadableBody(HttpMessageNotReadableException exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + // The framework message embeds the handler signature, so keep it server-side only. + log.warn("OTLP/HTTP {} rejected, unreadable body: {}", request.getRequestURI(), messageOf(exception)); + return respond(HttpStatus.BAD_REQUEST, Code.INVALID_ARGUMENT, UNREADABLE_BODY, + request, handlerMethod, false); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleUnexpected(Exception exception, + HttpServletRequest request, + HandlerMethod handlerMethod) { + if (exception instanceof ErrorResponse errorResponse) { + // Spring MVC's own request-level failures (unsupported method, unsupported media type, ...) + // already carry the right status; keep it instead of degrading a client error into a 500. + HttpStatus status = HttpStatus.valueOf(errorResponse.getStatusCode().value()); + log.warn("OTLP/HTTP {} rejected ({}): {}", request.getRequestURI(), status.value(), + messageOf(exception)); + return respond(status, status.is4xxClientError() ? Code.INVALID_ARGUMENT : Code.INTERNAL, + clientFacingDetail(errorResponse), request, handlerMethod, false); + } + // Never echo an unhandled exception message: it routinely carries handler signatures, package + // names and internal state that the exporter has no business seeing. + log.error("OTLP/HTTP {} failed unexpectedly: {}", request.getRequestURI(), messageOf(exception), + exception); + return respond(HttpStatus.INTERNAL_SERVER_ERROR, Code.INTERNAL, UNEXPECTED_FAILURE, + request, handlerMethod, false); + } + + /** + * Prefers the curated {@code ProblemDetail} text spring builds for the client over the raw + * exception message, which is written for a log line rather than for a response body. + */ + private static String clientFacingDetail(ErrorResponse errorResponse) { + String detail = errorResponse.getBody() == null ? null : errorResponse.getBody().getDetail(); + return StringUtils.hasText(detail) ? detail : CLIENT_ERROR; + } + + private static ResponseEntity respond(HttpStatus status, Code code, String message, + HttpServletRequest request, HandlerMethod handlerMethod, + boolean retryable) { + Status rpcStatus = Status.newBuilder().setCode(code.getNumber()).setMessage(message).build(); + boolean json = isJsonRequest(request); + ResponseEntity.BodyBuilder builder = ResponseEntity.status(status) + .contentType(json ? MediaType.APPLICATION_JSON : PROTOBUF); + if (retryable) { + builder.header(HttpHeaders.RETRY_AFTER, RETRY_AFTER_SECONDS); + } + if (handlerMethod != null + && LegacyOtlpLogRouteController.class.isAssignableFrom(handlerMethod.getBeanType())) { + LegacyOtlpLogRouteController.deprecated(builder); + } + return builder.body(json ? toJson(rpcStatus) : rpcStatus.toByteArray()); + } + + private static boolean isJsonRequest(HttpServletRequest request) { + String contentType = request.getContentType(); + if (contentType == null) { + return false; + } + try { + return MediaType.APPLICATION_JSON.includes(MediaType.parseMediaType(contentType)); + } catch (InvalidMediaTypeException exception) { + return false; + } + } + + private static byte[] toJson(Status status) { + String json; + try { + json = JsonFormat.printer().omittingInsignificantWhitespace().print(status); + } catch (InvalidProtocolBufferException exception) { + json = "{\"code\":" + status.getCode() + ",\"message\":\"" + + new String(JsonStringEncoder.getInstance().quoteAsString(status.getMessage())) + "\"}"; + } + return json.getBytes(StandardCharsets.UTF_8); + } + + private static String messageOf(Throwable throwable) { + return Objects.requireNonNullElse(throwable.getMessage(), UNKNOWN_ERROR); + } +} diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java new file mode 100644 index 0000000000..53e3e7bb86 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** Canonical OTLP/HTTP log endpoint backed by the HertzBeat log fan-out. */ +@RestController +@RequestMapping("/api/otlp/v1") +@Tag(name = "OTLP Log Controller") +public class OtlpLogController { + + private final OtlpLogIngestionService logIngestionService; + private final SignalWorkloadGuard workloadGuard; + + public OtlpLogController(OtlpLogIngestionService logIngestionService, SignalWorkloadGuard workloadGuard) { + this.logIngestionService = logIngestionService; + this.workloadGuard = workloadGuard; + } + + @PostMapping("/logs") + @Operation(summary = "Ingest OTLP logs") + public ResponseEntity logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { + return workloadGuard.execute(Workload.OTLP_WRITE, + () -> logIngestionService.ingestHttp(content, headers)); + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java similarity index 74% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java index a88e2d10eb..19294458e9 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java @@ -15,19 +15,16 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; -import java.nio.charset.StandardCharsets; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; @@ -55,12 +52,6 @@ public class OtlpSignalController { return forward("metrics", content, headers); } - @PostMapping("/logs") - @Operation(summary = "Ingest OTLP logs") - public ResponseEntity logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { - return forward("logs", content, headers); - } - @PostMapping("/traces") @Operation(summary = "Ingest OTLP traces") public ResponseEntity traces(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { @@ -71,10 +62,4 @@ public class OtlpSignalController { return workloadGuard.execute(Workload.OTLP_WRITE, () -> signalForwarder.forwardHttp(signal, content, headers)); } - - @ExceptionHandler(IllegalArgumentException.class) - public ResponseEntity invalidPayload(IllegalArgumentException exception) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(exception.getMessage().getBytes(StandardCharsets.UTF_8)); - } } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java similarity index 83% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java index 3cac628fc8..1a0830cee9 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -31,9 +31,14 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.client.RestClientException; -/** Maps bounded signal workload rejection to a retryable HTTP response. */ -@RestControllerAdvice(basePackages = "org.apache.hertzbeat.log.controller") -@Order(Ordered.HIGHEST_PRECEDENCE) +/** + * Maps bounded signal workload rejection to a retryable HTTP response. + * + *

Ordered just behind {@link OtlpHttpExceptionHandler}: the OTLP ingestion controllers need + * {@code google.rpc.Status} bodies, every other observability controller gets the {@code Message} envelope. + */ +@RestControllerAdvice(basePackages = "org.apache.hertzbeat.observability.controller") +@Order(Ordered.HIGHEST_PRECEDENCE + 1) public class SignalWorkloadExceptionHandler { @ExceptionHandler(IllegalArgumentException.class) diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java similarity index 91% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java index de95425c17..70be734dad 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -26,19 +26,21 @@ import org.apache.hertzbeat.common.entity.dto.observability.SignalPage; import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail; import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** Public entity-free query APIs for OTLP metrics and traces. */ @RestController @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") +@RequestMapping(path = "/api/observability", produces = "application/json") @Tag(name = "Three Signal Query Controller") public class ThreeSignalQueryController { @@ -50,7 +52,7 @@ public class ThreeSignalQueryController { this.workloadGuard = workloadGuard; } - @GetMapping(path = "/api/ingestion/otlp/metrics/console", produces = "application/json") + @GetMapping("/metrics/query") @Operation(summary = "Query OTLP metrics") public ResponseEntity> metrics( @RequestParam(value = "query", required = false) String query, @@ -69,7 +71,7 @@ public class ThreeSignalQueryController { filter, groupBy, aggregation, step, operationName)))); } - @GetMapping(path = "/api/ingestion/otlp/metrics/inventory", produces = "application/json") + @GetMapping("/metrics/inventory") @Operation(summary = "List OTLP metric names") public ResponseEntity> metricInventory( @RequestParam(value = "start", required = false) Long start, @@ -82,7 +84,7 @@ public class ThreeSignalQueryController { queryService.metricInventory(start, end, serviceName, serviceNamespace, environment, limit)))); } - @GetMapping(path = "/api/traces/list", produces = "application/json") + @GetMapping("/traces") @Operation(summary = "Query traces") public ResponseEntity>> traces( @RequestParam(value = "start", required = false) Long start, @@ -102,7 +104,7 @@ public class ThreeSignalQueryController { environment, operationName, minDurationMs, maxDurationMs, pageIndex, pageSize)))); } - @GetMapping(path = "/api/traces/stats/overview", produces = "application/json") + @GetMapping("/traces/overview") @Operation(summary = "Get trace overview") public ResponseEntity> traceOverview( @RequestParam(value = "start", required = false) Long start, @@ -120,14 +122,14 @@ public class ThreeSignalQueryController { environment, operationName, minDurationMs, maxDurationMs)))); } - @GetMapping(path = "/api/traces/{traceId}", produces = "application/json") + @GetMapping("/traces/{traceId}") @Operation(summary = "Get trace detail") public ResponseEntity> traceDetail(@PathVariable("traceId") String traceId) { return workloadGuard.execute(Workload.TRACES, () -> ResponseEntity.ok(Message.success(queryService.traceDetail(traceId)))); } - @GetMapping(path = "/api/traces/{traceId}/spans", produces = "application/json") + @GetMapping("/traces/{traceId}/spans") @Operation(summary = "Get trace spans") public ResponseEntity> traceSpans(@PathVariable("traceId") String traceId) { return workloadGuard.execute(Workload.TRACES, diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java index b42f68aa8a..6e476c1f3f 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import lombok.Data; import lombok.AllArgsConstructor; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java similarity index 99% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java index 78d499cc45..b8def56026 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import jakarta.annotation.PreDestroy; import lombok.AllArgsConstructor; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java new file mode 100644 index 0000000000..c351fd3474 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +/** + * Canonical OTLP log ingestion boundary shared by the HTTP and gRPC transports. + */ +public interface OtlpLogIngestionService { + + /** + * Decode and ingest an OTLP/HTTP log request. + * + * @param content encoded OTLP request body + * @param headers request headers + * @return an OTLP response encoded for the request content type + */ + ResponseEntity ingestHttp(byte[] content, HttpHeaders headers); + + /** + * Decode and ingest an OTLP/gRPC protobuf request. + * + * @param content encoded {@code ExportLogsServiceRequest} + * @return encoded {@code ExportLogsServiceResponse} + */ + byte[] ingestProtobuf(byte[] content); +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java index b2305fafc3..1cf555b8d0 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java index 40f43b1cbb..7e1d985007 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; /** Raised when a signal workload exceeds its isolated concurrency budget. */ public class SignalQueryRejectedException extends RuntimeException { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java index f60677ad15..0d2ada58de 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import java.time.Duration; import java.util.EnumMap; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java new file mode 100644 index 0000000000..4dbc8fef87 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service.impl; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.zip.GZIPInputStream; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +/** Default OTLP log ingestion service backed by the HertzBeat log fan-out. */ +@Service +public class DefaultOtlpLogIngestionService implements OtlpLogIngestionService { + + private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf"); + private static final byte[] JSON_RESPONSE = "{}".getBytes(StandardCharsets.UTF_8); + private static final byte[] PROTOBUF_RESPONSE = ExportLogsServiceResponse.getDefaultInstance().toByteArray(); + + private final OtlpLogProtocolAdapter protocolAdapter; + + public DefaultOtlpLogIngestionService(OtlpLogProtocolAdapter protocolAdapter) { + this.protocolAdapter = protocolAdapter; + } + + @Override + public ResponseEntity ingestHttp(byte[] content, HttpHeaders headers) { + HttpHeaders safeHeaders = headers == null ? HttpHeaders.EMPTY : headers; + byte[] normalizedContent = maybeDecompress(content, safeHeaders); + MediaType contentType = safeHeaders.getContentType(); + if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) { + protocolAdapter.ingest(new String(normalizedContent, StandardCharsets.UTF_8)); + return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(JSON_RESPONSE); + } + protocolAdapter.ingestBinary(normalizedContent); + return ResponseEntity.ok().contentType(PROTOBUF).body(PROTOBUF_RESPONSE); + } + + @Override + public byte[] ingestProtobuf(byte[] content) { + protocolAdapter.ingestBinary(content); + return PROTOBUF_RESPONSE; + } + + private byte[] maybeDecompress(byte[] content, HttpHeaders headers) { + byte[] safeContent = content == null ? new byte[0] : content; + List encodings = headers.get(HttpHeaders.CONTENT_ENCODING); + if (encodings == null || encodings.stream().noneMatch(this::containsGzipEncoding)) { + return safeContent; + } + try (GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(safeContent)); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + input.transferTo(output); + return output.toByteArray(); + } catch (IOException exception) { + throw new IllegalArgumentException("Malformed gzip OTLP log payload", exception); + } + } + + private boolean containsGzipEncoding(String value) { + if (value == null) { + return false; + } + for (String encoding : value.split(",")) { + if ("gzip".equalsIgnoreCase(encoding.trim())) { + return true; + } + } + return false; + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java similarity index 62% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java index 766c58f1ec..0fb2486544 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; @@ -34,20 +34,15 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.zip.GZIPInputStream; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; import org.apache.hertzbeat.common.util.JsonUtil; -import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.springframework.web.client.RestTemplate; /** Greptime native OTLP forwarder for metrics, logs, and traces. */ @Service @@ -55,25 +50,11 @@ import org.springframework.web.client.RestTemplate; public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { private static final String PROTOBUF = "application/x-protobuf"; - private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name"; - private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name"; - private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name"; - private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name"; - private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name"; - private static final String GREPTIME_PROMOTE_RESOURCE_HEADER = - "X-Greptime-OTLP-Metric-Promote-Resource-Attrs"; - private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of( - "service.name", "service.namespace", "service.version", "deployment.environment.name", - "host.name", "k8s.namespace.name", "k8s.pod.name")); private static final Set SIGNALS = Set.of("metrics", "logs", "traces"); - private final GreptimeProperties greptimeProperties; - private final RestTemplate restTemplate; + private final OtlpSignalStorage signalStorage; - public GreptimeOtlpSignalForwarder(GreptimeProperties greptimeProperties, - @Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE) - RestTemplate restTemplate) { - this.greptimeProperties = greptimeProperties; - this.restTemplate = restTemplate; + public GreptimeOtlpSignalForwarder(OtlpSignalStorage signalStorage) { + this.signalStorage = signalStorage; } @Override @@ -81,10 +62,11 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { HttpHeaders safeHeaders = requestHeaders == null ? new HttpHeaders() : requestHeaders; byte[] normalized = maybeDecompress(content, safeHeaders); MediaType contentType = safeHeaders.getContentType(); - byte[] protobuf = contentType != null && MediaType.APPLICATION_JSON.includes(contentType) - ? jsonToProtobuf(signal, normalized) : validateProtobuf(signal, normalized); - byte[] response = forwardProtobuf(signal, protobuf); - if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) { + boolean json = contentType != null && MediaType.APPLICATION_JSON.includes(contentType); + // Both branches yield already-validated protobuf, so write directly instead of re-parsing via forwardProtobuf. + byte[] protobuf = json ? jsonToProtobuf(signal, normalized) : validateProtobuf(signal, normalized); + byte[] response = signalStorage.writeProtobuf(signal, protobuf); + if (json) { return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON) .body("{}".getBytes(StandardCharsets.UTF_8)); } @@ -93,40 +75,8 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { @Override public byte[] forwardProtobuf(String signal, byte[] content) { - String normalizedSignal = normalizeSignal(signal); - byte[] validContent = validateProtobuf(normalizedSignal, content); - HttpHeaders headers = greptimeHeaders(normalizedSignal); - ResponseEntity response = restTemplate.exchange( - endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal), - HttpMethod.POST, - new HttpEntity<>(validContent, headers), - byte[].class); - if (!response.getStatusCode().is2xxSuccessful()) { - throw new IllegalStateException("GreptimeDB rejected OTLP " + normalizedSignal); - } - return response.getBody() == null ? new byte[0] : response.getBody(); - } - - private HttpHeaders greptimeHeaders(String signal) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.parseMediaType(PROTOBUF)); - headers.setAccept(List.of(MediaType.parseMediaType(PROTOBUF))); - headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database()) - ? greptimeProperties.database() : "public"); - if ("metrics".equals(signal)) { - headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES); - } else if ("traces".equals(signal)) { - headers.set(GREPTIME_TRACE_TABLE_HEADER, "hzb_traces"); - headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1"); - } else { - headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME); - headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1"); - } - if (StringUtils.hasText(greptimeProperties.username()) && StringUtils.hasText(greptimeProperties.password())) { - String credentials = greptimeProperties.username() + ":" + greptimeProperties.password(); - headers.setBasicAuth(Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8))); - } - return headers; + byte[] validContent = validateProtobuf(signal, content); + return signalStorage.writeProtobuf(signal, validContent); } private byte[] jsonToProtobuf(String signal, byte[] content) { @@ -213,7 +163,4 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { return normalized; } - private String endpoint(String base, String path) { - return StringUtils.trimTrailingCharacter(base, '/') + path; - } } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java similarity index 79% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java index 46978af182..93db9e7e0b 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java @@ -15,8 +15,11 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; @@ -29,14 +32,16 @@ import io.opentelemetry.proto.logs.v1.ScopeLogs; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.queue.CommonDataQueue; -import org.apache.hertzbeat.log.notice.LogSseManager; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; +import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.Set; /** * Adapter for OpenTelemetry OTLP/HTTP log ingestion. @@ -46,9 +51,10 @@ import java.util.Map; */ @Slf4j @Service -public class OtlpLogProtocolAdapter implements LogProtocolAdapter { +public class OtlpLogProtocolAdapter { - private static final String PROTOCOL_NAME = "otlp"; + private static final Set OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId"); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final CommonDataQueue commonDataQueue; private final LogSseManager logSseManager; @@ -58,7 +64,6 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter { this.logSseManager = logSseManager; } - @Override public void ingest(String content) { if (content == null || content.isEmpty()) { log.warn("Received empty OTLP JSON log payload - skip processing."); @@ -66,7 +71,7 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter { } ExportLogsServiceRequest.Builder builder = ExportLogsServiceRequest.newBuilder(); try { - JsonFormat.parser().ignoringUnknownFields().merge(content, builder); + JsonFormat.parser().ignoringUnknownFields().merge(normalizeOtlpJson(content), builder); ExportLogsServiceRequest request = builder.build(); processLogsRequest(request, "JSON"); } catch (InvalidProtocolBufferException e) { @@ -274,8 +279,58 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter { return hexString.toString(); } - @Override - public String supportProtocol() { - return PROTOCOL_NAME; + /** + * Accept the hexadecimal IDs emitted by the HertzBeat onboarding probe while preserving + * standard base64-encoded OTLP/JSON IDs. + */ + private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException { + try { + JsonNode root = OBJECT_MAPPER.readTree(content); + normalizeOtlpHexEncodedIds(root); + return OBJECT_MAPPER.writeValueAsString(root); + } catch (Exception exception) { + throw new InvalidProtocolBufferException("Failed to normalize OTLP JSON: " + exception.getMessage()); + } } -} \ No newline at end of file + + private void normalizeOtlpHexEncodedIds(JsonNode node) { + if (node == null) { + return; + } + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + objectNode.fieldNames().forEachRemaining(fieldName -> { + JsonNode child = objectNode.get(fieldName); + if (OTLP_HEX_ID_FIELDS.contains(fieldName) && child != null && child.isTextual()) { + String normalized = tryConvertHexToBase64(child.asText()); + if (normalized != null) { + objectNode.put(fieldName, normalized); + } + } else { + normalizeOtlpHexEncodedIds(child); + } + }); + return; + } + if (node.isArray()) { + node.forEach(this::normalizeOtlpHexEncodedIds); + } + } + + private String tryConvertHexToBase64(String value) { + if (value == null || value.isBlank() || (value.length() & 1) != 0) { + return null; + } + for (int i = 0; i < value.length(); i++) { + if (Character.digit(value.charAt(i), 16) < 0) { + return null; + } + } + try { + return Base64.getEncoder().encodeToString(HexFormat.of().parseHex(value)); + } catch (IllegalArgumentException exception) { + return null; + } + } + +} diff --git a/hertzbeat-otel/src/main/resources/META-INF/spring.factories b/hertzbeat-observability/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports similarity index 86% rename from hertzbeat-otel/src/main/resources/META-INF/spring.factories rename to hertzbeat-observability/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index cb9d601c41..c0b56f3c65 100644 --- a/hertzbeat-otel/src/main/resources/META-INF/spring.factories +++ b/hertzbeat-observability/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -13,5 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -org.apache.hertzbeat.otel.config.LogAutoConfiguration \ No newline at end of file +org.apache.hertzbeat.observability.config.ObservabilityAutoConfiguration diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java similarity index 58% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java index 41245fe391..e3b6422cab 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.observability.config; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.grpc.ManagedChannel; @@ -33,15 +34,21 @@ import io.grpc.StatusRuntimeException; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.stub.MetadataUtils; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import java.net.InetAddress; +import java.net.ServerSocket; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,10 +57,12 @@ class OtlpGrpcServerConfigTest { private Server server; private ManagedChannel channel; + private OtlpLogIngestionService logIngestionService; @BeforeEach void setUp() throws Exception { OtlpSignalForwarder forwarder = mock(OtlpSignalForwarder.class); + logIngestionService = mock(OtlpLogIngestionService.class); SignalWorkloadGuard guard = mock(SignalWorkloadGuard.class); OtlpAccessTokenValidator validator = token -> "probe-token".equals(token) ? null : "Invalid token"; when(guard.execute(eq(SignalWorkloadGuard.Workload.OTLP_WRITE), any())).thenAnswer(invocation -> { @@ -61,10 +70,14 @@ class OtlpGrpcServerConfigTest { return action.get(); }); when(forwarder.forwardProtobuf(eq("metrics"), any())).thenReturn(new byte[0]); + when(logIngestionService.ingestProtobuf(any())).thenReturn(new byte[0]); server = NettyServerBuilder.forPort(0) .addService(ServerInterceptors.intercept( new OtlpGrpcServerConfig.MetricsService(forwarder, guard), new OtlpGrpcServerConfig.BearerTokenInterceptor(validator))) + .addService(ServerInterceptors.intercept( + new OtlpGrpcServerConfig.LogsService(logIngestionService, guard), + new OtlpGrpcServerConfig.BearerTokenInterceptor(validator))) .build().start(); channel = NettyChannelBuilder.forAddress("127.0.0.1", server.getPort()).usePlaintext().build(); } @@ -86,6 +99,18 @@ class OtlpGrpcServerConfigTest { assertThat(response).isEqualTo(ExportMetricsServiceResponse.getDefaultInstance()); } + @Test + void shouldRouteAuthorizedOtlpGrpcLogsThroughLogIngestionService() { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER), "Bearer probe-token"); + ExportLogsServiceResponse response = LogsServiceGrpc.newBlockingStub(channel) + .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers)) + .export(ExportLogsServiceRequest.getDefaultInstance()); + + assertThat(response).isEqualTo(ExportLogsServiceResponse.getDefaultInstance()); + verify(logIngestionService).ingestProtobuf(any()); + } + @Test void shouldRejectMissingOtlpGrpcBearerToken() { assertThatThrownBy(() -> MetricsServiceGrpc.newBlockingStub(channel) @@ -104,4 +129,46 @@ class OtlpGrpcServerConfigTest { assertThat(OtlpGrpcServerConfig.toGrpcStatus(new IllegalStateException("storage"))) .isEqualTo(Status.UNAVAILABLE); } + + /** + * A bind failure must cost the gRPC channel only, never the whole application: HTTP ingestion + * serves the same signals on the main port, so an optional side channel has no business keeping + * the monitoring system from starting. + */ + @Test + void shouldGiveUpTheGrpcListenerInsteadOfTheApplicationWhenThePortIsTaken() throws Exception { + // Bind the same address the runner will use: on bsd a wildcard socket does not stop a + // listener from taking 127.0.0.1 on the same port. + try (ServerSocket occupied = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + OtlpGrpcServerConfig.OtlpGrpcServerRunner runner = runnerOn(occupied.getLocalPort()); + + runner.start(); + + assertThat(runner.isRunning()).isFalse(); + runner.stop(); + } + } + + @Test + void shouldReportTheGrpcListenerRunningOnceItBinds() throws Exception { + int freePort; + try (ServerSocket probe = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + freePort = probe.getLocalPort(); + } + OtlpGrpcServerConfig.OtlpGrpcServerRunner runner = runnerOn(freePort); + + runner.start(); + try { + assertThat(runner.isRunning()).isTrue(); + } finally { + runner.stop(); + } + } + + private OtlpGrpcServerConfig.OtlpGrpcServerRunner runnerOn(int port) { + return new OtlpGrpcServerConfig.OtlpGrpcServerRunner("127.0.0.1", port, + mock(OtlpSignalForwarder.class), mock(OtlpLogIngestionService.class), + mock(SignalWorkloadGuard.class), + new OtlpGrpcServerConfig.BearerTokenInterceptor(token -> null)); + } } diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteControllerTest.java new file mode 100644 index 0000000000..4d5a3017c2 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteControllerTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import com.google.rpc.Code; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +/** Deprecated 1.8.x OTLP log route alias contract tests. */ +@ExtendWith(MockitoExtension.class) +class LegacyOtlpLogRouteControllerTest { + + @Mock + private OtlpLogIngestionService logIngestionService; + + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + mockMvc = MockMvcBuilders.standaloneSetup( + new LegacyOtlpLogRouteController(logIngestionService, new SignalWorkloadGuard())) + .setControllerAdvice(new OtlpHttpExceptionHandler()) + .build(); + } + + @Test + void legacyOtlpRouteShouldForwardToTheCanonicalLogFanOutWithDeprecationHeaders() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isOk()) + .andExpect(header().string("Deprecation", "true")) + .andExpect(header().string("Link", "; rel=\"successor-version\"")) + .andExpect(content().string("{}")); + + verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class)); + } + + @Test + void legacyIngestOtlpRouteShouldForwardToTheCanonicalLogFanOut() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/ingest/OTLP") + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isOk()) + .andExpect(header().string("Deprecation", "true")); + + verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class)); + } + + @Test + void legacyIngestRouteShouldRejectNonOtlpProtocolsWithoutTouchingTheFanOut() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/ingest/vector") + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Deprecation", "true")); + + verifyNoInteractions(logIngestionService); + } + + @Test + void legacyRouteShouldPreserveCanonicalErrorStatus() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenThrow(new IllegalArgumentException("Malformed OTLP logs JSON payload")); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/logs/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{")) + .andExpect(status().isBadRequest()) + .andExpect(header().string("Deprecation", "true")) + .andExpect(header().string("Link", "; rel=\"successor-version\"")) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber())) + .andExpect(jsonPath("$.message").value("Malformed OTLP logs JSON payload")); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java similarity index 94% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java index 8bd4939e0b..29e73e0db3 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.when; @@ -61,7 +61,7 @@ class LogManagerControllerTest { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000") ) .andDo(print()) @@ -77,7 +77,7 @@ class LogManagerControllerTest { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000") ) .andExpect(status().isOk()) @@ -91,7 +91,7 @@ class LogManagerControllerTest { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000") ) .andExpect(status().isOk()) diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java similarity index 91% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java index 8dd39de96d..902ca5142e 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -32,7 +32,7 @@ import java.util.Map; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.log.LogEntry; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -69,8 +69,8 @@ class LogQueryControllerTest { void shouldReturnFriendlyFailureWhenLogQueryUnsupported() throws Exception { when(historyDataReader.supportsLogQuery()).thenReturn(false); - for (String path : List.of("/api/logs/list", "/api/logs/stats/overview", - "/api/logs/stats/trace-coverage", "/api/logs/stats/trend")) { + for (String path : List.of("/api/observability/logs", "/api/observability/logs/overview", + "/api/observability/logs/trace-coverage", "/api/observability/logs/trend")) { mockMvc.perform(MockMvcRequestBuilders.get(path)) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) @@ -108,7 +108,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/list") + .get("/api/observability/logs") .param("start", "1734005477000") .param("end", "1734005478000") .param("traceId", "trace123") @@ -144,7 +144,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/list") + .get("/api/observability/logs") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -157,7 +157,7 @@ class LogQueryControllerTest { when(historyDataReader.countObservabilityLogs(any())).thenReturn(0L); when(historyDataReader.queryObservabilityLogs(any(), any(), any())).thenReturn(List.of()); - mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs") .param("pageIndex", "-1") .param("pageSize", "1000")) .andExpect(status().isOk()) @@ -172,7 +172,7 @@ class LogQueryControllerTest { when(historyDataReader.countObservabilityLogs(any())) .thenThrow(new IllegalArgumentException("Invalid resource filter expression")); - mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs") .param("resource", "bad-expression")) .andExpect(status().isBadRequest()) .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)); @@ -186,7 +186,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/overview") + .get("/api/observability/logs/overview") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -205,7 +205,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/overview") + .get("/api/observability/logs/overview") .param("start", "1734005477000") .param("end", "1734005478000") ) @@ -221,7 +221,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trace-coverage") + .get("/api/observability/logs/trace-coverage") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -238,7 +238,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trend") + .get("/api/observability/logs/trend") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -251,7 +251,7 @@ class LogQueryControllerTest { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trend") + .get("/api/observability/logs/trend") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java similarity index 92% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java index 5528ec4f92..302e0e925a 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; @@ -24,8 +24,8 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -64,7 +64,7 @@ class LogSseControllerTest { @Test void testSubscribeWithoutFilters() throws Exception { // When: A request is made to the subscribe endpoint without any parameters - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().isOk()); @@ -86,7 +86,7 @@ class LogSseControllerTest { .standaloneSetup(new LogSseController(realEmitterManager)) .build(); - MvcResult result = realMockMvc.perform(get("/api/logs/sse/subscribe") + MvcResult result = realMockMvc.perform(get("/api/observability/logs/stream") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().isOk()) .andExpect(request().asyncStarted()) @@ -109,7 +109,7 @@ class LogSseControllerTest { String spanId = "abcdef1234567890"; // When: A request is made with all filter parameters - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .param("severityText", severityText) .param("severityNumber", severityNumber) .param("traceId", traceId) @@ -130,7 +130,7 @@ class LogSseControllerTest { @Test void testSubscribeWithInvalidSeverityNumber() throws Exception { // When: A request is made with a non-integer value for severityNumber - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .param("severityNumber", "not-a-number") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().is4xxClientError()); diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java similarity index 96% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java index e406213d11..dd6b54cb85 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandlerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandlerTest.java new file mode 100644 index 0000000000..c3892a549f --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpHttpExceptionHandlerTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import com.google.protobuf.util.JsonFormat; +import com.google.rpc.Code; +import com.google.rpc.Status; +import java.nio.charset.StandardCharsets; +import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.mock.http.MockHttpInputMessage; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.method.HandlerMethod; + +/** OTLP/HTTP failure response contract tests. */ +class OtlpHttpExceptionHandlerTest { + + private final OtlpHttpExceptionHandler handler = new OtlpHttpExceptionHandler(); + + @Test + void shouldEncodeRpcStatusAsJsonForJsonRequests() throws Exception { + ResponseEntity response = handler.handleInvalidPayload( + new IllegalArgumentException("Malformed OTLP metrics JSON payload"), + request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode()); + assertEquals(MediaType.APPLICATION_JSON, response.getHeaders().getContentType()); + assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + Status status = parseJson(response.getBody()); + assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode()); + assertEquals("Malformed OTLP metrics JSON payload", status.getMessage()); + } + + @Test + void shouldEncodeRpcStatusAsProtobufForProtobufRequests() throws Exception { + ResponseEntity response = handler.handleInvalidPayload( + new IllegalArgumentException("Malformed OTLP metrics protobuf payload"), + request("application/x-protobuf"), null); + + assertEquals(OtlpHttpExceptionHandler.PROTOBUF, response.getHeaders().getContentType()); + Status status = Status.parseFrom(response.getBody()); + assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode()); + assertEquals("Malformed OTLP metrics protobuf payload", status.getMessage()); + } + + @Test + void shouldFallBackToProtobufWhenContentTypeIsMissingOrUnparseable() throws Exception { + for (String contentType : new String[] {null, "not a media type"}) { + ResponseEntity response = handler.handleInvalidPayload( + new IllegalArgumentException("bad"), request(contentType), null); + + assertEquals(OtlpHttpExceptionHandler.PROTOBUF, response.getHeaders().getContentType()); + assertEquals("bad", Status.parseFrom(response.getBody()).getMessage()); + } + } + + @Test + void shouldNotFailWhenExceptionMessageIsNull() throws Exception { + ResponseEntity response = handler.handleInvalidPayload( + new IllegalArgumentException(), request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode()); + assertEquals("Unknown error", parseJson(response.getBody()).getMessage()); + } + + @Test + void shouldReturnRetryableTooManyRequestsWhenOverloaded() throws Exception { + ResponseEntity response = handler.handleOverloaded( + new SignalQueryRejectedException("OTLP_WRITE"), request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.TOO_MANY_REQUESTS, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + assertEquals(Code.RESOURCE_EXHAUSTED.getNumber(), parseJson(response.getBody()).getCode()); + } + + @Test + void shouldReturnRetryableServiceUnavailableWhenStorageIsDown() throws Exception { + ResponseEntity response = handler.handleStorageUnavailable( + new StorageUnavailableException("GreptimeDB log storage is unavailable", new RuntimeException()), + request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + Status status = parseJson(response.getBody()); + assertEquals(Code.UNAVAILABLE.getNumber(), status.getCode()); + assertEquals("GreptimeDB log storage is unavailable", status.getMessage()); + } + + @Test + void shouldHideTransportDetailsForGreptimeClientFailures() throws Exception { + ResponseEntity response = handler.handleStorageClientFailure( + new HttpServerErrorException(HttpStatus.BAD_GATEWAY, "upstream http://greptime:4000 exploded"), + request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + assertEquals("GreptimeDB storage is unavailable", parseJson(response.getBody()).getMessage()); + } + + @Test + void shouldReturnInternalErrorStatusForUnexpectedFailures() throws Exception { + ResponseEntity response = handler.handleUnexpected( + new NullPointerException("boom"), request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, response.getStatusCode()); + assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + Status status = parseJson(response.getBody()); + assertEquals(Code.INTERNAL.getNumber(), status.getCode()); + // The exporter gets a fixed phrase; the real cause only reaches the server log. + assertEquals("Unexpected OTLP ingestion failure", status.getMessage()); + assertFalse(status.getMessage().contains("boom")); + } + + /** + * A missing body is raised while spring resolves the arguments, so it never reaches the + * controller and is not an {@code ErrorResponse}. It must still answer 400 rather than 500: + * OTLP exporters replay 5xx, so a permanently malformed client would retry forever. + */ + @Test + void shouldRejectUnreadableBodyAsNonRetryableClientError() throws Exception { + HttpMessageNotReadableException exception = new HttpMessageNotReadableException( + "Required request body is missing: public org.springframework.http.ResponseEntity " + + "org.apache.hertzbeat.observability.controller.OtlpLogController.logs(byte[])", + new MockHttpInputMessage(new byte[0])); + + ResponseEntity response = handler.handleUnreadableBody( + exception, request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.BAD_REQUEST, response.getStatusCode()); + assertNull(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)); + Status status = parseJson(response.getBody()); + assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode()); + assertEquals("Malformed or missing OTLP request body", status.getMessage()); + // The framework message names the handler method; that must not travel back to the caller. + assertFalse(status.getMessage().contains("org.apache.hertzbeat")); + } + + @Test + void shouldKeepSpringMvcClientErrorStatusInsteadOfInternalError() throws Exception { + ResponseEntity response = handler.handleUnexpected( + new HttpRequestMethodNotSupportedException("GET"), request(MediaType.APPLICATION_JSON_VALUE), null); + + assertEquals(HttpStatus.METHOD_NOT_ALLOWED, response.getStatusCode()); + assertEquals(Code.INVALID_ARGUMENT.getNumber(), parseJson(response.getBody()).getCode()); + } + + @Test + void shouldKeepDeprecationHeadersOnLegacyRouteErrors() throws Exception { + LegacyOtlpLogRouteController legacy = new LegacyOtlpLogRouteController( + mock(OtlpLogIngestionService.class), new SignalWorkloadGuard()); + HandlerMethod legacyHandler = new HandlerMethod(legacy, + LegacyOtlpLogRouteController.class.getMethod("legacyOtlpLogs", byte[].class, HttpHeaders.class)); + + ResponseEntity response = handler.handleInvalidPayload( + new IllegalArgumentException("bad"), request(MediaType.APPLICATION_JSON_VALUE), legacyHandler); + + assertEquals("true", response.getHeaders().getFirst("Deprecation")); + assertTrue(response.getHeaders().getFirst("Link").contains(LegacyOtlpLogRouteController.CANONICAL_LOGS_ROUTE)); + } + + private static MockHttpServletRequest request(String contentType) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/otlp/v1/metrics"); + if (contentType != null) { + request.setContentType(contentType); + } + return request; + } + + private static Status parseJson(byte[] body) throws Exception { + Status.Builder builder = Status.newBuilder(); + JsonFormat.parser().merge(new String(body, StandardCharsets.UTF_8), builder); + return builder.build(); + } +} diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java new file mode 100644 index 0000000000..72be224135 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import com.google.rpc.Code; +import com.google.rpc.Status; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +/** OTLP/HTTP log route contract tests. */ +@ExtendWith(MockitoExtension.class) +class OtlpLogControllerTest { + + @Mock + private OtlpLogIngestionService logIngestionService; + + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + mockMvc = MockMvcBuilders.standaloneSetup( + new OtlpLogController(logIngestionService, new SignalWorkloadGuard())) + .setControllerAdvice(new OtlpHttpExceptionHandler()) + .build(); + } + + @Test + void shouldRouteLogsToTheLogFanOut() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isOk()); + + verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class)); + } + + @Test + void shouldReturnBadRequestForMalformedOtlpPayload() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenThrow(new IllegalArgumentException("Malformed OTLP logs JSON payload")); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{")) + .andExpect(status().isBadRequest()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber())) + .andExpect(jsonPath("$.message").value("Malformed OTLP logs JSON payload")); + } + + @Test + void shouldReturnBinaryRpcStatusForMalformedProtobufPayload() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenThrow(new IllegalArgumentException("Malformed OTLP logs protobuf payload")); + + byte[] body = mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(OtlpHttpExceptionHandler.PROTOBUF) + .content(new byte[] {1, 2, 3})) + .andExpect(status().isBadRequest()) + .andExpect(content().contentType(OtlpHttpExceptionHandler.PROTOBUF)) + .andReturn().getResponse().getContentAsByteArray(); + + Status status = Status.parseFrom(body); + assertEquals(Code.INVALID_ARGUMENT.getNumber(), status.getCode()); + assertEquals("Malformed OTLP logs protobuf payload", status.getMessage()); + } + + /** + * End to end guard for the missing body path: it is resolved before the controller runs, so only a + * request that actually goes through the dispatcher proves the advice turns it into a 400 with a + * google.rpc.Status body that does not name the handler method. + */ + @Test + void shouldRejectMissingBodyWithClientErrorInsteadOfServerError() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber())) + .andExpect(jsonPath("$.message").value("Malformed or missing OTLP request body")) + .andExpect(content().string(not(containsString("org.apache.hertzbeat")))); + + verifyNoInteractions(logIngestionService); + } +} diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java new file mode 100644 index 0000000000..1039b46312 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +class OtlpLogIngestionOwnershipTest { + + @Test + void httpAndGrpcLogsShouldUseTheLogFanOutInsteadOfTheRawGreptimeForwarder() throws Exception { + String httpController = Files.readString(Path.of( + "src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java")); + String grpcConfig = Files.readString(Path.of( + "src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java")); + + assertTrue(httpController.contains("OtlpLogIngestionService")); + assertTrue(httpController.contains("logIngestionService.ingestHttp(content, headers)")); + assertFalse(httpController.contains("forward(\"logs\"")); + + assertTrue(grpcConfig.contains("OtlpLogIngestionService")); + assertTrue(grpcConfig.contains("logIngestionService.ingestProtobuf(request.toByteArray())")); + assertFalse(grpcConfig.contains("forwardProtobuf(\"logs\"")); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java similarity index 58% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java index a3e48a14e0..dc7d861af2 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java @@ -15,17 +15,23 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import com.google.rpc.Code; +import com.google.rpc.Status; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -37,6 +43,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.client.ResourceAccessException; /** OTLP/HTTP route contract tests. */ @ExtendWith(MockitoExtension.class) @@ -50,21 +57,25 @@ class OtlpSignalControllerTest { @BeforeEach void setUp() { mockMvc = MockMvcBuilders.standaloneSetup( - new OtlpSignalController(signalForwarder, new SignalWorkloadGuard())).build(); + new OtlpSignalController(signalForwarder, new SignalWorkloadGuard())) + .setControllerAdvice(new OtlpHttpExceptionHandler()) + .build(); } @Test - void shouldRouteAllThreeSignals() throws Exception { + void shouldRouteGreptimeSignals() throws Exception { when(signalForwarder.forwardHttp(any(), any(), any())) .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); - for (String signal : new String[] {"metrics", "logs", "traces"}) { + for (String signal : new String[] {"metrics", "traces"}) { mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/" + signal) .contentType(MediaType.APPLICATION_JSON) .content("{}")) .andExpect(status().isOk()); verify(signalForwarder).forwardHttp(eq(signal), any(), any(HttpHeaders.class)); } + + verifyNoMoreInteractions(signalForwarder); } @Test @@ -76,6 +87,26 @@ class OtlpSignalControllerTest { .contentType(MediaType.APPLICATION_JSON) .content("{")) .andExpect(status().isBadRequest()) - .andExpect(content().string("Malformed OTLP metrics JSON payload")); + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.code").value(Code.INVALID_ARGUMENT.getNumber())) + .andExpect(jsonPath("$.message").value("Malformed OTLP metrics JSON payload")); + } + + @Test + void shouldReturnRetryableServiceUnavailableWhenGreptimeWriteFails() throws Exception { + when(signalForwarder.forwardHttp(eq("traces"), any(), any())) + .thenThrow(new ResourceAccessException("connect refused: http://greptime:4000")); + + byte[] body = mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/traces") + .contentType(OtlpHttpExceptionHandler.PROTOBUF) + .content(new byte[] {1, 2, 3})) + .andExpect(status().isServiceUnavailable()) + .andExpect(header().string(HttpHeaders.RETRY_AFTER, "1")) + .andExpect(content().contentType(OtlpHttpExceptionHandler.PROTOBUF)) + .andReturn().getResponse().getContentAsByteArray(); + + Status status = Status.parseFrom(body); + assertEquals(Code.UNAVAILABLE.getNumber(), status.getCode()); + assertEquals("GreptimeDB storage is unavailable", status.getMessage()); } } diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java similarity index 78% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java index 9e3a32fa6d..ef9db09a82 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hertzbeat.common.entity.dto.Message; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.junit.jupiter.api.Test; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -54,6 +55,16 @@ class SignalWorkloadExceptionHandlerTest { Order order = SignalWorkloadExceptionHandler.class.getAnnotation(Order.class); assertNotNull(order); - assertEquals(Ordered.HIGHEST_PRECEDENCE, order.value()); + assertTrue(order.value() < Ordered.LOWEST_PRECEDENCE); + } + + @Test + void shouldYieldToTheOtlpHttpAdvice() { + Order order = SignalWorkloadExceptionHandler.class.getAnnotation(Order.class); + Order otlpOrder = OtlpHttpExceptionHandler.class.getAnnotation(Order.class); + + assertNotNull(order); + assertNotNull(otlpOrder); + assertTrue(otlpOrder.value() < order.value()); } } diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java similarity index 91% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java index a98c9fdd2e..260e671474 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -30,8 +30,8 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail; import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -62,7 +62,7 @@ class ThreeSignalQueryControllerTest { "prod", null, null, null, 60, null)) .thenReturn(new OtlpMetricsConsole("http_server_duration", 1000L, 2000L, 60, List.of())); - mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/console") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/query") .param("query", "http_server_duration") .param("start", "1000") .param("end", "2000") @@ -80,7 +80,7 @@ class ThreeSignalQueryControllerTest { when(queryService.metricInventory(1000L, 2000L, "checkout", "payments", "prod", 50)) .thenReturn(new OtlpMetricsInventory(List.of("http_server_duration"))); - mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/inventory") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/inventory") .param("start", "1000") .param("end", "2000") .param("serviceName", "checkout") @@ -102,7 +102,7 @@ class ThreeSignalQueryControllerTest { "SERVER", "failed", 10_000_000L, 1000L, Map.of(), Map.of(), List.of()); when(queryService.traceDetail("trace-1")).thenReturn(new TraceDetail(item, List.of(span))); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces") .param("start", "1000") .param("end", "2000") .param("traceId", "trace-1") @@ -116,7 +116,7 @@ class ThreeSignalQueryControllerTest { .andExpect(status().isOk()) .andExpect(jsonPath("$.data.content[0].traceId").value("trace-1")); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/trace-1")) + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/trace-1")) .andExpect(status().isOk()) .andExpect(jsonPath("$.data.spans[0].spanId").value("span-1")); } @@ -127,7 +127,7 @@ class ThreeSignalQueryControllerTest { "GET /cart", 1L, 100L)) .thenReturn(new TraceOverview(4, 1, 0.25, 12.5, 24.0)); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/stats/overview") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/overview") .param("start", "1000") .param("end", "2000") .param("traceId", "trace-1") diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java index b06284d9ba..1845e6bc5b 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.junit.jupiter.api.BeforeEach; diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java index 231e8af877..2616507a30 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.junit.jupiter.api.AfterEach; diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java similarity index 92% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java index d8c92d7625..d0c496af05 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -25,8 +25,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Limits; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Limits; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.junit.jupiter.api.Test; /** Tests bounded signal workload isolation and recovery. */ diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java new file mode 100644 index 0000000000..791bc61656 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.verify; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +@ExtendWith(MockitoExtension.class) +class DefaultOtlpLogIngestionServiceTest { + + @Mock + private OtlpLogProtocolAdapter protocolAdapter; + + private DefaultOtlpLogIngestionService service; + + @BeforeEach + void setUp() { + service = new DefaultOtlpLogIngestionService(protocolAdapter); + } + + @Test + void shouldIngestJsonAndReturnJsonOtlpResponse() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + var response = service.ingestHttp("{}".getBytes(StandardCharsets.UTF_8), headers); + + verify(protocolAdapter).ingest("{}"); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); + assertThat(response.getBody()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void shouldIngestProtobufAndReturnProtobufOtlpResponse() { + byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + + var response = service.ingestHttp(request, headers); + + verify(protocolAdapter).ingestBinary(request); + assertThat(response.getHeaders().getContentType()) + .isEqualTo(MediaType.parseMediaType("application/x-protobuf")); + assertThat(response.getBody()).isEmpty(); + } + + @Test + void shouldDecompressCommaSeparatedGzipContentEncoding() throws Exception { + byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + headers.add(HttpHeaders.CONTENT_ENCODING, "identity, gzip"); + + service.ingestHttp(gzip(request), headers); + + verify(protocolAdapter).ingestBinary(request); + } + + @Test + void shouldRejectMalformedGzipBeforeFanOut() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + headers.set(HttpHeaders.CONTENT_ENCODING, "gzip"); + + assertThatThrownBy(() -> service.ingestHttp(new byte[] {1, 2, 3}, headers)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Malformed gzip OTLP log payload"); + } + + private byte[] gzip(byte[] content) throws Exception { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(output)) { + gzip.write(content); + gzip.finish(); + return output.toByteArray(); + } + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java similarity index 63% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java index 07b4cd1815..c5a03ed6ff 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -28,33 +28,27 @@ import static org.mockito.Mockito.when; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import java.nio.charset.StandardCharsets; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestTemplate; /** Greptime native OTLP forwarding tests. */ @ExtendWith(MockitoExtension.class) class GreptimeOtlpSignalForwarderTest { @Mock - private RestTemplate restTemplate; + private OtlpSignalStorage signalStorage; @Test - void shouldConvertJsonAndForwardMetricsWithResourcePromotion() { - when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), - any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + void shouldConvertJsonBeforeWritingMetrics() { + when(signalStorage.writeProtobuf(eq("metrics"), any())).thenReturn(new byte[0]); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); @@ -62,21 +56,15 @@ class GreptimeOtlpSignalForwarderTest { headers); assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); - verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), - request.capture(), eq(byte[].class)); - assertThat(request.getValue().getHeaders().getFirst("X-Greptime-OTLP-Metric-Promote-Resource-Attrs")) - .contains("service.name", "deployment.environment.name"); - assertThat(request.getValue().getBody()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray()); + ArgumentCaptor content = ArgumentCaptor.forClass(byte[].class); + verify(signalStorage).writeProtobuf(eq("metrics"), content.capture()); + assertThat(content.getValue()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray()); } @Test void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception { - when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), - any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + when(signalStorage.writeProtobuf(eq("traces"), any())).thenReturn(new byte[0]); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); String json = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{" @@ -86,10 +74,9 @@ class GreptimeOtlpSignalForwarderTest { forwarder.forwardHttp("traces", json.getBytes(StandardCharsets.UTF_8), headers); - ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); - verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), - request.capture(), eq(byte[].class)); - ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(request.getValue().getBody()); + ArgumentCaptor content = ArgumentCaptor.forClass(byte[].class); + verify(signalStorage).writeProtobuf(eq("traces"), content.capture()); + ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(content.getValue()); assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getTraceId().toByteArray()) .containsExactly(java.util.HexFormat.of().parseHex("0123456789abcdef0123456789abcdef")); assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getParentSpanId().toByteArray()) @@ -98,29 +85,25 @@ class GreptimeOtlpSignalForwarderTest { @Test void shouldRejectMalformedJsonBeforeCallingGreptime() { - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); assertThatThrownBy(() -> forwarder.forwardHttp("metrics", "{".getBytes(StandardCharsets.UTF_8), headers)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Malformed OTLP JSON payload"); - verifyNoInteractions(restTemplate); + verifyNoInteractions(signalStorage); } @Test void shouldRejectMalformedProtobufBeforeCallingGreptime() { - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); assertThatThrownBy(() -> forwarder.forwardHttp("traces", new byte[] {(byte) 0xff}, headers)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Malformed OTLP traces protobuf payload"); - verifyNoInteractions(restTemplate); + verifyNoInteractions(signalStorage); } } diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java similarity index 94% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java index 0a570155bf..bcd9b150e1 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import com.google.protobuf.ByteString; import com.google.protobuf.util.JsonFormat; @@ -29,7 +29,7 @@ import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.common.v1.InstrumentationScope; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.queue.CommonDataQueue; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -211,6 +211,31 @@ class OtlpLogProtocolAdapterTest { assertEquals(1, capturedEntry.getTraceFlags()); } + @Test + void testIngestWithHexEncodedTraceAndSpanIds() { + String otlpPayload = """ + { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "body": {"stringValue": "hex id probe"}, + "traceId": "1234567890abcdef1234567890abcdef", + "spanId": "1234567890abcdef" + }] + }] + }] + } + """; + + adapter.ingest(otlpPayload); + + ArgumentCaptor> listCaptor = ArgumentCaptor.forClass(List.class); + verify(commonDataQueue).sendLogEntryToStorageBatch(listCaptor.capture()); + LogEntry capturedEntry = listCaptor.getValue().getFirst(); + assertEquals("1234567890abcdef1234567890abcdef", capturedEntry.getTraceId()); + assertEquals("1234567890abcdef", capturedEntry.getSpanId()); + } + @Test void testIngestWithInvalidJsonContent() { String invalidJson = "{ invalid json content }"; diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java new file mode 100644 index 0000000000..7a0b87a11e --- /dev/null +++ b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.otel.config; + +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.context.annotation.ComponentScan; + +/** + * HertzBeat self-telemetry auto configuration. + */ +@AutoConfiguration +@ComponentScan(basePackageClasses = OpenTelemetryConfig.class) +public class OpenTelemetryAutoConfiguration { +} diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java index 134de32621..1cebd09e56 100644 --- a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java +++ b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java @@ -51,8 +51,8 @@ public class OpenTelemetryConfig { private static final String HERTZBEAT_SERVICE_NAME = "HertzBeat"; private static final String DEFAULT_GREPTIME_DB_NAME = "public"; - private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_logs"; - private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_traces"; + private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_internal_logs"; + private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_internal_traces"; private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name"; private static final String GREPTIME_LOG_TABLE_NAME_HEADER = "X-Greptime-Log-Table-Name"; private static final String GREPTIME_TRACE_TABLE_NAME_HEADER = "X-Greptime-Trace-Table-Name"; @@ -144,4 +144,4 @@ public class OpenTelemetryConfig { return sdkLoggerProviderBuilder.addLogRecordProcessor(batchLogProcessor); }); } -} \ No newline at end of file +} diff --git a/hertzbeat-otel/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/hertzbeat-otel/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..3103cc0a41 --- /dev/null +++ b/hertzbeat-otel/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hertzbeat.otel.config.OpenTelemetryAutoConfiguration diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index 425dd4e1f6..8e13abbd72 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -346,6 +346,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index 9485a76b54..e8aed38e31 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -88,8 +88,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -111,12 +109,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] - /api/account/token===get===[admin] - /api/account/token/**===post===[admin] - /api/account/token/**===delete===[admin] @@ -141,7 +139,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java index 19401144d7..c88cafe194 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java @@ -19,6 +19,7 @@ package org.apache.hertzbeat.startup; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import jakarta.annotation.Resource; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.AlerterWorkerPool; @@ -43,8 +44,8 @@ import org.apache.hertzbeat.common.config.CommonProperties; import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue; import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.alert.service.impl.TencentSmsClientImpl; -import org.apache.hertzbeat.log.controller.OtlpSignalController; -import org.apache.hertzbeat.log.controller.ThreeSignalQueryController; +import org.apache.hertzbeat.observability.controller.OtlpSignalController; +import org.apache.hertzbeat.observability.controller.ThreeSignalQueryController; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; import org.apache.hertzbeat.warehouse.controller.MetricsDataController; import org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb.IotDbDataStorage; @@ -54,6 +55,8 @@ import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisDataStorage; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.context.ApplicationContext; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; /** * Manager Test @@ -113,4 +116,14 @@ class ContextTest extends AbstractSpringIntegrationTest { assertThrows(NoSuchBeanDefinitionException.class, () -> ctx.getBean(ThreeSignalQueryController.class)); } + @Test + void canonicalOtlpLogRouteShouldBeAvailableWithoutGreptime() { + RequestMappingHandlerMapping handlerMapping = ctx.getBean( + "requestMappingHandlerMapping", RequestMappingHandlerMapping.class); + + assertTrue(handlerMapping.getHandlerMethods().keySet().stream() + .anyMatch(mapping -> mapping.getPatternValues().contains("/api/otlp/v1/logs") + && mapping.getMethodsCondition().getMethods().contains(RequestMethod.POST))); + } + } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java new file mode 100644 index 0000000000..e7aa9895f7 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +class EntityFreeObservabilityTransitionContractTest { + + private static final Path REPOSITORY_ROOT = repositoryRoot(); + + @Test + void reactorAndRuntimeShouldOwnOneObservabilityModule() throws IOException { + String rootPom = Files.readString(REPOSITORY_ROOT.resolve("pom.xml")); + String managerPom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-manager/pom.xml")); + + assertTrue(rootPom.contains("hertzbeat-observability")); + assertFalse(rootPom.contains("hertzbeat-log")); + assertTrue(managerPom.contains("hertzbeat-observability")); + assertFalse(managerPom.contains("hertzbeat-log")); + assertFalse(Files.exists(REPOSITORY_ROOT.resolve("hertzbeat-log"))); + } + + @Test + void productionSourcesShouldExposeOnlyTheCanonicalEntityFreeContract() throws IOException { + Path sourceRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main/java"); + assertTrue(Files.isDirectory(sourceRoot)); + + // The deprecated 1.8.x OTLP log ingestion aliases live in exactly one class so the + // 2.0 removal is a single file delete plus the matching Sureness rules. + Path legacyAliasController = sourceRoot.resolve( + "org/apache/hertzbeat/observability/controller/LegacyOtlpLogRouteController.java"); + assertTrue(Files.isRegularFile(legacyAliasController)); + String legacyAliasSource = Files.readString(legacyAliasController); + assertTrue(legacyAliasSource.contains("@Deprecated(since = \"1.9.0\", forRemoval = true)")); + assertTrue(legacyAliasSource.contains("/api/logs/otlp/v1/logs")); + assertTrue(legacyAliasSource.contains("/api/logs/ingest/{protocol}")); + assertTrue(legacyAliasSource.contains("logIngestionService.ingestHttp(content, headers)")); + + String productionSources; + try (Stream sources = Files.walk(sourceRoot)) { + productionSources = sources + .filter(path -> path.toString().endsWith(".java")) + .filter(path -> !path.equals(legacyAliasController)) + .map(EntityFreeObservabilityTransitionContractTest::readSource) + .reduce("", (left, right) -> left + '\n' + right); + } + + assertTrue(productionSources.contains("/api/otlp/v1")); + assertTrue(productionSources.contains("/api/observability")); + assertFalse(productionSources.contains("/api/logs/otlp")); + assertFalse(productionSources.contains("/api/logs/ingest")); + assertFalse(productionSources.contains("/api/logs")); + assertFalse(productionSources.contains("/api/ingestion/otlp")); + assertFalse(productionSources.contains("/api/traces")); + assertFalse(productionSources.contains("org.apache.hertzbeat.manager.service.entity")); + assertFalse(productionSources.contains("HERTZBEAT_ENTITY_ID")); + assertFalse(productionSources.contains("HERTZBEAT_ENTITY_TYPE")); + assertFalse(productionSources.contains("EntityObservability")); + assertFalse(productionSources.contains("EntityTrace")); + assertFalse(productionSources.contains("OtlpEntity")); + } + + @Test + void warehouseShouldOwnGreptimeSignalStorageSchemaAndQueries() throws IOException { + Path warehouseRoot = REPOSITORY_ROOT.resolve("hertzbeat-warehouse/src/main"); + Path observabilityRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main"); + + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve("resources/greptime/tables/hertzbeat_traces.sql"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml"))); + + assertFalse(Files.exists(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/config/GreptimeSignalInitializer.java"))); + assertFalse(Files.exists(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/service/impl/GreptimeThreeSignalQueryService.java"))); + assertFalse(Files.exists(observabilityRoot.resolve("resources/greptime"))); + + String warehousePom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-warehouse/pom.xml")); + assertFalse(warehousePom.contains("hertzbeat-observability")); + String observabilityForwarder = Files.readString(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java")); + assertTrue(observabilityForwarder.contains("OtlpSignalStorage")); + assertFalse(observabilityForwarder.contains("RestTemplate")); + } + + @Test + void currentSecurityConfigurationsShouldProtectOnlyCanonicalObservabilityRoutes() throws IOException { + // The runtime configuration is the source of truth; pinning its observability surface + // once here keeps all nine copies from drifting together unnoticed. + SurenessRules baseline = observabilityRules("hertzbeat-startup/src/main/resources/sureness.yml"); + assertEquals(Set.of( + "/api/otlp/v1/**===post===[admin,user]", + // deprecated 1.8.x ingestion aliases keep the canonical write roles + "/api/logs/otlp/**===post===[admin,user]", + "/api/logs/ingest/**===post===[admin,user]", + "/api/observability/logs===delete===[admin]", + "/api/observability/**===get===[admin,user,guest]", + "/api/alert/sse/**===get===[admin,user,guest]", + "/api/manager/sse/**===get===[admin,user,guest]"), + baseline.resourceRole()); + assertEquals(Set.of(), baseline.excludedResource()); + + // Every deployable and test copy must carry exactly the baseline rules on this surface, + // so a partial edit that skips a file fails on the file that was missed. + for (String relativePath : new String[] { + "hertzbeat-manager/src/test/resources/sureness.yml", + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml", + "script/sureness.yml", + "script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml" + }) { + SurenessRules copy = observabilityRules(relativePath); + assertEquals(baseline.resourceRole(), copy.resourceRole(), relativePath); + assertEquals(baseline.excludedResource(), copy.excludedResource(), relativePath); + } + } + + private record SurenessRules(Set resourceRole, Set excludedResource) { + } + + @SuppressWarnings("unchecked") + private static SurenessRules observabilityRules(String relativePath) throws IOException { + Map document = new Yaml().load(Files.readString(REPOSITORY_ROOT.resolve(relativePath))); + return new SurenessRules( + observabilitySubset((List) document.get("resourceRole")), + observabilitySubset((List) document.get("excludedResource"))); + } + + private static Set observabilitySubset(List rules) { + // The route prefixes this transition owns, plus the sse streams scoped alongside it. + List observabilityRoutePrefixes = List.of( + "/api/otlp", "/api/observability", "/api/logs", "/api/traces", + "/api/ingestion", "/api/alert/sse", "/api/manager/sse"); + return rules.stream() + .filter(rule -> observabilityRoutePrefixes.stream().anyMatch(rule::startsWith)) + .collect(Collectors.toSet()); + } + + @Test + void productAndSelfTelemetryShouldUseSeparateSignalTables() throws IOException { + String traceSchema = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql")); + assertTrue(traceSchema.contains("CREATE TABLE IF NOT EXISTS hertzbeat_traces")); + assertFalse(traceSchema.contains("CREATE TABLE IF NOT EXISTS hzb_traces")); + + String selfTelemetry = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java")); + assertTrue(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hzb_internal_logs\"")); + assertTrue(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hzb_internal_traces\"")); + assertFalse(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hertzbeat_logs\"")); + assertFalse(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hertzbeat_traces\"")); + } + + @Test + void currentClientsDocsAndProbesShouldUseCanonicalObservabilityRoutes() throws IOException { + for (String relativePath : new String[] { + "web-app/src/app/service/log.service.ts", + "web-app/src/app/service/observability.service.ts", + "web-app/src/app/routes/log/log-stream/log-stream.component.ts", + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml" + }) { + String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath)); + assertFalse(content.contains("/api/logs"), relativePath); + assertFalse(content.contains("/logs/list"), relativePath); + assertFalse(content.contains("/logs/stats"), relativePath); + assertFalse(content.contains("/ingestion/otlp/metrics"), relativePath); + assertFalse(content.contains("/traces/list"), relativePath); + assertFalse(content.contains("/traces/stats"), relativePath); + } + + // Integration docs must lead with the canonical route; the 1.8.x ingestion paths may only + // appear inside the upgrade notice that marks them as deprecated aliases. + for (String relativePath : new String[] { + "web-app/src/assets/doc/log-integration/otlp.en-US.md", + "web-app/src/assets/doc/log-integration/otlp.zh-CN.md", + "home/docs/help/log_integration.md", + "home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md" + }) { + String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath)); + assertTrue(content.contains("POST /api/otlp/v1/logs"), relativePath); + assertTrue(content.contains("/api/otlp/v1/logs"), relativePath); + assertTrue(content.contains("Deprecation: true"), relativePath); + assertFalse(content.contains("logs_endpoint: http://{hertzbeat_host}:1157/api/logs/"), relativePath); + assertFalse(content.contains("/logs/list"), relativePath); + assertFalse(content.contains("/logs/stats"), relativePath); + assertFalse(content.contains("/ingestion/otlp/metrics"), relativePath); + assertFalse(content.contains("/traces/list"), relativePath); + assertFalse(content.contains("/traces/stats"), relativePath); + } + + // The upgrade guide must carry the 1.8.x -> 1.9.0 route table so operators see the break. + for (String relativePath : new String[] { + "home/docs/start/upgrade.md", + "home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md" + }) { + String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath)); + assertTrue(content.contains("`POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs`"), relativePath); + assertTrue(content.contains("`GET /api/logs/list` | `GET /api/observability/logs`"), relativePath); + assertTrue(content.contains("`GET /api/traces/**` | `GET /api/observability/traces/**`"), relativePath); + } + + String logService = Files.readString(REPOSITORY_ROOT.resolve("web-app/src/app/service/log.service.ts")); + assertTrue(logService.contains("/observability/logs")); + String observabilityService = Files.readString( + REPOSITORY_ROOT.resolve("web-app/src/app/service/observability.service.ts")); + assertTrue(observabilityService.contains("/observability/metrics/query")); + assertTrue(observabilityService.contains("/observability/traces")); + String streamComponent = Files.readString(REPOSITORY_ROOT.resolve( + "web-app/src/app/routes/log/log-stream/log-stream.component.ts")); + assertTrue(streamComponent.contains("/api/observability/logs/stream")); + String vectorConfig = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml")); + assertTrue(vectorConfig.contains("/api/otlp/v1/logs")); + } + + private static String readSource(Path path) { + try { + return Files.readString(path); + } catch (IOException exception) { + throw new IllegalStateException("Failed to read " + path, exception); + } + } + + private static Path repositoryRoot() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null && !Files.isRegularFile(candidate.resolve("mvnw"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException("Unable to locate repository root"); + } + return candidate; + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessSseRuleTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessSseRuleTest.java index f8dba6e5e1..8ee9c3ecbc 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessSseRuleTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessSseRuleTest.java @@ -39,7 +39,8 @@ import org.yaml.snakeyaml.Yaml; * `curl -N` stayed subscribed and received every alert the deployment raised - internal * hostnames, addresses, metric values and alert content - because * `AlertNoticeDispatch` broadcasts each alert to every subscriber with no per subscriber - * filtering. `/api/logs/sse/**` was already scoped this way; these now match it. + * filtering. The log stream at `/api/observability/logs/stream` was already scoped this + * way; these now match it. */ class SurenessSseRuleTest { @@ -98,6 +99,6 @@ class SurenessSseRuleTest { @Test void theLogStreamScopingIsUnchanged() { assertEquals("[admin,user,guest]", - roleTree.searchPathFilterRoles("/api/logs/sse/subscribe" + SEPARATOR + "get")); + roleTree.searchPathFilterRoles("/api/observability/logs/stream" + SEPARATOR + "get")); } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java index 49ef4a867e..be12696063 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/security/SurenessUnruledEndpointTest.java @@ -78,21 +78,21 @@ class SurenessUnruledEndpointTest { @Test void deletingLogsIsRestrictedToAdmin() { - assertEquals("[admin]", rolesFor("/api/logs", "delete")); + assertEquals("[admin]", rolesFor("/api/observability/logs", "delete")); } @Test void readingLogsStaysOpenToEveryRole() { - assertEquals("[admin,user,guest]", rolesFor("/api/logs/list", "get")); + assertEquals("[admin,user,guest]", rolesFor("/api/observability/logs", "get")); } /** - * Matches how the sibling ingestion routes `/api/otlp/**` and `/api/logs/ingest/**` - * are already scoped, so a low privileged account can no longer forge log records. + * Authenticated operators may write telemetry to the canonical OTLP boundary; + * guests may not. */ @Test void ingestingOtlpLogsRequiresAtLeastUser() { - assertEquals("[admin,user]", rolesFor("/api/logs/otlp/v1/logs", "post")); + assertEquals("[admin,user]", rolesFor("/api/otlp/v1/logs", "post")); } /** diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java index ff8ef5ca20..3ec129a4e9 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java @@ -19,7 +19,9 @@ package org.apache.hertzbeat.warehouse.config; import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.SignConstants; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; /** @@ -31,5 +33,6 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG + SignConstants.DOT + ConfigConstants.FunctionModuleConstants.WAREHOUSE) +@EnableConfigurationProperties(GreptimeProperties.class) public class WarehouseAutoConfiguration { } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java index fc0c325e40..d186941601 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java @@ -72,6 +72,8 @@ public interface WarehouseConstants { String LOG_TABLE_NAME = "hertzbeat_logs"; + String TRACE_TABLE_NAME = "hertzbeat_traces"; + String GREPTIME_QUERY_REST_TEMPLATE = "greptimeQueryRestTemplate"; String GREPTIME_WRITE_REST_TEMPLATE = "greptimeWriteRestTemplate"; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java similarity index 56% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java index a33e861c00..a9d27b8861 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java @@ -15,27 +15,17 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.warehouse.service; -/** - * Adapter interface for ingesting logs pushed via different protocols - * (e.g. OTLP, Loki, Filebeat, Vector). - * Implementations should: - * 1. Parse raw HTTP payload of their protocol. - * 2. Convert data to LogEntry. - * 3. Forward / persist it to downstream pipeline. - */ -public interface LogProtocolAdapter { +/** Storage boundary for validated OTLP protobuf signals. */ +public interface OtlpSignalStorage { /** - * Ingest log payload pushed from external system. + * Persist one validated OTLP export request. * - * @param content raw request body string + * @param signal metrics, logs, or traces + * @param content encoded OTLP protobuf request + * @return encoded OTLP protobuf response */ - void ingest(String content); - - /** - * Identifier of the protocol this adapter supports ("otlp", "vector", etc.) - */ - String supportProtocol(); -} \ No newline at end of file + byte[] writeProtobuf(String signal, byte[] content); +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java index 5b5409f252..8c1f0fb4a0 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.warehouse.service; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsConsole; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsInventory; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java similarity index 97% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java index dd85d7d2c5..5eb1b4ddc6 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.warehouse.service.impl; import java.net.URI; import java.time.Instant; @@ -40,9 +40,9 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanEvent; import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.beans.factory.annotation.Qualifier; @@ -63,13 +63,14 @@ import tools.jackson.core.type.TypeReference; @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService { - private static final String TRACE_TABLE = "hzb_traces"; + private static final String TRACE_TABLE = WarehouseConstants.TRACE_TABLE_NAME; private static final Pattern SAFE_IDENTIFIER = Pattern.compile("[A-Za-z_:][A-Za-z0-9_:.-]*"); private static final Pattern SAFE_LABEL = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); private static final Set AGGREGATIONS = Set.of("sum", "avg", "min", "max", "count"); private static final long DEFAULT_WINDOW_MILLIS = 30 * 60 * 1000L; private static final int DEFAULT_STEP_SECONDS = 30; private static final int MAX_PAGE_SIZE = 200; + private static final long MAX_DURATION_MS = Long.MAX_VALUE / 1_000_000L; private final GreptimeProperties greptimeProperties; private final GreptimeSqlQueryExecutor sqlQueryExecutor; private final RestTemplate restTemplate; @@ -178,7 +179,7 @@ public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService } String sql = "SELECT *, COUNT(*) OVER () AS total_count FROM (" + grouped + ") trace_page ORDER BY start_time DESC LIMIT " + effectiveSize - + " OFFSET " + effectivePage * effectiveSize; + + " OFFSET " + (long) effectivePage * effectiveSize; List> rows = sqlQueryExecutor.execute(sql); long total = rows.isEmpty() ? 0 : asLong(rows.getFirst().get("total_count")); return new SignalPage<>(rows.stream().map(this::toTraceListItem).toList(), effectivePage, effectiveSize, total); @@ -335,14 +336,18 @@ public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService addFlattenedResourceFilter(filters, "resource_attributes.service.namespace", serviceNamespace); addFlattenedResourceFilter(filters, "resource_attributes.deployment.environment.name", environment); if (minDurationMs != null) { - filters.add("duration_nano >= " + Math.max(0, minDurationMs) * 1_000_000L); + filters.add("duration_nano >= " + clampDurationMs(minDurationMs) * 1_000_000L); } if (maxDurationMs != null) { - filters.add("duration_nano <= " + Math.max(0, maxDurationMs) * 1_000_000L); + filters.add("duration_nano <= " + clampDurationMs(maxDurationMs) * 1_000_000L); } return String.join(" AND ", filters); } + private long clampDurationMs(long durationMs) { + return Math.min(Math.max(0, durationMs), MAX_DURATION_MS); + } + private void addTextFilter(List filters, String column, String value) { if (StringUtils.hasText(value)) { filters.add(column + " = '" + escapeSql(value) + "'"); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java index d0132de5b1..5ccbe4b84a 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java @@ -101,6 +101,10 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { private static final String LABEL_KEY_END_TIME = "end"; private static final String LABEL_KEY_TS = "ts"; private static final int LOG_BATCH_SIZE = 500; + private static final Map OTLP_RESOURCE_KEY_ALIASES = Map.of( + "service_name", "service.name", + "service_namespace", "service.namespace", + "deployment_environment_name", "deployment.environment.name"); private GreptimeDB greptimeDb; @@ -756,7 +760,7 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { addJsonCondition(resourceConditions, "deployment.environment.name", filter.environment()); if (StringUtils.hasText(filter.resourceFilter())) { for (ResourceFilterExpression.Clause clause : ResourceFilterExpression.parse(filter.resourceFilter())) { - String attribute = "json_get_string(resource, '$[\"" + clause.key() + "\"]')"; + String attribute = resourceAttributeExpression(clause.key()); switch (clause.operator()) { case EQUALS -> resourceConditions.add(attribute + " = '" + safeString(clause.value()) + "'"); case NOT_EQUALS -> resourceConditions.add(attribute + " <> '" + safeString(clause.value()) + "'"); @@ -774,10 +778,20 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { private void addJsonCondition(List conditions, String key, String value) { if (StringUtils.hasText(value)) { - conditions.add("json_get_string(resource, '$[\"" + key + "\"]') = '" + safeString(value) + "'"); + conditions.add(resourceAttributeExpression(key) + " = '" + safeString(value) + "'"); } } + private String resourceAttributeExpression(String key) { + String normalizedKey = key.replace('.', '_'); + String canonical = "json_get_string(resource, '$[\"" + key + "\"]')"; + if (normalizedKey.equals(key)) { + return canonical; + } + String normalized = "json_get_string(resource, '$[\"" + normalizedKey + "\"]')"; + return "COALESCE(" + canonical + ", " + normalized + ")"; + } + private static long msToNs(Long ms) { return ms * 1_000_000L; } @@ -857,7 +871,8 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { Object bodyObj = parseJsonMaybe(row.get("body")); Map attributes = castToMap(parseJsonMaybe(row.get("attributes"))); - Map resource = castToMap(parseJsonMaybe(row.get("resource"))); + Map resource = canonicalizeOtlpResource( + castToMap(parseJsonMaybe(row.get("resource")))); LogEntry entry = LogEntry.builder() .timeUnixNano(castToLong(row.get("time_unix_nano"))) @@ -913,6 +928,19 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { return null; } + private static Map canonicalizeOtlpResource(Map resource) { + if (resource == null || resource.isEmpty()) { + return resource; + } + Map canonical = new LinkedHashMap<>(resource); + OTLP_RESOURCE_KEY_ALIASES.forEach((normalized, dotted) -> { + if (!canonical.containsKey(dotted) && resource.containsKey(normalized)) { + canonical.put(dotted, resource.get(normalized)); + } + }); + return canonical; + } + private static Long castToLong(Object obj) { if (obj == null) return null; if (obj instanceof Number n) return n.longValue(); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java new file mode 100644 index 0000000000..654eb5e21b --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.http.HttpStatusCode; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +/** GreptimeDB storage implementation for validated OTLP protobuf requests. */ +@Slf4j +@Service +@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") +public class GreptimeOtlpSignalStorage implements OtlpSignalStorage { + + private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf"); + private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name"; + private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name"; + private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name"; + private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name"; + private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name"; + private static final String GREPTIME_PROMOTE_RESOURCE_HEADER = + "X-Greptime-OTLP-Metric-Promote-Resource-Attrs"; + private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of( + "service.name", "service.namespace", "service.version", "deployment.environment.name", + "host.name", "k8s.namespace.name", "k8s.pod.name")); + private static final Set SIGNALS = Set.of("metrics", "logs", "traces"); + /** + * 4xx codes that still mean "come back later" rather than "this payload is wrong", so the + * batch must keep its retryable semantics instead of being dropped by the exporter. + */ + private static final Set RETRYABLE_CLIENT_ERRORS = Set.of(408, 425, 429); + /** Upper bound on how much of a GreptimeDB error body travels back to the exporter. */ + private static final int MAX_REJECTION_DETAIL_LENGTH = 200; + + private final GreptimeProperties greptimeProperties; + private final RestTemplate restTemplate; + + public GreptimeOtlpSignalStorage(GreptimeProperties greptimeProperties, + @Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE) + RestTemplate restTemplate) { + this.greptimeProperties = greptimeProperties; + this.restTemplate = restTemplate; + } + + @Override + public byte[] writeProtobuf(String signal, byte[] content) { + String normalizedSignal = normalizeSignal(signal); + HttpHeaders headers = greptimeHeaders(normalizedSignal); + ResponseEntity response; + try { + response = restTemplate.exchange( + endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal), + HttpMethod.POST, + new HttpEntity<>(content == null ? new byte[0] : content, headers), + byte[].class); + } catch (HttpClientErrorException exception) { + if (isRetryable(exception.getStatusCode())) { + // Let it stay a RestClientException so the ingestion boundary answers with a retryable + // 503 and the exporter replays the batch instead of discarding it. + throw exception; + } + // Any other 4xx means the payload itself was rejected: surface it as a client error instead of + // a retryable storage failure so exporters stop retrying a request that can never succeed. + // The full body only goes to the log; the exporter gets a bounded excerpt. + log.warn("GreptimeDB rejected OTLP {} with {}: {}", normalizedSignal, + exception.getStatusCode().value(), exception.getResponseBodyAsString(StandardCharsets.UTF_8)); + throw new IllegalArgumentException(rejectionMessage(normalizedSignal, exception), exception); + } + if (!response.getStatusCode().is2xxSuccessful()) { + throw new RestClientException("GreptimeDB returned unexpected OTLP " + normalizedSignal + + " status " + response.getStatusCode().value()); + } + // 3xx, 5xx, and transport failures remain RestClientException and keep their retryable semantics. + return response.getBody() == null ? new byte[0] : response.getBody(); + } + + private static boolean isRetryable(HttpStatusCode statusCode) { + return RETRYABLE_CLIENT_ERRORS.contains(statusCode.value()); + } + + private static String rejectionMessage(String signal, HttpClientErrorException exception) { + String detail = exception.getResponseBodyAsString(StandardCharsets.UTF_8); + if (!StringUtils.hasText(detail)) { + detail = exception.getStatusText(); + } + return "GreptimeDB rejected OTLP " + signal + " (" + exception.getStatusCode().value() + "): " + + truncate(detail.strip()); + } + + /** + * GreptimeDB error bodies can carry long internal detail, and this text ends up in a + * {@code google.rpc.Status} message and in a grpc status description, where an oversized value + * can overflow the trailer budget and cost the client the error itself. + */ + private static String truncate(String detail) { + return detail.length() <= MAX_REJECTION_DETAIL_LENGTH + ? detail : detail.substring(0, MAX_REJECTION_DETAIL_LENGTH) + "..."; + } + + private HttpHeaders greptimeHeaders(String signal) { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(PROTOBUF); + headers.setAccept(List.of(PROTOBUF)); + headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database()) + ? greptimeProperties.database() : "public"); + if ("metrics".equals(signal)) { + headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES); + } else if ("traces".equals(signal)) { + headers.set(GREPTIME_TRACE_TABLE_HEADER, WarehouseConstants.TRACE_TABLE_NAME); + headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1"); + } else { + headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME); + headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1"); + } + if (StringUtils.hasText(greptimeProperties.username()) + && StringUtils.hasText(greptimeProperties.password())) { + headers.setBasicAuth(greptimeProperties.username(), greptimeProperties.password(), + StandardCharsets.UTF_8); + } + return headers; + } + + private String normalizeSignal(String signal) { + String normalized = StringUtils.hasText(signal) ? signal.toLowerCase(Locale.ROOT) : ""; + if (!SIGNALS.contains(normalized)) { + throw new IllegalArgumentException("Unsupported OTLP signal"); + } + return normalized; + } + + private String endpoint(String base, String path) { + return StringUtils.trimTrailingCharacter(base, '/') + path; + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java similarity index 94% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java index 24c8725457..2db940ea0f 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import java.nio.charset.StandardCharsets; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -43,7 +42,7 @@ import org.springframework.web.client.RestTemplate; @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeSignalInitializer { - private static final String TRACE_SCHEMA = "greptime/tables/hzb_traces.sql"; + private static final String TRACE_SCHEMA = "greptime/tables/hertzbeat_traces.sql"; private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql"; private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml"; private final GreptimeProperties greptimeProperties; @@ -65,9 +64,9 @@ public class GreptimeSignalInitializer { String traceSchema = new ClassPathResource(TRACE_SCHEMA) .getContentAsString(StandardCharsets.UTF_8).strip(); sqlQueryExecutor.execute(StringUtils.trimTrailingCharacter(traceSchema, ';')); - sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " + sqlQueryExecutor.execute("ALTER TABLE " + WarehouseConstants.TRACE_TABLE_NAME + " ADD COLUMN IF NOT EXISTS " + "\"resource_attributes.service.namespace\" STRING NULL"); - sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " + sqlQueryExecutor.execute("ALTER TABLE " + WarehouseConstants.TRACE_TABLE_NAME + " ADD COLUMN IF NOT EXISTS " + "\"resource_attributes.deployment.environment.name\" STRING NULL"); String logSchema = new ClassPathResource(LOG_SCHEMA) .getContentAsString(StandardCharsets.UTF_8).strip(); diff --git a/hertzbeat-log/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml b/hertzbeat-warehouse/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml similarity index 100% rename from hertzbeat-log/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml rename to hertzbeat-warehouse/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml diff --git a/hertzbeat-log/src/main/resources/greptime/tables/hertzbeat_logs.sql b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_logs.sql similarity index 100% rename from hertzbeat-log/src/main/resources/greptime/tables/hertzbeat_logs.sql rename to hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_logs.sql diff --git a/hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql similarity index 97% rename from hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql rename to hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql index c0d62a0a44..0a1fdeae4b 100644 --- a/hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql +++ b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql @@ -13,7 +13,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -CREATE TABLE IF NOT EXISTS hzb_traces ( +CREATE TABLE IF NOT EXISTS hertzbeat_traces ( "timestamp" TIMESTAMP(9) TIME INDEX, "timestamp_end" TIMESTAMP(9) NULL, "duration_nano" BIGINT UNSIGNED NULL, diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java index 5a377ac196..563091104b 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.warehouse.service.impl; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java index 3de76446b6..91e4ccac62 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java @@ -53,6 +53,7 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.arrow.ArrowCell; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; +import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; @@ -296,6 +297,38 @@ class GreptimeDbDataStorageTest { } } + @Test + void shouldQueryNormalizedOtlpResourceKeysAndRestoreCanonicalNames() { + try (MockedStatic mockedStatic = mockStatic(GreptimeDB.class)) { + mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb); + greptimeDbDataStorage = new GreptimeDbDataStorage( + greptimeProperties, restTemplate, greptimeSqlQueryExecutor); + Map row = new HashMap<>(); + row.put("time_unix_nano", System.nanoTime()); + row.put("severity_text", "INFO"); + row.put("body", "OTLP resource proof"); + row.put("resource", Map.of( + "service_name", "checkout-api", + "service_namespace", "storefront", + "deployment_environment_name", "preview")); + when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(List.of(row)); + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + + List result = greptimeDbDataStorage.queryObservabilityLogs( + new LogQueryFilter(null, null, null, null, null, null, null, + "checkout-api", "storefront", "preview", null), + 0, 20); + + verify(greptimeSqlQueryExecutor).execute(sqlCaptor.capture()); + assertTrue(sqlCaptor.getValue().contains("service_name")); + assertTrue(sqlCaptor.getValue().contains("service_namespace")); + assertTrue(sqlCaptor.getValue().contains("deployment_environment_name")); + assertEquals("checkout-api", result.getFirst().getResource().get("service.name")); + assertEquals("storefront", result.getFirst().getResource().get("service.namespace")); + assertEquals("preview", result.getFirst().getResource().get("deployment.environment.name")); + } + } + @Test void testBatchDeleteLogsWithValidList() { diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java new file mode 100644 index 0000000000..6b2a70b782 --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +@ExtendWith(MockitoExtension.class) +class GreptimeOtlpSignalStorageTest { + + @Mock + private RestTemplate restTemplate; + + private GreptimeOtlpSignalStorage storage; + + @BeforeEach + void setUp() { + storage = new GreptimeOtlpSignalStorage( + new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000/", + "public", "greptime", "secret"), + restTemplate); + } + + @Test + void shouldWriteMetricsWithResourcePromotionAndAuthentication() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenReturn(ResponseEntity.ok("response".getBytes(StandardCharsets.UTF_8))); + + byte[] response = storage.writeProtobuf("metrics", new byte[] {1, 2}); + + ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), + request.capture(), eq(byte[].class)); + assertThat(request.getValue().getHeaders().getFirst( + "X-Greptime-OTLP-Metric-Promote-Resource-Attrs")) + .contains("service.name", "deployment.environment.name"); + assertThat(request.getValue().getHeaders().getFirst("Authorization")).startsWith("Basic "); + assertThat(request.getValue().getBody()).containsExactly(1, 2); + assertThat(response).isEqualTo("response".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void shouldSelectWarehouseOwnedTraceAndLogSchemas() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenReturn(ResponseEntity.ok(new byte[0])); + + storage.writeProtobuf("traces", new byte[0]); + storage.writeProtobuf("logs", new byte[0]); + + ArgumentCaptor> requests = ArgumentCaptor.forClass(HttpEntity.class); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), + requests.capture(), eq(byte[].class)); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/logs"), eq(HttpMethod.POST), + requests.capture(), eq(byte[].class)); + assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Trace-Table-Name")) + .isEqualTo("hertzbeat_traces"); + assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Pipeline-Name")) + .isEqualTo("greptime_trace_v1"); + assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Table-Name")) + .isEqualTo("hertzbeat_logs"); + assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Pipeline-Name")) + .isEqualTo("hertzbeat_otlp_log_v1"); + } + + @Test + void shouldRejectUnsupportedSignalBeforeStorageCall() { + assertThatThrownBy(() -> storage.writeProtobuf("profiles", new byte[0])) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported OTLP signal"); + } + + @Test + void shouldSurfaceGreptimeRejectionAsClientErrorWithResponseBody() { + byte[] body = "{\"error\":\"pipeline hertzbeat_otlp_log_v1 not found\"}".getBytes(StandardCharsets.UTF_8); + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpClientErrorException.create(HttpStatus.BAD_REQUEST, "Bad Request", + new HttpHeaders(), body, StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0])) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("GreptimeDB rejected OTLP logs (400): " + + "{\"error\":\"pipeline hertzbeat_otlp_log_v1 not found\"}") + .hasCauseInstanceOf(HttpClientErrorException.class); + } + + @Test + void shouldFallBackToStatusTextWhenGreptimeRejectionHasNoBody() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpClientErrorException.create(HttpStatus.UNAUTHORIZED, "Unauthorized", + new HttpHeaders(), new byte[0], StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("metrics", new byte[0])) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("GreptimeDB rejected OTLP metrics (401): Unauthorized"); + } + + @Test + void shouldKeepServerErrorsRetryable() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpServerErrorException.create(HttpStatus.SERVICE_UNAVAILABLE, "Service Unavailable", + new HttpHeaders(), new byte[0], StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("traces", new byte[0])) + .isInstanceOf(HttpServerErrorException.class); + } + + @Test + void shouldKeepUnexpectedRedirectResponsesRetryable() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenReturn(ResponseEntity.status(HttpStatus.TEMPORARY_REDIRECT).body(new byte[0])); + + assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0])) + .isInstanceOf(RestClientException.class) + .hasMessage("GreptimeDB returned unexpected OTLP logs status 307"); + } + + /** + * 429 and 408 are 4xx codes that still mean "try again", so converting them into a client error + * would make the exporter drop a batch GreptimeDB was willing to take a moment later. + */ + @Test + void shouldKeepGreptimeBackPressureRetryable() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpClientErrorException.create(HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", + new HttpHeaders(), new byte[0], StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0])) + .isInstanceOf(HttpClientErrorException.class); + } + + @Test + void shouldKeepGreptimeRequestTimeoutRetryable() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpClientErrorException.create(HttpStatus.REQUEST_TIMEOUT, "Request Timeout", + new HttpHeaders(), new byte[0], StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("metrics", new byte[0])) + .isInstanceOf(HttpClientErrorException.class); + } + + /** + * The message travels back to the exporter inside a google.rpc.Status and, on the grpc transport, + * inside a status description that shares the trailer budget, so it must stay bounded. + */ + @Test + void shouldTruncateOversizedGreptimeRejectionDetail() { + byte[] body = ("x".repeat(5000)).getBytes(StandardCharsets.UTF_8); + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenThrow(HttpClientErrorException.create(HttpStatus.BAD_REQUEST, "Bad Request", + new HttpHeaders(), body, StandardCharsets.UTF_8)); + + assertThatThrownBy(() -> storage.writeProtobuf("logs", new byte[0])) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageEndingWith("...") + .satisfies(thrown -> assertThat(thrown.getMessage().length()).isLessThan(300)); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java similarity index 90% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java index 1d21fc7a63..7c9c1b65a8 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import org.mockito.InOrder; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -71,11 +70,11 @@ class GreptimeSignalInitializerTest { InOrder sqlOrder = org.mockito.Mockito.inOrder(sqlQueryExecutor); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( - "CREATE TABLE IF NOT EXISTS hzb_traces")); + "CREATE TABLE IF NOT EXISTS hertzbeat_traces")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( - "ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\"")); + "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\"")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( - "ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\"")); + "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\"")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( "CREATE TABLE IF NOT EXISTS hertzbeat_logs")); sqlOrder.verify(sqlQueryExecutor).execute("SELECT 1 AS ready"); diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java similarity index 95% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java index 1e29119ec4..7859196365 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import static org.assertj.core.api.Assertions.assertThat; diff --git a/home/docs/help/log_integration.md b/home/docs/help/log_integration.md index ffdd3aaaa9..b17c0cc50f 100644 --- a/home/docs/help/log_integration.md +++ b/home/docs/help/log_integration.md @@ -11,6 +11,14 @@ keywords: [open source monitoring, log integration, log management, multi-source The log integration feature is currently in Beta (experimental) stage. There may be potential defects and limitations. The feature is under active development and iteration. ::: +:::warning Upgrading from 1.8.x? The ingestion path changed +The 1.8.x endpoint `POST /api/logs/otlp/v1/logs` (and `POST /api/logs/ingest/otlp`) is replaced by `POST /api/otlp/v1/logs`. Update the `logs_endpoint` of every OpenTelemetry Collector / SDK exporter that points at HertzBeat. On 1.9.x the old paths still work as deprecated aliases (the response carries `Deprecation: true` and HertzBeat logs a warning); they are removed in 2.0. The query paths `/api/logs/**`, `/api/traces/**` and `/api/ingestion/otlp/**` moved to `/api/observability/**` with no alias. See the [Version Upgrade Guide](../start/upgrade) for the full old/new path table. +::: + +:::info HertzBeat 1.9.0 transition +Metrics, logs, and traces share `/api/otlp/v1/{signal}` for ingestion and `/api/observability/**` for queries. This release intentionally does not create or bind Entity records from telemetry. External OTLP signal tables are also separate from HertzBeat's internal self-telemetry tables. +::: + ## Core Capabilities - **Multi-source Log Integration**: Support receiving log data from mainstream platforms such as OpenTelemetry, Filebeat, Vector, Loki @@ -36,9 +44,41 @@ You can view specific integration methods and configuration examples through Her HertzBeat provides the following interface for receiving OTLP log data: ```text -POST /api/logs/otlp/v1/logs +POST /api/otlp/v1/logs ``` +### OTLP/gRPC Endpoint + +HertzBeat also runs an OTLP/gRPC listener when GreptimeDB storage is enabled, accepting metrics, logs +and traces. It expects the same `Authorization: Bearer {token}` credential as the HTTP endpoint. + +```text +{hertzbeat_host}:14317 +``` + +The port is 14317 on every deployment - the docker images publish it unchanged, so there is no +container-versus-host translation to remember. + +It is deliberately not the OpenTelemetry standard 4317: an OTel Collector, Jaeger or Tempo on the +same host normally holds that port already, and a clash on a published port stops the container from +starting at all. HertzBeat serves OTLP/HTTP on its own port too, so this is consistent with the rest +of the product rather than an exception. + +To use 4317 anyway, or to turn the listener off, set it in `application.yml` or through the matching +environment variables (and update the port mapping in `docker-compose.yaml` to match): + +```yaml +hertzbeat: + otlp: + grpc: + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} +``` + +If the port cannot be bound, HertzBeat logs the failure and starts without gRPC ingestion; OTLP/HTTP +on `/api/otlp/v1` keeps working. + ### Request Configuration #### Request Headers @@ -112,7 +152,7 @@ Add HertzBeat as a log export target in the OpenTelemetry Collector configuratio ```yaml exporters: otlphttp: - logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs + logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs compression: none encoding: json headers: diff --git a/home/docs/start/upgrade.md b/home/docs/start/upgrade.md index e7363fc424..ffa38aca9a 100644 --- a/home/docs/start/upgrade.md +++ b/home/docs/start/upgrade.md @@ -14,6 +14,92 @@ Apache HertzBeat's metadata information is stored in H2 or Mysql, PostgreSQL rel **You need to save and back up the data files of the database and monitoring templates yml files before upgrading** +## Breaking Changes In 1.9.0 + +### Observability (OTLP / logs / traces) API paths moved + +1.9.0 consolidates the 1.8.x log module into `hertzbeat-observability`. Metrics, logs and traces now share one ingestion prefix (`/api/otlp/v1/{signal}`) and one query prefix (`/api/observability/**`). Any OpenTelemetry Collector, Vector, SDK exporter, script or dashboard that was configured against a 1.8.x path must be updated. + +| 1.8.x path | 1.9.0 path | Status in 1.9.x | +|---|---|---| +| `POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs` | **Deprecated alias kept**, still works, responds with `Deprecation: true`; removed in 2.0 | +| `POST /api/logs/ingest/otlp` | `POST /api/otlp/v1/logs` | **Deprecated alias kept**, still works, responds with `Deprecation: true`; removed in 2.0 | +| `POST /api/logs/ingest/{other protocol}` | — | Removed (`400`), only `otlp` ever had an adapter | +| `GET /api/logs/list` | `GET /api/observability/logs` | Removed (`404`) | +| `GET /api/logs/stats/overview` | `GET /api/observability/logs/overview` | Removed (`404`) | +| `GET /api/logs/stats/trace-coverage` | `GET /api/observability/logs/trace-coverage` | Removed (`404`) | +| `GET /api/logs/stats/trend` | `GET /api/observability/logs/trend` | Removed (`404`) | +| `GET /api/logs/sse/subscribe` | `GET /api/observability/logs/stream` | Removed (`404`); the new route requires an authenticated `admin/user/guest` instead of anonymous access | +| `DELETE /api/logs` | `DELETE /api/observability/logs` | Removed (`404`) | +| `GET /api/traces/**` | `GET /api/observability/traces/**` | Removed (`404`) | +| `GET /api/ingestion/otlp/metrics/console` | `GET /api/observability/metrics/query` | Removed (`404`) | +| `GET /api/ingestion/otlp/metrics/inventory` | `GET /api/observability/metrics/inventory` | Removed (`404`) | + +Recommended upgrade steps: + +- Before upgrading, search your collector / exporter configuration for `/api/logs/` and change it to `/api/otlp/v1/logs`. OTLP HTTP exporters treat a `404` as a permanent error and silently drop the batch, so a stale path shows up only as "logs stopped arriving". +- If you cannot change the exporters in the same maintenance window, the two ingestion aliases above keep accepting data on 1.9.x. Watch the HertzBeat log for `Deprecated OTLP log route ... was called` warnings and migrate before 2.0. +- If you use a customised `sureness.yml`, add `/api/otlp/v1/**===post===[admin,user]` and `/api/observability/**===get===[admin,user,guest]` (see the packaged `sureness.yml`); the old `/api/logs/**`, `/api/traces/**` and `/api/ingestion/otlp/**` rules can be dropped once your exporters are migrated. + +### New OTLP/gRPC listener on port 14317 + +When `warehouse.store.greptime.enabled=true`, 1.9.0 additionally starts an OTLP/gRPC listener on +`0.0.0.0:14317` so exporters can push metrics, logs and traces over gRPC. The packaged Dockerfile and +the docker-compose files publish it unchanged, so the port is the same on every deployment. + +- **It is not the OpenTelemetry standard 4317.** An OTel Collector, Jaeger or Tempo on the same host + normally holds 4317 already, and a clash on a published port makes `docker compose up` fail + outright. HertzBeat serves OTLP/HTTP on its own port as well, so 14317 is consistent with the rest + of the product. +- Existing deployments gain one newly bound port. If your firewall or security policy enumerates + listening ports, add 14317. +- A port that cannot be bound does **not** stop HertzBeat: the failure is logged and the process + starts without gRPC ingestion, while OTLP/HTTP on `/api/otlp/v1` keeps working. +- To move the listener to 4317, or disable it, set these in `application.yml` or through the matching + environment variables, and update the docker-compose port mapping to match: + + ```yaml + hertzbeat: + otlp: + grpc: + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} + ``` + +- If you deploy with the Helm chart, note that the chart is maintained in `apache/hertzbeat-helm-chart`; + check that its release exposes 14317 before relying on gRPC ingestion there. + +### GreptimeDB signal tables renamed + +When `warehouse.store.greptime.enabled=true`, HertzBeat writes two different kinds of telemetry to GreptimeDB: the traces and logs **you** send it over OTLP, and its **own** runtime logs and traces shipped via OpenTelemetry. On 1.8.x both kinds of traces landed in the same `hzb_traces` table. 1.9.0 separates them, which renames one product table and both self-monitoring tables: + +| Data | 1.8.x table | 1.9.0 table | +|---|---|---| +| Product OTLP traces (the traces page, trace queries) | `hzb_traces` | `hertzbeat_traces` | +| Product OTLP logs (the logs page, log alerting, SQL editor) | `hertzbeat_logs` | `hertzbeat_logs` (unchanged) | +| HertzBeat internal logs (self-monitoring) | `hzb_logs` | `hzb_internal_logs` | +| HertzBeat internal traces (self-monitoring) | `hzb_traces` | `hzb_internal_traces` | + +- **The traces page will be empty for data ingested before the upgrade.** 1.9.0 creates `hertzbeat_traces` and queries only that table, so spans written to `hzb_traces` on 1.8.x are no longer visible in the UI until you copy them over. +- The product log table `hertzbeat_logs` is **not** renamed; historical logs ingested on 1.8.x remain queryable with no action needed. +- No automatic migration is performed. The old `hzb_logs` / `hzb_traces` tables are left untouched but no longer receive new data. Once 1.9.0 has created the new tables you can copy the history manually, for example: + + ```sql + INSERT INTO hzb_internal_logs SELECT * FROM hzb_logs; + ``` + + Copying traces needs more care, because `hzb_traces` holds your spans and HertzBeat's own spans mixed together. Filter by service so the self-monitoring spans do not end up in the product table: + + ```sql + -- keep only your own services; HertzBeat's self-telemetry uses service.name = 'HertzBeat' + INSERT INTO hertzbeat_traces SELECT * FROM hzb_traces WHERE service_name <> 'HertzBeat'; + INSERT INTO hzb_internal_traces SELECT * FROM hzb_traces WHERE service_name = 'HertzBeat'; + ``` + + Otherwise you can `DROP` the old tables when the retention no longer matters. +- If you have dashboards or ad-hoc SQL against `hzb_logs` / `hzb_traces`, point them at the new table names. + ## Upgrade For Docker Deploy 1. If using custom monitoring templates diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md index 9e3a29bf6d..6e5af7b36b 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md @@ -11,6 +11,14 @@ keywords: [开源监控, 日志集成, 日志管理, 多源日志] 日志集成功能目前处于 Beta（实验性）阶段，可能存在潜在缺陷和局限性。该功能正在积极开发和迭代中。 ::: +:::warning 从 1.8.x 升级？接收路径已变更 +1.8.x 的接收端点 `POST /api/logs/otlp/v1/logs`（以及 `POST /api/logs/ingest/otlp`）已由 `POST /api/otlp/v1/logs` 取代，请更新所有指向 HertzBeat 的 OpenTelemetry Collector / SDK exporter 的 `logs_endpoint`。1.9.x 仍保留旧路径作为 deprecated 别名（响应带 `Deprecation: true`，HertzBeat 日志会打印告警），2.0 将移除。查询侧的 `/api/logs/**`、`/api/traces/**`、`/api/ingestion/otlp/**` 已迁移到 `/api/observability/**`，没有别名。完整的新旧路径对照表见[版本升级指南](../start/upgrade)。 +::: + +:::info HertzBeat 1.9.0 过渡版本 +指标、日志和链路统一通过 `/api/otlp/v1/{signal}` 接收，并通过 `/api/observability/**` 查询。该版本不会根据遥测数据创建或绑定 Entity；外部 OTLP 三信号表也与 HertzBeat 自身遥测表分开存储。 +::: + ## 核心能力 - **多源日志接入**：支持从 OpenTelemetry、Filebeat、Vector、Loki 等主流平台接收日志数据 @@ -36,9 +44,34 @@ HertzBeat 当前已支持以下协议进行日志数据接入： HertzBeat 提供以下接口用于接收 OTLP 日志数据： ```text -POST /api/logs/otlp/v1/logs +POST /api/otlp/v1/logs ``` +### OTLP/gRPC 端点 + +启用 GreptimeDB 存储时，HertzBeat 同时会启动一个 OTLP/gRPC 监听器，接收指标、日志与链路。凭证与 HTTP 端点一致，同样使用 `Authorization: Bearer {token}`。 + +```text +{hertzbeat_host}:14317 +``` + +所有部署方式下都是这一个端口——Docker 镜像原样发布，不存在"容器内一个、宿主上另一个"的换算。 + +这里刻意没有使用 OpenTelemetry 标准的 4317：同机的 OTel Collector、Jaeger 或 Tempo 通常已经占着该端口，而已发布端口一旦冲突，容器会直接起不来。HertzBeat 的 OTLP/HTTP 同样走自有端口，因此这个选择与产品其余部分是一致的，并非特例。 + +如果确实想用 4317，或想关闭该监听器，可在 `application.yml` 中配置，或使用对应的环境变量（同时记得把 `docker-compose.yaml` 里的端口映射改成一致）： + +```yaml +hertzbeat: + otlp: + grpc: + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} +``` + +若端口无法绑定，HertzBeat 会记录错误并在没有 gRPC 接收能力的情况下继续启动，`/api/otlp/v1` 上的 OTLP/HTTP 不受影响。 + ### 请求配置 #### 请求头 @@ -112,7 +145,7 @@ POST /api/logs/otlp/v1/logs ```yaml exporters: otlphttp: - logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs + logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs compression: none encoding: json headers: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md index 12b328a0c9..d36d3698f2 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/upgrade.md @@ -14,6 +14,83 @@ HertzBeat 的元数据信息保存在 H2 或 Mysql, PostgreSQL 关系型数据 **升级前您需要保存备份好数据库的数据文件和监控模板文件** +## 1.9.0 不兼容变更 + +### 可观测（OTLP / 日志 / 链路）接口路径变更 + +1.9.0 将 1.8.x 的日志模块合并为 `hertzbeat-observability`，指标、日志、链路统一使用 `/api/otlp/v1/{signal}` 接收、`/api/observability/**` 查询。所有按 1.8.x 路径配置的 OpenTelemetry Collector、Vector、SDK exporter、脚本或看板都需要更新。 + +| 1.8.x 路径 | 1.9.0 路径 | 1.9.x 状态 | +|---|---|---| +| `POST /api/logs/otlp/v1/logs` | `POST /api/otlp/v1/logs` | **保留为 deprecated 别名**，仍可用，响应带 `Deprecation: true`；2.0 移除 | +| `POST /api/logs/ingest/otlp` | `POST /api/otlp/v1/logs` | **保留为 deprecated 别名**，仍可用，响应带 `Deprecation: true`；2.0 移除 | +| `POST /api/logs/ingest/{其他协议}` | — | 已移除（`400`），历史上只有 `otlp` 有适配器 | +| `GET /api/logs/list` | `GET /api/observability/logs` | 已移除（`404`） | +| `GET /api/logs/stats/overview` | `GET /api/observability/logs/overview` | 已移除（`404`） | +| `GET /api/logs/stats/trace-coverage` | `GET /api/observability/logs/trace-coverage` | 已移除（`404`） | +| `GET /api/logs/stats/trend` | `GET /api/observability/logs/trend` | 已移除（`404`） | +| `GET /api/logs/sse/subscribe` | `GET /api/observability/logs/stream` | 已移除（`404`）；新路径需要 `admin/user/guest` 登录，不再匿名放行 | +| `DELETE /api/logs` | `DELETE /api/observability/logs` | 已移除（`404`） | +| `GET /api/traces/**` | `GET /api/observability/traces/**` | 已移除（`404`） | +| `GET /api/ingestion/otlp/metrics/console` | `GET /api/observability/metrics/query` | 已移除（`404`） | +| `GET /api/ingestion/otlp/metrics/inventory` | `GET /api/observability/metrics/inventory` | 已移除（`404`） | + +建议的升级步骤： + +- 升级前在 collector / exporter 配置中搜索 `/api/logs/`，改为 `/api/otlp/v1/logs`。OTLP HTTP exporter 会把 `404` 视为永久错误并静默丢弃该批数据，路径过期的表现只是"日志突然没了"。 +- 如果无法在同一维护窗口内改完 exporter，上表两条接收别名在 1.9.x 仍然可用；请关注 HertzBeat 日志中的 `Deprecated OTLP log route ... was called` 告警并在 2.0 之前完成迁移。 +- 如果使用了自定义 `sureness.yml`，请补充 `/api/otlp/v1/**===post===[admin,user]` 与 `/api/observability/**===get===[admin,user,guest]`（参考安装包内的 `sureness.yml`）；旧的 `/api/logs/**`、`/api/traces/**`、`/api/ingestion/otlp/**` 规则在 exporter 迁移完成后即可删除。 + +### 新增 OTLP/gRPC 监听端口 14317 + +当 `warehouse.store.greptime.enabled=true` 时，1.9.0 会额外启动一个 OTLP/gRPC 监听器，绑定 `0.0.0.0:14317`，供 exporter 通过 gRPC 推送指标、日志与链路。官方 Dockerfile 与 docker-compose 原样发布该端口，因此所有部署方式下端口一致。 + +- **这里没有使用 OpenTelemetry 标准的 4317。** 同机的 OTel Collector、Jaeger 或 Tempo 通常已经占着 4317，而已发布端口一旦冲突，`docker compose up` 会直接失败。HertzBeat 的 OTLP/HTTP 同样走自有端口，因此 14317 与产品其余部分是一致的。 +- 存量部署升级后会多出一个监听端口。如果你的防火墙或安全策略按端口清单管理，请把 14317 加进去。 +- 端口绑定失败**不会**导致 HertzBeat 启动失败：失败会被记录到日志，进程在没有 gRPC 接收能力的情况下继续启动，`/api/otlp/v1` 上的 OTLP/HTTP 不受影响。 +- 如需把监听器改到 4317 或关闭它，可在 `application.yml` 中配置，或使用对应的环境变量，并同步修改 docker-compose 的端口映射： + + ```yaml + hertzbeat: + otlp: + grpc: + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} + ``` + +- 使用 Helm 部署时请注意：Chart 维护在 `apache/hertzbeat-helm-chart` 仓库，依赖 gRPC 接入前请先确认其发布版本已暴露 14317。 + +### GreptimeDB 信号表改名 + +当 `warehouse.store.greptime.enabled=true` 时，GreptimeDB 里同时存着两类遥测数据：**你**通过 OTLP 推送的日志与链路，以及 HertzBeat 通过 OpenTelemetry 写入的**自身**运行日志与链路。1.8.x 中两类链路数据落在同一张 `hzb_traces` 表里，1.9.0 将其拆开，因此一张产品表和两张自监控表都改了名： + +| 数据 | 1.8.x 表名 | 1.9.0 表名 | +|---|---|---| +| 产品 OTLP 链路（链路页面、链路查询） | `hzb_traces` | `hertzbeat_traces` | +| 产品 OTLP 日志（日志页面、日志告警、SQL 编辑器） | `hertzbeat_logs` | `hertzbeat_logs`（不变） | +| HertzBeat 自身日志（自监控） | `hzb_logs` | `hzb_internal_logs` | +| HertzBeat 自身链路（自监控） | `hzb_traces` | `hzb_internal_traces` | + +- **升级前接入的链路数据在链路页面上会是空的。** 1.9.0 只创建并查询 `hertzbeat_traces`，1.8.x 期间写入 `hzb_traces` 的 span 在手动迁移之前不会显示在界面上。 +- 产品日志表 `hertzbeat_logs` **没有**改名，1.8.x 期间接入的历史日志升级后无需任何操作即可正常查询。 +- 不做自动迁移。旧的 `hzb_logs` / `hzb_traces` 表会原样保留但不再写入新数据。在 1.9.0 建好新表后可手动迁移历史数据，例如： + + ```sql + INSERT INTO hzb_internal_logs SELECT * FROM hzb_logs; + ``` + + 迁移链路数据需要更谨慎：`hzb_traces` 里混着你的 span 和 HertzBeat 自身的 span，需按服务名过滤，避免把自监控数据灌进产品表： + + ```sql + -- 只保留业务服务；HertzBeat 自监控使用 service.name = 'HertzBeat' + INSERT INTO hertzbeat_traces SELECT * FROM hzb_traces WHERE service_name <> 'HertzBeat'; + INSERT INTO hzb_internal_traces SELECT * FROM hzb_traces WHERE service_name = 'HertzBeat'; + ``` + + 如果不需要历史数据，待保留期过后直接 `DROP` 旧表即可。 +- 如果有看板或临时 SQL 直接查询 `hzb_logs` / `hzb_traces`，请改为新表名。 + ## Docker部署方式的升级 1. 若使用了自定义监控模板 diff --git a/mcp-servers/mcp-log-server/README.md b/mcp-servers/mcp-log-server/README.md index 6b790a16bc..4821220a8b 100644 --- a/mcp-servers/mcp-log-server/README.md +++ b/mcp-servers/mcp-log-server/README.md @@ -8,7 +8,7 @@ ```sql SELECT timestamp, severity_text, body -FROM hzb_logs +FROM hzb_internal_logs WHERE <结构化过滤条件> ORDER BY timestamp DESC LIMIT <1-100> @@ -36,7 +36,7 @@ LIMIT <1-100> } ``` -当前 `hzb_logs` 表没有 `monitorId` 字段，因此本接口不提供无效的监控 ID 过滤。如果后续需要该能力，应先在 OpenTelemetry 日志写入链中定义并提取统一的监控 ID 字段。 +当前 `hzb_internal_logs` 表没有 `monitorId` 字段，因此本接口不提供无效的监控 ID 过滤。如果后续需要该能力，应先在 OpenTelemetry 日志写入链中定义并提取统一的监控 ID 字段。 ## GreptimeDB 账号 diff --git a/mcp-servers/mcp-log-server/src/main/java/org/apache/hertzbeat/mcp/log/server/service/LogService.java b/mcp-servers/mcp-log-server/src/main/java/org/apache/hertzbeat/mcp/log/server/service/LogService.java index d87154561f..eed4310495 100644 --- a/mcp-servers/mcp-log-server/src/main/java/org/apache/hertzbeat/mcp/log/server/service/LogService.java +++ b/mcp-servers/mcp-log-server/src/main/java/org/apache/hertzbeat/mcp/log/server/service/LogService.java @@ -48,7 +48,7 @@ import java.util.Set; @Slf4j public class LogService { - private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_logs"; + private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_internal_logs"; private static final String TIMESTAMP_COLUMN = "timestamp"; private static final String SEVERITY_TEXT_COLUMN = "severity_text"; private static final String BODY_COLUMN = "body"; diff --git a/mcp-servers/mcp-log-server/src/test/java/org/apache/hertzbeat/mcp/server/service/LogServiceTest.java b/mcp-servers/mcp-log-server/src/test/java/org/apache/hertzbeat/mcp/server/service/LogServiceTest.java index 9608222888..5826625a5e 100644 --- a/mcp-servers/mcp-log-server/src/test/java/org/apache/hertzbeat/mcp/server/service/LogServiceTest.java +++ b/mcp-servers/mcp-log-server/src/test/java/org/apache/hertzbeat/mcp/server/service/LogServiceTest.java @@ -45,7 +45,7 @@ import org.springframework.test.web.client.MockRestServiceServer; */ class LogServiceTest { - private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_logs"; + private static final String BASE_QUERY = "SELECT timestamp, severity_text, body FROM hzb_internal_logs"; @Test void shouldBuildDefaultReadOnlyQuery() { @@ -70,13 +70,13 @@ class LogServiceTest { @Test void shouldBuildQueryFromValidatedFilters() { String query = LogService.buildQuery( - " error ", " x'); DELETE FROM hzb_logs; -- ", 1L, 2L, 10); + " error ", " x'); DELETE FROM hzb_internal_logs; -- ", 1L, 2L, 10); assertThat(query).isEqualTo(BASE_QUERY + " WHERE timestamp >= 1000000" + " AND timestamp <= 2000000" + " AND severity_text = 'ERROR'" - + " AND matches_term(body, 'x''); DELETE FROM hzb_logs; --')" + + " AND matches_term(body, 'x''); DELETE FROM hzb_internal_logs; --')" + " ORDER BY timestamp DESC LIMIT 10"); } diff --git a/pom.xml b/pom.xml index 1def3df5ae..4a08a84ce7 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ hertzbeat-otel hertzbeat-e2e hertzbeat-base - hertzbeat-log + hertzbeat-observability hertzbeat-ai @@ -269,7 +269,7 @@ org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability ${hertzbeat.version} diff --git a/script/application.yml b/script/application.yml index 425dd4e1f6..8e13abbd72 100644 --- a/script/application.yml +++ b/script/application.yml @@ -346,6 +346,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index f141461fb4..2528df16bd 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -246,6 +246,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml index 4fb6e2f1f5..a13913e44c 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,8 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -130,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml index 1beb96ca8d..0094eed6d0 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml @@ -84,5 +84,8 @@ services: ports: - "1157:1157" - "1158:1158" + # OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local + # OTel Collector normally holds - and a clash on a published port stops the container. + - "14317:14317" networks: - hertzbeat diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index f6aa53215a..e9b8f00d39 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -243,6 +243,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml index 4fb6e2f1f5..a13913e44c 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,8 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -130,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml index fb474162c4..321798368b 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml @@ -83,5 +83,8 @@ services: ports: - "1157:1157" - "1158:1158" + # OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local + # OTel Collector normally holds - and a clash on a published port stops the container. + - "14317:14317" networks: - hertzbeat diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index 413bdb3718..9f55355696 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -246,6 +246,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml index 4fb6e2f1f5..a13913e44c 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,8 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -130,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml index f127a4198c..45e66a1efe 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml @@ -83,5 +83,8 @@ services: ports: - "1157:1157" - "1158:1158" + # OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local + # OTel Collector normally holds - and a clash on a published port stops the container. + - "14317:14317" networks: - hertzbeat diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index 2f59761b65..7cbbc60802 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -243,6 +243,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml index 5f170b6469..37174e7ecc 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,12 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -134,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml index 0e905fda9e..011ded3475 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml @@ -109,5 +109,8 @@ services: ports: - "1157:1157" - "1158:1158" + # OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local + # OTel Collector normally holds - and a clash on a published port stops the container. + - "14317:14317" networks: - hertzbeat diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index 9189445de0..88b58c489b 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -245,6 +245,17 @@ grafana: password: admin hertzbeat: + otlp: + grpc: + # OTLP/gRPC ingestion listener, started when greptime storage is enabled. Point exporters at + # this port on every deployment - docker publishes it unchanged. + # 14317 rather than the OpenTelemetry standard 4317, which an OTel Collector on the same host + # normally holds already. Set 4317 here if you want the standard port and know it is free. + # A port that cannot be bound only disables gRPC ingestion - OTLP/HTTP on /api/otlp/v1 keeps + # working either way. + enabled: ${HERTZBEAT_OTLP_GRPC_ENABLED:true} + host: ${HERTZBEAT_OTLP_GRPC_HOST:0.0.0.0} + port: ${HERTZBEAT_OTLP_GRPC_PORT:14317} collector: mysql: # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml index 4fb6e2f1f5..a13913e44c 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,8 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -130,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml index 60fe377d0d..f1f5cfae09 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml @@ -88,5 +88,8 @@ services: ports: - "1157:1157" - "1158:1158" + # OTLP/gRPC ingestion. 14317 rather than the OpenTelemetry standard 4317, which a local + # OTel Collector normally holds - and a clash on a published port stops the container. + - "14317:14317" networks: - hertzbeat diff --git a/script/docker/server/Dockerfile b/script/docker/server/Dockerfile index d30ce13ac6..386ea01a64 100644 --- a/script/docker/server/Dockerfile +++ b/script/docker/server/Dockerfile @@ -32,7 +32,7 @@ ENV LANG=en_US.UTF-8 ADD apache-hertzbeat-1.*-docker-bin.tar.gz /opt/ -EXPOSE 1157 1158 22 +EXPOSE 1157 1158 14317 22 WORKDIR /opt/hertzbeat/ diff --git a/script/sureness.yml b/script/sureness.yml index 5f170b6469..37174e7ecc 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -84,8 +84,6 @@ resourceRole: # route forwards a raw promql expression straight to the time series database - /api/warehouse/**===get===[admin,user,guest] - /api/warehouse/query===post===[admin] - - /api/logs/otlp/**===post===[admin,user] - - /api/logs===delete===[admin] - /api/v2/alerts===post===[admin,user] - /api/status/page/**===get===[admin,user,guest] - /api/status/page/**===post===[admin,user] @@ -107,12 +105,12 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + # deprecated 1.8.x OTLP log aliases, forwarded to /api/otlp/v1/logs, removed in 2.0 + - /api/logs/otlp/**===post===[admin,user] - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # The OpenAPI document is a map of every route, parameter and model, so it is # scoped like any other administrative resource instead of being anonymous - /v3/api-docs/**===get===[admin] @@ -134,7 +132,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* # web ui resource diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.html b/web-app/src/app/routes/log/log-manage/log-manage.component.html index d70c4a7fd0..d8a48ee67a 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.html +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.html @@ -15,111 +15,189 @@ limitations under the License. --> -

-

{{ 'observability.logs.title' | i18n }}

{{ 'observability.logs.subtitle' | i18n }}

- -
- - - - - - - - - - +
+ +
+ +
- -
{{ 'observability.logs.total' | i18n }}{{ overview.totalCount || 0 }}
ERROR{{ overview.errorCount || 0 }}
WARN{{ overview.warnCount || 0 }}
{{ 'observability.logs.with-trace' | i18n }}{{ overview.traceCoverage?.withTrace || 0 }}
+ + + +
+ + + + +
+ {{ 'observability.filter.advanced' | i18n }} +
+ + + +
+
+
+ +
+ +
+
{{ 'observability.logs.total' | i18n }}{{ overview.totalCount || 0 }}
-

{{ 'observability.logs.trend' | i18n }}

{{ total }}
ERROR{{ overview.errorCount || 0 }}
-
-

{{ 'observability.logs.results' | i18n }}

{{ total }}
WARN{{ overview.warnCount || 0 }}
- {{ 'observability.logs.with-trace' | i18n }}{{ overview.traceCoverage?.withTrace || 0 }}
- {{ 'observability.column.time' | i18n }}{{ 'observability.logs.severity' | i18n }}{{ 'observability.column.service' | i18n }}{{ 'observability.logs.message' | i18n }}Trace ID + +
+
+
+
+

{{ 'observability.logs.trend' | i18n }}

+ {{ total }} +
+
+
+
+ +
+
+ +
+
+
+

{{ 'observability.logs.results' | i18n }}

+ {{ total }} +
+
+ - {{ (log.timeUnixNano || 0) / 1000000 | date : 'yyyy-MM-dd HH:mm:ss.SSS' }}{{ log.severityText || log.severityNumber || '-' }}{{ service(log) }}{{ bodyText(log) }}- - + {{ 'observability.column.time' | i18n }}{{ 'observability.logs.severity' | i18n }}{{ 'observability.column.service' | i18n }}{{ 'observability.logs.message' | i18n }}Trace ID + {{ (log.timeUnixNano || 0) / 1000000 | date : 'yyyy-MM-dd HH:mm:ss.SSS' }}{{ log.severityText || log.severityNumber || '-' }}{{ service(log) }}{{ bodyText(log) }}- + +
+ 
{{ detailJson() }}
 - - +
+ +
- -
+ + diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.less b/web-app/src/app/routes/log/log-manage/log-manage.component.less index e37a150eaf..91c729f4fd 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.less +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.less @@ -1,116 +1,44 @@ @import '~src/styles/theme'; -.signal-page { - min-height: 100%; -} - -.page-heading { - display: flex; - gap: 24px; - align-items: flex-start; - justify-content: space-between; - padding-bottom: 18px; - border-bottom: 1px solid @border-color-split; -} - -.page-heading h1 { - margin: 0 0 5px; - font-weight: 600; - font-size: 24px; -} - -.page-heading p { - margin: 0; - color: @text-color-secondary; -} - .heading-actions { display: flex; - gap: 16px; + gap: 12px; align-items: center; } .mode-switch { display: flex; - gap: 0; + gap: 2px; + padding: 3px; + background: @background-color-light; + border: 1px solid @border-color-split; + border-radius: 6px; } -.mode-switch button + button { - margin-left: -1px; +.mode-switch button { + min-width: 62px; + border: 0; + box-shadow: none; } -.query-toolbar { - display: grid; - grid-template-columns: repeat(4, minmax(150px, 1fr)); - gap: 8px; - align-items: center; - padding: 16px 0; +.logs-query-fields { + grid-template-columns: repeat(4, minmax(0, 1fr)); } -.query-toolbar app-signal-time-range { - grid-column: 1 / -1; +.logs-advanced-filter-grid { + grid-template-columns: repeat(3, minmax(0, 1fr)); } .resource-expression { width: 100%; } -.overview-strip { - display: grid; - grid-template-columns: repeat(4, 1fr); - margin-bottom: 16px; - border: 1px solid @border-color-split; -} - -.overview-strip div { - display: flex; - flex-direction: column; - gap: 5px; - padding: 10px 16px; - border-right: 1px solid @border-color-split; -} - -.overview-strip div:last-child { - border-right: 0; -} - -.overview-strip span { - color: @text-color-secondary; - font-size: 12px; -} - -.overview-strip strong { - font-size: 20px; -} - -.chart-region { - min-height: 240px; - border: 1px solid @border-color-split; +.logs-overview { + grid-template-columns: repeat(4, minmax(0, 1fr)); } .trend-chart { - height: 240px; -} - -.chart-region nz-empty { - display: block; - padding-top: 48px; -} - -.section-title { - display: flex; - gap: 8px; - align-items: center; - margin: 20px 0 10px; -} - -.section-title h2 { - margin: 0; - font-size: 16px; -} - -.section-title span { - color: @text-color-secondary; + height: 196px; } .message-cell { @@ -121,12 +49,16 @@ } .mono { - max-width: 220px; + max-width: 260px; overflow: hidden; font-family: SFMono-Regular, Consolas, monospace; text-overflow: ellipsis; } +.stream-evidence { + margin-top: 16px; +} + .drawer-actions { display: flex; gap: 8px; @@ -141,12 +73,13 @@ pre { color: #d7e0ea; background: #0d1117; } -@media (max-width: 1400px) { - .query-toolbar { - grid-template-columns: repeat(4, minmax(150px, 1fr)); + +@media (max-width: 1100px) { + .logs-query-fields { + grid-template-columns: repeat(2, minmax(0, 1fr)); } - .query-toolbar > button:last-child { - grid-column: -2 / -1; + .logs-advanced-filter-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); } } diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.ts b/web-app/src/app/routes/log/log-manage/log-manage.component.ts index 97cda52845..3c14708879 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.ts +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.ts @@ -44,6 +44,7 @@ import { } from '../../observability/signal-query-context'; import { SignalStorageGuideComponent } from '../../observability/signal-storage-guide.component'; import { SignalTimeRangeComponent } from '../../observability/signal-time-range.component'; +import { SignalWorkspaceComponent } from '../../observability/signal-workspace.component'; import { LogStreamComponent } from '../log-stream/log-stream.component'; type LogViewMode = 'query' | 'stream'; @@ -72,6 +73,7 @@ export function trendBucketMillis(value: string): number { SignalNavigationComponent, SignalStorageGuideComponent, SignalTimeRangeComponent, + SignalWorkspaceComponent, LogStreamComponent ], templateUrl: './log-manage.component.html', diff --git a/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts b/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts index b3e6908e32..08d1c7f151 100644 --- a/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts +++ b/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts @@ -54,7 +54,7 @@ describe('LogStreamComponent', () => { it('should authenticate the log stream request', () => { expect(fetchSpy).toHaveBeenCalledWith( - '/api/logs/sse/subscribe', + '/api/observability/logs/stream', jasmine.objectContaining({ method: 'GET', headers: jasmine.objectContaining({ diff --git a/web-app/src/app/routes/log/log-stream/log-stream.component.ts b/web-app/src/app/routes/log/log-stream/log-stream.component.ts index c1c0393712..61330b1aa3 100644 --- a/web-app/src/app/routes/log/log-stream/log-stream.component.ts +++ b/web-app/src/app/routes/log/log-stream/log-stream.component.ts @@ -163,7 +163,7 @@ export class LogStreamComponent implements OnInit, OnDestroy, AfterViewInit { // Build filter parameters const filterParams = this.buildFilterParams(); - const url = `/api/logs/sse/subscribe${filterParams ? `?${filterParams}` : ''}`; + const url = `/api/observability/logs/stream${filterParams ? `?${filterParams}` : ''}`; const token = this.localStorageService.getAuthorizationToken(); const headers: Record = { Accept: 'text/event-stream', diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.html b/web-app/src/app/routes/metrics/metrics-manage.component.html index 03dc7be18f..f2f3cde54f 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.html +++ b/web-app/src/app/routes/metrics/metrics-manage.component.html @@ -15,45 +15,60 @@ limitations under the License. --> -
-
-
-

{{ 'observability.metrics.title' | i18n }}

-

{{ 'observability.metrics.subtitle' | i18n }}

-
-
- - {{ series.length }} {{ 'observability.metrics.series' | i18n }} -
-
+ +
+ +
-
- - - - {{ name }} - - -
- +
-
+
{{ 'observability.metrics.total-series' | i18n }}{{ series.length }}
-
-

{{ 'observability.metrics.trend' | i18n }}

{{ series.length }} -
+
+
+
+
+

{{ 'observability.metrics.trend' | i18n }}

+ {{ series.length }} {{ 'observability.metrics.series' | i18n }} +
+
+
+
+ +
+
+ +
+
-
-
- -
- -
- -
- -
-

{{ 'observability.metrics.samples' | i18n }}

- {{ rows.length }} -
- - {{ 'observability.column.time' | i18n }}{{ 'observability.column.value' | i18n }}{{ 'observability.column.labels' | i18n }} - - - {{ row.timestamp | date : 'yyyy-MM-dd HH:mm:ss' }}{{ row.value }}{{ labelText(row.labels) }} - - - - - - -
-
{{ label.key }}
{{ label.value }}
 +
+
+

{{ 'observability.metrics.samples' | i18n }}

+ {{ rows.length }} +
+
+ + {{ 'observability.column.time' | i18n }}{{ 'observability.column.value' | i18n }}{{ 'observability.column.labels' | i18n }} -
- - -
+ + + {{ row.timestamp | date : 'yyyy-MM-dd HH:mm:ss' }}{{ row.value }}{{ labelText(row.labels) }} + + + + + + + +
+
{{ label.key }}
{{ label.value }}
 +
+ + + + diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.less b/web-app/src/app/routes/metrics/metrics-manage.component.less index 277be38e3a..7994969001 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.less +++ b/web-app/src/app/routes/metrics/metrics-manage.component.less @@ -1,95 +1,31 @@ @import '~src/styles/theme'; -.signal-page { - min-height: 100%; +.metrics-query-fields { + grid-template-columns: minmax(320px, 1fr) 150px 112px; } -.page-heading { - display: flex; - align-items: flex-start; - justify-content: space-between; - padding-bottom: 18px; - border-bottom: 1px solid @border-color-split; -} - -.page-heading h1 { - margin: 0 0 5px; - font-weight: 600; - font-size: 24px; -} - -.page-heading p, -.result-count { - margin: 0; - color: @text-color-secondary; -} - -.heading-actions { - display: flex; - gap: 16px; - align-items: center; -} - -.query-toolbar { - display: grid; - grid-template-columns: minmax(400px, 1fr) 140px auto; - gap: 8px; - align-items: center; - padding: 16px 0; -} - -.query-toolbar app-signal-time-range { - grid-column: 1 / -1; +.metrics-overview { + grid-template-columns: repeat(4, minmax(0, 1fr)); } .promql-input { width: 100%; } -.chart-region { - min-height: 240px; - background: @component-background; - border: 1px solid @border-color-split; -} - .metric-chart { - height: 240px; + height: 196px; } -.overview-strip { - display: grid; - grid-template-columns: repeat(4, 1fr); - margin-bottom: 0; - border: 1px solid @border-color-split; -} - -.overview-strip div { - display: flex; - flex-direction: column; - gap: 5px; - padding: 10px 16px; - border-right: 1px solid @border-color-split; -} - -.overview-strip div:last-child { - border-right: 0; -} - -.overview-strip span { - color: @text-color-secondary; - font-size: 12px; -} - -.overview-strip strong { - font-size: 20px; +.loading-region { + opacity: 0.55; } .series-strip { display: flex; gap: 6px; - padding: 8px 0; + padding: 8px 12px; overflow-x: auto; - border-bottom: 1px solid @border-color-split; + border-top: 1px solid @border-color-split; } .series-strip button { @@ -100,31 +36,6 @@ text-overflow: ellipsis; } -.chart-region nz-empty { - display: block; - padding-top: 60px; -} - -.loading-region { - opacity: 0.55; -} - -.section-title { - display: flex; - gap: 8px; - align-items: center; - margin: 18px 0 10px; -} - -.section-title h2 { - margin: 0; - font-size: 16px; -} - -.section-title span { - color: @text-color-secondary; -} - .mono { max-width: 640px; overflow: hidden; @@ -149,12 +60,9 @@ .detail-list dt { color: @text-color-secondary; } -@media (max-width: 1400px) { - .query-toolbar { - grid-template-columns: minmax(320px, 1fr) 130px auto; - } - .query-toolbar > button:last-child { - grid-column: -2 / -1; +@media (max-width: 1000px) { + .metrics-query-fields { + grid-template-columns: minmax(260px, 1fr) 130px 104px; } } diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.ts b/web-app/src/app/routes/metrics/metrics-manage.component.ts index a6d8dfe069..61242623dd 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.ts +++ b/web-app/src/app/routes/metrics/metrics-manage.component.ts @@ -42,6 +42,7 @@ import { } from '../observability/signal-query-context'; import { SignalStorageGuideComponent } from '../observability/signal-storage-guide.component'; import { SignalTimeRangeComponent } from '../observability/signal-time-range.component'; +import { SignalWorkspaceComponent } from '../observability/signal-workspace.component'; interface MetricTableRow { timestamp: number; @@ -64,7 +65,8 @@ interface MetricTableRow { NzAutocompleteModule, SignalNavigationComponent, SignalStorageGuideComponent, - SignalTimeRangeComponent + SignalTimeRangeComponent, + SignalWorkspaceComponent ], templateUrl: './metrics-manage.component.html', styleUrl: './metrics-manage.component.less' diff --git a/web-app/src/app/routes/observability/signal-navigation.component.ts b/web-app/src/app/routes/observability/signal-navigation.component.ts index c1bd3389fe..d60b1615b2 100644 --- a/web-app/src/app/routes/observability/signal-navigation.component.ts +++ b/web-app/src/app/routes/observability/signal-navigation.component.ts @@ -31,10 +31,12 @@ type SignalKind = 'metrics' | 'logs' | 'traces'; standalone: true, imports: [I18nPipe, RouterModule, SharedModule], template: ` -