diff --git a/documentation/spring-boot-docs/src/docs/antora/modules/reference/pages/actuator/metrics.adoc b/documentation/spring-boot-docs/src/docs/antora/modules/reference/pages/actuator/metrics.adoc index 7b9b7573d4b..a7172e80c78 100644 --- a/documentation/spring-boot-docs/src/docs/antora/modules/reference/pages/actuator/metrics.adoc +++ b/documentation/spring-boot-docs/src/docs/antora/modules/reference/pages/actuator/metrics.adoc @@ -530,6 +530,8 @@ If an `OtlpMetricsSender` bean is available, it will be configured on the `OtlpM https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exemplars[OTLP Exemplars] are also supported. To enable this feature, an javadoc:io.micrometer.registry.otlp.ExemplarContextProvider[] bean should be present. If you use xref:actuator/tracing.adoc[Micrometer Tracing], this will be auto-configured for you. +By default, only sampled traces are selected as exemplars. +You can control this behavior using the configprop:management.tracing.exemplars.filter[] property. @@ -556,6 +558,9 @@ https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage[Pr To enable this feature, a javadoc:io.prometheus.metrics.tracer.common.SpanContext[] bean should be present. If you're using the deprecated Prometheus simpleclient support and want to enable that feature, a javadoc:io.prometheus.client.exemplars.tracer.common.SpanContextSupplier[] bean should be present. If you use {url-micrometer-tracing-docs}[Micrometer Tracing], this will be auto-configured for you, but you can always create your own if you want. +By default, only sampled traces are selected as exemplars. +You can control this behavior using the configprop:management.tracing.exemplars.filter[] property. +The value `always-on` is not supported with Prometheus. Please check the https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage[Prometheus Docs], since this feature needs to be explicitly enabled on Prometheus' side, and it is only supported using the https://github.com/OpenObservability/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars[OpenMetrics] format. For ephemeral or batch jobs that may not exist long enough to be scraped, you can use https://github.com/prometheus/pushgateway[Prometheus Pushgateway] support to expose the metrics to Prometheus. diff --git a/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/OtlpExemplarsAutoConfigurationTests.java b/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/OtlpExemplarsAutoConfigurationTests.java index ac3f0161a8b..a963d0c2ede 100644 --- a/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/OtlpExemplarsAutoConfigurationTests.java +++ b/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/OtlpExemplarsAutoConfigurationTests.java @@ -94,6 +94,43 @@ class OtlpExemplarsAutoConfigurationTests { }); } + @Test + void otlpOutputShouldContainExemplarsWhenFilterIsAlwaysOnAndSpanIsNotSampled() { + this.contextRunner.withUserConfiguration(TracingConfiguration.class) + .withPropertyValues("management.tracing.sampling.probability=0.0", + "management.tracing.exemplars.filter=always-on") + .run((context) -> { + assertThat(context).hasSingleBean(ExemplarContextProvider.class); + ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class); + Observation.start("test.observation", observationRegistry).stop(); + OtlpMeterRegistry otlpMeterRegistry = context.getBean(OtlpMeterRegistry.class); + TestOtlpMetricsSender metricsSender = context.getBean(TestOtlpMetricsSender.class); + otlpMeterRegistry.close(); + assertThat(metricsSender.getOtlpRequest()).containsOnlyOnce("name: \"test.observation\"") + .containsOnlyOnce("exemplars") + .containsOnlyOnce("span_id") + .containsOnlyOnce("trace_id"); + }); + } + + @Test + void otlpOutputShouldNotContainExemplarsWhenFilterIsAlwaysOff() { + this.contextRunner.withUserConfiguration(TracingConfiguration.class) + .withPropertyValues("management.tracing.exemplars.filter=always-off") + .run((context) -> { + assertThat(context).hasSingleBean(ExemplarContextProvider.class); + ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class); + Observation.start("test.observation", observationRegistry).stop(); + OtlpMeterRegistry otlpMeterRegistry = context.getBean(OtlpMeterRegistry.class); + TestOtlpMetricsSender metricsSender = context.getBean(TestOtlpMetricsSender.class); + otlpMeterRegistry.close(); + assertThat(metricsSender.getOtlpRequest()).containsOnlyOnce("name: \"test.observation\"") + .doesNotContain("exemplars") + .doesNotContain("span_id") + .doesNotContain("trace_id"); + }); + } + @Configuration(proxyBeanMethods = false) private static final class CustomConfiguration { diff --git a/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/PrometheusExemplarsAutoConfigurationTests.java b/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/PrometheusExemplarsAutoConfigurationTests.java index b7803982284..652ec872e59 100644 --- a/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/PrometheusExemplarsAutoConfigurationTests.java +++ b/module/spring-boot-micrometer-tracing-brave/src/test/java/org/springframework/boot/micrometer/tracing/brave/autoconfigure/PrometheusExemplarsAutoConfigurationTests.java @@ -32,6 +32,7 @@ import io.prometheus.metrics.tracer.common.SpanContext; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException; import org.springframework.boot.micrometer.metrics.autoconfigure.MetricsAutoConfiguration; import org.springframework.boot.micrometer.metrics.autoconfigure.export.prometheus.PrometheusMetricsExportAutoConfiguration; import org.springframework.boot.micrometer.observation.autoconfigure.ObservationAutoConfiguration; @@ -148,6 +149,33 @@ class PrometheusExemplarsAutoConfigurationTests { }); } + @Test + void shouldFailWhenFilterIsAlwaysOn() { + this.contextRunner.withUserConfiguration(TracingConfiguration.class) + .withPropertyValues("management.tracing.exemplars.filter=always-on") + .run((context) -> assertThat(context).hasFailed() + .getFailure() + .rootCause() + .isInstanceOf(InvalidConfigurationPropertyValueException.class) + .hasMessageContaining( + "Property management.tracing.exemplars.filter with value 'always-on' is invalid: Prometheus doesn't support the 'always-on' exemplar filter.")); + } + + @Test + void prometheusOpenMetricsOutputShouldNotContainExemplarsWhenFilterIsAlwaysOff() { + this.contextRunner.withUserConfiguration(TracingConfiguration.class) + .withPropertyValues("management.tracing.exemplars.filter=always-off") + .run((context) -> { + assertThat(context).hasSingleBean(SpanContext.class); + ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class); + Observation.start("test.observation", observationRegistry).stop(); + PrometheusMeterRegistry prometheusMeterRegistry = context.getBean(PrometheusMeterRegistry.class); + String openMetricsOutput = prometheusMeterRegistry.scrape(OpenMetricsTextFormatWriter.CONTENT_TYPE); + assertThat(openMetricsOutput).contains("test_observation_seconds_bucket"); + assertThat(openMetricsOutput).doesNotContain("span_id").doesNotContain("trace_id"); + }); + } + @Configuration(proxyBeanMethods = false) private static final class CustomConfiguration { diff --git a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/TracingProperties.java b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/TracingProperties.java index 7a5378a18b0..0bda60865f4 100644 --- a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/TracingProperties.java +++ b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/TracingProperties.java @@ -48,6 +48,11 @@ public class TracingProperties { */ private final Propagation propagation = new Propagation(); + /** + * Exemplars configuration. + */ + private final Exemplars exemplars = new Exemplars(); + public Sampling getSampling() { return this.sampling; } @@ -60,6 +65,10 @@ public class TracingProperties { return this.propagation; } + public Exemplars getExemplars() { + return this.exemplars; + } + public static class Sampling { /** @@ -249,4 +258,44 @@ public class TracingProperties { } + /** + * Exemplars configuration. + */ + public static class Exemplars { + + /** + * Filter which exemplars are selected. ALWAYS_ON is not supported when using + * Prometheus. + */ + private Filter filter = Filter.SAMPLED_TRACES; + + public Filter getFilter() { + return this.filter; + } + + public void setFilter(Filter filter) { + this.filter = filter; + } + + public enum Filter { + + /** + * Always select exemplars, regardless of whether the span is sampled. + */ + ALWAYS_ON, + + /** + * Never select exemplars. + */ + ALWAYS_OFF, + + /** + * Only select exemplars from sampled traces. + */ + SAMPLED_TRACES + + } + + } + } diff --git a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/OtlpExemplarsAutoConfiguration.java b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/OtlpExemplarsAutoConfiguration.java index 848efd2b957..bc646102d2b 100644 --- a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/OtlpExemplarsAutoConfiguration.java +++ b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/OtlpExemplarsAutoConfiguration.java @@ -29,7 +29,10 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.micrometer.tracing.autoconfigure.MicrometerTracingAutoConfiguration; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter; import org.springframework.context.annotation.Bean; import org.springframework.lang.Contract; import org.springframework.util.function.SingletonSupplier; @@ -39,6 +42,7 @@ import org.springframework.util.function.SingletonSupplier; * Exemplars with Micrometer Tracing. * * @author Jonatan Ivanov + * @author Moritz Halbritter * @since 4.1.0 */ @AutoConfiguration( @@ -46,12 +50,14 @@ import org.springframework.util.function.SingletonSupplier; after = MicrometerTracingAutoConfiguration.class) @ConditionalOnBean(Tracer.class) @ConditionalOnClass({ Tracer.class, ExemplarContextProvider.class }) +@EnableConfigurationProperties(TracingProperties.class) public final class OtlpExemplarsAutoConfiguration { @Bean @ConditionalOnMissingBean - ExemplarContextProvider exemplarContextProvider(ObjectProvider tracerProvider) { - return new LazyTracingExemplarContextProvider(tracerProvider); + ExemplarContextProvider exemplarContextProvider(ObjectProvider tracerProvider, + TracingProperties properties) { + return new LazyTracingExemplarContextProvider(tracerProvider, properties.getExemplars().getFilter()); } /** @@ -64,14 +70,17 @@ public final class OtlpExemplarsAutoConfiguration { private final SingletonSupplier tracer; - LazyTracingExemplarContextProvider(ObjectProvider tracerProvider) { + private final Filter filter; + + LazyTracingExemplarContextProvider(ObjectProvider tracerProvider, Filter filter) { this.tracer = SingletonSupplier.of(tracerProvider::getObject); + this.filter = filter; } @Override public @Nullable OtlpExemplarContext getExemplarContext() { Span span = this.tracer.obtain().currentSpan(); - if (isSampled(span)) { + if (isExemplar(span)) { TraceContext context = span.context(); return new OtlpExemplarContext(context.traceId(), context.spanId()); } @@ -79,10 +88,18 @@ public final class OtlpExemplarsAutoConfiguration { } @Contract("null -> false") - private boolean isSampled(@Nullable Span span) { + private boolean isExemplar(@Nullable Span span) { if (span == null) { return false; } + return switch (this.filter) { + case ALWAYS_ON -> true; + case ALWAYS_OFF -> false; + case SAMPLED_TRACES -> isSampled(span); + }; + } + + private boolean isSampled(Span span) { Boolean sampled = span.context().sampled(); return sampled != null && sampled; } diff --git a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/PrometheusExemplarsAutoConfiguration.java b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/PrometheusExemplarsAutoConfiguration.java index a98b081e61b..8b2164d7880 100644 --- a/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/PrometheusExemplarsAutoConfiguration.java +++ b/module/spring-boot-micrometer-tracing/src/main/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/PrometheusExemplarsAutoConfiguration.java @@ -27,7 +27,11 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException; import org.springframework.boot.micrometer.tracing.autoconfigure.MicrometerTracingAutoConfiguration; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter; import org.springframework.context.annotation.Bean; import org.springframework.util.function.SingletonSupplier; @@ -36,6 +40,7 @@ import org.springframework.util.function.SingletonSupplier; * Micrometer Tracing. * * @author Jonatan Ivanov + * @author Moritz Halbritter * @since 4.0.0 */ @AutoConfiguration( @@ -43,12 +48,13 @@ import org.springframework.util.function.SingletonSupplier; after = MicrometerTracingAutoConfiguration.class) @ConditionalOnBean(Tracer.class) @ConditionalOnClass({ Tracer.class, SpanContext.class }) +@EnableConfigurationProperties(TracingProperties.class) public final class PrometheusExemplarsAutoConfiguration { @Bean @ConditionalOnMissingBean - SpanContext spanContext(ObjectProvider tracerProvider) { - return new LazyTracingSpanContext(tracerProvider); + SpanContext spanContext(ObjectProvider tracerProvider, TracingProperties properties) { + return new LazyTracingSpanContext(tracerProvider, properties.getExemplars().getFilter()); } /** @@ -60,8 +66,15 @@ public final class PrometheusExemplarsAutoConfiguration { private final SingletonSupplier tracer; - LazyTracingSpanContext(ObjectProvider tracerProvider) { + private final Filter filter; + + LazyTracingSpanContext(ObjectProvider tracerProvider, Filter filter) { + if (filter == Filter.ALWAYS_ON) { + throw new InvalidConfigurationPropertyValueException("management.tracing.exemplars.filter", "always-on", + "Prometheus doesn't support the 'always-on' exemplar filter."); + } this.tracer = SingletonSupplier.of(tracerProvider::getObject); + this.filter = filter; } @Override @@ -82,14 +95,22 @@ public final class PrometheusExemplarsAutoConfiguration { if (currentSpan == null) { return false; } - Boolean sampled = currentSpan.context().sampled(); - return sampled != null && sampled; + return switch (this.filter) { + case ALWAYS_ON -> throw new UnsupportedOperationException("ALWAYS_ON filter is not supported"); + case ALWAYS_OFF -> false; + case SAMPLED_TRACES -> isSampled(currentSpan); + }; } @Override public void markCurrentSpanAsExemplar() { } + private boolean isSampled(Span span) { + Boolean sampled = span.context().sampled(); + return sampled != null && sampled; + } + private @Nullable Span currentSpan() { return this.tracer.obtain().currentSpan(); } diff --git a/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/LazyTracingExemplarContextProviderTests.java b/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/LazyTracingExemplarContextProviderTests.java index 3c7ae921983..fbeba665762 100644 --- a/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/LazyTracingExemplarContextProviderTests.java +++ b/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/otlp/LazyTracingExemplarContextProviderTests.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.BeansException; import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter; import org.springframework.boot.micrometer.tracing.autoconfigure.otlp.OtlpExemplarsAutoConfiguration.LazyTracingExemplarContextProvider; import static org.assertj.core.api.Assertions.assertThat; @@ -62,8 +63,8 @@ class LazyTracingExemplarContextProviderTests { }; - private final LazyTracingExemplarContextProvider contextProvider = new LazyTracingExemplarContextProvider( - this.objectProvider); + private LazyTracingExemplarContextProvider contextProvider = new LazyTracingExemplarContextProvider( + this.objectProvider, Filter.SAMPLED_TRACES); @Test void whenCurrentSpanIsNullThenExemplarContextIsNull() { @@ -146,4 +147,32 @@ class LazyTracingExemplarContextProviderTests { assertThat(this.contextProvider.getExemplarContext()).isNull(); } + @Test + void whenFilterIsAlwaysOnAndSpanIsNotSampledThenExemplarContextIsNotNull() { + this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_ON); + Span span = mock(Span.class); + given(this.tracer.currentSpan()).willReturn(span); + TraceContext traceContext = mock(TraceContext.class); + given(traceContext.sampled()).willReturn(false); + given(span.context()).willReturn(traceContext); + assertThat(this.contextProvider.getExemplarContext()).isNotNull(); + } + + @Test + void whenFilterIsAlwaysOnAndCurrentSpanIsNullThenExemplarContextIsNull() { + this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_ON); + assertThat(this.contextProvider.getExemplarContext()).isNull(); + } + + @Test + void whenFilterIsAlwaysOffAndSpanIsSampledThenExemplarContextIsNull() { + this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_OFF); + Span span = mock(Span.class); + given(this.tracer.currentSpan()).willReturn(span); + TraceContext traceContext = mock(TraceContext.class); + given(traceContext.sampled()).willReturn(true); + given(span.context()).willReturn(traceContext); + assertThat(this.contextProvider.getExemplarContext()).isNull(); + } + } diff --git a/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/LazyTracingSpanContextTests.java b/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/LazyTracingSpanContextTests.java index b7a6d7e48eb..870b8c46031 100644 --- a/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/LazyTracingSpanContextTests.java +++ b/module/spring-boot-micrometer-tracing/src/test/java/org/springframework/boot/micrometer/tracing/autoconfigure/prometheus/LazyTracingSpanContextTests.java @@ -24,9 +24,12 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.BeansException; import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException; +import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter; import org.springframework.boot.micrometer.tracing.autoconfigure.prometheus.PrometheusExemplarsAutoConfiguration.LazyTracingSpanContext; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.mock; @@ -63,7 +66,7 @@ class LazyTracingSpanContextTests { }; - private final LazyTracingSpanContext spanContext = new LazyTracingSpanContext(this.objectProvider); + private LazyTracingSpanContext spanContext = new LazyTracingSpanContext(this.objectProvider, Filter.SAMPLED_TRACES); @Test void whenCurrentSpanIsNullThenSpanIdIsNull() { @@ -148,4 +151,21 @@ class LazyTracingSpanContextTests { assertThat(this.spanContext.isCurrentSpanSampled()).isFalse(); } + @Test + void whenFilterIsAlwaysOnThenConstructorThrows() { + assertThatExceptionOfType(InvalidConfigurationPropertyValueException.class) + .isThrownBy(() -> new LazyTracingSpanContext(this.objectProvider, Filter.ALWAYS_ON)); + } + + @Test + void whenFilterIsAlwaysOffAndSpanIsSampledThenSampledIsFalse() { + this.spanContext = new LazyTracingSpanContext(this.objectProvider, Filter.ALWAYS_OFF); + Span span = mock(Span.class); + given(this.tracer.currentSpan()).willReturn(span); + TraceContext traceContext = mock(TraceContext.class); + given(traceContext.sampled()).willReturn(true); + given(span.context()).willReturn(traceContext); + assertThat(this.spanContext.isCurrentSpanSampled()).isFalse(); + } + }