Add more fine-grained support for selecting metric exemplars

Closes gh-49572
2026-10-01 14:59:50 +00:00 · 2026-03-12 11:58:05 +01:00
parent bf1b0f6e32
commit 5ca8e0956a
8 changed files with 219 additions and 13 deletions
@@ -530,6 +530,8 @@ If an `OtlpMetricsSender` bean is available, it will be configured on the `OtlpM
 https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exemplars[OTLP Exemplars] are also supported.
 To enable this feature, an javadoc:io.micrometer.registry.otlp.ExemplarContextProvider[] bean should be present.
 If you use xref:actuator/tracing.adoc[Micrometer Tracing], this will be auto-configured for you.
+By default, only sampled traces are selected as exemplars.
+You can control this behavior using the configprop:management.tracing.exemplars.filter[] property.



@@ -556,6 +558,9 @@ https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage[Pr
 To enable this feature, a javadoc:io.prometheus.metrics.tracer.common.SpanContext[] bean should be present.
 If you're using the deprecated Prometheus simpleclient support and want to enable that feature, a javadoc:io.prometheus.client.exemplars.tracer.common.SpanContextSupplier[] bean should be present.
 If you use {url-micrometer-tracing-docs}[Micrometer Tracing], this will be auto-configured for you, but you can always create your own if you want.
+By default, only sampled traces are selected as exemplars.
+You can control this behavior using the configprop:management.tracing.exemplars.filter[] property.
+The value `always-on` is not supported with Prometheus.
 Please check the https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage[Prometheus Docs], since this feature needs to be explicitly enabled on Prometheus' side, and it is only supported using the https://github.com/OpenObservability/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars[OpenMetrics] format.

 For ephemeral or batch jobs that may not exist long enough to be scraped, you can use https://github.com/prometheus/pushgateway[Prometheus Pushgateway] support to expose the metrics to Prometheus.
@@ -94,6 +94,43 @@ class OtlpExemplarsAutoConfigurationTests {
 		});
 	}

+	@Test
+	void otlpOutputShouldContainExemplarsWhenFilterIsAlwaysOnAndSpanIsNotSampled() {
+		this.contextRunner.withUserConfiguration(TracingConfiguration.class)
+			.withPropertyValues("management.tracing.sampling.probability=0.0",
+					"management.tracing.exemplars.filter=always-on")
+			.run((context) -> {
+				assertThat(context).hasSingleBean(ExemplarContextProvider.class);
+				ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class);
+				Observation.start("test.observation", observationRegistry).stop();
+				OtlpMeterRegistry otlpMeterRegistry = context.getBean(OtlpMeterRegistry.class);
+				TestOtlpMetricsSender metricsSender = context.getBean(TestOtlpMetricsSender.class);
+				otlpMeterRegistry.close();
+				assertThat(metricsSender.getOtlpRequest()).containsOnlyOnce("name: \"test.observation\"")
+					.containsOnlyOnce("exemplars")
+					.containsOnlyOnce("span_id")
+					.containsOnlyOnce("trace_id");
+			});
+	}
+
+	@Test
+	void otlpOutputShouldNotContainExemplarsWhenFilterIsAlwaysOff() {
+		this.contextRunner.withUserConfiguration(TracingConfiguration.class)
+			.withPropertyValues("management.tracing.exemplars.filter=always-off")
+			.run((context) -> {
+				assertThat(context).hasSingleBean(ExemplarContextProvider.class);
+				ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class);
+				Observation.start("test.observation", observationRegistry).stop();
+				OtlpMeterRegistry otlpMeterRegistry = context.getBean(OtlpMeterRegistry.class);
+				TestOtlpMetricsSender metricsSender = context.getBean(TestOtlpMetricsSender.class);
+				otlpMeterRegistry.close();
+				assertThat(metricsSender.getOtlpRequest()).containsOnlyOnce("name: \"test.observation\"")
+					.doesNotContain("exemplars")
+					.doesNotContain("span_id")
+					.doesNotContain("trace_id");
+			});
+	}
+
 	@Configuration(proxyBeanMethods = false)
 	private static final class CustomConfiguration {

@@ -32,6 +32,7 @@ import io.prometheus.metrics.tracer.common.SpanContext;
 import org.junit.jupiter.api.Test;

 import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException;
 import org.springframework.boot.micrometer.metrics.autoconfigure.MetricsAutoConfiguration;
 import org.springframework.boot.micrometer.metrics.autoconfigure.export.prometheus.PrometheusMetricsExportAutoConfiguration;
 import org.springframework.boot.micrometer.observation.autoconfigure.ObservationAutoConfiguration;
@@ -148,6 +149,33 @@ class PrometheusExemplarsAutoConfigurationTests {
 		});
 	}

+	@Test
+	void shouldFailWhenFilterIsAlwaysOn() {
+		this.contextRunner.withUserConfiguration(TracingConfiguration.class)
+			.withPropertyValues("management.tracing.exemplars.filter=always-on")
+			.run((context) -> assertThat(context).hasFailed()
+				.getFailure()
+				.rootCause()
+				.isInstanceOf(InvalidConfigurationPropertyValueException.class)
+				.hasMessageContaining(
+						"Property management.tracing.exemplars.filter with value 'always-on' is invalid: Prometheus doesn't support the 'always-on' exemplar filter."));
+	}
+
+	@Test
+	void prometheusOpenMetricsOutputShouldNotContainExemplarsWhenFilterIsAlwaysOff() {
+		this.contextRunner.withUserConfiguration(TracingConfiguration.class)
+			.withPropertyValues("management.tracing.exemplars.filter=always-off")
+			.run((context) -> {
+				assertThat(context).hasSingleBean(SpanContext.class);
+				ObservationRegistry observationRegistry = context.getBean(ObservationRegistry.class);
+				Observation.start("test.observation", observationRegistry).stop();
+				PrometheusMeterRegistry prometheusMeterRegistry = context.getBean(PrometheusMeterRegistry.class);
+				String openMetricsOutput = prometheusMeterRegistry.scrape(OpenMetricsTextFormatWriter.CONTENT_TYPE);
+				assertThat(openMetricsOutput).contains("test_observation_seconds_bucket");
+				assertThat(openMetricsOutput).doesNotContain("span_id").doesNotContain("trace_id");
+			});
+	}
+
 	@Configuration(proxyBeanMethods = false)
 	private static final class CustomConfiguration {

@@ -48,6 +48,11 @@ public class TracingProperties {
 	 */
 	private final Propagation propagation = new Propagation();

+	/**
+	 * Exemplars configuration.
+	 */
+	private final Exemplars exemplars = new Exemplars();
+
 	public Sampling getSampling() {
 		return this.sampling;
 	}
@@ -60,6 +65,10 @@ public class TracingProperties {
 		return this.propagation;
 	}

+	public Exemplars getExemplars() {
+		return this.exemplars;
+	}
+
 	public static class Sampling {

 		/**
@@ -249,4 +258,44 @@ public class TracingProperties {

 	}

+	/**
+	 * Exemplars configuration.
+	 */
+	public static class Exemplars {
+
+		/**
+		 * Filter which exemplars are selected. ALWAYS_ON is not supported when using
+		 * Prometheus.
+		 */
+		private Filter filter = Filter.SAMPLED_TRACES;
+
+		public Filter getFilter() {
+			return this.filter;
+		}
+
+		public void setFilter(Filter filter) {
+			this.filter = filter;
+		}
+
+		public enum Filter {
+
+			/**
+			 * Always select exemplars, regardless of whether the span is sampled.
+			 */
+			ALWAYS_ON,
+
+			/**
+			 * Never select exemplars.
+			 */
+			ALWAYS_OFF,
+
+			/**
+			 * Only select exemplars from sampled traces.
+			 */
+			SAMPLED_TRACES
+
+		}
+
+	}
+
 }
@@ -29,7 +29,10 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.boot.micrometer.tracing.autoconfigure.MicrometerTracingAutoConfiguration;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter;
 import org.springframework.context.annotation.Bean;
 import org.springframework.lang.Contract;
 import org.springframework.util.function.SingletonSupplier;
@@ -39,6 +42,7 @@ import org.springframework.util.function.SingletonSupplier;
 * Exemplars with Micrometer Tracing.
 *
 * @author Jonatan Ivanov
+ * @author Moritz Halbritter
 * @since 4.1.0
 */
@AutoConfiguration(
@@ -46,12 +50,14 @@ import org.springframework.util.function.SingletonSupplier;
 		after = MicrometerTracingAutoConfiguration.class)
@ConditionalOnBean(Tracer.class)
@ConditionalOnClass({ Tracer.class, ExemplarContextProvider.class })
+@EnableConfigurationProperties(TracingProperties.class)
 public final class OtlpExemplarsAutoConfiguration {

 	@Bean
 	@ConditionalOnMissingBean
-	ExemplarContextProvider exemplarContextProvider(ObjectProvider<Tracer> tracerProvider) {
-		return new LazyTracingExemplarContextProvider(tracerProvider);
+	ExemplarContextProvider exemplarContextProvider(ObjectProvider<Tracer> tracerProvider,
+			TracingProperties properties) {
+		return new LazyTracingExemplarContextProvider(tracerProvider, properties.getExemplars().getFilter());
 	}

 	/**
@@ -64,14 +70,17 @@ public final class OtlpExemplarsAutoConfiguration {

 		private final SingletonSupplier<Tracer> tracer;

-		LazyTracingExemplarContextProvider(ObjectProvider<Tracer> tracerProvider) {
+		private final Filter filter;
+
+		LazyTracingExemplarContextProvider(ObjectProvider<Tracer> tracerProvider, Filter filter) {
 			this.tracer = SingletonSupplier.of(tracerProvider::getObject);
+			this.filter = filter;
 		}

 		@Override
 		public @Nullable OtlpExemplarContext getExemplarContext() {
 			Span span = this.tracer.obtain().currentSpan();
-			if (isSampled(span)) {
+			if (isExemplar(span)) {
 				TraceContext context = span.context();
 				return new OtlpExemplarContext(context.traceId(), context.spanId());
 			}
@@ -79,10 +88,18 @@ public final class OtlpExemplarsAutoConfiguration {
 		}

 		@Contract("null -> false")
-		private boolean isSampled(@Nullable Span span) {
+		private boolean isExemplar(@Nullable Span span) {
 			if (span == null) {
 				return false;
 			}
+			return switch (this.filter) {
+				case ALWAYS_ON -> true;
+				case ALWAYS_OFF -> false;
+				case SAMPLED_TRACES -> isSampled(span);
+			};
+		}
+
+		private boolean isSampled(Span span) {
 			Boolean sampled = span.context().sampled();
 			return sampled != null && sampled;
 		}
@@ -27,7 +27,11 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException;
 import org.springframework.boot.micrometer.tracing.autoconfigure.MicrometerTracingAutoConfiguration;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter;
 import org.springframework.context.annotation.Bean;
 import org.springframework.util.function.SingletonSupplier;

@@ -36,6 +40,7 @@ import org.springframework.util.function.SingletonSupplier;
 * Micrometer Tracing.
 *
 * @author Jonatan Ivanov
+ * @author Moritz Halbritter
 * @since 4.0.0
 */
@AutoConfiguration(
@@ -43,12 +48,13 @@ import org.springframework.util.function.SingletonSupplier;
 		after = MicrometerTracingAutoConfiguration.class)
@ConditionalOnBean(Tracer.class)
@ConditionalOnClass({ Tracer.class, SpanContext.class })
+@EnableConfigurationProperties(TracingProperties.class)
 public final class PrometheusExemplarsAutoConfiguration {

 	@Bean
 	@ConditionalOnMissingBean
-	SpanContext spanContext(ObjectProvider<Tracer> tracerProvider) {
-		return new LazyTracingSpanContext(tracerProvider);
+	SpanContext spanContext(ObjectProvider<Tracer> tracerProvider, TracingProperties properties) {
+		return new LazyTracingSpanContext(tracerProvider, properties.getExemplars().getFilter());
 	}

 	/**
@@ -60,8 +66,15 @@ public final class PrometheusExemplarsAutoConfiguration {

 		private final SingletonSupplier<Tracer> tracer;

-		LazyTracingSpanContext(ObjectProvider<Tracer> tracerProvider) {
+		private final Filter filter;
+
+		LazyTracingSpanContext(ObjectProvider<Tracer> tracerProvider, Filter filter) {
+			if (filter == Filter.ALWAYS_ON) {
+				throw new InvalidConfigurationPropertyValueException("management.tracing.exemplars.filter", "always-on",
+						"Prometheus doesn't support the 'always-on' exemplar filter.");
+			}
 			this.tracer = SingletonSupplier.of(tracerProvider::getObject);
+			this.filter = filter;
 		}

 		@Override
@@ -82,14 +95,22 @@ public final class PrometheusExemplarsAutoConfiguration {
 			if (currentSpan == null) {
 				return false;
 			}
-			Boolean sampled = currentSpan.context().sampled();
-			return sampled != null && sampled;
+			return switch (this.filter) {
+				case ALWAYS_ON -> throw new UnsupportedOperationException("ALWAYS_ON filter is not supported");
+				case ALWAYS_OFF -> false;
+				case SAMPLED_TRACES -> isSampled(currentSpan);
+			};
 		}

 		@Override
 		public void markCurrentSpanAsExemplar() {
 		}

+		private boolean isSampled(Span span) {
+			Boolean sampled = span.context().sampled();
+			return sampled != null && sampled;
+		}
+
 		private @Nullable Span currentSpan() {
 			return this.tracer.obtain().currentSpan();
 		}
@@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test;

 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter;
 import org.springframework.boot.micrometer.tracing.autoconfigure.otlp.OtlpExemplarsAutoConfiguration.LazyTracingExemplarContextProvider;

 import static org.assertj.core.api.Assertions.assertThat;
@@ -62,8 +63,8 @@ class LazyTracingExemplarContextProviderTests {

 	};

-	private final LazyTracingExemplarContextProvider contextProvider = new LazyTracingExemplarContextProvider(
-			this.objectProvider);
+	private LazyTracingExemplarContextProvider contextProvider = new LazyTracingExemplarContextProvider(
+			this.objectProvider, Filter.SAMPLED_TRACES);

 	@Test
 	void whenCurrentSpanIsNullThenExemplarContextIsNull() {
@@ -146,4 +147,32 @@ class LazyTracingExemplarContextProviderTests {
 		assertThat(this.contextProvider.getExemplarContext()).isNull();
 	}

+	@Test
+	void whenFilterIsAlwaysOnAndSpanIsNotSampledThenExemplarContextIsNotNull() {
+		this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_ON);
+		Span span = mock(Span.class);
+		given(this.tracer.currentSpan()).willReturn(span);
+		TraceContext traceContext = mock(TraceContext.class);
+		given(traceContext.sampled()).willReturn(false);
+		given(span.context()).willReturn(traceContext);
+		assertThat(this.contextProvider.getExemplarContext()).isNotNull();
+	}
+
+	@Test
+	void whenFilterIsAlwaysOnAndCurrentSpanIsNullThenExemplarContextIsNull() {
+		this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_ON);
+		assertThat(this.contextProvider.getExemplarContext()).isNull();
+	}
+
+	@Test
+	void whenFilterIsAlwaysOffAndSpanIsSampledThenExemplarContextIsNull() {
+		this.contextProvider = new LazyTracingExemplarContextProvider(this.objectProvider, Filter.ALWAYS_OFF);
+		Span span = mock(Span.class);
+		given(this.tracer.currentSpan()).willReturn(span);
+		TraceContext traceContext = mock(TraceContext.class);
+		given(traceContext.sampled()).willReturn(true);
+		given(span.context()).willReturn(traceContext);
+		assertThat(this.contextProvider.getExemplarContext()).isNull();
+	}
+
 }
@@ -24,9 +24,12 @@ import org.junit.jupiter.api.Test;

 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.boot.context.properties.source.InvalidConfigurationPropertyValueException;
+import org.springframework.boot.micrometer.tracing.autoconfigure.TracingProperties.Exemplars.Filter;
 import org.springframework.boot.micrometer.tracing.autoconfigure.prometheus.PrometheusExemplarsAutoConfiguration.LazyTracingSpanContext;

 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.mockito.BDDMockito.given;
 import static org.mockito.Mockito.mock;

@@ -63,7 +66,7 @@ class LazyTracingSpanContextTests {

 	};

-	private final LazyTracingSpanContext spanContext = new LazyTracingSpanContext(this.objectProvider);
+	private LazyTracingSpanContext spanContext = new LazyTracingSpanContext(this.objectProvider, Filter.SAMPLED_TRACES);

 	@Test
 	void whenCurrentSpanIsNullThenSpanIdIsNull() {
@@ -148,4 +151,21 @@ class LazyTracingSpanContextTests {
 		assertThat(this.spanContext.isCurrentSpanSampled()).isFalse();
 	}

+	@Test
+	void whenFilterIsAlwaysOnThenConstructorThrows() {
+		assertThatExceptionOfType(InvalidConfigurationPropertyValueException.class)
+			.isThrownBy(() -> new LazyTracingSpanContext(this.objectProvider, Filter.ALWAYS_ON));
+	}
+
+	@Test
+	void whenFilterIsAlwaysOffAndSpanIsSampledThenSampledIsFalse() {
+		this.spanContext = new LazyTracingSpanContext(this.objectProvider, Filter.ALWAYS_OFF);
+		Span span = mock(Span.class);
+		given(this.tracer.currentSpan()).willReturn(span);
+		TraceContext traceContext = mock(TraceContext.class);
+		given(traceContext.sampled()).willReturn(true);
+		given(span.context()).willReturn(traceContext);
+		assertThat(this.spanContext.isCurrentSpanSampled()).isFalse();
+	}
+
 }