Metrics Monitoring

@Component
public class MeterExamples {

    private final MeterRegistry registry;
    private final AtomicInteger inFlight = new AtomicInteger();   // long-lived field: safe for a gauge

    public MeterExamples(MeterRegistry registry) {
        this.registry = registry;
        // Gauge: current state, read at scrape time.
        registry.gauge("orders.in.flight", inFlight);
    }

    void examples() {
        // Counter: things that happened.
        registry.counter("orders.placed", "channel", "web").increment();

        // Timer: how long something took (also gives you a count).
        Timer.builder("payment.call.duration")
            .register(registry)
            .record(Duration.ofMillis(180));

        // DistributionSummary: how big, for anything that is not time.
        DistributionSummary.builder("upload.size")
            .baseUnit("bytes")
            .register(registry)
            .record(52_431);
    }
}

<!-- pom.xml -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
    <groupId>io.micrometer</groupId>
    <artifactId>micrometer-registry-prometheus</artifactId>
    <scope>runtime</scope>
</dependency>

# application.yml
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus
  metrics:
    tags:
      application: ${spring.application.name}
      env: ${APP_ENV:local}

# prometheus.yml (scrape config)
scrape_configs:
  - job_name: orders-service
    metrics_path: /actuator/prometheus
    scrape_interval: 15s
    static_configs:
      - targets: ["orders-service:8080"]

# application.yml
management:
  metrics:
    distribution:
      percentiles-histogram:
        http.server.requests: true
        orders.placement.duration: true      # must match the Timer name exactly
      slo:
        http.server.requests: 100ms, 250ms, 500ms, 1s

# Useful built-in meters (names as seen in Actuator; Prometheus rewrites the dots):
#   jvm.memory.used, jvm.memory.max, jvm.gc.pause, jvm.threads.live
#   hikaricp.connections.active / .idle / .pending / .max
#   process.cpu.usage, system.cpu.usage, process.uptime
#   cache.gets{result=hit|miss}, cache.evictions
#   http.server.requests, http.client.requests

# PromQL you will actually use
# Error ratio over 5 minutes:
sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
  / sum(rate(http_server_requests_seconds_count[5m]))

# p99 latency across all instances (note the sum by (le), which is required):
histogram_quantile(0.99,
  sum by (le) (rate(http_server_requests_seconds_bucket[5m])))

# Threads waiting for a DB connection (the pool is too small if this is above 0 for long):
max(hikaricp_connections_pending)

@Service
public class OrderMetrics {

    private final MeterRegistry registry;
    private final AtomicInteger inFlight = new AtomicInteger();
    private final LongTaskTimer batchTimer;
    private final Timer placementTimer;

    public OrderMetrics(MeterRegistry registry) {
        this.registry = registry;

        Gauge.builder("orders.in.flight", inFlight, AtomicInteger::get)
            .description("Orders currently being processed")
            .register(registry);

        this.batchTimer = LongTaskTimer.builder("batch.job.active")
            .description("Batch jobs currently running")
            .register(registry);                       // use the injected registry, not Metrics.globalRegistry

        this.placementTimer = Timer.builder("orders.placement.duration")
            .description("End-to-end order placement time")
            .publishPercentileHistogram()              // aggregatable across instances
            .register(registry);
    }

    public OrderResponse place(CreateOrderRequest req, Supplier<OrderResponse> work) {
        inFlight.incrementAndGet();
        try {
            OrderResponse result = placementTimer.record(work);
            registry.counter("orders.placed", "channel", req.channel()).increment();
            registry.summary("orders.value.cents").record(req.totalCents());
            return result;
        } catch (PaymentException e) {
            registry.counter("orders.failed", "reason", "payment_failed").increment();
            throw e;
        } finally {
            inFlight.decrementAndGet();
        }
    }

    public void runBatch(Runnable job) {
        LongTaskTimer.Sample sample = batchTimer.start();
        try { job.run(); } finally { sample.stop(); }
    }
}

// Why the LongTaskTimer is created in the constructor: a field initialiser such as
//   private final LongTaskTimer t = LongTaskTimer.builder(...).register(meterRegistry);
// runs before Lombok's generated constructor assigns meterRegistry, so it would hit a null.

@Configuration
public class MetricsConfig {

    @Bean TimedAspect timedAspect(MeterRegistry registry) { return new TimedAspect(registry); }

    @Bean CountedAspect countedAspect(MeterRegistry registry) { return new CountedAspect(registry); }
}

@Service
public class ProductService {

    @Timed(value = "product.search.duration",
           description = "Product search time",
           histogram = true)
    public List<ProductResponse> search(ProductSearchRequest request) { ... }

    // Counts successes and failures separately (result and exception tags are added for you).
    @Counted(value = "product.views", description = "Product detail views")
    public ProductDetailResponse getDetail(Long id) { ... }

    public ProductDetailResponse featured() {
        return getDetail(1L);      // self-invocation: the annotation is NOT applied here
    }
}

// Boot 3 alternative: one annotation, metrics plus a trace span.
// @Bean ObservedAspect observedAspect(ObservationRegistry r) { return new ObservedAspect(r); }
// @Observed(name = "product.search", contextualName = "product-search")

@Configuration
public class MeterFilters {

    // Cap a risky tag: once more than 100 distinct values appear, stop creating new series.
    @Bean
    MeterFilter limitUriCardinality() {
        return MeterFilter.maximumAllowableTags("http.client.requests", "uri", 100, MeterFilter.deny());
    }

    // Drop a metric you do not want to pay for.
    @Bean
    MeterFilter dropNoisyMetric() {
        return MeterFilter.denyNameStartsWith("jvm.buffer");
    }
}

// Good tags:   channel=web|mobile|api   status=success|failed   region=eu-west-1
// Bad tags:    userId=48213   orderId=A-10293   path=/orders/48213   exception message text

class OrderMetricsTest {

    private final SimpleMeterRegistry registry = new SimpleMeterRegistry();
    private final OrderMetrics metrics = new OrderMetrics(registry);

    @Test
    void successfulOrderIncrementsCounterAndRecordsTiming() {
        metrics.place(new CreateOrderRequest("web", 2_500), () -> new OrderResponse("ok"));

        assertThat(registry.get("orders.placed").tag("channel", "web").counter().count())
            .isEqualTo(1.0);
        assertThat(registry.get("orders.placement.duration").timer().count())
            .isEqualTo(1);
        assertThat(registry.get("orders.in.flight").gauge().value())
            .isZero();                                  // gauge returns to 0 after completion
    }

    @Test
    void paymentFailureIsCountedAndInFlightIsReleased() {
        assertThatThrownBy(() ->
            metrics.place(new CreateOrderRequest("web", 2_500), () -> { throw new PaymentException("declined"); }))
            .isInstanceOf(PaymentException.class);

        assertThat(registry.get("orders.failed").tag("reason", "payment_failed").counter().count())
            .isEqualTo(1.0);
        assertThat(registry.get("orders.in.flight").gauge().value()).isZero();
    }
}