Micrometer và Prometheus trong Spring Boot: Metric Actuator trên Production

<!-- pom.xml -->
<!-- Actuator + Micrometer + Prometheus (Spring Boot 4.x) -->
<dependencies>
    <!-- Spring Boot Actuator: monitoring endpoints, includes Micrometer -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>

    <!-- Micrometer Registry Prometheus (Prometheus Java client 1.x) -->
    <!-- Exposes metrics in Prometheus format at /actuator/prometheus -->
    <dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
    </dependency>

    <!-- AspectJ for @Timed and @Counted -->
    <!-- Spring Boot 4 renamed spring-boot-starter-aop to spring-boot-starter-aspectj -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-aspectj</artifactId>
    </dependency>
</dependencies>

# application.yml
# Actuator configuration for production (Spring Boot 3.4+ access model)
management:
  endpoints:
    # Every endpoint is inaccessible unless granted below
    access:
      default: none
    web:
      exposure:
        # Endpoints exposed over HTTP
        # health, info, prometheus are the minimum for monitoring
        include: health,info,prometheus,metrics,env,loggers
      base-path: /actuator
  endpoint:
    health:
      access: read-only
      show-details: when-authorized
      show-components: when-authorized
    info:
      access: read-only
    prometheus:
      access: read-only
    metrics:
      access: read-only
    env:
      access: read-only
      # Mask sensitive values for anonymous callers
      show-values: when-authorized
    loggers:
      # unrestricted allows POST to change a log level at runtime
      access: unrestricted

// Securing Actuator endpoints (Spring Boot 4, Spring Security 7)
package com.example.monitoring.config;

import static org.springframework.security.web.access.IpAddressAuthorizationManager.hasIpAddress;

import org.springframework.boot.security.autoconfigure.actuate.web.servlet.EndpointRequest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.web.SecurityFilterChain;

@Configuration(proxyBeanMethods = false)
public class ActuatorSecurityConfig {

    @Bean
    SecurityFilterChain actuatorSecurityFilterChain(HttpSecurity http) throws Exception {
        return http
            .securityMatcher(EndpointRequest.toAnyEndpoint())
            .authorizeHttpRequests(auth -> auth
                // Health and info public for load balancers
                .requestMatchers(EndpointRequest.to("health", "info")).permitAll()
                // Prometheus accessible from the internal network only
                .requestMatchers(EndpointRequest.to("prometheus")).access(hasIpAddress("10.0.0.0/8"))
                // Other endpoints restricted to admins
                .anyRequest().hasRole("ACTUATOR_ADMIN")
            )
            .httpBasic(basic -> {})
            .build();
    }
}

// Custom business metrics service
package com.example.monitoring.metrics;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import org.springframework.stereotype.Service;

import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;

@Service
public class OrderMetricsService {

    // Counter for orders created
    private final Counter ordersCreatedCounter;
    // Timer to measure processing duration
    private final Timer orderProcessingTimer;
    // Atomic value for pending orders gauge
    private final AtomicInteger pendingOrdersCount = new AtomicInteger(0);

    public OrderMetricsService(MeterRegistry registry) {
        // Exported as orders_created_total: the _total suffix is added by the registry
        this.ordersCreatedCounter = Counter.builder("orders.created")
            .description("Total number of orders created")
            .tag("channel", "web")
            .register(registry);

        // Timer with histogram buckets: percentiles computed server-side in PromQL
        this.orderProcessingTimer = Timer.builder("orders.processing.duration")
            .description("Order processing duration")
            .publishPercentileHistogram()
            .register(registry);

        // Gauge linked to atomic value
        // Read on each scrape
        Gauge.builder("orders.pending.count", pendingOrdersCount, AtomicInteger::get)
            .description("Number of orders pending processing")
            .register(registry);
    }

    public void recordOrderCreated() {
        ordersCreatedCounter.increment();
        pendingOrdersCount.incrementAndGet();
    }

    public void recordOrderProcessed(Runnable processingLogic) {
        // Measures execution duration
        orderProcessingTimer.record(processingLogic);
        pendingOrdersCount.decrementAndGet();
    }

    public <T> T recordOrderProcessedWithResult(Supplier<T> processingLogic) {
        return orderProcessingTimer.record(processingLogic);
    }
}

// Automatic instrumentation with annotations
package com.example.monitoring.service;

import io.micrometer.core.annotation.Counted;
import io.micrometer.core.annotation.Timed;
import org.springframework.stereotype.Service;

@Service
public class PaymentService {

    private final PaymentValidator paymentValidator;

    public PaymentService(PaymentValidator paymentValidator) {
        this.paymentValidator = paymentValidator;
    }

    // @Timed creates a Timer
    // Measures each call and publishes count, sum, max and histogram buckets
    @Timed(
        value = "payment.process.duration",
        description = "Payment processing duration",
        histogram = true
    )
    public PaymentResult processPayment(PaymentRequest request) {
        // Call through another bean: the proxy intercepts it
        paymentValidator.validate(request);
        return executePayment(request);
    }

    // @Counted increments a counter on each call
    // Useful for discrete events
    @Counted(
        value = "payment.refunds",
        description = "Total number of refunds"
    )
    public void refundPayment(String transactionId) {
        // Refund logic
    }
}

# application.yml
# Enables TimedAspect, CountedAspect and the other Micrometer annotation aspects
management:
  observations:
    annotations:
      enabled: true

// Health indicator for payment gateway (Spring Boot 4 packages)
package com.example.monitoring.health;

import org.springframework.boot.health.contributor.Health;
import org.springframework.boot.health.contributor.HealthIndicator;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClient;

import java.time.Duration;
import java.time.Instant;

@Component
public class PaymentGatewayHealthIndicator implements HealthIndicator {

    private final RestClient restClient;
    private final String gatewayHealthUrl;

    public PaymentGatewayHealthIndicator(RestClient.Builder restClientBuilder) {
        this.restClient = restClientBuilder.build();
        this.gatewayHealthUrl = "https://api.payment-gateway.com/health";
    }

    @Override
    public Health health() {
        Instant start = Instant.now();

        try {
            // Call gateway health endpoint
            var response = restClient.get()
                .uri(gatewayHealthUrl)
                .retrieve()
                .toBodilessEntity();

            Duration responseTime = Duration.between(start, Instant.now());

            if (response.getStatusCode().is2xxSuccessful()) {
                return Health.up()
                    .withDetail("responseTime", responseTime.toMillis() + "ms")
                    .withDetail("statusCode", response.getStatusCode().value())
                    .build();
            } else {
                return Health.down()
                    .withDetail("statusCode", response.getStatusCode().value())
                    .withDetail("reason", "Unexpected status code")
                    .build();
            }
        } catch (Exception e) {
            Duration responseTime = Duration.between(start, Instant.now());

            return Health.down()
                .withDetail("error", e.getClass().getSimpleName())
                .withDetail("message", e.getMessage())
                .withDetail("responseTime", responseTime.toMillis() + "ms")
                .build();
        }
    }
}

# application.yml
# Health groups configuration for Kubernetes
management:
  endpoint:
    health:
      probes:
        # true by default in Spring Boot 4
        enabled: true
        # Also serve the groups at /livez and /readyz on the main server port
        add-additional-paths: true
      group:
        # Readiness: can the application receive traffic?
        readiness:
          include: readinessState,db
        # Custom group for critical dependencies
        critical:
          include: db,paymentGateway
          show-details: when-authorized

# kubernetes-deployment.yml
# Kubernetes probes configuration
spec:
  containers:
    - name: order-service
      livenessProbe:
        httpGet:
          path: /livez
          port: 8080
        initialDelaySeconds: 30
        periodSeconds: 10
        failureThreshold: 3
      readinessProbe:
        httpGet:
          path: /readyz
          port: 8080
        initialDelaySeconds: 10
        periodSeconds: 5
        failureThreshold: 3

# prometheus.yml
# Prometheus configuration for Spring Boot
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'spring-boot-apps'
    metrics_path: '/actuator/prometheus'
    scrape_interval: 10s
    static_configs:
      - targets:
          - 'order-service:9090'
          - 'payment-service:9090'
          - 'inventory-service:9090'
    # Relabeling to strip the port from the instance label
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        regex: '([^:]+):\d+'
        replacement: '${1}'

  # Kubernetes service discovery
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Only scrape pods with annotation
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)

# PromQL queries for JVM monitoring

# Heap memory usage, summed across heap pools
sum by (instance) (jvm_memory_used_bytes{area="heap"})

# Heap usage percentage
sum by (instance) (jvm_memory_used_bytes{area="heap"})
  / sum by (instance) (jvm_memory_max_bytes{area="heap"}) * 100

# Live threads
jvm_threads_live_threads

# Garbage collection: seconds spent paused per second
rate(jvm_gc_pause_seconds_sum[5m])

# GC pauses per minute
rate(jvm_gc_pause_seconds_count[1m]) * 60

# CPU used by the JVM process
process_cpu_usage

# Active database connections
hikaricp_connections_active

# Connection pool utilization
hikaricp_connections_active / hikaricp_connections_max * 100

// Additional JVM metrics
package com.example.monitoring.metrics;

import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.MeterBinder;
import org.springframework.stereotype.Component;

import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;

@Component
public class CustomJvmMetrics implements MeterBinder {

    @Override
    public void bindTo(MeterRegistry registry) {
        OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();

        // System load average
        Gauge.builder("system.load.average", osBean, OperatingSystemMXBean::getSystemLoadAverage)
            .description("System load average over 1 minute")
            .register(registry);

        // Available processors count
        Gauge.builder("system.cpu.count", osBean, OperatingSystemMXBean::getAvailableProcessors)
            .description("Number of available processors")
            .register(registry);

        // Application uptime
        Gauge.builder("application.uptime.seconds",
                ManagementFactory.getRuntimeMXBean(),
                bean -> bean.getUptime() / 1000.0)
            .description("Application uptime in seconds")
            .register(registry);
    }
}

{
  "annotations": {
    "list": []
  },
  "panels": [
    {
      "title": "Request Rate",
      "type": "timeseries",
      "targets": [
        {
          "expr": "sum by (method, uri, status) (rate(http_server_requests_seconds_count{application=\"$application\"}[5m]))",
          "legendFormat": "{{method}} {{uri}} - {{status}}"
        }
      ]
    },
    {
      "title": "Response Time P99",
      "type": "timeseries",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum by (le, method, uri) (rate(http_server_requests_seconds_bucket{application=\"$application\"}[5m])))",
          "legendFormat": "{{method}} {{uri}}"
        }
      ]
    },
    {
      "title": "Error Rate",
      "type": "stat",
      "targets": [
        {
          "expr": "sum(rate(http_server_requests_seconds_count{application=\"$application\",status=~\"5..\"}[5m])) / sum(rate(http_server_requests_seconds_count{application=\"$application\"}[5m])) * 100"
        }
      ]
    }
  ]
}

# alerting-rules.yml
# Alert rules for Spring Boot applications
groups:
  - name: spring-boot-alerts
    rules:
      # Alert if application is down
      - alert: ApplicationDown
        expr: up{job="spring-boot-apps"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Application {{ $labels.instance }} is down"
          description: "{{ $labels.instance }} has been down for more than 1 minute"

      # Alert on HTTP error rate
      - alert: HighErrorRate
        expr: |
          sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m])) by (application)
          /
          sum(rate(http_server_requests_seconds_count[5m])) by (application)
          > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High error rate on {{ $labels.application }}"
          description: "Error rate is {{ $value | humanizePercentage }}"

      # Alert on P99 latency, aggregated per application
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum by (le, application) (rate(http_server_requests_seconds_bucket[5m]))
          ) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High latency on {{ $labels.application }}"
          description: "P99 latency is {{ $value | humanizeDuration }}"

      # Heap memory alert
      - alert: HighHeapUsage
        expr: |
          sum by (instance) (jvm_memory_used_bytes{area="heap"})
          / sum by (instance) (jvm_memory_max_bytes{area="heap"})
          > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High heap memory usage on {{ $labels.instance }}"
          description: "Heap usage is at {{ $value | humanizePercentage }}"

      # Database connection pool exhausted alert
      - alert: DatabaseConnectionPoolExhausted
        expr: |
          hikaricp_connections_active
          / hikaricp_connections_max
          > 0.9
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Database connection pool nearly exhausted"
          description: "{{ $value | humanizePercentage }} of connections in use"

      # Excessive GC alert
      - alert: HighGCPause
        expr: |
          rate(jvm_gc_pause_seconds_sum[5m])
          / rate(jvm_gc_pause_seconds_count[5m])
          > 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High GC pause time"
          description: "Average GC pause is {{ $value | humanizeDuration }}"

# application.yml
# HTTP metrics configuration
management:
  metrics:
    distribution:
      # Histogram buckets for server-side percentiles
      percentiles-histogram:
        http.server.requests: true
      # SLO buckets: count requests faster than each boundary
      slo:
        http.server.requests: 100ms, 500ms, 1s, 2s
    tags:
      # Common tags added to every meter
      application: ${spring.application.name}
      environment: ${spring.profiles.active:default}

// Adds custom tags to http.server.requests (Spring Boot 3.0+)
package com.example.monitoring.config;

import io.micrometer.common.KeyValue;
import io.micrometer.common.KeyValues;
import org.springframework.http.server.observation.DefaultServerRequestObservationConvention;
import org.springframework.http.server.observation.ServerRequestObservationContext;
import org.springframework.stereotype.Component;

import jakarta.servlet.http.HttpServletRequest;

@Component
public class TenantObservationConvention extends DefaultServerRequestObservationConvention {

    @Override
    public KeyValues getLowCardinalityKeyValues(ServerRequestObservationContext context) {
        HttpServletRequest request = context.getCarrier();
        String userId = request.getHeader("X-User-Id");
        String tenantId = request.getHeader("X-Tenant-Id");

        // Keep the default tags and add two bounded ones
        return super.getLowCardinalityKeyValues(context).and(
            KeyValue.of("user.type", userId != null ? "authenticated" : "anonymous"),
            KeyValue.of("tenant", tenantId != null ? tenantId : "default")
        );
    }
}

# application.yml
# HikariCP pool configuration
spring:
  datasource:
    hikari:
      # Becomes the "pool" tag on hikaricp_* metrics
      pool-name: OrderServicePool
      maximum-pool-size: 20
      minimum-idle: 5
      connection-timeout: 30000
      idle-timeout: 600000
      max-lifetime: 1800000

<!-- pom.xml -->
<!-- JDBC observations: 2.x for Spring Boot 4, 1.x for Spring Boot 3 -->
<dependency>
    <groupId>net.ttddyy.observation</groupId>
    <artifactId>datasource-micrometer-spring-boot</artifactId>
    <version>2.0.1</version>
</dependency>

// AVOID: unbounded cardinality
package com.example.monitoring.antipattern;

@Service
public class AntiPatternHighCardinality {

    private final MeterRegistry registry;

    // BAD: userId creates one series per user
    public void trackUserAction(String userId, String action) {
        Counter.builder("user.actions")
            .tag("userId", userId)  // Millions of possible values
            .tag("action", action)
            .register(registry)
            .increment();
    }
}

// Controlled cardinality
package com.example.monitoring.bestpractice;

@Service
public class GoodPracticeCardinality {

    private final MeterRegistry registry;

    // GOOD: user category instead of ID
    public void trackUserAction(User user, String action) {
        Counter.builder("user.actions")
            .tag("userType", user.getSubscriptionType())  // FREE, PREMIUM, ENTERPRISE
            .tag("action", action)
            .register(registry)
            .increment();
    }

    // GOOD: let a timer histogram bucket durations
    public void trackResponseTime(long responseTimeMs) {
        Timer.builder("response.time")
            .publishPercentileHistogram()
            .register(registry)
            .record(Duration.ofMillis(responseTimeMs));
    }
}

# application-production.yml
# Production configuration (Spring Boot 4)
management:
  server:
    # Separate port for management endpoints, not exposed publicly
    port: 9090
  endpoints:
    access:
      default: none
    web:
      exposure:
        include: health,info,prometheus
  endpoint:
    health:
      access: read-only
      show-details: when-authorized
      probes:
        add-additional-paths: true
    info:
      access: read-only
    prometheus:
      access: read-only
  observations:
    annotations:
      enabled: true
  metrics:
    distribution:
      percentiles-histogram:
        http.server.requests: true
      minimum-expected-value:
        http.server.requests: 1ms
      maximum-expected-value:
        http.server.requests: 30s
    tags:
      application: ${spring.application.name}
      environment: production
      version: ${app.version:unknown}