observability & production monitoring

// GOOD: structured, traceable, searchable
log.info("[traceId={}] | event=orderCreated | orderId={} | userId={} | amount={}",
   RequestContext.getRequestId(), order.getId(), userId, amount);
 
// GOOD: separate data fields by |
log.info("[traceId={}] | event=paymentProcessed | transId={} | status={} | duration={}ms",
   traceId, transactionId, status, duration);
 
// BAD: unstructured, hard to parse
log.info("Order " + order.getId() + " created by user " + userId);
 
// BAD: logging entire objects (PII risk, noise)
log.info("Processing request: {}", requestDTO); // ← dumps all fields

// Set once in filter → available in all logs within request
try {
    MDC.put("traceId", requestId);
    MDC.put("userId", jwt.getSubject());
    MDC.put("service", "order-service");
    chain.doFilter(request, response);
} finally {
    MDC.clear();  // thread trả về pool, không clear thì request sau "thừa kế" userId cũ
}
 
// Log4j2 pattern includes MDC automatically
// %X{traceId} %X{userId} trong log pattern

<Configuration>
   <Properties>
       <Property name="LOG_PATTERN">
           %d{yyyy-MM-dd HH:mm:ss.SSS} [%pid] [%t] %-5level %logger{36}.%M(%L) - %msg%n
       </Property>
   </Properties>
 
   <Appenders>
       <RollingFile name="File" fileName="logs/application.log"
                    filePattern="logs/application-%d{yyyy-MM-dd}-%i.log">
           <PatternLayout pattern="${LOG_PATTERN}"/>
           <Policies>
               <TimeBasedTriggeringPolicy interval="1"/>
               <SizeBasedTriggeringPolicy size="100MB"/>
           </Policies>
           <DefaultRolloverStrategy max="30"/>
       </RollingFile>
   </Appenders>
 
   <Loggers>
       <!-- Async logger for performance (cần thêm dependency com.lmax:disruptor) -->
       <AsyncRoot level="info" includeLocation="true">
           <AppenderRef ref="File"/>
       </AsyncRoot>
 
       <!-- Reduce noise from frameworks -->
       <Logger name="org.hibernate.SQL" level="info"/> <!-- đổi sang debug ở dev để xem SQL -->
       <Logger name="org.springframework.web" level="warn"/>
       <Logger name="com.netflix.eureka" level="warn"/>
   </Loggers>
</Configuration>

management:
  endpoints:
    web:
      exposure:
        include: health, info, prometheus, metrics
  endpoint:
    health:
      show-details: when-authorized
  metrics:
    tags:
      application: ${spring.application.name}
      environment: ${spring.profiles.active:dev}
    distribution:
      percentiles-histogram:
        http.server.requests: true # cần cho histogram_quantile ở phần alert
  prometheus:
    metrics:
      export:
        enabled: true # mặc định đã true khi có micrometer-registry-prometheus

@Service
public class OrderService {  // không dùng @RequiredArgsConstructor: đã có constructor tự viết
   private final MeterRegistry meterRegistry;
   private final Counter orderCreatedCounter;
   private final Timer orderProcessingTimer;
 
   public OrderService(MeterRegistry registry) {
       this.meterRegistry = registry;
       this.orderCreatedCounter = Counter.builder("orders.created")
           .description("Total orders created")
           .tag("service", "order-service")
           .register(registry);
       this.orderProcessingTimer = Timer.builder("orders.processing.duration")
           .description("Order processing time")
           .publishPercentiles(0.5, 0.95, 0.99)
           .register(registry);
   }
 
   public Order createOrder(CreateOrderDTO dto) {
       return orderProcessingTimer.record(() -> {
           Order order = processOrder(dto);
           orderCreatedCounter.increment();
           meterRegistry.counter("orders.created.by_type",
               "type", order.getType().name()).increment();
           return order;
       });
   }
}

// Counter — only goes up
Counter.builder("emails.sent")
   .tag("type", "confirmation")
   .register(registry)
   .increment();
 
// Gauge — current state
Gauge.builder("queue.size", queue, Queue::size)
   .register(registry);
 
// Timer — measure duration
Timer.builder("db.query.duration")
   .tag("query", "findByStatus")
   .publishPercentiles(0.5, 0.95, 0.99)
   .register(registry);
 
// Distribution Summary — value distribution
DistributionSummary.builder("http.request.size")
   .baseUnit("bytes")
   .publishPercentiles(0.5, 0.95)
   .register(registry)
   .record(requestBody.length);

@Component
public class ExternalServiceHealthIndicator implements HealthIndicator {
 
   @Override
   public Health health() {
       try {
           boolean reachable = checkExternalService();
           if (reachable) {
               return Health.up()
                   .withDetail("service", "payment-gateway")
                   .withDetail("responseTime", "45ms")
                   .build();
           }
           return Health.down()
               .withDetail("service", "payment-gateway")
               .withDetail("error", "Connection refused")
               .build();
       } catch (Exception e) {
           return Health.down(e).build();
       }
   }
}

management:
  tracing:
    sampling:
      probability: 0.1 # 10% in prod, 1.0 in dev
    propagation:
      type: w3c # W3C TraceContext headers
 
  # Export to collector (Spring Boot 3.x + micrometer-tracing-bridge-otel + opentelemetry-exporter-otlp)
  # Spring Boot 4 đổi thành management.opentelemetry.tracing.export.otlp.endpoint
  otlp:
    tracing:
      endpoint: http://otel-collector:4318/v1/traces
  opentelemetry:
    resource-attributes:
      deployment.environment: ${spring.profiles.active:dev}
# service.name lấy từ spring.application.name

@Service
@RequiredArgsConstructor
public class PaymentService {
   private final ObservationRegistry observationRegistry;
 
   public PaymentResult processPayment(PaymentRequest request) {
       return Observation.createNotStarted("payment.process", observationRegistry)
           .lowCardinalityKeyValue("payment.type", request.getType().name())
           .highCardinalityKeyValue("payment.id", request.getId().toString())
           .observe(() -> {
               // Auto-creates span with timing
               validate(request);
               PaymentResult result = callGateway(request);
               persistResult(result);
               return result;
           });
   }
}
 
// Or with Tracer directly
@Service
@RequiredArgsConstructor
public class ImportService {
   private final Tracer tracer;
 
   public void importBatch(List<Record> records) {
       Span span = tracer.nextSpan().name("import.batch").start();
       try (Tracer.SpanInScope ws = tracer.withSpan(span)) {
           span.tag("batch.size", String.valueOf(records.size()));
           records.forEach(this::importRecord);
           span.tag("batch.result", "success");
       } catch (Exception e) {
           span.error(e);
           throw e;
       } finally {
           span.end();
       }
   }
}

// HTTP client — auto-propagated via WebClient/RestClient instrumentation
// RabbitMQ — propagated via message headers
// Manual propagation (e.g., custom protocol): dùng bean io.micrometer.tracing.propagation.Propagator,
// đừng tự ghép header.
// traceId() chỉ là một phần; header W3C đầy đủ là traceparent = 00-<traceId>-<spanId>-<flags>
Map<String, String> headers = new HashMap<>();
Span current = tracer.currentSpan();  // có thể null nếu không nằm trong span nào
if (current != null) {
    propagator.inject(current.context(), headers, Map::put);  // điền "traceparent" (+ "tracestate")
}
// Gửi các header này kèm request; bên nhận dùng propagator.extract(...) để nối tiếp trace

# Error rate > 5% for 5 minutes
- alert: HighErrorRate
  expr: |
    sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
    / sum(rate(http_server_requests_seconds_count[5m])) > 0.05
  for: 5m
  labels:
    severity: P2
 
# P95 latency > 2 seconds
- alert: HighLatency
  expr: |
    histogram_quantile(0.95, sum by (le) (rate(http_server_requests_seconds_bucket[5m]))) > 2
  for: 5m
  labels:
    severity: P2
 
# DB connection pool exhausted
- alert: DbPoolExhausted
  expr: hikaricp_connections_pending > 0
  for: 2m
  labels:
    severity: P1
 
# Service health check down
- alert: ServiceDown
  expr: up{job="order-service"} == 0
  for: 1m
  labels:
    severity: P1
 
# JVM heap > 85%
- alert: HighMemoryUsage
  expr: |
    sum by (instance) (jvm_memory_used_bytes{area="heap"})
    / sum by (instance) (jvm_memory_max_bytes{area="heap"}) > 0.85
  for: 10m
  labels:
    severity: P3
 
# Queue growing (messages not consumed)
- alert: QueueBacklog
  expr: rabbitmq_queue_messages > 10000
  for: 10m
  labels:
    severity: P3

# Change log level without restart via Actuator
curl -X POST http://service:8080/actuator/loggers/com.tungdadev \
 -H 'Content-Type: application/json' \
 -d '{"configuredLevel": "DEBUG"}'
 
# Revert
curl -X POST http://service:8080/actuator/loggers/com.tungdadev \
 -H 'Content-Type: application/json' \
 -d '{"configuredLevel": "INFO"}'

# Via Actuator
curl http://service:8080/actuator/threaddump
 
# Look for:
# - BLOCKED threads (chờ lock; deadlock khi các thread chờ vòng tròn lẫn nhau,
#   jstack/jcmd Thread.print sẽ in "Found one Java-level deadlock")
# - WAITING on same lock (contention)
# - Many threads in same stack (bottleneck)

# Via Actuator
curl -o heapdump.hprof http://service:8080/actuator/heapdump
 
# Analyze with Eclipse MAT or VisualVM
# Look for:
# - Largest retained objects
# - Objects growing over time
# - Unclosed resources (connections, streams)