broad-announce/deploy/prometheus/rules/nats.yml

groups:
  - name: nats_resource_limits
    interval: 30s
    rules:
      # F2 (M11 NATS investigation): alert when NATS JetStream storage
      # gets close to the server-level max_storage cap. The M11 2026-06-16
      # finding was that storage silently filled to the cap (5.46 GiB
      # default) and the broker started rejecting publishes, but the
      # receive metric kept showing green. This alert would have fired
      # well before the system went red.
      #
      # Threshold: 80% of max_storage for 5 min. F1 raised max_storage
      # to 10 GiB (deploy/nats/nats.conf), so 80% = 8 GiB used. Real
      # usage after F1 should be ~1.1 GiB (ALERTS 1 GiB + DELIVERIES
      # 100 MiB + DLQ 10 MiB).
      - alert: NatsJetStreamStorageHigh
        expr: |
          (jetstream_stats_storage / on() jetstream_config_max_storage) > 0.80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "NATS JetStream storage > 80% of max_storage"
          description: |
            NATS JetStream storage is at {{ $value | humanizePercentage }} of
            max_storage. With stream-level MaxBytes caps from F1, the
            streams self-trim, so anything above 80% indicates either
            unexpected growth or a misconfigured stream. Check the
            ALERTS / DELIVERIES / DLQ stream state via the NATS
            monitoring endpoint (http://nats:8222/jsz?streams=true).

      - alert: NatsJetStreamStorageCritical
        expr: |
          (jetstream_stats_storage / on() jetstream_config_max_storage) > 0.95
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "NATS JetStream storage > 95% of max_storage"
          description: |
            NATS is about to enter the 'resource limits exceeded' state and
            start rejecting publishes. The M11 NATS investigation documents
            this failure mode in detail (M11_NATS_INVESTIGATION.md).

  - name: nats_publish_path
    interval: 30s
    rules:
      # F2: alert when ingestd's NATS publish success rate is materially
      # below the receive rate. This catches the "system looks healthy
      # but publishes are silently failing" class of bug that the M11
      # 10-min soak missed.
      - alert: IngestdNatsPublishErrorsHigh
        expr: |
          (
            sum(rate(ba_ingestd_nats_publish_total{result="error"}[5m]))
            /
            sum(rate(ba_ingestd_nats_publish_total[5m]))
          ) > 0.05
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "ingestd NATS publish error rate > 5%"
          description: |
            More than 5% of ingestd NATS publish attempts are failing.
            Check the ingestd logs and the NATS server logs. The M11
            NATS investigation is the playbook for diagnosing this.

      - alert: IngestdReceivePublishMismatch
        expr: |
          (
            sum(rate(ba_ingestd_alerts_received_total{transport="grpc",result="accepted"}[5m]))
            -
            sum(rate(ba_ingestd_nats_publish_total{result="ok"}[5m]))
          ) > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "ingestd receive rate > publish OK rate"
          description: |
            The gRPC receive path is accepting alerts at a rate more than
            100/s higher than the NATS publish path is acknowledging. This
            indicates the publish path is broken even if individual publish
            failures are within tolerance. The M11 10-min soak was a false
            positive because it only checked the receive rate.