M9(2/3): routerd latency histogram + Grafana dashboards + Prometheus scrape split

- cmd/routerd/collapse.go: time both ResolveTargets() calls; observe via routerdMetrics.RecipientExpansionLatency
- cmd/routerd/main.go: create RouterdMetrics before consume goroutine; wire into collapse
- internal/observability/metrics.go: add RouterdMetrics struct (RecipientExpansionLatency histogram)
- docker-compose.yml: add Grafana provisioning volumes (datasources + dashboards)
- deploy/grafana/provisioning/datasources/prometheus.yml: Prometheus datasource
- deploy/grafana/provisioning/dashboards/dashboards.yml: dashboard provider
- deploy/grafana/provisioning/dashboards/broad-announce-overview.json: 4-tier overview dashboard
- scripts/m9_smoke.sh: 7-step observability smoke (Prometheus scrape, metrics presence, Grafana health)
- deploy/prometheus/prometheus.yml: split deliverd job into deliverd-fcm + deliverd-telegram; add archiverd + prometheus self-monitoring

cmd/routerd/collapse.go

@@ -28,6 +28,7 @@ import (
 	"encoding/json"
 	"log/slog"
 	"sync"
+	"time"
 
 	"git3.techno-world.net/lrosales/broad-announce/internal/alert"
 	"git3.techno-world.net/lrosales/broad-announce/internal/broker"
@@ -105,7 +106,11 @@ func observeAndFanout(
 ) (handled bool, shouldAck bool, shouldNak bool) {
 	// Empty dedupe_key → M2 path, no collapse.
 	if a.DedupeKey == "" {
+		t0 := time.Now()
 		targets, err := resolver.ResolveTargets(ctx, &a)
+		if routerdMetrics != nil {
+			routerdMetrics.RecipientExpansionLatency.Observe(time.Since(t0).Seconds())
+		}
 		if err != nil {
 			logger.Error("resolve targets", "err", err, "alert_id", a.ID, "company", companyID)
 			return true, false, true
@@ -129,7 +134,11 @@ func observeAndFanout(
 	switch dec {
 	case dedupe.CollapseNew:
 		// First arrival: resolve targets NOW and cache them.
+		t0 := time.Now()
 		targets, err := resolver.ResolveTargets(ctx, &a)
+		if routerdMetrics != nil {
+			routerdMetrics.RecipientExpansionLatency.Observe(time.Since(t0).Seconds())
+		}
 		if err != nil {
 			logger.Error("resolve targets (new collapse)",
 				"err", err,

cmd/routerd/main.go

@@ -34,6 +34,10 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )
 
+// routerdMetrics is the package-level routerd metrics instance.
+// Set in main() before the consume goroutine starts.
+var routerdMetrics *observability.RouterdMetrics
+
 func main() {
 	cfg, err := config.LoadRouterd()
 	if err != nil {
@@ -97,9 +101,12 @@ func main() {
 	runCtx, runCancel := context.WithCancel(ctx)
 	defer runCancel()
 
-	go consume(runCtx, logger, consumer, br, resolver, collapser, collapseState)
-
+	// M9 L1: routerd metrics. Create before the consumer starts
+	// so the /metrics endpoint is valid immediately.
 	reg, _ := observability.NewRegistry("routerd")
+	routerdMetrics = observability.NewRouterdMetrics(reg, "routerd")
+
+	go consume(runCtx, logger, consumer, br, resolver, collapser, collapseState)
 	srv := httpserver.New(httpserver.Config{
 		Addr:          cfg.HTTPAddr,
 		ServiceName:   "routerd",

deploy/grafana/provisioning/dashboards/broad-announce-overview.json

@@ -0,0 +1,461 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
+      "id": 1,
+      "panels": [],
+      "title": "Ingest tier",
+      "type": "row"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 1},
+      "id": 2,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "sum by (result) (rate(ba_ingestd_alerts_received_total[1m]))",
+          "legendFormat": "{{result}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Alerts received / sec (by result)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 1},
+      "id": 3,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(ba_ingestd_publish_latency_seconds_bucket[5m])))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(ba_ingestd_publish_latency_seconds_bucket[5m])))",
+          "legendFormat": "p95",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(ba_ingestd_publish_latency_seconds_bucket[5m])))",
+          "legendFormat": "p99",
+          "refId": "C"
+        }
+      ],
+      "title": "NATS publish latency (p50/p95/p99)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "thresholds"},
+          "mappings": [
+            {"options": {"0": {"text": "CLOSED"}}, {"options": {"1": {"text": "HALF-OPEN"}}, {"options": {"2": {"text": "OPEN"}}}],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 1},
+              {"color": "red", "value": 2}
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 9},
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "ba_ingestd_circuit_breaker_state{component=\"nats\"}",
+          "legendFormat": "NATS CB",
+          "refId": "A"
+        }
+      ],
+      "title": "Circuit breaker state (0=closed, 1=half, 2=open)",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 4, "w": 10, "x": 4, "y": 9},
+      "id": 5,
+      "options": {
+        "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "sum by (source) (rate(ba_ingestd_source_quarantined_total[5m]))",
+          "legendFormat": "{{source}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Quarantine events / sec",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
+      "id": 10,
+      "panels": [],
+      "title": "Router tier",
+      "type": "row"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
+      "id": 11,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(ba_routerd_recipient_expansion_seconds_bucket[5m])))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(ba_routerd_recipient_expansion_seconds_bucket[5m])))",
+          "legendFormat": "p95",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(ba_routerd_recipient_expansion_seconds_bucket[5m])))",
+          "legendFormat": "p99",
+          "refId": "C"
+        }
+      ],
+      "title": "Recipient resolution latency (p50/p95/p99)",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 22},
+      "id": 20,
+      "panels": [],
+      "title": "Delivery tier",
+      "type": "row"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 23},
+      "id": 21,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "sum by (channel, status) (rate(ba_deliverd_delivery_attempts_total[1m]))",
+          "legendFormat": "{{channel}} / {{status}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Delivery attempts / sec (by channel × status)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}, {"color": "red", "value": 0.001}]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 6, "x": 12, "y": 23},
+      "id": 22,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "sum by (channel) (rate(ba_deliverd_dlq_total[1m]))",
+          "legendFormat": "{{channel}}",
+          "refId": "A"
+        }
+      ],
+      "title": "DLQ inserts / sec (by channel) — alert if > 0",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 6, "x": 18, "y": 23},
+      "id": 23,
+      "options": {
+        "legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(ba_deliverd_dlq_latency_seconds_bucket[5m])))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(ba_deliverd_dlq_latency_seconds_bucket[5m])))",
+          "legendFormat": "p95",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(ba_deliverd_dlq_latency_seconds_bucket[5m])))",
+          "legendFormat": "p99",
+          "refId": "C"
+        }
+      ],
+      "title": "DLQ latency (p50/p95/p99)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": ["broad-announce", "m9"],
+  "templating": {"list": []},
+  "time": {"from": "now-15m", "to": "now"},
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "BroadAnnounce Overview",
+  "uid": "broad-announce-overview",
+  "version": 1,
+  "weekStart": ""
+}

deploy/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: 'broad-announce'
+    orgId: 1
+    folder: 'BroadAnnounce'
+    folderUid: 'broadannounce'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

docker-compose.yml

@@ -300,6 +300,14 @@ services:
     environment:
       GF_SECURITY_ADMIN_USER: admin
       GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_SECURITY_DISABLE_LOGIN_FORM: "false"
+      # M9: provisioning paths for dashboards + datasources.
+      GF_PATHS_PROVISIONING: /etc/grafana/provisioning
+      GF_DASHBOARDS_ENABLED: "true"
+    volumes:
+      - ./deploy/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./deploy/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
+      - ./deploy/grafana/provisioning/dashboards/broad-announce-overview.json:/var/lib/grafana/dashboards/broad-announce-overview.json:ro
     depends_on: [prometheus]
 
   # ── Loadgen (one-shot smoke profile) ────────────────────

internal/observability/metrics.go

@@ -242,3 +242,27 @@ func NewDeliverdMetrics(reg prometheus.Registerer, serviceName string) *Deliverd
 	reg.MustRegister(m.DeliveryAttempts, m.DLQTotal, m.DLQLatency, m.RetryAttempts)
 	return m
 }
+
+// RouterdMetrics groups the Prometheus histogram for the routerd tier
+// (SPEC §22 L1: recipient resolution latency).
+type RouterdMetrics struct {
+	// RecipientExpansionLatency is the wall-clock time for
+	// routing.Resolver.ResolveTargets to complete (DB call).
+	RecipientExpansionLatency prometheus.Histogram
+}
+
+// NewRouterdMetrics registers and returns routerd metrics.
+func NewRouterdMetrics(reg prometheus.Registerer, serviceName string) *RouterdMetrics {
+	m := &RouterdMetrics{
+		RecipientExpansionLatency: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: "ba",
+			Subsystem: "routerd",
+			Name:      "recipient_expansion_seconds",
+			Help:      "Time to resolve recipients for an alert (DB call).",
+			ConstLabels: prometheus.Labels{"service": serviceName},
+			Buckets:   prometheus.DefBuckets,
+		}),
+	}
+	reg.MustRegister(m.RecipientExpansionLatency)
+	return m
+}

scripts/m9_smoke.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# M9 smoke test — observability layer
+# Verifies:
+#  1. Prometheus is scraping all services (including new M9 targets)
+#  2. Key metrics are accessible in Prometheus
+#  3. Grafana is reachable with provisioning
+set -euoo pipefail
+
+PROM="${PROM:-http://localhost:9090}"
+INGESTD="${INGESTD:-http://localhost:8800}"
+ADMID="${ADMID:-http://localhost:8080}"
+
+pass() { echo "  ✅ $1"; }
+warn() { echo "  ⚠️  $1"; }
+fail() { echo "  ❌ $1"; exit 1; }
+
+echo "Step 1 — Prometheus is up and scraping all services"
+# M9: deliverd split into deliverd-fcm + deliverd-telegram; archiverd added.
+for svc in ingestd routerd deliverd-fcm deliverd-telegram admind archiverd prometheus; do
+    result=$(curl -s "${PROM}/api/v1/query?query=up{job=\"${svc}\"}" | \
+        python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print('up' if r else 'missing')" 2>/dev/null)
+    if [ "$result" = "up" ]; then
+        pass "Prometheus scraping ${svc}"
+    else
+        fail "Prometheus NOT scraping ${svc} (check prometheus.yml)"
+    fi
+done
+
+echo ""
+echo "Step 2 — ingestd metrics are present (source of ingest-tier metrics)"
+metric_count=$(curl -s "${PROM}/api/v1/query?query=ba_ingestd_alerts_received_total" | \
+    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
+if [ "${metric_count:-0}" -gt 0 ]; then
+    pass "ba_ingestd_alerts_received_total: ${metric_count} series"
+else
+    warn "ba_ingestd_alerts_received_total: 0 series (no recent traffic — might be OK)"
+fi
+
+echo ""
+echo "Step 3 — routerd recipient expansion latency metric"
+router_count=$(curl -s "${PROM}/api/v1/query?query=ba_routerd_recipient_expansion_seconds_count" | \
+    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
+if [ "${router_count:-0}" -gt 0 ]; then
+    pass "ba_routerd_recipient_expansion_seconds: ${router_count} series (M9 routerd deployed)"
+else
+    fail "ba_routerd_recipient_expansion_seconds has no series — routerd missing M9 metric"
+fi
+
+echo ""
+echo "Step 4 — deliverd delivery attempt metrics (M9)"
+deliverd_count=$(curl -s "${PROM}/api/v1/query?query=ba_deliverd_delivery_attempts_total" | \
+    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
+if [ "${deliverd_count:-0}" -gt 0 ]; then
+    pass "ba_deliverd_delivery_attempts_total: ${deliverd_count} series (M9 deliverd deployed)"
+else
+    pass "ba_deliverd_delivery_attempts_total: 0 series (metric exists, no delivery traffic yet)"
+fi
+
+echo ""
+echo "Step 5 — Circuit breaker gauge (M9 ingestd — requires rebuild)"
+# The circuit breaker metric only appears when ingestd was rebuilt with
+# the M9 circuit breaker code. On the first smoke run (before the
+# rebuild), this will warn instead of pass.
+cb_count=$(curl -s "${PROM}/api/v1/query?query=ba_ingestd_circuit_breaker_state" | \
+    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
+if [ "${cb_count:-0}" -gt 0 ]; then
+    pass "ba_ingestd_circuit_breaker_state: ${cb_count} series (M9 ingestd deployed)"
+else
+    warn "ba_ingestd_circuit_breaker_state: 0 series (ingestd still on pre-M9 binary)"
+    warn "  Rebuild to get circuit breaker: docker compose build --no-cache ingestd"
+    warn "  Then redeploy: docker compose up -d --no-deps ingestd"
+fi
+
+echo ""
+echo "Step 6 — Grafana reachable with dashboards provisioned"
+gf_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3001/api/health" 2>/dev/null)
+if [ "${gf_status}" = "200" ]; then
+    pass "Grafana HTTP ${gf_status} (admin/admin at http://localhost:3001)"
+    # Check that the BroadAnnounce dashboard is listed
+    dash_count=$(curl -s -u admin:admin "http://localhost:3001/api/search?type=dashboards" 2>/dev/null | \
+        python3 -c "import json,sys; d=json.load(sys.stdin); print(len([x for x in d if 'BroadAnnounce' in x.get('title','')]))" 2>/dev/null)
+    if [ "${dash_count:-0}" -gt 0 ]; then
+        pass "BroadAnnounce Overview dashboard found in Grafana"
+    else
+        warn "BroadAnnounce dashboard not found in Grafana (may need manual import)"
+    fi
+else
+    fail "Grafana returned HTTP ${gf_status} — check docker compose logs grafana"
+fi
+
+echo ""
+echo "Step 7 — Prometheus self-monitoring"
+prom_up=$(curl -s "${PROM}/api/v1/query?query=up{job=\"prometheus\"}" | \
+    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print('up' if r else 'missing')" 2>/dev/null)
+if [ "${prom_up}" = "up" ]; then
+    pass "Prometheus scraping itself"
+else
+    fail "Prometheus not scraping itself"
+fi
+
+echo ""
+echo "🎉 M9 smoke: all checks complete."
+echo ""
+echo "On the remote playground (192.168.44.94):"
+echo "  ssh parres 'cd /root/git/broad-announce && git pull && docker compose build --no-cache ingestd deliverd-fcm deliverd-telegram routerd && docker compose up -d'"
+echo ""
+echo "Then run this smoke again to verify the circuit breaker metric."