broad-announce/scripts/m9_smoke.sh

#!/usr/bin/env bash
# M9 smoke test — observability layer
# Verifies:
#  1. Prometheus is scraping all services (including new M9 targets)
#  2. Key metrics are accessible in Prometheus
#  3. Grafana is reachable with provisioning
set -euoo pipefail

PROM="${PROM:-http://localhost:9090}"
INGESTD="${INGESTD:-http://localhost:8800}"
ADMID="${ADMID:-http://localhost:8080}"

pass() { echo "  ✅ $1"; }
warn() { echo "  ⚠️  $1"; }
fail() { echo "  ❌ $1"; exit 1; }

echo "Step 1 — Prometheus is up and scraping all services"
# M9: deliverd split into deliverd-fcm + deliverd-telegram; archiverd added.
for svc in ingestd routerd deliverd-fcm deliverd-telegram admind archiverd prometheus; do
    result=$(curl -s "${PROM}/api/v1/query?query=up{job=\"${svc}\"}" | \
        python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print('up' if r else 'missing')" 2>/dev/null)
    if [ "$result" = "up" ]; then
        pass "Prometheus scraping ${svc}"
    else
        fail "Prometheus NOT scraping ${svc} (check prometheus.yml)"
    fi
done

echo ""
echo "Step 2 — ingestd metrics are present (source of ingest-tier metrics)"
metric_count=$(curl -s "${PROM}/api/v1/query?query=ba_ingestd_alerts_received_total" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
if [ "${metric_count:-0}" -gt 0 ]; then
    pass "ba_ingestd_alerts_received_total: ${metric_count} series"
else
    warn "ba_ingestd_alerts_received_total: 0 series (no recent traffic — might be OK)"
fi

echo ""
echo "Step 3 — routerd recipient expansion latency metric"
router_count=$(curl -s "${PROM}/api/v1/query?query=ba_routerd_recipient_expansion_seconds_count" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
if [ "${router_count:-0}" -gt 0 ]; then
    pass "ba_routerd_recipient_expansion_seconds: ${router_count} series (M9 routerd deployed)"
else
    fail "ba_routerd_recipient_expansion_seconds has no series — routerd missing M9 metric"
fi

echo ""
echo "Step 4 — deliverd delivery attempt metrics (M9)"
deliverd_count=$(curl -s "${PROM}/api/v1/query?query=ba_deliverd_delivery_attempts_total" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
if [ "${deliverd_count:-0}" -gt 0 ]; then
    pass "ba_deliverd_delivery_attempts_total: ${deliverd_count} series (M9 deliverd deployed)"
else
    pass "ba_deliverd_delivery_attempts_total: 0 series (metric exists, no delivery traffic yet)"
fi

echo ""
echo "Step 5 — Circuit breaker gauge (M9 ingestd — requires rebuild)"
# The circuit breaker metric only appears when ingestd was rebuilt with
# the M9 circuit breaker code. On the first smoke run (before the
# rebuild), this will warn instead of pass.
cb_count=$(curl -s "${PROM}/api/v1/query?query=ba_ingestd_circuit_breaker_state" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print(len(r))" 2>/dev/null)
if [ "${cb_count:-0}" -gt 0 ]; then
    pass "ba_ingestd_circuit_breaker_state: ${cb_count} series (M9 ingestd deployed)"
else
    warn "ba_ingestd_circuit_breaker_state: 0 series (ingestd still on pre-M9 binary)"
    warn "  Rebuild to get circuit breaker: docker compose build --no-cache ingestd"
    warn "  Then redeploy: docker compose up -d --no-deps ingestd"
fi

echo ""
echo "Step 6 — Grafana reachable with dashboards provisioned"
gf_status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3001/api/health" 2>/dev/null)
if [ "${gf_status}" = "200" ]; then
    pass "Grafana HTTP ${gf_status} (admin/admin at http://localhost:3001)"
    # Check that the BroadAnnounce dashboard is listed
    dash_count=$(curl -s -u admin:admin "http://localhost:3001/api/search?type=dashboards" 2>/dev/null | \
        python3 -c "import json,sys; d=json.load(sys.stdin); print(len([x for x in d if 'BroadAnnounce' in x.get('title','')]))" 2>/dev/null)
    if [ "${dash_count:-0}" -gt 0 ]; then
        pass "BroadAnnounce Overview dashboard found in Grafana"
    else
        warn "BroadAnnounce dashboard not found in Grafana (may need manual import)"
    fi
else
    fail "Grafana returned HTTP ${gf_status} — check docker compose logs grafana"
fi

echo ""
echo "Step 7 — Prometheus self-monitoring"
prom_up=$(curl -s "${PROM}/api/v1/query?query=up{job=\"prometheus\"}" | \
    python3 -c "import json,sys; d=json.load(sys.stdin); r=d.get('data',{}).get('result',[]); print('up' if r else 'missing')" 2>/dev/null)
if [ "${prom_up}" = "up" ]; then
    pass "Prometheus scraping itself"
else
    fail "Prometheus not scraping itself"
fi

echo ""
echo "🎉 M9 smoke: all checks complete."
echo ""
echo "On the remote playground (192.168.44.94):"
echo "  ssh parres 'cd /root/git/broad-announce && git pull && docker compose build --no-cache ingestd deliverd-fcm deliverd-telegram routerd && docker compose up -d'"
echo ""
echo "Then run this smoke again to verify the circuit breaker metric."