#!/usr/bin/env bash # Live M3 smoke test. Run from repo root: # bash scripts/m3_smoke.sh # # Walks through the scenarios in M3_VERIFICATION.md: # # Step 2 — warning to db-prod-04 → Alice only (Alice pre-linked, Bob not # yet linked, Carol unlinked + quiet hours). # Expected: 1 fcm (Alice) + 1 telegram (Alice). # Step 3 — critical to db-prod-04 → Alice + Bob (fcm). Bob is unlinked # so no telegram for him. # Step 4 — Bob runs /start acme-bob-002 via faketgmd admin queue. # Expect reply + telegram_user_id linked. # Step 5 — critical to db-prod-04 again. Bob is now linked. # Expected: 1 fcm (Alice) + 1 fcm (Bob) + 1 telegram (Alice) + # 1 telegram (Bob). # Step 6 — inminent_colapse to rack-b. Carol's quiet hours (00:00-23:59 # UTC) are bypassed. Bob is at min_severity=critical so he # passes. Expected: 1 fcm + 1 telegram for Alice, Bob, Carol # → 6 deliveries. # Step 7 — Bob /preferences. Confirm 1 subscription row, channel_mask # fcm+telegram, min=critical. # Step 8 — faketgmd /admin/sent should have 3 telegram messages # (one per Alice/Bob/Carol from step 6 plus 2 from step 5). # # Exit code is the number of failed checks. set -e cd "$(dirname "$0")/.." SECRET=s3cret-acme INGEST=http://localhost:8800/v1/ingest TELEGRAM_BOT_TOKEN=fake-tg-bot-token-acme-001 FAKETGMD=http://localhost:8830 PG="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t" send() { # send [var_out] # Echoes the response; if var_out is set, stores the alert_id in it. local body="$1" label="$2" var_out="$3" local ts=$(date +%s) local sig=$(printf '%s.%s' "$ts" "$body" | openssl dgst -sha256 -hmac "$SECRET" -hex | awk '{print $2}') local resp resp=$(curl -s -X POST "$INGEST" \ -H "Content-Type: application/json" \ -H "X-BA-Signature: t=$ts,v1=$sig" \ --data "$body") echo "[$label] $resp" if [[ -n "$var_out" ]]; then # alert_id is the first JSON string of the form "00000..." local aid aid=$(printf '%s' "$resp" | python3 -c 'import sys,json,re; s=sys.stdin.read(); m=re.search(r"\"alert_id\":\"([0-9a-f]+)\"", s); print(m.group(1) if m else "")') printf -v "$var_out" '%s' "$aid" fi } queue() { local text="$1" user_id="$2" chat_id="$3" curl -s -X POST "$FAKETGMD/admin/queue" \ -H "Content-Type: application/json" \ --data "$(cat </dev/null $PG -c "UPDATE individuals SET telegram_chat_id = NULL, telegram_user_id = NULL WHERE id = 'ind-acme-003';" >/dev/null $PG -c "UPDATE subscriptions SET min_severity = 'critical' WHERE individual_id = 'ind-acme-002' AND source_id = 'prom-prod';" >/dev/null $PG -c "TRUNCATE deliveries;" >/dev/null # Reset faketgmd's queue + sent (and its in-memory per-bot offset). curl -sS -X POST "$FAKETGMD/admin/reset" >/dev/null # Restart telegramd so its in-process long-poll offset goes back to 0. # (In production, Telegram's getUpdates tracks the offset server-side; # faketgmd is a fake that needs the bot to be re-started for clean state.) docker compose restart telegramd >/dev/null # Wait for telegramd to come up and load its bot list. for i in 1 2 3 4 5 6 7 8 9 10; do if curl -sS http://localhost:8822/health 2>/dev/null | grep -q '"status":"ok"'; then sleep 1 # let it issue the first getUpdates long-poll break fi sleep 1 done echo echo "▸ Pre-flight: state" echo " individuals:" $PG -c "SELECT id, telegram_chat_id, telegram_user_id, telegram_invite_code FROM individuals ORDER BY id;" | sed 's/^/ /' echo " subscriptions:" $PG -c "SELECT individual_id, min_severity, channel_mask, status FROM subscriptions WHERE source_id='prom-prod' ORDER BY individual_id;" | sed 's/^/ /' echo " deliveries (should be 0):" $PG -c "SELECT COUNT(*) FROM deliveries;" | sed 's/^/ /' echo " faketgmd sent (should be 0):" curl -sS "$FAKETGMD/admin/sent" | sed 's/^/ /' echo # Helper: count rows in deliveries for a specific alert_id. rows_for() { $PG -c "SELECT COUNT(*) FROM deliveries WHERE alert_id = '$1';" } rows_total() { $PG -c "SELECT COUNT(*) FROM deliveries;"; } rows_channel() { $PG -c "SELECT COUNT(*) FROM deliveries WHERE channel='$1' AND status='sent';"; } rows_channel_total() { $PG -c "SELECT COUNT(*) FROM deliveries WHERE channel='$1';"; } # ── Step 2: warning → db-prod-04 → Alice only ───────────────── echo "▸ Step 2: POST warning (Alice only; Bob unlinked; Carol quiet hours)" send '{"company_id":"acme-001","source_id":"prom-prod","severity":"warning","category":"storage","title":"Disk 80% full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s2"}' "step 2" aid2 sleep 4 n=$(rows_for "$aid2") [[ "$n" == "2" ]] && pass "2 deliveries for step-2 alert (1 fcm + 1 telegram for Alice)" || fail "expected 2 deliveries for step 2, got $n" fb=$(rows_channel fcm) tb=$(rows_channel telegram) [[ "$fb" == "1" && "$tb" == "1" ]] && pass "running totals: fcm=1, telegram=1" || fail "fcm=$fb, telegram=$tb (expected 1,1)" # ── Step 3: critical → db-prod-04 → Alice + Bob (fcm only) ───── echo echo "▸ Step 3: POST critical (Bob unlinked, telegram skipped for him)" send '{"company_id":"acme-001","source_id":"prom-prod","severity":"critical","category":"storage","title":"Disk full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s3"}' "step 3" aid3 sleep 4 n=$(rows_for "$aid3") [[ "$n" == "3" ]] && pass "3 deliveries for step-3 alert (Alice fcm+telegram, Bob fcm only)" || fail "expected 3 deliveries for step 3, got $n" fb=$(rows_channel fcm) tb=$(rows_channel telegram) [[ "$fb" == "3" && "$tb" == "2" ]] && pass "running totals: fcm=3, telegram=2 (Bob telegram blocked by missing link, Alice has 1+1)" || fail "fcm=$fb, telegram=$tb (expected 3,2)" # ── Step 4: Bob /start acme-bob-002 via faketgmd queue ───────── echo echo "▸ Step 4: Bob runs /start acme-bob-002" queue "/start acme-bob-002" 900002 2002 sleep 4 linked=$($PG -c "SELECT telegram_user_id FROM individuals WHERE id = 'ind-acme-002';") [[ "$linked" == "900002" ]] && pass "Bob linked (telegram_user_id=900002)" || fail "Bob telegram_user_id='$linked' (expected 900002)" bot_reply=$(curl -sS "$FAKETGMD/admin/sent" | grep -c '"text":"Linked' || true) [[ "$bot_reply" -ge 1 ]] && pass "faketgmd saw bot's 'Linked' reply" || fail "no 'Linked' reply from bot" # ── Step 5: critical → db-prod-04 again; Bob telegram should now fire ─ echo echo "▸ Step 5: POST critical (Bob now linked, telegram should fire for him)" send '{"company_id":"acme-001","source_id":"prom-prod","severity":"critical","category":"storage","title":"Disk full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s5"}' "step 5" aid5 sleep 4 n=$(rows_for "$aid5") [[ "$n" == "4" ]] && pass "4 deliveries for step-5 alert (2 fcm + 2 telegram for Alice + Bob)" || fail "expected 4 deliveries for step 5, got $n" fb=$(rows_channel fcm) tb=$(rows_channel telegram) [[ "$fb" == "5" && "$tb" == "4" ]] && pass "running totals: fcm=5, telegram=4 (Bob now receives telegram)" || fail "fcm=$fb, telegram=$tb (expected 5,4)" # ── Step 6: inminent_colapse → all three via fcm, but Carol unlinked (no telegram) ─ echo echo "▸ Step 6: POST inminent_colapse (Carol's quiet hours bypassed → fcm, but Carol unlinked → no telegram)" send '{"company_id":"acme-001","source_id":"prom-prod","severity":"inminent_colapse","category":"power","title":"PDU overload imminent","body":"rack-B","data":{"host":"rack-b"},"dedupe_key":"m3-s6"}' "step 6" aid6 sleep 4 n=$(rows_for "$aid6") [[ "$n" == "5" ]] && pass "5 deliveries for step-6 alert (3 fcm + 2 telegram: Alice, Bob fcm+telegram; Carol fcm only — never linked)" || fail "expected 5 deliveries for step 6, got $n" fb=$(rows_channel fcm) tb=$(rows_channel telegram) [[ "$fb" == "8" && "$tb" == "6" ]] && pass "running totals: fcm=8, telegram=6" || fail "fcm=$fb, telegram=$tb (expected 8,6)" # ── Step 7: Bob /preferences ──────────────────────────────────── echo echo "▸ Step 7: Bob runs /preferences" queue "/preferences" 900002 2002 sleep 3 prefs=$(curl -sS "$FAKETGMD/admin/sent" | grep -c '"text":"Your subscriptions' || true) [[ "$prefs" -ge 1 ]] && pass "faketgmd saw bot's preferences reply" || fail "no preferences reply from bot" # ── Step 8: faketgmd sent count ───────────────────────────────── echo echo "▸ Step 8: faketgmd totals" sent=$(curl -sS "$FAKETGMD/admin/sent" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["count"])') [[ "$sent" -ge 5 ]] && pass "faketgmd received $sent sendMessage calls (>=5 expected: 1 alice/step2 + 1 alice/step3 + 1 linked-reply + 1 alice/step5 + 1 bob/step5 + 3 step6 + 1 prefs-reply = 8)" || fail "faketgmd sent count=$sent, expected >=5" echo echo "═══════════════════════════════════════════════════════" echo " Per-individual / per-channel delivery table" echo "═══════════════════════════════════════════════════════" $PG -c "SELECT individual_id, channel, status, COUNT(*) AS n FROM deliveries GROUP BY 1,2,3 ORDER BY 1,2;" | sed 's/^/ /' echo if [[ "$fails" -eq 0 ]]; then echo "✅ M3 smoke: 0 failures" exit 0 else echo "❌ M3 smoke: $fails failure(s)" exit $fails fi