lrosales
/
broad-announce


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
							#!/usr/bin/env bash
# Live M3 smoke test. Run from repo root:
#   bash scripts/m3_smoke.sh
#
# Walks through the scenarios in M3_VERIFICATION.md:
#
#   Step 2 — warning to db-prod-04 → Alice only (Alice pre-linked, Bob not
#            yet linked, Carol unlinked + quiet hours).
#            Expected: 1 fcm (Alice) + 1 telegram (Alice).
#   Step 3 — critical to db-prod-04 → Alice + Bob (fcm). Bob is unlinked
#            so no telegram for him.
#   Step 4 — Bob runs /start acme-bob-002 via faketgmd admin queue.
#            Expect reply + telegram_user_id linked.
#   Step 5 — critical to db-prod-04 again. Bob is now linked.
#            Expected: 1 fcm (Alice) + 1 fcm (Bob) + 1 telegram (Alice) +
#            1 telegram (Bob).
#   Step 6 — inminent_colapse to rack-b. Carol's quiet hours (00:00-23:59
#            UTC) are bypassed. Bob is at min_severity=critical so he
#            passes. Expected: 1 fcm + 1 telegram for Alice, Bob, Carol
#            → 6 deliveries.
#   Step 7 — Bob /preferences. Confirm 1 subscription row, channel_mask
#            fcm+telegram, min=critical.
#   Step 8 — faketgmd /admin/sent should have 3 telegram messages
#            (one per Alice/Bob/Carol from step 6 plus 2 from step 5).
#
# Exit code is the number of failed checks.

set -e
cd "$(dirname "$0")/.."

SECRET=s3cret-acme
INGEST=http://localhost:8800/v1/ingest
TELEGRAM_BOT_TOKEN=fake-tg-bot-token-acme-001
FAKETGMD=http://localhost:8830
PG="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t"

send() {
  # send <body> <label> [var_out]
  # Echoes the response; if var_out is set, stores the alert_id in it.
  local body="$1" label="$2" var_out="$3"
  local ts=$(date +%s)
  local sig=$(printf '%s.%s' "$ts" "$body" | openssl dgst -sha256 -hmac "$SECRET" -hex | awk '{print $2}')
  local resp
  resp=$(curl -s -X POST "$INGEST" \
    -H "Content-Type: application/json" \
    -H "X-BA-Signature: t=$ts,v1=$sig" \
    --data "$body")
  echo "[$label] $resp"
  if [[ -n "$var_out" ]]; then
    # alert_id is the first JSON string of the form "00000..."
    local aid
    aid=$(printf '%s' "$resp" | python3 -c 'import sys,json,re; s=sys.stdin.read(); m=re.search(r"\"alert_id\":\"([0-9a-f]+)\"", s); print(m.group(1) if m else "")')
    printf -v "$var_out" '%s' "$aid"
  fi
}

queue() {
  local text="$1" user_id="$2" chat_id="$3"
  curl -s -X POST "$FAKETGMD/admin/queue" \
    -H "Content-Type: application/json" \
    --data "$(cat <<JSON
{"bot_token":"$TELEGRAM_BOT_TOKEN","user_id":$user_id,"chat_id":$chat_id,"text":"$text","first_name":"Bob SRE"}
JSON
)"
  echo
}

fails=0
pass() { echo "  ✅ $*"; }
fail() { echo "  ❌ $*"; fails=$((fails+1)); }

echo "═══════════════════════════════════════════════════════"
echo "  M3 smoke test — broad-announce"
echo "═══════════════════════════════════════════════════════"

echo
echo "▸ Pre-flight: reset state and restart telegramd so its long-poll offset clears"
# Reset DB to M3 baseline: Bob unlinked, Bob subscription min=critical.
$PG -c "UPDATE individuals SET telegram_chat_id = NULL, telegram_user_id = NULL, telegram_invite_code = 'acme-bob-002' WHERE id = 'ind-acme-002';" >/dev/null
$PG -c "UPDATE individuals SET telegram_chat_id = NULL, telegram_user_id = NULL WHERE id = 'ind-acme-003';" >/dev/null
$PG -c "UPDATE subscriptions SET min_severity = 'critical' WHERE individual_id = 'ind-acme-002' AND source_id = 'prom-prod';" >/dev/null
$PG -c "TRUNCATE deliveries;" >/dev/null
# Reset faketgmd's queue + sent (and its in-memory per-bot offset).
curl -sS -X POST "$FAKETGMD/admin/reset" >/dev/null
# Restart telegramd so its in-process long-poll offset goes back to 0.
# (In production, Telegram's getUpdates tracks the offset server-side;
#  faketgmd is a fake that needs the bot to be re-started for clean state.)
docker compose restart telegramd >/dev/null
# Wait for telegramd to come up and load its bot list.
for i in 1 2 3 4 5 6 7 8 9 10; do
  if curl -sS http://localhost:8822/health 2>/dev/null | grep -q '"status":"ok"'; then
    sleep 1   # let it issue the first getUpdates long-poll
    break
  fi
  sleep 1
done

echo
echo "▸ Pre-flight: state"
echo "  individuals:"
$PG -c "SELECT id, telegram_chat_id, telegram_user_id, telegram_invite_code FROM individuals ORDER BY id;" | sed 's/^/    /'
echo "  subscriptions:"
$PG -c "SELECT individual_id, min_severity, channel_mask, status FROM subscriptions WHERE source_id='prom-prod' ORDER BY individual_id;" | sed 's/^/    /'
echo "  deliveries (should be 0):"
$PG -c "SELECT COUNT(*) FROM deliveries;" | sed 's/^/    /'
echo "  faketgmd sent (should be 0):"
curl -sS "$FAKETGMD/admin/sent" | sed 's/^/    /'
echo

# Helper: count rows in deliveries for a specific alert_id.
rows_for() {
  $PG -c "SELECT COUNT(*) FROM deliveries WHERE alert_id = '$1';"
}
rows_total() { $PG -c "SELECT COUNT(*) FROM deliveries;"; }
rows_channel() { $PG -c "SELECT COUNT(*) FROM deliveries WHERE channel='$1' AND status='sent';"; }
rows_channel_total() { $PG -c "SELECT COUNT(*) FROM deliveries WHERE channel='$1';"; }

# ── Step 2: warning → db-prod-04 → Alice only ─────────────────
echo "▸ Step 2: POST warning (Alice only; Bob unlinked; Carol quiet hours)"
send '{"company_id":"acme-001","source_id":"prom-prod","severity":"warning","category":"storage","title":"Disk 80% full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s2"}' "step 2" aid2
sleep 4
n=$(rows_for "$aid2")
[[ "$n" == "2" ]] && pass "2 deliveries for step-2 alert (1 fcm + 1 telegram for Alice)" || fail "expected 2 deliveries for step 2, got $n"
fb=$(rows_channel fcm)
tb=$(rows_channel telegram)
[[ "$fb" == "1" && "$tb" == "1" ]] && pass "running totals: fcm=1, telegram=1" || fail "fcm=$fb, telegram=$tb (expected 1,1)"

# ── Step 3: critical → db-prod-04 → Alice + Bob (fcm only) ─────
echo
echo "▸ Step 3: POST critical (Bob unlinked, telegram skipped for him)"
send '{"company_id":"acme-001","source_id":"prom-prod","severity":"critical","category":"storage","title":"Disk full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s3"}' "step 3" aid3
sleep 4
n=$(rows_for "$aid3")
[[ "$n" == "3" ]] && pass "3 deliveries for step-3 alert (Alice fcm+telegram, Bob fcm only)" || fail "expected 3 deliveries for step 3, got $n"
fb=$(rows_channel fcm)
tb=$(rows_channel telegram)
[[ "$fb" == "3" && "$tb" == "2" ]] && pass "running totals: fcm=3, telegram=2 (Bob telegram blocked by missing link, Alice has 1+1)" || fail "fcm=$fb, telegram=$tb (expected 3,2)"

# ── Step 4: Bob /start acme-bob-002 via faketgmd queue ─────────
echo
echo "▸ Step 4: Bob runs /start acme-bob-002"
queue "/start acme-bob-002" 900002 2002
sleep 4
linked=$($PG -c "SELECT telegram_user_id FROM individuals WHERE id = 'ind-acme-002';")
[[ "$linked" == "900002" ]] && pass "Bob linked (telegram_user_id=900002)" || fail "Bob telegram_user_id='$linked' (expected 900002)"
bot_reply=$(curl -sS "$FAKETGMD/admin/sent" | grep -c '"text":"Linked' || true)
[[ "$bot_reply" -ge 1 ]] && pass "faketgmd saw bot's 'Linked' reply" || fail "no 'Linked' reply from bot"

# ── Step 5: critical → db-prod-04 again; Bob telegram should now fire ─
echo
echo "▸ Step 5: POST critical (Bob now linked, telegram should fire for him)"
send '{"company_id":"acme-001","source_id":"prom-prod","severity":"critical","category":"storage","title":"Disk full","body":"db-prod-04","data":{"host":"db-prod-04"},"dedupe_key":"m3-s5"}' "step 5" aid5
sleep 4
n=$(rows_for "$aid5")
[[ "$n" == "4" ]] && pass "4 deliveries for step-5 alert (2 fcm + 2 telegram for Alice + Bob)" || fail "expected 4 deliveries for step 5, got $n"
fb=$(rows_channel fcm)
tb=$(rows_channel telegram)
[[ "$fb" == "5" && "$tb" == "4" ]] && pass "running totals: fcm=5, telegram=4 (Bob now receives telegram)" || fail "fcm=$fb, telegram=$tb (expected 5,4)"

# ── Step 6: inminent_colapse → all three via fcm, but Carol unlinked (no telegram) ─
echo
echo "▸ Step 6: POST inminent_colapse (Carol's quiet hours bypassed → fcm, but Carol unlinked → no telegram)"
send '{"company_id":"acme-001","source_id":"prom-prod","severity":"inminent_colapse","category":"power","title":"PDU overload imminent","body":"rack-B","data":{"host":"rack-b"},"dedupe_key":"m3-s6"}' "step 6" aid6
sleep 4
n=$(rows_for "$aid6")
[[ "$n" == "5" ]] && pass "5 deliveries for step-6 alert (3 fcm + 2 telegram: Alice, Bob fcm+telegram; Carol fcm only — never linked)" || fail "expected 5 deliveries for step 6, got $n"
fb=$(rows_channel fcm)
tb=$(rows_channel telegram)
[[ "$fb" == "8" && "$tb" == "6" ]] && pass "running totals: fcm=8, telegram=6" || fail "fcm=$fb, telegram=$tb (expected 8,6)"

# ── Step 7: Bob /preferences ────────────────────────────────────
echo
echo "▸ Step 7: Bob runs /preferences"
queue "/preferences" 900002 2002
sleep 3
prefs=$(curl -sS "$FAKETGMD/admin/sent" | grep -c '"text":"Your subscriptions' || true)
[[ "$prefs" -ge 1 ]] && pass "faketgmd saw bot's preferences reply" || fail "no preferences reply from bot"

# ── Step 8: faketgmd sent count ─────────────────────────────────
echo
echo "▸ Step 8: faketgmd totals"
sent=$(curl -sS "$FAKETGMD/admin/sent" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["count"])')
[[ "$sent" -ge 5 ]] && pass "faketgmd received $sent sendMessage calls (>=5 expected: 1 alice/step2 + 1 alice/step3 + 1 linked-reply + 1 alice/step5 + 1 bob/step5 + 3 step6 + 1 prefs-reply = 8)" || fail "faketgmd sent count=$sent, expected >=5"

echo
echo "═══════════════════════════════════════════════════════"
echo "  Per-individual / per-channel delivery table"
echo "═══════════════════════════════════════════════════════"
$PG -c "SELECT individual_id, channel, status, COUNT(*) AS n FROM deliveries GROUP BY 1,2,3 ORDER BY 1,2;" | sed 's/^/  /'

echo
if [[ "$fails" -eq 0 ]]; then
  echo "✅  M3 smoke: 0 failures"
  exit 0
else
  echo "❌  M3 smoke: $fails failure(s)"
  exit $fails
fi