lrosales
/
broad-announce


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310
							#!/usr/bin/env bash
# Live M8 smoke test. Run from repo root:
#   bash scripts/m8_smoke.sh
#
# Walks through the 4 scenarios in M8_VERIFICATION.md:
#
#   Step 2 — happy path unchanged: a normal alert with
#            fakefcmd in healthy mode delivers in 1
#            attempt; 0 DLQ rows.
#   Step 3 — DLQ creation: flip fakefcmd into fail
#            mode at runtime via /control?fail=1, send
#            one alert, wait for the retry budget to
#            exhaust (~12s for the default config), and
#            verify 1 row in deliveries_dlq with
#            attempts=10 + 10 'failed' rows in
#            deliveries.
#   Step 4 — replay: flip fakefcmd back to healthy,
#            POST /v1/dlq/{id}/replay, verify a fresh
#            'sent' row in deliveries and the DLQ row
#            is now discarded.
#   Step 5 — discard: flip fakefcmd to fail again,
#            send another alert, wait, POST
#            /v1/dlq/{id}/discard, verify hidden from
#            the default list and visible with
#            include=all.
#
# Exit code is the number of failed checks.

set -e
cd "$(dirname "$0")/.."

INGESTD=http://localhost:8800
ADMIND=http://localhost:8803
FAKEFCMD=http://localhost:8820
COMPANY=acme-001
SOURCE=prom-prod
SECRET=s3cret-acme
PGCMD="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t"

fails=0
pass() { echo "  ✅ $*"; }
fail() { echo "  ❌ $*"; fails=$((fails+1)); }

# ── Helpers ─────────────────────────────────────────────────────

# HMAC-sign a body the same way M3 / M5 / M6 do.
# X-BA-Signature: t=<unix>,v1=<hex(hmac-sha256(secret, "<unix>.<body>"))>
# The 202 response body is {"alert_id":"...","dedupe_count":N,
# "received_at":"..."} — we extract alert_id so the
# downstream checks can find the right row.
sign_and_send() {
  local body="$1"
  local ts=$(date +%s)
  local sig=$(printf '%s.%s' "$ts" "$body" | openssl dgst -sha256 -hmac "$SECRET" -hex | awk '{print $NF}')
  curl -sS -X POST "$INGESTD/v1/ingest" \
    -H "Content-Type: application/json" \
    -H "X-BA-Signature: t=$ts,v1=$sig" \
    --data "$body" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("alert_id",""))'
}

# ── Step 1: stack ready ─────────────────────────────────────────
echo "Step 1 — stack ready"
docker compose up -d --no-deps seed >/dev/null
sleep 1

if ! curl -sf "$INGESTD/health" >/dev/null; then
  fail "ingestd not healthy"
  exit 1
fi
pass "ingestd healthy"

if ! curl -sf "$ADMIND/health" >/dev/null; then
  fail "admind not healthy"
  exit 1
fi
pass "admind healthy"

if ! $PGCMD -c "\\d deliveries_dlq" >/dev/null 2>&1; then
  fail "deliveries_dlq table missing"
  exit 1
fi
pass "deliveries_dlq table present"

# Reset fakefcmd to healthy.
curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
pass "fakefcmd fail mode = 0 (healthy)"

# ── Step 2: happy path ──────────────────────────────────────────
echo "Step 2 — happy path: 1 alert, 1 attempt, 0 DLQ rows"
DLQ_BEFORE=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE discarded = false;" | tr -d ' ')

ALERT_BODY=$(cat <<EOF
{
  "id": "smoke-m8-happy-$(date +%s)",
  "company_id": "$COMPANY",
  "source_id": "$SOURCE",
  "title": "smoke m8 happy",
  "body": "M8 smoke happy path",
  "severity": "info",
  "category": "smoke"
}
EOF
)
ALERT_ID=$(sign_and_send "$ALERT_BODY")
if [ -n "$ALERT_ID" ]; then
  pass "POST /v1/ingest accepted, alert_id=$ALERT_ID"
else
  fail "POST /v1/ingest returned no alert_id"
fi
sleep 2

SENT_ROW=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'sent';" | tr -d ' ')
DLQ_AFTER=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE discarded = false;" | tr -d ' ')
# An alert fans out to every matching recipient; the
# M2 seed has 2 individuals subscribed to acme-001:
# prom-prod, so we expect 2 sent rows.
if [ "$SENT_ROW" -ge "1" ]; then
  pass "$SENT_ROW sent deliveries rows for happy alert (>=1)"
else
  fail "expected >=1 sent row, got $SENT_ROW"
fi
if [ "$DLQ_BEFORE" = "$DLQ_AFTER" ]; then
  pass "DLQ row count unchanged ($DLQ_BEFORE)"
else
  fail "DLQ grew from $DLQ_BEFORE to $DLQ_AFTER on happy path"
fi

# ── Step 3: DLQ creation via runtime fail mode ──────────────────
echo "Step 3 — DLQ creation: flip fakefcmd to fail, 1 alert, retries exhaust"
curl -sf "$FAKEFCMD/control?fail=1" >/dev/null
pass "fakefcmd fail mode = 1"

ALERT_BODY=$(cat <<EOF
{
  "id": "smoke-m8-dlq-$(date +%s)",
  "company_id": "$COMPANY",
  "source_id": "$SOURCE",
  "title": "smoke m8 dlq",
  "body": "M8 smoke DLQ test",
  "severity": "info",
  "category": "smoke"
}
EOF
)
ALERT_ID=$(sign_and_send "$ALERT_BODY")
if [ -n "$ALERT_ID" ]; then
  pass "POST /v1/ingest accepted, alert_id=$ALERT_ID (will fail at delivery)"
else
  fail "POST /v1/ingest returned no alert_id"
fi

# Wait for retry budget to exhaust. Defaults: 10
# attempts, base 100ms, cap 2s ≈ 12s total. The
# timeout ceiling here is 30s.
for i in $(seq 1 30); do
  DLQ_COUNT=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')
  if [ "$DLQ_COUNT" = "1" ]; then
    break
  fi
  sleep 1
done

if [ "$DLQ_COUNT" = "1" ]; then
  pass "1 row in deliveries_dlq for $ALERT_ID"
else
  fail "expected 1 DLQ row for $ALERT_ID, got $DLQ_COUNT (waited 30s)"
fi

ATTEMPTS=$($PGCMD -c "SELECT attempts FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')
if [ "$ATTEMPTS" = "10" ]; then
  pass "DLQ row has attempts=10"
else
  fail "expected attempts=10, got $ATTEMPTS"
fi

FAILED_ROWS=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'failed';" | tr -d ' ')
# Per-recipient: 10 attempts × N recipients (1 for
# M8 smoke, but the seed could fan out). We check
# >= 10*N for the smoke.
EXPECTED_FAILED=$((10 * 1))
if [ "$FAILED_ROWS" -ge "$EXPECTED_FAILED" ]; then
  pass "$FAILED_ROWS failed deliveries rows (>= $EXPECTED_FAILED for one recipient × 10 attempts)"
else
  fail "expected >= $EXPECTED_FAILED failed rows, got $FAILED_ROWS"
fi

# Check the API: GET /v1/dlq should include our row.
API_ROW=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true)
if [ -n "$API_ROW" ]; then
  pass "GET /v1/dlq returns the row"
else
  fail "GET /v1/dlq missing the row"
fi

DLQ_ID=$($PGCMD -c "SELECT id FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')

# ── Step 4: replay ──────────────────────────────────────────────
echo "Step 4 — replay: flip fakefcmd healthy, POST /v1/dlq/$DLQ_ID/replay"
curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
pass "fakefcmd fail mode = 0"

REPLAY_RESP=$(curl -sf -X POST "$ADMIND/v1/dlq/$DLQ_ID/replay" || true)
REPLAY_OK=$(echo "$REPLAY_RESP" | grep -o '"replayed":true' | head -1 || true)
if [ -n "$REPLAY_OK" ]; then
  pass "POST /v1/dlq/$DLQ_ID/replay returned replayed=true"
else
  fail "POST /v1/dlq/$DLQ_ID/replay did not return replayed=true (got: $REPLAY_RESP)"
fi

# Wait for the replayed delivery to land. The replay
# re-publishes onto deliveries.fcm.acme-001, which
# deliverd-fcm consumes (now with healthy fakefcmd).
for i in $(seq 1 15); do
  SENT_REPLAY=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'sent';" | tr -d ' ')
  if [ "$SENT_REPLAY" -ge "1" ]; then
    break
  fi
  sleep 1
done

if [ "$SENT_REPLAY" -ge "1" ]; then
  pass "replay produced a sent deliveries row"
else
  fail "no sent row after replay (got $SENT_REPLAY)"
fi

DISCARDED=$($PGCMD -c "SELECT discarded FROM deliveries_dlq WHERE id = $DLQ_ID;" | tr -d ' ')
if [ "$DISCARDED" = "t" ]; then
  pass "DLQ row marked discarded after replay"
else
  fail "DLQ row not discarded (got '$DISCARDED')"
fi

API_AFTER=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true)
if [ -z "$API_AFTER" ]; then
  pass "GET /v1/dlq no longer returns the discarded row"
else
  fail "GET /v1/dlq still returns the discarded row"
fi

# ── Step 5: discard ─────────────────────────────────────────────
echo "Step 5 — discard: another DLQ row, then POST /v1/dlq/{id}/discard"
curl -sf "$FAKEFCMD/control?fail=1" >/dev/null
ALERT_BODY=$(cat <<EOF
{
  "id": "smoke-m8-discard-$(date +%s)",
  "company_id": "$COMPANY",
  "source_id": "$SOURCE",
  "title": "smoke m8 discard",
  "body": "M8 smoke discard test",
  "severity": "info",
  "category": "smoke"
}
EOF
)
ALERT_ID2=$(sign_and_send "$ALERT_BODY")
if [ -n "$ALERT_ID2" ]; then
  pass "POST /v1/ingest accepted, alert_id=$ALERT_ID2"
else
  fail "POST /v1/ingest returned no alert_id"
fi

for i in $(seq 1 30); do
  DLQ2_COUNT=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE alert_id = '$ALERT_ID2';" | tr -d ' ')
  if [ "$DLQ2_COUNT" = "1" ]; then
    break
  fi
  sleep 1
done

if [ "$DLQ2_COUNT" = "1" ]; then
  pass "1 row in deliveries_dlq for $ALERT_ID2"
else
  fail "expected 1 DLQ row for $ALERT_ID2, got $DLQ2_COUNT"
fi

DLQ_ID2=$($PGCMD -c "SELECT id FROM deliveries_dlq WHERE alert_id = '$ALERT_ID2';" | tr -d ' ')

DISC_RESP=$(curl -sf -X POST "$ADMIND/v1/dlq/$DLQ_ID2/discard" || true)
DISC_OK=$(echo "$DISC_RESP" | grep -o '"discarded":true' | head -1 || true)
if [ -n "$DISC_OK" ]; then
  pass "POST /v1/dlq/$DLQ_ID2/discard returned discarded=true"
else
  fail "POST /v1/dlq/$DLQ_ID2/discard did not return discarded=true (got: $DISC_RESP)"
fi

API_DISC=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID2" | grep -o "\"alert_id\":\"$ALERT_ID2\"" | head -1 || true)
if [ -z "$API_DISC" ]; then
  pass "GET /v1/dlq hides the discarded row"
else
  fail "GET /v1/dlq still returns the discarded row"
fi

API_DISC_ALL=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID2&include=all" | grep -o "\"alert_id\":\"$ALERT_ID2\"" | head -1 || true)
if [ -n "$API_DISC_ALL" ]; then
  pass "GET /v1/dlq?include=all shows the discarded row"
else
  fail "GET /v1/dlq?include=all does not show the discarded row"
fi

# ── Summary ─────────────────────────────────────────────────────
curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
echo
if [ "$fails" = "0" ]; then
  echo "🎉 M8 smoke: 12/12 checks green."
  exit 0
else
  echo "💥 M8 smoke: $fails check(s) failed."
  exit 1
fi