#!/usr/bin/env bash # Live M8 smoke test. Run from repo root: # bash scripts/m8_smoke.sh # # Walks through the 4 scenarios in M8_VERIFICATION.md: # # Step 2 — happy path unchanged: a normal alert with # fakefcmd in healthy mode delivers in 1 # attempt; 0 DLQ rows. # Step 3 — DLQ creation: flip fakefcmd into fail # mode at runtime via /control?fail=1, send # one alert, wait for the retry budget to # exhaust (~12s for the default config), and # verify 1 row in deliveries_dlq with # attempts=10 + 10 'failed' rows in # deliveries. # Step 4 — replay: flip fakefcmd back to healthy, # POST /v1/dlq/{id}/replay, verify a fresh # 'sent' row in deliveries and the DLQ row # is now discarded. # Step 5 — discard: flip fakefcmd to fail again, # send another alert, wait, POST # /v1/dlq/{id}/discard, verify hidden from # the default list and visible with # include=all. # # Exit code is the number of failed checks. set -e cd "$(dirname "$0")/.." INGESTD=http://localhost:8800 ADMIND=http://localhost:8803 FAKEFCMD=http://localhost:8820 COMPANY=acme-001 SOURCE=prom-prod SECRET=s3cret-acme PGCMD="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t" fails=0 pass() { echo " ✅ $*"; } fail() { echo " ❌ $*"; fails=$((fails+1)); } # ── Helpers ───────────────────────────────────────────────────── # HMAC-sign a body the same way M3 / M5 / M6 do. # X-BA-Signature: t=,v1=."))> # The 202 response body is {"alert_id":"...","dedupe_count":N, # "received_at":"..."} — we extract alert_id so the # downstream checks can find the right row. sign_and_send() { local body="$1" local ts=$(date +%s) local sig=$(printf '%s.%s' "$ts" "$body" | openssl dgst -sha256 -hmac "$SECRET" -hex | awk '{print $NF}') curl -sS -X POST "$INGESTD/v1/ingest" \ -H "Content-Type: application/json" \ -H "X-BA-Signature: t=$ts,v1=$sig" \ --data "$body" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("alert_id",""))' } # ── Step 1: stack ready ───────────────────────────────────────── echo "Step 1 — stack ready" docker compose up -d --no-deps seed >/dev/null sleep 1 if ! curl -sf "$INGESTD/health" >/dev/null; then fail "ingestd not healthy" exit 1 fi pass "ingestd healthy" if ! curl -sf "$ADMIND/health" >/dev/null; then fail "admind not healthy" exit 1 fi pass "admind healthy" if ! $PGCMD -c "\\d deliveries_dlq" >/dev/null 2>&1; then fail "deliveries_dlq table missing" exit 1 fi pass "deliveries_dlq table present" # Reset fakefcmd to healthy. curl -sf "$FAKEFCMD/control?fail=0" >/dev/null pass "fakefcmd fail mode = 0 (healthy)" # ── Step 2: happy path ────────────────────────────────────────── echo "Step 2 — happy path: 1 alert, 1 attempt, 0 DLQ rows" DLQ_BEFORE=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE discarded = false;" | tr -d ' ') ALERT_BODY=$(cat <=1)" else fail "expected >=1 sent row, got $SENT_ROW" fi if [ "$DLQ_BEFORE" = "$DLQ_AFTER" ]; then pass "DLQ row count unchanged ($DLQ_BEFORE)" else fail "DLQ grew from $DLQ_BEFORE to $DLQ_AFTER on happy path" fi # ── Step 3: DLQ creation via runtime fail mode ────────────────── echo "Step 3 — DLQ creation: flip fakefcmd to fail, 1 alert, retries exhaust" curl -sf "$FAKEFCMD/control?fail=1" >/dev/null pass "fakefcmd fail mode = 1" ALERT_BODY=$(cat <= 10*N for the smoke. EXPECTED_FAILED=$((10 * 1)) if [ "$FAILED_ROWS" -ge "$EXPECTED_FAILED" ]; then pass "$FAILED_ROWS failed deliveries rows (>= $EXPECTED_FAILED for one recipient × 10 attempts)" else fail "expected >= $EXPECTED_FAILED failed rows, got $FAILED_ROWS" fi # Check the API: GET /v1/dlq should include our row. API_ROW=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true) if [ -n "$API_ROW" ]; then pass "GET /v1/dlq returns the row" else fail "GET /v1/dlq missing the row" fi DLQ_ID=$($PGCMD -c "SELECT id FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ') # ── Step 4: replay ────────────────────────────────────────────── echo "Step 4 — replay: flip fakefcmd healthy, POST /v1/dlq/$DLQ_ID/replay" curl -sf "$FAKEFCMD/control?fail=0" >/dev/null pass "fakefcmd fail mode = 0" REPLAY_RESP=$(curl -sf -X POST "$ADMIND/v1/dlq/$DLQ_ID/replay" || true) REPLAY_OK=$(echo "$REPLAY_RESP" | grep -o '"replayed":true' | head -1 || true) if [ -n "$REPLAY_OK" ]; then pass "POST /v1/dlq/$DLQ_ID/replay returned replayed=true" else fail "POST /v1/dlq/$DLQ_ID/replay did not return replayed=true (got: $REPLAY_RESP)" fi # Wait for the replayed delivery to land. The replay # re-publishes onto deliveries.fcm.acme-001, which # deliverd-fcm consumes (now with healthy fakefcmd). for i in $(seq 1 15); do SENT_REPLAY=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'sent';" | tr -d ' ') if [ "$SENT_REPLAY" -ge "1" ]; then break fi sleep 1 done if [ "$SENT_REPLAY" -ge "1" ]; then pass "replay produced a sent deliveries row" else fail "no sent row after replay (got $SENT_REPLAY)" fi DISCARDED=$($PGCMD -c "SELECT discarded FROM deliveries_dlq WHERE id = $DLQ_ID;" | tr -d ' ') if [ "$DISCARDED" = "t" ]; then pass "DLQ row marked discarded after replay" else fail "DLQ row not discarded (got '$DISCARDED')" fi API_AFTER=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true) if [ -z "$API_AFTER" ]; then pass "GET /v1/dlq no longer returns the discarded row" else fail "GET /v1/dlq still returns the discarded row" fi # ── Step 5: discard ───────────────────────────────────────────── echo "Step 5 — discard: another DLQ row, then POST /v1/dlq/{id}/discard" curl -sf "$FAKEFCMD/control?fail=1" >/dev/null ALERT_BODY=$(cat </dev/null echo if [ "$fails" = "0" ]; then echo "🎉 M8 smoke: 12/12 checks green." exit 0 else echo "💥 M8 smoke: $fails check(s) failed." exit 1 fi