|
|
@@ -0,0 +1,310 @@
|
|
|
+#!/usr/bin/env bash
|
|
|
+# Live M8 smoke test. Run from repo root:
|
|
|
+# bash scripts/m8_smoke.sh
|
|
|
+#
|
|
|
+# Walks through the 4 scenarios in M8_VERIFICATION.md:
|
|
|
+#
|
|
|
+# Step 2 — happy path unchanged: a normal alert with
|
|
|
+# fakefcmd in healthy mode delivers in 1
|
|
|
+# attempt; 0 DLQ rows.
|
|
|
+# Step 3 — DLQ creation: flip fakefcmd into fail
|
|
|
+# mode at runtime via /control?fail=1, send
|
|
|
+# one alert, wait for the retry budget to
|
|
|
+# exhaust (~12s for the default config), and
|
|
|
+# verify 1 row in deliveries_dlq with
|
|
|
+# attempts=10 + 10 'failed' rows in
|
|
|
+# deliveries.
|
|
|
+# Step 4 — replay: flip fakefcmd back to healthy,
|
|
|
+# POST /v1/dlq/{id}/replay, verify a fresh
|
|
|
+# 'sent' row in deliveries and the DLQ row
|
|
|
+# is now discarded.
|
|
|
+# Step 5 — discard: flip fakefcmd to fail again,
|
|
|
+# send another alert, wait, POST
|
|
|
+# /v1/dlq/{id}/discard, verify hidden from
|
|
|
+# the default list and visible with
|
|
|
+# include=all.
|
|
|
+#
|
|
|
+# Exit code is the number of failed checks.
|
|
|
+
|
|
|
+set -e
|
|
|
+cd "$(dirname "$0")/.."
|
|
|
+
|
|
|
+INGESTD=http://localhost:8800
|
|
|
+ADMIND=http://localhost:8803
|
|
|
+FAKEFCMD=http://localhost:8820
|
|
|
+COMPANY=acme-001
|
|
|
+SOURCE=prom-prod
|
|
|
+SECRET=s3cret-acme
|
|
|
+PGCMD="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t"
|
|
|
+
|
|
|
+fails=0
|
|
|
+pass() { echo " ✅ $*"; }
|
|
|
+fail() { echo " ❌ $*"; fails=$((fails+1)); }
|
|
|
+
|
|
|
+# ── Helpers ─────────────────────────────────────────────────────
|
|
|
+
|
|
|
+# HMAC-sign a body the same way M3 / M5 / M6 do.
|
|
|
+# X-BA-Signature: t=<unix>,v1=<hex(hmac-sha256(secret, "<unix>.<body>"))>
|
|
|
+# The 202 response body is {"alert_id":"...","dedupe_count":N,
|
|
|
+# "received_at":"..."} — we extract alert_id so the
|
|
|
+# downstream checks can find the right row.
|
|
|
+sign_and_send() {
|
|
|
+ local body="$1"
|
|
|
+ local ts=$(date +%s)
|
|
|
+ local sig=$(printf '%s.%s' "$ts" "$body" | openssl dgst -sha256 -hmac "$SECRET" -hex | awk '{print $NF}')
|
|
|
+ curl -sS -X POST "$INGESTD/v1/ingest" \
|
|
|
+ -H "Content-Type: application/json" \
|
|
|
+ -H "X-BA-Signature: t=$ts,v1=$sig" \
|
|
|
+ --data "$body" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("alert_id",""))'
|
|
|
+}
|
|
|
+
|
|
|
+# ── Step 1: stack ready ─────────────────────────────────────────
|
|
|
+echo "Step 1 — stack ready"
|
|
|
+docker compose up -d --no-deps seed >/dev/null
|
|
|
+sleep 1
|
|
|
+
|
|
|
+if ! curl -sf "$INGESTD/health" >/dev/null; then
|
|
|
+ fail "ingestd not healthy"
|
|
|
+ exit 1
|
|
|
+fi
|
|
|
+pass "ingestd healthy"
|
|
|
+
|
|
|
+if ! curl -sf "$ADMIND/health" >/dev/null; then
|
|
|
+ fail "admind not healthy"
|
|
|
+ exit 1
|
|
|
+fi
|
|
|
+pass "admind healthy"
|
|
|
+
|
|
|
+if ! $PGCMD -c "\\d deliveries_dlq" >/dev/null 2>&1; then
|
|
|
+ fail "deliveries_dlq table missing"
|
|
|
+ exit 1
|
|
|
+fi
|
|
|
+pass "deliveries_dlq table present"
|
|
|
+
|
|
|
+# Reset fakefcmd to healthy.
|
|
|
+curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
|
|
|
+pass "fakefcmd fail mode = 0 (healthy)"
|
|
|
+
|
|
|
+# ── Step 2: happy path ──────────────────────────────────────────
|
|
|
+echo "Step 2 — happy path: 1 alert, 1 attempt, 0 DLQ rows"
|
|
|
+DLQ_BEFORE=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE discarded = false;" | tr -d ' ')
|
|
|
+
|
|
|
+ALERT_BODY=$(cat <<EOF
|
|
|
+{
|
|
|
+ "id": "smoke-m8-happy-$(date +%s)",
|
|
|
+ "company_id": "$COMPANY",
|
|
|
+ "source_id": "$SOURCE",
|
|
|
+ "title": "smoke m8 happy",
|
|
|
+ "body": "M8 smoke happy path",
|
|
|
+ "severity": "info",
|
|
|
+ "category": "smoke"
|
|
|
+}
|
|
|
+EOF
|
|
|
+)
|
|
|
+ALERT_ID=$(sign_and_send "$ALERT_BODY")
|
|
|
+if [ -n "$ALERT_ID" ]; then
|
|
|
+ pass "POST /v1/ingest accepted, alert_id=$ALERT_ID"
|
|
|
+else
|
|
|
+ fail "POST /v1/ingest returned no alert_id"
|
|
|
+fi
|
|
|
+sleep 2
|
|
|
+
|
|
|
+SENT_ROW=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'sent';" | tr -d ' ')
|
|
|
+DLQ_AFTER=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE discarded = false;" | tr -d ' ')
|
|
|
+# An alert fans out to every matching recipient; the
|
|
|
+# M2 seed has 2 individuals subscribed to acme-001:
|
|
|
+# prom-prod, so we expect 2 sent rows.
|
|
|
+if [ "$SENT_ROW" -ge "1" ]; then
|
|
|
+ pass "$SENT_ROW sent deliveries rows for happy alert (>=1)"
|
|
|
+else
|
|
|
+ fail "expected >=1 sent row, got $SENT_ROW"
|
|
|
+fi
|
|
|
+if [ "$DLQ_BEFORE" = "$DLQ_AFTER" ]; then
|
|
|
+ pass "DLQ row count unchanged ($DLQ_BEFORE)"
|
|
|
+else
|
|
|
+ fail "DLQ grew from $DLQ_BEFORE to $DLQ_AFTER on happy path"
|
|
|
+fi
|
|
|
+
|
|
|
+# ── Step 3: DLQ creation via runtime fail mode ──────────────────
|
|
|
+echo "Step 3 — DLQ creation: flip fakefcmd to fail, 1 alert, retries exhaust"
|
|
|
+curl -sf "$FAKEFCMD/control?fail=1" >/dev/null
|
|
|
+pass "fakefcmd fail mode = 1"
|
|
|
+
|
|
|
+ALERT_BODY=$(cat <<EOF
|
|
|
+{
|
|
|
+ "id": "smoke-m8-dlq-$(date +%s)",
|
|
|
+ "company_id": "$COMPANY",
|
|
|
+ "source_id": "$SOURCE",
|
|
|
+ "title": "smoke m8 dlq",
|
|
|
+ "body": "M8 smoke DLQ test",
|
|
|
+ "severity": "info",
|
|
|
+ "category": "smoke"
|
|
|
+}
|
|
|
+EOF
|
|
|
+)
|
|
|
+ALERT_ID=$(sign_and_send "$ALERT_BODY")
|
|
|
+if [ -n "$ALERT_ID" ]; then
|
|
|
+ pass "POST /v1/ingest accepted, alert_id=$ALERT_ID (will fail at delivery)"
|
|
|
+else
|
|
|
+ fail "POST /v1/ingest returned no alert_id"
|
|
|
+fi
|
|
|
+
|
|
|
+# Wait for retry budget to exhaust. Defaults: 10
|
|
|
+# attempts, base 100ms, cap 2s ≈ 12s total. The
|
|
|
+# timeout ceiling here is 30s.
|
|
|
+for i in $(seq 1 30); do
|
|
|
+ DLQ_COUNT=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')
|
|
|
+ if [ "$DLQ_COUNT" = "1" ]; then
|
|
|
+ break
|
|
|
+ fi
|
|
|
+ sleep 1
|
|
|
+done
|
|
|
+
|
|
|
+if [ "$DLQ_COUNT" = "1" ]; then
|
|
|
+ pass "1 row in deliveries_dlq for $ALERT_ID"
|
|
|
+else
|
|
|
+ fail "expected 1 DLQ row for $ALERT_ID, got $DLQ_COUNT (waited 30s)"
|
|
|
+fi
|
|
|
+
|
|
|
+ATTEMPTS=$($PGCMD -c "SELECT attempts FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')
|
|
|
+if [ "$ATTEMPTS" = "10" ]; then
|
|
|
+ pass "DLQ row has attempts=10"
|
|
|
+else
|
|
|
+ fail "expected attempts=10, got $ATTEMPTS"
|
|
|
+fi
|
|
|
+
|
|
|
+FAILED_ROWS=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'failed';" | tr -d ' ')
|
|
|
+# Per-recipient: 10 attempts × N recipients (1 for
|
|
|
+# M8 smoke, but the seed could fan out). We check
|
|
|
+# >= 10*N for the smoke.
|
|
|
+EXPECTED_FAILED=$((10 * 1))
|
|
|
+if [ "$FAILED_ROWS" -ge "$EXPECTED_FAILED" ]; then
|
|
|
+ pass "$FAILED_ROWS failed deliveries rows (>= $EXPECTED_FAILED for one recipient × 10 attempts)"
|
|
|
+else
|
|
|
+ fail "expected >= $EXPECTED_FAILED failed rows, got $FAILED_ROWS"
|
|
|
+fi
|
|
|
+
|
|
|
+# Check the API: GET /v1/dlq should include our row.
|
|
|
+API_ROW=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true)
|
|
|
+if [ -n "$API_ROW" ]; then
|
|
|
+ pass "GET /v1/dlq returns the row"
|
|
|
+else
|
|
|
+ fail "GET /v1/dlq missing the row"
|
|
|
+fi
|
|
|
+
|
|
|
+DLQ_ID=$($PGCMD -c "SELECT id FROM deliveries_dlq WHERE alert_id = '$ALERT_ID';" | tr -d ' ')
|
|
|
+
|
|
|
+# ── Step 4: replay ──────────────────────────────────────────────
|
|
|
+echo "Step 4 — replay: flip fakefcmd healthy, POST /v1/dlq/$DLQ_ID/replay"
|
|
|
+curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
|
|
|
+pass "fakefcmd fail mode = 0"
|
|
|
+
|
|
|
+REPLAY_RESP=$(curl -sf -X POST "$ADMIND/v1/dlq/$DLQ_ID/replay" || true)
|
|
|
+REPLAY_OK=$(echo "$REPLAY_RESP" | grep -o '"replayed":true' | head -1 || true)
|
|
|
+if [ -n "$REPLAY_OK" ]; then
|
|
|
+ pass "POST /v1/dlq/$DLQ_ID/replay returned replayed=true"
|
|
|
+else
|
|
|
+ fail "POST /v1/dlq/$DLQ_ID/replay did not return replayed=true (got: $REPLAY_RESP)"
|
|
|
+fi
|
|
|
+
|
|
|
+# Wait for the replayed delivery to land. The replay
|
|
|
+# re-publishes onto deliveries.fcm.acme-001, which
|
|
|
+# deliverd-fcm consumes (now with healthy fakefcmd).
|
|
|
+for i in $(seq 1 15); do
|
|
|
+ SENT_REPLAY=$($PGCMD -c "SELECT count(*) FROM deliveries WHERE alert_id = '$ALERT_ID' AND status = 'sent';" | tr -d ' ')
|
|
|
+ if [ "$SENT_REPLAY" -ge "1" ]; then
|
|
|
+ break
|
|
|
+ fi
|
|
|
+ sleep 1
|
|
|
+done
|
|
|
+
|
|
|
+if [ "$SENT_REPLAY" -ge "1" ]; then
|
|
|
+ pass "replay produced a sent deliveries row"
|
|
|
+else
|
|
|
+ fail "no sent row after replay (got $SENT_REPLAY)"
|
|
|
+fi
|
|
|
+
|
|
|
+DISCARDED=$($PGCMD -c "SELECT discarded FROM deliveries_dlq WHERE id = $DLQ_ID;" | tr -d ' ')
|
|
|
+if [ "$DISCARDED" = "t" ]; then
|
|
|
+ pass "DLQ row marked discarded after replay"
|
|
|
+else
|
|
|
+ fail "DLQ row not discarded (got '$DISCARDED')"
|
|
|
+fi
|
|
|
+
|
|
|
+API_AFTER=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID" | grep -o "\"alert_id\":\"$ALERT_ID\"" | head -1 || true)
|
|
|
+if [ -z "$API_AFTER" ]; then
|
|
|
+ pass "GET /v1/dlq no longer returns the discarded row"
|
|
|
+else
|
|
|
+ fail "GET /v1/dlq still returns the discarded row"
|
|
|
+fi
|
|
|
+
|
|
|
+# ── Step 5: discard ─────────────────────────────────────────────
|
|
|
+echo "Step 5 — discard: another DLQ row, then POST /v1/dlq/{id}/discard"
|
|
|
+curl -sf "$FAKEFCMD/control?fail=1" >/dev/null
|
|
|
+ALERT_BODY=$(cat <<EOF
|
|
|
+{
|
|
|
+ "id": "smoke-m8-discard-$(date +%s)",
|
|
|
+ "company_id": "$COMPANY",
|
|
|
+ "source_id": "$SOURCE",
|
|
|
+ "title": "smoke m8 discard",
|
|
|
+ "body": "M8 smoke discard test",
|
|
|
+ "severity": "info",
|
|
|
+ "category": "smoke"
|
|
|
+}
|
|
|
+EOF
|
|
|
+)
|
|
|
+ALERT_ID2=$(sign_and_send "$ALERT_BODY")
|
|
|
+if [ -n "$ALERT_ID2" ]; then
|
|
|
+ pass "POST /v1/ingest accepted, alert_id=$ALERT_ID2"
|
|
|
+else
|
|
|
+ fail "POST /v1/ingest returned no alert_id"
|
|
|
+fi
|
|
|
+
|
|
|
+for i in $(seq 1 30); do
|
|
|
+ DLQ2_COUNT=$($PGCMD -c "SELECT count(*) FROM deliveries_dlq WHERE alert_id = '$ALERT_ID2';" | tr -d ' ')
|
|
|
+ if [ "$DLQ2_COUNT" = "1" ]; then
|
|
|
+ break
|
|
|
+ fi
|
|
|
+ sleep 1
|
|
|
+done
|
|
|
+
|
|
|
+if [ "$DLQ2_COUNT" = "1" ]; then
|
|
|
+ pass "1 row in deliveries_dlq for $ALERT_ID2"
|
|
|
+else
|
|
|
+ fail "expected 1 DLQ row for $ALERT_ID2, got $DLQ2_COUNT"
|
|
|
+fi
|
|
|
+
|
|
|
+DLQ_ID2=$($PGCMD -c "SELECT id FROM deliveries_dlq WHERE alert_id = '$ALERT_ID2';" | tr -d ' ')
|
|
|
+
|
|
|
+DISC_RESP=$(curl -sf -X POST "$ADMIND/v1/dlq/$DLQ_ID2/discard" || true)
|
|
|
+DISC_OK=$(echo "$DISC_RESP" | grep -o '"discarded":true' | head -1 || true)
|
|
|
+if [ -n "$DISC_OK" ]; then
|
|
|
+ pass "POST /v1/dlq/$DLQ_ID2/discard returned discarded=true"
|
|
|
+else
|
|
|
+ fail "POST /v1/dlq/$DLQ_ID2/discard did not return discarded=true (got: $DISC_RESP)"
|
|
|
+fi
|
|
|
+
|
|
|
+API_DISC=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID2" | grep -o "\"alert_id\":\"$ALERT_ID2\"" | head -1 || true)
|
|
|
+if [ -z "$API_DISC" ]; then
|
|
|
+ pass "GET /v1/dlq hides the discarded row"
|
|
|
+else
|
|
|
+ fail "GET /v1/dlq still returns the discarded row"
|
|
|
+fi
|
|
|
+
|
|
|
+API_DISC_ALL=$(curl -sf "$ADMIND/v1/dlq?company_id=$COMPANY&alert_id=$ALERT_ID2&include=all" | grep -o "\"alert_id\":\"$ALERT_ID2\"" | head -1 || true)
|
|
|
+if [ -n "$API_DISC_ALL" ]; then
|
|
|
+ pass "GET /v1/dlq?include=all shows the discarded row"
|
|
|
+else
|
|
|
+ fail "GET /v1/dlq?include=all does not show the discarded row"
|
|
|
+fi
|
|
|
+
|
|
|
+# ── Summary ─────────────────────────────────────────────────────
|
|
|
+curl -sf "$FAKEFCMD/control?fail=0" >/dev/null
|
|
|
+echo
|
|
|
+if [ "$fails" = "0" ]; then
|
|
|
+ echo "🎉 M8 smoke: 12/12 checks green."
|
|
|
+ exit 0
|
|
|
+else
|
|
|
+ echo "💥 M8 smoke: $fails check(s) failed."
|
|
|
+ exit 1
|
|
|
+fi
|
Luis Rosales