#!/usr/bin/env bash # Live M6.5 smoke test. Run from repo root: # bash scripts/m6.5_smoke.sh # # Walks through the 4 scenarios in M6.5_VERIFICATION.md: # # Step 2 — single collapse: 100 alerts with the same # --dedupe-key → faketgmd /admin/sent shows # count=1, text contains '(×100)'. (This is the # headline M6.5 win: the recipient gets ONE # message, not 100.) # Step 3 — tail visibility: the WS tail still shows # every arrival (operators see the storm live), # even though only 1 delivery went out. # Step 4 — max-wait re-flush: a 6s continuous burst # re-flushes every 2s. The recipient gets 3+ # messages, each with the count-so-far. # Step 5 — per-source isolation: same --dedupe-key from # acme (prom-prod) and globex (grafana) does # NOT collapse into one — separate state, two # messages. # Step 6 — empty key passthrough: the M2 path is # preserved — alerts with empty dedupe_key # deliver immediately. (The loadgen normal # profile gives each alert a unique key, so # 3 alerts produce 3 deliveries — i.e. no # collapse happens for keys we never see twice.) # # Exit code is the number of failed checks. set -e cd "$(dirname "$0")/.." INGESTD_METRICS=http://localhost:8800/metrics FAKETGMD=http://localhost:8830 WS_INGEST=ws://localhost:8800/v1/ingest/ws WS_TAIL=ws://localhost:8800/v1/tail/ws SRC_ACME=acme-001:prom-prod:s3cret-acme SRC_GLOBEX=globex-002:grafana:s3cret-globex TAIL_TOKEN=tail-dev-token-please-change-in-prod BA_FLUSH_MS=${BA_ROUTERD_DEDUPE_FLUSH_MS:-2000} fails=0 pass() { echo " ✅ $*"; } fail() { echo " ❌ $*"; fails=$((fails+1)); } # Seed globex company + source + individual + subscription. # Done once at the top of the script (idempotent), not in # reset_state — the schema's foreign keys make per-call seed # noisy. Step 5 needs globex to actually have a recipient. PGCMD="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t" $PGCMD -c \ "INSERT INTO companies (id, name) VALUES ('globex-002', 'Globex Corp') ON CONFLICT (id) DO NOTHING;" \ >/dev/null 2>&1 || true $PGCMD -c \ "INSERT INTO sources (id, company_id, name) VALUES ('grafana', 'globex-002', 'Globex Grafana') ON CONFLICT (company_id, id) DO NOTHING;" \ >/dev/null 2>&1 || true $PGCMD -c \ "INSERT INTO individuals (id, company_id, full_name, telegram_chat_id) VALUES ('ind-globex-001', 'globex-002', 'Globex Test', 2001) ON CONFLICT (id) DO UPDATE SET telegram_chat_id = 2001;" \ >/dev/null 2>&1 || true $PGCMD -c \ "INSERT INTO subscriptions (individual_id, company_id, source_id, channel_mask, min_severity, tz, status) VALUES ('ind-globex-001', 'globex-002', 'grafana', '[\"fcm\",\"telegram\"]'::jsonb, '', 'UTC', 'active') ON CONFLICT (individual_id, source_id) DO UPDATE SET channel_mask = '[\"fcm\",\"telegram\"]'::jsonb;" \ >/dev/null 2>&1 || true reset_state() { curl -sS -X POST $FAKETGMD/admin/reset >/dev/null # deliveries are kept (we want to count them across steps) } # ── Setup: build loadgen-ws and tailcount ── if [[ ! -x /tmp/loadgen-ws-m65 ]]; then echo "▸ Building /tmp/loadgen-ws-m65" (cd loadgen && CGO_ENABLED=0 go build -o /tmp/loadgen-ws-m65 ./cmd/ws) fi if [[ ! -x /tmp/tailcount ]]; then echo "▸ Building /tmp/tailcount" (cd testfakes/tailcount && CGO_ENABLED=0 go build -o /tmp/tailcount) fi # ───────────────────────────────────────────────────────────────── echo "── M6.5 smoke — Router-level dedupe collapse ──" echo "" # ── Step 2: single collapse (100 alerts → 1 message) ───── echo "── Step 2: 100 alerts with same --dedupe-key → 1 message, (×100) ──" reset_state # Drain the dedupe state for our key. We use a timestamp # suffix so each run is isolated. KEY="m65-step2-$(date +%s%N)" /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \ --count 100 --rate 50 --dedupe-key "$KEY" 2>&1 | tail -2 # Wait for the router's flush (BA_ROUTERD_DEDUPE_FLUSH_MS) # plus a margin for the deliverer round-trip. sleep_secs=$(awk "BEGIN { print int($BA_FLUSH_MS/1000) + 3 }") sleep "$sleep_secs" count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null) if [[ "$count" == "1" ]]; then pass "faketgmd /admin/sent count = 1 (100 alerts collapsed to 1 delivery)" else fail "faketgmd /admin/sent count = $count (expected 1)" fi last_text=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); items=d.get("items",[]); print(items[-1]["text"] if items else "")' 2>/dev/null) if echo "$last_text" | grep -q '(×100)'; then pass "faketgmd last message contains '(×100)'" else fail "faketgmd last message: $last_text (expected '(×100)' suffix)" fi # ── Step 3: tail still shows the storm ──────────────────── echo "" echo "── Step 3: WS tail sees every arrival (operators see the storm live) ──" reset_state KEY="m65-step3-$(date +%s%N)" # Subscribe to the tail in the background, filtered to the # title pattern that the loadgen's makeAlert puts in when # --dedupe-key is set ("LG M5 burst #N"). The compact # tail event includes title but NOT dedupe_key, so we filter # on the title. /tmp/tailcount -token "$TAIL_TOKEN" -company "acme-001" \ -filter "burst $KEY" -timeout 12s > /tmp/m65-step3-tail.out 2>&1 & TAIL_PID=$! # Give the tail 1.5s to connect. sleep 1.5 /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \ --count 25 --rate 50 --dedupe-key "$KEY" 2>&1 | tail -1 # Wait for the tailcount to finish. wait $TAIL_PID 2>/dev/null || true tail_count=$(grep -oE 'count=[0-9]+' /tmp/m65-step3-tail.out | tail -1 | cut -d= -f2) tail_count=${tail_count:-0} if [[ $tail_count -ge 20 ]]; then pass "tail subscriber saw $tail_count events (≥20 of 25 alerts streamed live)" else fail "tail saw $tail_count events (expected ≥20, output=$(cat /tmp/m65-step3-tail.out))" fi # And the recipient still only got 1. sleep "$sleep_secs" count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null) if [[ "$count" == "1" ]]; then pass "faketgmd count = 1 (collapse still applied — tail ≠ delivery)" else fail "faketgmd count = $count (expected 1)" fi # ── Step 4: max-wait re-flush (continuous stream) ──────── echo "" echo "── Step 4: 6s continuous burst at 30/s → ≥3 re-flushes ──" reset_state KEY="m65-step4-$(date +%s%N)" /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \ --count 180 --rate 30 --dedupe-key "$KEY" 2>&1 | tail -1 # 180 / 30 = 6 seconds of streaming, well above 3 flush windows # at BA_ROUTERD_DEDUPE_FLUSH_MS=2000. sleep "$sleep_secs" count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null) if [[ $count -ge 2 && $count -le 5 ]]; then pass "faketgmd count = $count (expected 2..5 re-flushes over 6s @ flushMs=$BA_FLUSH_MS)" else fail "faketgmd count = $count (expected 2..5 re-flushes)" fi # ── Step 5: per-source isolation ───────────────────────── echo "" echo "── Step 5: same --dedupe-key from acme and globex → 2 separate collapses ──" reset_state KEY="m65-step5-$(date +%s%N)" # 5 from acme, 5 from globex, same key. # The two sources have separate Collapser state, so each # gets 1 delivery. Total: 2 messages. # acme subscription is on 'prom-prod'; globex is on 'grafana'. /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \ --count 5 --rate 5 --dedupe-key "$KEY" 2>&1 | tail -1 sleep 1 /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_GLOBEX" \ --count 5 --rate 5 --dedupe-key "$KEY" 2>&1 | tail -1 sleep "$sleep_secs" count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null) # We expect 2: one from acme's chat (1001) and one from # globex's chat (2001). The faketgmd log groups by # (bot_token, chat_id), so 2 distinct recipients show # up as 2 items in /admin/sent. if [[ "$count" == "2" ]]; then pass "faketgmd count = 2 (one per source — per-source isolation holds)" else fail "faketgmd count = $count (expected 2: one per source)" fi # Verify the messages are from different chats. chats=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); items=d.get("items",[]); print(",".join(sorted(set(str(i.get("chat_id")) for i in items))))' 2>/dev/null) if [[ "$chats" == "1001,2001" ]]; then pass "chat_ids = $chats (acme=1001, globex=2001)" else fail "chat_ids = $chats (expected 1001,2001)" fi # ── Step 6: passthrough — unique keys don't collapse ────── echo "" echo "── Step 6: loadgen normal profile (unique keys per alert) → no collapse ──" reset_state # The loadgen's normal profile gives each alert a unique # dedupe_key (lg-m5-0, lg-m5-1, ...) — 30% share, but with # idx>0 and only 3 alerts, in practice they're all unique. # The Collapser's Passthrough path is for empty dedupe_key; # the unique-keys path uses CollapseNew, which still # publishes after the first observe. So 3 alerts → 3 # messages (one per key). /tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \ --count 3 --rate 5 2>&1 | tail -1 sleep "$sleep_secs" count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null) if [[ "$count" == "3" ]]; then pass "faketgmd count = 3 (3 unique keys → 3 deliveries, no collapse)" else fail "faketgmd count = $count (expected 3)" fi # ── Summary ───────────────────────────────────────────────── echo "" if [[ $fails -eq 0 ]]; then echo "🟢 M6.5 smoke PASS — all checks green" exit 0 else echo "🔴 M6.5 smoke FAIL — $fails check(s) failed" exit $fails fi