|
@@ -0,0 +1,236 @@
|
|
|
|
|
+#!/usr/bin/env bash
|
|
|
|
|
+# Live M6.5 smoke test. Run from repo root:
|
|
|
|
|
+# bash scripts/m6.5_smoke.sh
|
|
|
|
|
+#
|
|
|
|
|
+# Walks through the 4 scenarios in M6.5_VERIFICATION.md:
|
|
|
|
|
+#
|
|
|
|
|
+# Step 2 — single collapse: 100 alerts with the same
|
|
|
|
|
+# --dedupe-key → faketgmd /admin/sent shows
|
|
|
|
|
+# count=1, text contains '(×100)'. (This is the
|
|
|
|
|
+# headline M6.5 win: the recipient gets ONE
|
|
|
|
|
+# message, not 100.)
|
|
|
|
|
+# Step 3 — tail visibility: the WS tail still shows
|
|
|
|
|
+# every arrival (operators see the storm live),
|
|
|
|
|
+# even though only 1 delivery went out.
|
|
|
|
|
+# Step 4 — max-wait re-flush: a 6s continuous burst
|
|
|
|
|
+# re-flushes every 2s. The recipient gets 3+
|
|
|
|
|
+# messages, each with the count-so-far.
|
|
|
|
|
+# Step 5 — per-source isolation: same --dedupe-key from
|
|
|
|
|
+# acme (prom-prod) and globex (grafana) does
|
|
|
|
|
+# NOT collapse into one — separate state, two
|
|
|
|
|
+# messages.
|
|
|
|
|
+# Step 6 — empty key passthrough: the M2 path is
|
|
|
|
|
+# preserved — alerts with empty dedupe_key
|
|
|
|
|
+# deliver immediately. (The loadgen normal
|
|
|
|
|
+# profile gives each alert a unique key, so
|
|
|
|
|
+# 3 alerts produce 3 deliveries — i.e. no
|
|
|
|
|
+# collapse happens for keys we never see twice.)
|
|
|
|
|
+#
|
|
|
|
|
+# Exit code is the number of failed checks.
|
|
|
|
|
+
|
|
|
|
|
+set -e
|
|
|
|
|
+cd "$(dirname "$0")/.."
|
|
|
|
|
+
|
|
|
|
|
+INGESTD_METRICS=http://localhost:8800/metrics
|
|
|
|
|
+FAKETGMD=http://localhost:8830
|
|
|
|
|
+WS_INGEST=ws://localhost:8800/v1/ingest/ws
|
|
|
|
|
+WS_TAIL=ws://localhost:8800/v1/tail/ws
|
|
|
|
|
+SRC_ACME=acme-001:prom-prod:s3cret-acme
|
|
|
|
|
+SRC_GLOBEX=globex-002:grafana:s3cret-globex
|
|
|
|
|
+TAIL_TOKEN=tail-dev-token-please-change-in-prod
|
|
|
|
|
+BA_FLUSH_MS=${BA_ROUTERD_DEDUPE_FLUSH_MS:-2000}
|
|
|
|
|
+
|
|
|
|
|
+fails=0
|
|
|
|
|
+pass() { echo " ✅ $*"; }
|
|
|
|
|
+fail() { echo " ❌ $*"; fails=$((fails+1)); }
|
|
|
|
|
+
|
|
|
|
|
+# Seed globex company + source + individual + subscription.
|
|
|
|
|
+# Done once at the top of the script (idempotent), not in
|
|
|
|
|
+# reset_state — the schema's foreign keys make per-call seed
|
|
|
|
|
+# noisy. Step 5 needs globex to actually have a recipient.
|
|
|
|
|
+PGCMD="docker exec -i broad-announce-postgres-1 psql -U ba -d ba -A -t"
|
|
|
|
|
+$PGCMD -c \
|
|
|
|
|
+ "INSERT INTO companies (id, name) VALUES ('globex-002', 'Globex Corp') ON CONFLICT (id) DO NOTHING;" \
|
|
|
|
|
+ >/dev/null 2>&1 || true
|
|
|
|
|
+$PGCMD -c \
|
|
|
|
|
+ "INSERT INTO sources (id, company_id, name) VALUES ('grafana', 'globex-002', 'Globex Grafana') ON CONFLICT (company_id, id) DO NOTHING;" \
|
|
|
|
|
+ >/dev/null 2>&1 || true
|
|
|
|
|
+$PGCMD -c \
|
|
|
|
|
+ "INSERT INTO individuals (id, company_id, full_name, telegram_chat_id) VALUES ('ind-globex-001', 'globex-002', 'Globex Test', 2001) ON CONFLICT (id) DO UPDATE SET telegram_chat_id = 2001;" \
|
|
|
|
|
+ >/dev/null 2>&1 || true
|
|
|
|
|
+$PGCMD -c \
|
|
|
|
|
+ "INSERT INTO subscriptions (individual_id, company_id, source_id, channel_mask, min_severity, tz, status) VALUES ('ind-globex-001', 'globex-002', 'grafana', '[\"fcm\",\"telegram\"]'::jsonb, '', 'UTC', 'active') ON CONFLICT (individual_id, source_id) DO UPDATE SET channel_mask = '[\"fcm\",\"telegram\"]'::jsonb;" \
|
|
|
|
|
+ >/dev/null 2>&1 || true
|
|
|
|
|
+
|
|
|
|
|
+reset_state() {
|
|
|
|
|
+ curl -sS -X POST $FAKETGMD/admin/reset >/dev/null
|
|
|
|
|
+ # deliveries are kept (we want to count them across steps)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ── Setup: build loadgen-ws and tailcount ──
|
|
|
|
|
+if [[ ! -x /tmp/loadgen-ws-m65 ]]; then
|
|
|
|
|
+ echo "▸ Building /tmp/loadgen-ws-m65"
|
|
|
|
|
+ (cd loadgen && CGO_ENABLED=0 go build -o /tmp/loadgen-ws-m65 ./cmd/ws)
|
|
|
|
|
+fi
|
|
|
|
|
+if [[ ! -x /tmp/tailcount ]]; then
|
|
|
|
|
+ echo "▸ Building /tmp/tailcount"
|
|
|
|
|
+ (cd testfakes/tailcount && CGO_ENABLED=0 go build -o /tmp/tailcount)
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ─────────────────────────────────────────────────────────────────
|
|
|
|
|
+echo "── M6.5 smoke — Router-level dedupe collapse ──"
|
|
|
|
|
+echo ""
|
|
|
|
|
+
|
|
|
|
|
+# ── Step 2: single collapse (100 alerts → 1 message) ─────
|
|
|
|
|
+echo "── Step 2: 100 alerts with same --dedupe-key → 1 message, (×100) ──"
|
|
|
|
|
+reset_state
|
|
|
|
|
+# Drain the dedupe state for our key. We use a timestamp
|
|
|
|
|
+# suffix so each run is isolated.
|
|
|
|
|
+KEY="m65-step2-$(date +%s%N)"
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \
|
|
|
|
|
+ --count 100 --rate 50 --dedupe-key "$KEY" 2>&1 | tail -2
|
|
|
|
|
+# Wait for the router's flush (BA_ROUTERD_DEDUPE_FLUSH_MS)
|
|
|
|
|
+# plus a margin for the deliverer round-trip.
|
|
|
|
|
+sleep_secs=$(awk "BEGIN { print int($BA_FLUSH_MS/1000) + 3 }")
|
|
|
|
|
+sleep "$sleep_secs"
|
|
|
|
|
+
|
|
|
|
|
+count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null)
|
|
|
|
|
+if [[ "$count" == "1" ]]; then
|
|
|
|
|
+ pass "faketgmd /admin/sent count = 1 (100 alerts collapsed to 1 delivery)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd /admin/sent count = $count (expected 1)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+last_text=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); items=d.get("items",[]); print(items[-1]["text"] if items else "")' 2>/dev/null)
|
|
|
|
|
+if echo "$last_text" | grep -q '(×100)'; then
|
|
|
|
|
+ pass "faketgmd last message contains '(×100)'"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd last message: $last_text (expected '(×100)' suffix)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ── Step 3: tail still shows the storm ────────────────────
|
|
|
|
|
+echo ""
|
|
|
|
|
+echo "── Step 3: WS tail sees every arrival (operators see the storm live) ──"
|
|
|
|
|
+reset_state
|
|
|
|
|
+KEY="m65-step3-$(date +%s%N)"
|
|
|
|
|
+
|
|
|
|
|
+# Subscribe to the tail in the background, filtered to the
|
|
|
|
|
+# title pattern that the loadgen's makeAlert puts in when
|
|
|
|
|
+# --dedupe-key is set ("LG M5 burst <key> #N"). The compact
|
|
|
|
|
+# tail event includes title but NOT dedupe_key, so we filter
|
|
|
|
|
+# on the title.
|
|
|
|
|
+/tmp/tailcount -token "$TAIL_TOKEN" -company "acme-001" \
|
|
|
|
|
+ -filter "burst $KEY" -timeout 12s > /tmp/m65-step3-tail.out 2>&1 &
|
|
|
|
|
+TAIL_PID=$!
|
|
|
|
|
+
|
|
|
|
|
+# Give the tail 1.5s to connect.
|
|
|
|
|
+sleep 1.5
|
|
|
|
|
+
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \
|
|
|
|
|
+ --count 25 --rate 50 --dedupe-key "$KEY" 2>&1 | tail -1
|
|
|
|
|
+
|
|
|
|
|
+# Wait for the tailcount to finish.
|
|
|
|
|
+wait $TAIL_PID 2>/dev/null || true
|
|
|
|
|
+
|
|
|
|
|
+tail_count=$(grep -oE 'count=[0-9]+' /tmp/m65-step3-tail.out | tail -1 | cut -d= -f2)
|
|
|
|
|
+tail_count=${tail_count:-0}
|
|
|
|
|
+if [[ $tail_count -ge 20 ]]; then
|
|
|
|
|
+ pass "tail subscriber saw $tail_count events (≥20 of 25 alerts streamed live)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "tail saw $tail_count events (expected ≥20, output=$(cat /tmp/m65-step3-tail.out))"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# And the recipient still only got 1.
|
|
|
|
|
+sleep "$sleep_secs"
|
|
|
|
|
+count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null)
|
|
|
|
|
+if [[ "$count" == "1" ]]; then
|
|
|
|
|
+ pass "faketgmd count = 1 (collapse still applied — tail ≠ delivery)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd count = $count (expected 1)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ── Step 4: max-wait re-flush (continuous stream) ────────
|
|
|
|
|
+echo ""
|
|
|
|
|
+echo "── Step 4: 6s continuous burst at 30/s → ≥3 re-flushes ──"
|
|
|
|
|
+reset_state
|
|
|
|
|
+KEY="m65-step4-$(date +%s%N)"
|
|
|
|
|
+
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \
|
|
|
|
|
+ --count 180 --rate 30 --dedupe-key "$KEY" 2>&1 | tail -1
|
|
|
|
|
+# 180 / 30 = 6 seconds of streaming, well above 3 flush windows
|
|
|
|
|
+# at BA_ROUTERD_DEDUPE_FLUSH_MS=2000.
|
|
|
|
|
+sleep "$sleep_secs"
|
|
|
|
|
+
|
|
|
|
|
+count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null)
|
|
|
|
|
+if [[ $count -ge 2 && $count -le 5 ]]; then
|
|
|
|
|
+ pass "faketgmd count = $count (expected 2..5 re-flushes over 6s @ flushMs=$BA_FLUSH_MS)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd count = $count (expected 2..5 re-flushes)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ── Step 5: per-source isolation ─────────────────────────
|
|
|
|
|
+echo ""
|
|
|
|
|
+echo "── Step 5: same --dedupe-key from acme and globex → 2 separate collapses ──"
|
|
|
|
|
+reset_state
|
|
|
|
|
+KEY="m65-step5-$(date +%s%N)"
|
|
|
|
|
+# 5 from acme, 5 from globex, same key.
|
|
|
|
|
+# The two sources have separate Collapser state, so each
|
|
|
|
|
+# gets 1 delivery. Total: 2 messages.
|
|
|
|
|
+# acme subscription is on 'prom-prod'; globex is on 'grafana'.
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \
|
|
|
|
|
+ --count 5 --rate 5 --dedupe-key "$KEY" 2>&1 | tail -1
|
|
|
|
|
+sleep 1
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_GLOBEX" \
|
|
|
|
|
+ --count 5 --rate 5 --dedupe-key "$KEY" 2>&1 | tail -1
|
|
|
|
|
+sleep "$sleep_secs"
|
|
|
|
|
+
|
|
|
|
|
+count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null)
|
|
|
|
|
+# We expect 2: one from acme's chat (1001) and one from
|
|
|
|
|
+# globex's chat (2001). The faketgmd log groups by
|
|
|
|
|
+# (bot_token, chat_id), so 2 distinct recipients show
|
|
|
|
|
+# up as 2 items in /admin/sent.
|
|
|
|
|
+if [[ "$count" == "2" ]]; then
|
|
|
|
|
+ pass "faketgmd count = 2 (one per source — per-source isolation holds)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd count = $count (expected 2: one per source)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# Verify the messages are from different chats.
|
|
|
|
|
+chats=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); items=d.get("items",[]); print(",".join(sorted(set(str(i.get("chat_id")) for i in items))))' 2>/dev/null)
|
|
|
|
|
+if [[ "$chats" == "1001,2001" ]]; then
|
|
|
|
|
+ pass "chat_ids = $chats (acme=1001, globex=2001)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "chat_ids = $chats (expected 1001,2001)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ── Step 6: passthrough — unique keys don't collapse ──────
|
|
|
|
|
+echo ""
|
|
|
|
|
+echo "── Step 6: loadgen normal profile (unique keys per alert) → no collapse ──"
|
|
|
|
|
+reset_state
|
|
|
|
|
+# The loadgen's normal profile gives each alert a unique
|
|
|
|
|
+# dedupe_key (lg-m5-0, lg-m5-1, ...) — 30% share, but with
|
|
|
|
|
+# idx>0 and only 3 alerts, in practice they're all unique.
|
|
|
|
|
+# The Collapser's Passthrough path is for empty dedupe_key;
|
|
|
|
|
+# the unique-keys path uses CollapseNew, which still
|
|
|
|
|
+# publishes after the first observe. So 3 alerts → 3
|
|
|
|
|
+# messages (one per key).
|
|
|
|
|
+/tmp/loadgen-ws-m65 --target "$WS_INGEST" --api-key "$SRC_ACME" \
|
|
|
|
|
+ --count 3 --rate 5 2>&1 | tail -1
|
|
|
|
|
+sleep "$sleep_secs"
|
|
|
|
|
+
|
|
|
|
|
+count=$(curl -sS $FAKETGMD/admin/sent 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("count",0))' 2>/dev/null)
|
|
|
|
|
+if [[ "$count" == "3" ]]; then
|
|
|
|
|
+ pass "faketgmd count = 3 (3 unique keys → 3 deliveries, no collapse)"
|
|
|
|
|
+else
|
|
|
|
|
+ fail "faketgmd count = $count (expected 3)"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+# ── Summary ─────────────────────────────────────────────────
|
|
|
|
|
+echo ""
|
|
|
|
|
+if [[ $fails -eq 0 ]]; then
|
|
|
|
|
+ echo "🟢 M6.5 smoke PASS — all checks green"
|
|
|
|
|
+ exit 0
|
|
|
|
|
+else
|
|
|
|
|
+ echo "🔴 M6.5 smoke FAIL — $fails check(s) failed"
|
|
|
|
|
+ exit $fails
|
|
|
|
|
+fi
|