broad-announce/scripts/m10_smoke.py

#!/usr/bin/env python3
"""
m10_smoke.py — M10 soak test (5k/s, 10 min, zero DLQ, runaway-source isolation)

Usage:
  # Local docker-compose:
  docker compose --profile loadgen-m10 up -d
  python3 scripts/m10_smoke.py

  # Remote parres:
  ssh root@192.168.44.94 "cd /root/broad-announce && docker compose --profile loadgen-m10 up -d && python3 scripts/m10_smoke.py"

Exit code 0 = all green. Exit code 1 = assertion failed.

Step 1 — pre-flight
Step 2 — 5k/s soak (10 min, ramp 30s)
Step 3 — runaway-source fault injection (60s)
Step 4 — DLQ invariant
Step 5 — teardown (profile down)
"""
import subprocess
import sys
import time
import urllib.request
import urllib.parse
import json
import base64

sys.path.insert(0, __file__.rsplit("/", 1)[0])
import m10_lib as lib

PROM = "http://localhost:9090"
GRAFANA = "http://localhost:3001"
SOAK_DURATION_MIN = 10      # minutes
SOAK_RAMP_SEC = 30           # ramp-up seconds
RUNAWAY_DURATION_SEC = 60    # runaway fault injection duration
CLUSTER_TARGET = 5000       # alerts/sec cluster-wide target
P99_THRESHOLD = 5.0         # seconds — p99 must be under this
DLQ_EXPECTED = 0            # zero DLQ is the invariant
RATE_TOLERANCE = 0.05       # ±5%


def curl_json(url: str) -> dict | None:
    try:
        with urllib.request.urlopen(url, timeout=10) as r:
            return json.loads(r.read())
    except Exception:
        return None


def pass_(msg: str):
    print(f"  ✅ {msg}")


def warn_(msg: str):
    print(f"  ⚠️  {msg}")


def fail_(msg: str):
    print(f"  ❌ {msg}")
    sys.exit(1)


def step1_preflight() -> dict:
    """Verify all services are up and DLQ baseline is clean."""
    print("Step 1 — pre-flight")
    targets = lib.get_prometheus_targets()
    required = ["ingestd", "routerd", "deliverd-fcm", "deliverd-telegram",
                "admind", "archiverd", "prometheus",
                "loadgen-http-1", "loadgen-http-2", "loadgen-http-3"]
    all_up = True
    for svc in required:
        status = targets.get(svc, "0")
        if status == "1":
            pass_(f"{svc} is up")
        else:
            warn_(f"{svc} is {'not scraped' if status == '0' else status}")
            all_up = False

    # Check DLQ baseline
    dlq_now = lib.assert_dlq_count_equals(0, window_seconds=60)
    pass_(f"DLQ baseline clean: {dlq_now} rows")

    # Check circuit breaker is closed
    cb_state = lib.get_circuit_breaker_state("nats")
    cb_names = {0: "CLOSED", 1: "HALF-OPEN", 2: "OPEN"}
    if cb_state == 0:
        pass_(f"circuit breaker CLOSED (nats)")
    elif cb_state < 0:
        warn_(f"circuit breaker gauge not initialized yet (expected on cold start)")
    else:
        warn_(f"circuit breaker is {cb_names.get(cb_state, cb_state)} — may recover under load")

    if not all_up:
        fail_("not all services are up — fix before running M10 smoke")
    return targets


def step2_start_loadgen() -> subprocess.Popen:
    """Start the 3-instance loadgen cluster."""
    print("\nStep 2 — starting 3-instance loadgen cluster (5k/s)")
    proc = subprocess.Popen(
        ["docker", "compose", "--profile", "loadgen-m10", "up", "-d"],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
    )
    code = proc.wait()
    if code != 0:
        fail_(f"docker compose --profile loadgen-m10 up -d failed (exit {code})")
    pass_("3 loadgen-http instances started")
    # Wait for ramp-up
    print(f"  waiting {SOAK_RAMP_SEC}s for ramp-up to complete...", flush=True)
    time.sleep(SOAK_RAMP_SEC)
    pass_(f"ramp-up complete — now targeting {CLUSTER_TARGET}/s")
    return proc


def step2_monitor_soak() -> dict:
    """
    Monitor the soak: sample p99 + rate + DLQ every 30s.
    Fails fast on any breach.
    Returns dict of samples for the log.
    """
    print(f"\n  Monitoring soak for {SOAK_DURATION_MIN} minutes...")
    samples = []
    start = time.time()
    deadline = start + SOAK_DURATION_MIN * 60

    sample_interval = 30  # seconds between samples
    while time.time() < deadline:
        time.sleep(sample_interval)
        elapsed = int(time.time() - start) // 60

        try:
            rate = lib.assert_rate_near(CLUSTER_TARGET, tolerance=RATE_TOLERANCE, window_seconds=30)
            p99 = lib.assert_ingest_p99_under(P99_THRESHOLD, window_seconds=60)
            dlq = lib.assert_dlq_count_equals(DLQ_EXPECTED, window_seconds=60)
            print(f"  [{elapsed}m] rate={rate:.0f}/s p99={p99:.3f}s dlq={dlq}")
            samples.append({"elapsed_min": elapsed, "rate": rate, "p99": p99, "dlq": dlq})
        except AssertionError as e:
            fail_(f"soak breach at {elapsed}m: {e}")

    return samples


def step3_runaway_test() -> None:
    """
    Runaway-source fault injection.

    Scenario:
      - loadgen-http-1 (acme-001/prom-prod) is already running at 1700/s.
      - loadgen-http-4 starts, targeting the SAME company+source at 1000/s.
        Combined: ~2700/s for acme-001/prom-prod. Per-source cap is 100/s,
        so ingestd rate-limits ~2600/s back with HTTP 429.
      - acme-002 (loadgen-http-2) and acme-003 (loadgen-http-3) continue
        unaffected at ~1700/s each.


    Pass condition: p99 for acme-002 and acme-003 stays ≤ P99_THRESHOLD (5s)
    during the 60-second runaway window.
    """
    print(f"\nStep 3 — runaway-source fault injection ({RUNAWAY_DURATION_SEC}s)")

    # Sources that must remain healthy (acme-002 and acme-003 send via different
    # source_ids: prom-prod is hard-coded in loadgen, but loadgen-http-2 and
    # loadgen-http-3 each send as their own company, so the per-source metric
    # query uses company_id as the label on ba_ingestd_publish_latency_seconds).
    # NOTE: the source_id label on the histogram is the SourceID field from the
    # alert payload (always "prom-prod" in the current loadgen). The
    # company_id is in the metric labels as "company_id".
    # We check the aggregate p99 for all non-acme-001 companies.
    healthy_companies = ["acme-002", "acme-003"]

    # Start the rogue loadgen (loadgen-http-4 is in the loadgen-m10 profile).
    print("  starting loadgen-http-4 (rogue, 10× per-source cap)...")
    proc = subprocess.Popen(
        ["docker", "compose", "up", "-d", "loadgen-http-4"],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
        cwd="/root/broad-announce",
    )
    code = proc.wait()
    if code != 0:
        fail_("docker compose up -d loadgen-http-4 failed")
    pass_("loadgen-http-4 started")

    # Wait for ramp-up to complete (loadgen-http-4 uses 10s ramp-up).
    print("  waiting 15s for rogue ramp-up...", flush=True)
    time.sleep(15)

    # Sample per-source p99 every 10s for RUNAWAY_DURATION_SEC.
    print(f"  sampling p99 every 10s for {RUNAWAY_DURATION_SEC}s...")
    samples = []
    start = time.time()
    deadline = start + RUNAWAY_DURATION_SEC
    while time.time() < deadline:
        time.sleep(10)
        elapsed = int(time.time() - start)

        # Check each healthy company: p99 must stay under threshold.
        # We query the per-source latency histogram using the company_id label.
        # Each company sends at ~1700/s; the rogue does not affect these.
        all_ok = True
        for company in healthy_companies:
            try:
                p99 = _per_company_p99(company, window_seconds=30)
                print(f"    [{elapsed}s] {company} p99={p99:.3f}s")
                samples.append({"company": company, "elapsed": elapsed, "p99": p99})
                if p99 > P99_THRESHOLD:
                    all_ok = False
            except AssertionError:
                all_ok = False

        if not all_ok:
            # Print what we saw before failing
            for s in samples:
                marker = "❌" if s["p99"] > P99_THRESHOLD else "✅"
                print(f"  {marker} {s['company']} p99={s['p99']:.3f}s at {s['elapsed']}s")
            fail_(
                f"runaway-source p99 breach: one or more healthy companies exceeded "
                f"{P99_THRESHOLD}s p99 threshold during rogue injection"
            )

    pass_(
        f"all {len(healthy_companies)} healthy companies kept p99 ≤ {P99_THRESHOLD}s "
        f"throughout {RUNAWAY_DURATION_SEC}s rogue injection"
    )

    # Stop the rogue.
    print("  stopping loadgen-http-4 (rogue)...")
    r = subprocess.run(
        ["docker", "compose", "stop", "loadgen-http-4"],
        capture_output=True,
    )
    if r.returncode == 0:
        pass_("loadgen-http-4 stopped")
    else:
        warn_(f"failed to stop loadgen-http-4: {r.stderr.decode().strip()}")


def _per_company_p99(company_id: str, window_seconds: int = 60) -> float:
    """
    Return p99 publish latency for a specific company_id.
    Queries ba_ingestd_publish_latency_seconds_bucket with company_id label.
    Returns 0.0 if no data.
    """
    query = (
        f'histogram_quantile(0.99, '
        f'rate(ba_ingestd_publish_latency_seconds_bucket{{company_id="{company_id}"}}[{window_seconds}s]))'
    )
    results = lib.scrape(query)
    if not results:
        # No data yet — treat as 0 (pre-warm). Will be caught if still 0 at end.
        return 0.0
    return float(results[0]["value"][1])


def step4_dlq_invariant(samples: list[dict]) -> int:
    """Assert zero DLQ rows for the entire soak window."""
    print("\nStep 4 — DLQ invariant check")
    # The soak samples already checked DLQ delta, but do a final absolute check.
    # Query the full soak window.
    soak_seconds = SOAK_DURATION_MIN * 60
    dlq = lib.assert_dlq_count_equals(DLQ_EXPECTED, window_seconds=soak_seconds)
    pass_(f"DLQ count over {SOAK_DURATION_MIN}min soak: {dlq} (expected 0)")
    return dlq


def step5_teardown(proc) -> None:
    """Bring down the loadgen cluster."""
    print("\nStep 5 — teardown")
    r = subprocess.run(
        ["docker", "compose", "--profile", "loadgen-m10", "down", "-v"],
        capture_output=True,
    )
    if r.returncode == 0:
        pass_("loadgen cluster torn down")
    else:
        warn_(f"teardown returned {r.returncode}: {r.stderr.decode().strip()}")


def print_summary(samples: list[dict], dlq_final: int) -> None:
    """Print a summary table of the soak run."""
    print("\n=== M10 Soak Summary ===")
    print(f"Duration:  {SOAK_DURATION_MIN} min")
    print(f"Target:    {CLUSTER_TARGET}/s (±{RATE_TOLERANCE*100:.0f}%)")
    print(f"p99 threshold: {P99_THRESHOLD}s")
    print()
    if samples:
        print(f"{'Time':>6}  {'Rate/s':>8}  {'p99(s)':>7}  {'DLQ':>4}")
        print("-" * 35)
        for s in samples:
            print(f"{s['elapsed_min']:>5}m  {s['rate']:>8.0f}  {s['p99']:>7.3f}  {s['dlq']:>4}")
    print()
    print(f"Final DLQ count: {dlq_final} (expected 0)")
    print()
    print("🎉 M10 smoke: all checks complete.")


def main():
    print(f"M10 Soak Test — target {CLUSTER_TARGET}/s for {SOAK_DURATION_MIN} min")
    print(f"p99 threshold: {P99_THRESHOLD}s | DLQ expected: {DLQ_EXPECTED}")
    print()

    try:
        step1_preflight()
        proc = step2_start_loadgen()
        samples = step2_monitor_soak()
        step3_runaway_test()
        dlq_final = step4_dlq_invariant(samples)
        step5_teardown(proc)
        print_summary(samples, dlq_final)
    except AssertionError as e:
        print(f"\n💥 M10 smoke FAILED: {e}")
        sys.exit(1)
    except KeyboardInterrupt:
        print("\nInterrupted.")
        sys.exit(1)


if __name__ == "__main__":
    main()