|
|
@@ -146,15 +146,113 @@ def step2_monitor_soak() -> dict:
|
|
|
def step3_runaway_test() -> None:
|
|
|
"""
|
|
|
Runaway-source fault injection.
|
|
|
- - Start a 4th loadgen instance firing at 10× per-source cap for the same company.
|
|
|
- - Verify p99 for the OTHER sources (acme-002, acme-003) stays under threshold.
|
|
|
- - The runaway (acme-001) can be anything.
|
|
|
+
|
|
|
+ Scenario:
|
|
|
+ - loadgen-http-1 (acme-001/prom-prod) is already running at 1700/s.
|
|
|
+ - loadgen-http-4 starts, targeting the SAME company+source at 1000/s.
|
|
|
+ Combined: ~2700/s for acme-001/prom-prod. Per-source cap is 100/s,
|
|
|
+ so ingestd rate-limits ~2600/s back with HTTP 429.
|
|
|
+ - acme-002 (loadgen-http-2) and acme-003 (loadgen-http-3) continue
|
|
|
+ unaffected at ~1700/s each.
|
|
|
+
|
|
|
+
|
|
|
+ Pass condition: p99 for acme-002 and acme-003 stays ≤ P99_THRESHOLD (5s)
|
|
|
+ during the 60-second runaway window.
|
|
|
"""
|
|
|
print(f"\nStep 3 — runaway-source fault injection ({RUNAWAY_DURATION_SEC}s)")
|
|
|
- print(" (not yet implemented — requires --rate override on loadgen-http-1)")
|
|
|
- warn_(f"runaway-source test skipped (W4 enhancement pending)")
|
|
|
- # TODO: spin up a 4th instance at 10× cap targeting acme-001
|
|
|
- # Expected: per_source_p99("acme-002") < 5s and per_source_p99("acme-003") < 5s
|
|
|
+
|
|
|
+ # Sources that must remain healthy (acme-002 and acme-003 send via different
|
|
|
+ # source_ids: prom-prod is hard-coded in loadgen, but loadgen-http-2 and
|
|
|
+ # loadgen-http-3 each send as their own company, so the per-source metric
|
|
|
+ # query uses company_id as the label on ba_ingestd_publish_latency_seconds).
|
|
|
+ # NOTE: the source_id label on the histogram is the SourceID field from the
|
|
|
+ # alert payload (always "prom-prod" in the current loadgen). The
|
|
|
+ # company_id is in the metric labels as "company_id".
|
|
|
+ # We check the aggregate p99 for all non-acme-001 companies.
|
|
|
+ healthy_companies = ["acme-002", "acme-003"]
|
|
|
+
|
|
|
+ # Start the rogue loadgen (loadgen-http-4 is in the loadgen-m10 profile).
|
|
|
+ print(" starting loadgen-http-4 (rogue, 10× per-source cap)...")
|
|
|
+ proc = subprocess.Popen(
|
|
|
+ ["docker", "compose", "up", "-d", "loadgen-http-4"],
|
|
|
+ stdout=subprocess.DEVNULL,
|
|
|
+ stderr=subprocess.DEVNULL,
|
|
|
+ cwd="/root/broad-announce",
|
|
|
+ )
|
|
|
+ code = proc.wait()
|
|
|
+ if code != 0:
|
|
|
+ fail_("docker compose up -d loadgen-http-4 failed")
|
|
|
+ pass_("loadgen-http-4 started")
|
|
|
+
|
|
|
+ # Wait for ramp-up to complete (loadgen-http-4 uses 10s ramp-up).
|
|
|
+ print(" waiting 15s for rogue ramp-up...", flush=True)
|
|
|
+ time.sleep(15)
|
|
|
+
|
|
|
+ # Sample per-source p99 every 10s for RUNAWAY_DURATION_SEC.
|
|
|
+ print(f" sampling p99 every 10s for {RUNAWAY_DURATION_SEC}s...")
|
|
|
+ samples = []
|
|
|
+ start = time.time()
|
|
|
+ deadline = start + RUNAWAY_DURATION_SEC
|
|
|
+ while time.time() < deadline:
|
|
|
+ time.sleep(10)
|
|
|
+ elapsed = int(time.time() - start)
|
|
|
+
|
|
|
+ # Check each healthy company: p99 must stay under threshold.
|
|
|
+ # We query the per-source latency histogram using the company_id label.
|
|
|
+ # Each company sends at ~1700/s; the rogue does not affect these.
|
|
|
+ all_ok = True
|
|
|
+ for company in healthy_companies:
|
|
|
+ try:
|
|
|
+ p99 = _per_company_p99(company, window_seconds=30)
|
|
|
+ print(f" [{elapsed}s] {company} p99={p99:.3f}s")
|
|
|
+ samples.append({"company": company, "elapsed": elapsed, "p99": p99})
|
|
|
+ if p99 > P99_THRESHOLD:
|
|
|
+ all_ok = False
|
|
|
+ except AssertionError:
|
|
|
+ all_ok = False
|
|
|
+
|
|
|
+ if not all_ok:
|
|
|
+ # Print what we saw before failing
|
|
|
+ for s in samples:
|
|
|
+ marker = "❌" if s["p99"] > P99_THRESHOLD else "✅"
|
|
|
+ print(f" {marker} {s['company']} p99={s['p99']:.3f}s at {s['elapsed']}s")
|
|
|
+ fail_(
|
|
|
+ f"runaway-source p99 breach: one or more healthy companies exceeded "
|
|
|
+ f"{P99_THRESHOLD}s p99 threshold during rogue injection"
|
|
|
+ )
|
|
|
+
|
|
|
+ pass_(
|
|
|
+ f"all {len(healthy_companies)} healthy companies kept p99 ≤ {P99_THRESHOLD}s "
|
|
|
+ f"throughout {RUNAWAY_DURATION_SEC}s rogue injection"
|
|
|
+ )
|
|
|
+
|
|
|
+ # Stop the rogue.
|
|
|
+ print(" stopping loadgen-http-4 (rogue)...")
|
|
|
+ r = subprocess.run(
|
|
|
+ ["docker", "compose", "stop", "loadgen-http-4"],
|
|
|
+ capture_output=True,
|
|
|
+ )
|
|
|
+ if r.returncode == 0:
|
|
|
+ pass_("loadgen-http-4 stopped")
|
|
|
+ else:
|
|
|
+ warn_(f"failed to stop loadgen-http-4: {r.stderr.decode().strip()}")
|
|
|
+
|
|
|
+
|
|
|
+def _per_company_p99(company_id: str, window_seconds: int = 60) -> float:
|
|
|
+ """
|
|
|
+ Return p99 publish latency for a specific company_id.
|
|
|
+ Queries ba_ingestd_publish_latency_seconds_bucket with company_id label.
|
|
|
+ Returns 0.0 if no data.
|
|
|
+ """
|
|
|
+ query = (
|
|
|
+ f'histogram_quantile(0.99, '
|
|
|
+ f'rate(ba_ingestd_publish_latency_seconds_bucket{{company_id="{company_id}"}}[{window_seconds}s]))'
|
|
|
+ )
|
|
|
+ results = lib.scrape(query)
|
|
|
+ if not results:
|
|
|
+ # No data yet — treat as 0 (pre-warm). Will be caught if still 0 at end.
|
|
|
+ return 0.0
|
|
|
+ return float(results[0]["value"][1])
|
|
|
|
|
|
|
|
|
def step4_dlq_invariant(samples: list[dict]) -> int:
|
Luis Rosales