|
|
@@ -2,11 +2,17 @@
|
|
|
// --mode normal only for now; other profiles land as the protocols
|
|
|
// come online.
|
|
|
//
|
|
|
-// Example:
|
|
|
+// Example (single instance):
|
|
|
//
|
|
|
// loadgen-http --target http://localhost:8800 \
|
|
|
// --api-key acme-001:prom-prod:s3cret \
|
|
|
-// --mode normal --rate 100 --duration 30s
|
|
|
+// --mode normal --rate 100 --duration 30s --ramp-up 10s
|
|
|
+//
|
|
|
+// Example (3-instance cluster hitting 5k/s):
|
|
|
+//
|
|
|
+// loadgen-http-1 --rate 1700 --duration 10m --ramp-up 30s --cluster-id m10-run
|
|
|
+// loadgen-http-2 --rate 1700 --duration 10m --ramp-up 30s --cluster-id m10-run
|
|
|
+// loadgen-http-3 --rate 1700 --duration 10m --ramp-up 30s --cluster-id m10-run
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
@@ -32,6 +38,7 @@ import (
|
|
|
"time"
|
|
|
|
|
|
"git3.techno-world.net/lrosales/broad-announce/internal/alert"
|
|
|
+ "git3.techno-world.net/lrosales/broad-announce/loadgen/internal/pacer"
|
|
|
)
|
|
|
|
|
|
func main() {
|
|
|
@@ -39,12 +46,15 @@ func main() {
|
|
|
target = flag.String("target", "http://localhost:8800", "ingestd base URL")
|
|
|
apiKey = flag.String("api-key", "", "company_id:source_id:secret")
|
|
|
mode = flag.String("mode", "normal", "profile: normal|burst|stress")
|
|
|
- rate = flag.Int("rate", 100, "target alerts/sec per instance")
|
|
|
+ rate = flag.Int("rate", 100, "target alerts/sec (after ramp-up)")
|
|
|
duration = flag.Duration("duration", 30*time.Second, "total run time")
|
|
|
+ rampUp = flag.Duration("ramp-up", 0, "linear ramp from 0 to --rate over this duration (default 0 = instant)")
|
|
|
conc = flag.Int("concurrency", 16, "concurrent HTTP requests in flight")
|
|
|
metrics = flag.String("metrics", ":8891", "Prometheus metrics listen addr (empty to disable)")
|
|
|
+ clusterID = flag.String("cluster-id", "default", "tag added as label on all loadgen metrics")
|
|
|
+ instance = flag.String("instance", "default", "loadgen instance name (e.g. loadgen-http-1)")
|
|
|
dedupePct = flag.Int("dedupe-pct", 30, "percent of alerts sharing a dedupe_key (normal mode)")
|
|
|
- dedupeKey = flag.String("dedupe-key", "", "force a specific dedupe_key on every alert (overrides --dedupe-pct; useful for M6 ×N smoke tests)")
|
|
|
+ dedupeKey = flag.String("dedupe-key", "", "force a specific dedupe_key on every alert (overrides --dedupe-pct)")
|
|
|
companyN = flag.Int("companies", 1, "how many fake companies to cycle through")
|
|
|
payloadB = flag.Int("payload-bytes", 256, "approximate payload size in bytes (Data map)")
|
|
|
)
|
|
|
@@ -59,22 +69,26 @@ func main() {
|
|
|
os.Exit(2)
|
|
|
}
|
|
|
company, source, secret := parts[0], parts[1], parts[2]
|
|
|
- _ = company
|
|
|
- _ = source
|
|
|
- _ = secret
|
|
|
|
|
|
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
|
|
+ logger.Info("starting",
|
|
|
+ "target", *target,
|
|
|
+ "rate", *rate,
|
|
|
+ "ramp_up", *rampUp,
|
|
|
+ "duration", *duration,
|
|
|
+ "concurrency", *conc,
|
|
|
+ "cluster_id", *clusterID,
|
|
|
+ "instance", *instance,
|
|
|
+ )
|
|
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
|
defer stop()
|
|
|
|
|
|
- limiter := newRateLimiter(*rate)
|
|
|
-
|
|
|
var (
|
|
|
- sent atomic.Uint64
|
|
|
- failed atomic.Uint64
|
|
|
- dupes atomic.Uint64
|
|
|
- rlHits atomic.Uint64
|
|
|
+ sent atomic.Uint64
|
|
|
+ failed atomic.Uint64
|
|
|
+ dupes atomic.Uint64
|
|
|
+ rlHits atomic.Uint64
|
|
|
)
|
|
|
|
|
|
// Spawn workers.
|
|
|
@@ -89,9 +103,6 @@ func main() {
|
|
|
if ctx.Err() != nil {
|
|
|
return
|
|
|
}
|
|
|
- if err := limiter.Wait(ctx); err != nil {
|
|
|
- return
|
|
|
- }
|
|
|
a := mkAlert(*mode, *companyN, *dedupePct, *dedupeKey, *payloadB)
|
|
|
if err := sendOne(ctx, client, *target, company, source, []byte(secret), a); err != nil {
|
|
|
failed.Add(1)
|
|
|
@@ -109,18 +120,21 @@ func main() {
|
|
|
}(i)
|
|
|
}
|
|
|
|
|
|
- // Producer.
|
|
|
+ // Producer with pacer (supports linear ramp-up).
|
|
|
producerCtx, cancelProducer := context.WithTimeout(ctx, *duration)
|
|
|
defer cancelProducer()
|
|
|
+
|
|
|
+ p := pacer.New(*rate, *rampUp)
|
|
|
+ tickCh, stopPacer := p.Tick(producerCtx)
|
|
|
+ defer stopPacer()
|
|
|
+
|
|
|
go func() {
|
|
|
- ticker := time.NewTicker(time.Second / time.Duration(*rate+1))
|
|
|
- defer ticker.Stop()
|
|
|
for {
|
|
|
select {
|
|
|
case <-producerCtx.Done():
|
|
|
close(jobs)
|
|
|
return
|
|
|
- case <-ticker.C:
|
|
|
+ case <-tickCh:
|
|
|
select {
|
|
|
case jobs <- struct{}{}:
|
|
|
default:
|
|
|
@@ -132,7 +146,7 @@ func main() {
|
|
|
|
|
|
// Metrics endpoint.
|
|
|
if *metrics != "" {
|
|
|
- go runMetrics(*metrics, &sent, &failed, &dupes, &rlHits)
|
|
|
+ go runMetrics(*metrics, *clusterID, *instance, *mode, &sent, &failed, &dupes, &rlHits)
|
|
|
}
|
|
|
|
|
|
// Status ticker.
|
|
|
@@ -164,17 +178,10 @@ func main() {
|
|
|
}
|
|
|
|
|
|
func mkAlert(mode string, companies, dedupePct int, dedupeKey string, payloadB int) alert.Alert {
|
|
|
- // Cycle through a few company_ids; the source is fixed for the
|
|
|
- // run (loadgen is one source).
|
|
|
companyID := fmt.Sprintf("acme-%03d", (rand.IntN(companies) + 1))
|
|
|
severity := pickSeverity(mode)
|
|
|
category := pickCategory(severity)
|
|
|
|
|
|
- // Dedupe: dedupe_pct % of alerts share a fixed "burst" key
|
|
|
- // within a minute. For the simpler "normal" mode, we just
|
|
|
- // include a stable dedupe key on a fraction of alerts.
|
|
|
- // M6: --dedupe-key forces a specific key on every alert,
|
|
|
- // which is what the ×N smoke test needs (one key, N copies).
|
|
|
dk := ""
|
|
|
if dedupeKey != "" {
|
|
|
dk = dedupeKey
|
|
|
@@ -182,7 +189,6 @@ func mkAlert(mode string, companies, dedupePct int, dedupeKey string, payloadB i
|
|
|
dk = fmt.Sprintf("burst:%s:probe", category)
|
|
|
}
|
|
|
|
|
|
- // Synthetic data roughly `payloadB` bytes.
|
|
|
data := map[string]string{
|
|
|
"host": fmt.Sprintf("host-%d", rand.IntN(100)),
|
|
|
"probe": category,
|
|
|
@@ -202,7 +208,6 @@ func mkAlert(mode string, companies, dedupePct int, dedupeKey string, payloadB i
|
|
|
}
|
|
|
|
|
|
func pickSeverity(mode string) alert.Severity {
|
|
|
- // 70/25/4/1 split for normal, even for stress
|
|
|
r := rand.IntN(100)
|
|
|
switch {
|
|
|
case r < 70:
|
|
|
@@ -278,49 +283,19 @@ func isRateLimited(err error) bool {
|
|
|
return ok
|
|
|
}
|
|
|
|
|
|
-// rateLimiter is a simple per-second pacer. Not a token bucket; for
|
|
|
-// loadgen we want "send at most N per second" averaged, not bursty.
|
|
|
-type rateLimiter struct {
|
|
|
- perSec int
|
|
|
- last time.Time
|
|
|
- mu sync.Mutex
|
|
|
-}
|
|
|
-
|
|
|
-func newRateLimiter(perSec int) *rateLimiter { return &rateLimiter{perSec: perSec} }
|
|
|
-
|
|
|
-func (r *rateLimiter) Wait(ctx context.Context) error {
|
|
|
- if r.perSec <= 0 {
|
|
|
- return nil
|
|
|
- }
|
|
|
- r.mu.Lock()
|
|
|
- defer r.mu.Unlock()
|
|
|
- now := time.Now()
|
|
|
- interval := time.Second / time.Duration(r.perSec)
|
|
|
- if d := interval - now.Sub(r.last); d > 0 {
|
|
|
- t := time.NewTimer(d)
|
|
|
- defer t.Stop()
|
|
|
- r.mu.Unlock()
|
|
|
- select {
|
|
|
- case <-ctx.Done():
|
|
|
- r.mu.Lock()
|
|
|
- return ctx.Err()
|
|
|
- case <-t.C:
|
|
|
- }
|
|
|
- r.mu.Lock()
|
|
|
- }
|
|
|
- r.last = time.Now()
|
|
|
- return nil
|
|
|
-}
|
|
|
-
|
|
|
-func runMetrics(addr string, sent, failed, dupes, rlHits *atomic.Uint64) {
|
|
|
+func runMetrics(addr, clusterID, instance, mode string, sent, failed, dupes, rlHits *atomic.Uint64) {
|
|
|
mux := http.NewServeMux()
|
|
|
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
|
|
fmt.Fprintf(w, "# HELP loadgen_alerts_sent_total Alerts successfully accepted.\n")
|
|
|
fmt.Fprintf(w, "# TYPE loadgen_alerts_sent_total counter\n")
|
|
|
- fmt.Fprintf(w, "loadgen_alerts_sent_total %d\n", sent.Load())
|
|
|
- fmt.Fprintf(w, "loadgen_alerts_failed_total %d\n", failed.Load())
|
|
|
- fmt.Fprintf(w, "loadgen_dedupe_hits_total %d\n", dupes.Load())
|
|
|
- fmt.Fprintf(w, "loadgen_rate_limited_total %d\n", rlHits.Load())
|
|
|
+ fmt.Fprintf(w, "loadgen_alerts_sent_total{profile=%q,instance=%q,cluster_id=%q,mode=%q} %d\n",
|
|
|
+ mode, instance, clusterID, mode, sent.Load())
|
|
|
+ fmt.Fprintf(w, "loadgen_alerts_failed_total{reason=%q,instance=%q,cluster_id=%q} %d\n",
|
|
|
+ "send", instance, clusterID, failed.Load())
|
|
|
+ fmt.Fprintf(w, "loadgen_dedupe_hits_total{instance=%q,cluster_id=%q} %d\n",
|
|
|
+ instance, clusterID, dupes.Load())
|
|
|
+ fmt.Fprintf(w, "loadgen_rate_limited_total{instance=%q,cluster_id=%q} %d\n",
|
|
|
+ instance, clusterID, rlHits.Load())
|
|
|
})
|
|
|
srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
|
|
_ = srv.ListenAndServe()
|
|
|
@@ -331,4 +306,4 @@ func max(a, b int) int {
|
|
|
return a
|
|
|
}
|
|
|
return b
|
|
|
-}
|
|
|
+}
|