|
@@ -47,6 +47,7 @@ import (
|
|
|
// populate the Sources map with entries keyed by "company_id:source_id".
|
|
// populate the Sources map with entries keyed by "company_id:source_id".
|
|
|
type SourceConfig struct {
|
|
type SourceConfig struct {
|
|
|
CompanyID string
|
|
CompanyID string
|
|
|
|
|
+ SourceID string
|
|
|
HMACSecret []byte // may be empty for transports that don't use HMAC
|
|
HMACSecret []byte // may be empty for transports that don't use HMAC
|
|
|
RateLimitPerSec int
|
|
RateLimitPerSec int
|
|
|
AllowedTargets []string // M2: allowed routing targets (pipeline ignores; caller enforces)
|
|
AllowedTargets []string // M2: allowed routing targets (pipeline ignores; caller enforces)
|
|
@@ -143,18 +144,18 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
// 5. Parse + validate.
|
|
// 5. Parse + validate.
|
|
|
var a alert.Alert
|
|
var a alert.Alert
|
|
|
if err := json.Unmarshal(body, &a); err != nil {
|
|
if err := json.Unmarshal(body, &a); err != nil {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("invalid").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "invalid").Inc()
|
|
|
return Reject("invalid_json", 400, err.Error())
|
|
return Reject("invalid_json", 400, err.Error())
|
|
|
}
|
|
}
|
|
|
if err := a.Validate(); err != nil {
|
|
if err := a.Validate(); err != nil {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("invalid").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "invalid").Inc()
|
|
|
return Reject("invalid", 400, err.Error())
|
|
return Reject("invalid", 400, err.Error())
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Source lookup.
|
|
// Source lookup.
|
|
|
src, ok := d.Sources[a.CompanyID+":"+a.SourceID]
|
|
src, ok := d.Sources[a.CompanyID+":"+a.SourceID]
|
|
|
if !ok {
|
|
if !ok {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("invalid").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "invalid").Inc()
|
|
|
return Reject("unknown_source", 401,
|
|
return Reject("unknown_source", 401,
|
|
|
fmt.Sprintf("no such source %s/%s", a.CompanyID, a.SourceID))
|
|
fmt.Sprintf("no such source %s/%s", a.CompanyID, a.SourceID))
|
|
|
}
|
|
}
|
|
@@ -162,7 +163,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
// 2. Quarantine check (M9 layer 7). Before we spend any CPU.
|
|
// 2. Quarantine check (M9 layer 7). Before we spend any CPU.
|
|
|
if d.Quarantine != nil {
|
|
if d.Quarantine != nil {
|
|
|
if banned, remaining, err := d.Quarantine.IsBanned(ctx, a.SourceID); err == nil && banned {
|
|
if banned, remaining, err := d.Quarantine.IsBanned(ctx, a.SourceID); err == nil && banned {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("quarantined").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "quarantined").Inc()
|
|
|
d.Logger.Warn("source quarantined",
|
|
d.Logger.Warn("source quarantined",
|
|
|
"source_id", a.SourceID,
|
|
"source_id", a.SourceID,
|
|
|
"company_id", a.CompanyID,
|
|
"company_id", a.CompanyID,
|
|
@@ -175,7 +176,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
|
|
|
|
|
// 6. Auth (transport-specific; gRPC skips by passing "").
|
|
// 6. Auth (transport-specific; gRPC skips by passing "").
|
|
|
if sig != "" && !verifyHMAC(sig, src.HMACSecret, body, now) {
|
|
if sig != "" && !verifyHMAC(sig, src.HMACSecret, body, now) {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("invalid").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "invalid").Inc()
|
|
|
return Reject("bad_signature", 401, "")
|
|
return Reject("bad_signature", 401, "")
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -213,7 +214,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
if ok, ttl, err := d.Limiter.Allow(ctx, "source:"+a.CompanyID+":"+a.SourceID, src.RateLimitPerSec); err != nil {
|
|
if ok, ttl, err := d.Limiter.Allow(ctx, "source:"+a.CompanyID+":"+a.SourceID, src.RateLimitPerSec); err != nil {
|
|
|
d.Logger.Warn("ratelimit redis error (failing open)", "err", err, "scope", "source")
|
|
d.Logger.Warn("ratelimit redis error (failing open)", "err", err, "scope", "source")
|
|
|
} else if !ok {
|
|
} else if !ok {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("rate_limited").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "rate_limited").Inc()
|
|
|
d.Metrics.RateLimitHits.WithLabelValues("source").Inc()
|
|
d.Metrics.RateLimitHits.WithLabelValues("source").Inc()
|
|
|
recordHit()
|
|
recordHit()
|
|
|
return Reject("rate_limited_source", 429, strconv.Itoa(int(ttl.Seconds())))
|
|
return Reject("rate_limited_source", 429, strconv.Itoa(int(ttl.Seconds())))
|
|
@@ -223,7 +224,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
// 4. Per-company rate limit (new alerts only).
|
|
// 4. Per-company rate limit (new alerts only).
|
|
|
if isNew {
|
|
if isNew {
|
|
|
if ok, ttl, _ := d.Limiter.Allow(ctx, "company:"+a.CompanyID, d.CompanyRatePerSec); !ok {
|
|
if ok, ttl, _ := d.Limiter.Allow(ctx, "company:"+a.CompanyID, d.CompanyRatePerSec); !ok {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("rate_limited").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "rate_limited").Inc()
|
|
|
d.Metrics.RateLimitHits.WithLabelValues("company").Inc()
|
|
d.Metrics.RateLimitHits.WithLabelValues("company").Inc()
|
|
|
recordHit()
|
|
recordHit()
|
|
|
return Reject("rate_limited_company", 429, strconv.Itoa(int(ttl.Seconds())))
|
|
return Reject("rate_limited_company", 429, strconv.Itoa(int(ttl.Seconds())))
|
|
@@ -239,7 +240,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
subject := broker.AlertsSubject(a.CompanyID)
|
|
subject := broker.AlertsSubject(a.CompanyID)
|
|
|
payload, err := json.Marshal(a)
|
|
payload, err := json.Marshal(a)
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("invalid").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "invalid").Inc()
|
|
|
return Reject("marshal_failed", 500, err.Error())
|
|
return Reject("marshal_failed", 500, err.Error())
|
|
|
}
|
|
}
|
|
|
start := time.Now()
|
|
start := time.Now()
|
|
@@ -253,7 +254,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
}
|
|
}
|
|
|
if publishErr != nil {
|
|
if publishErr != nil {
|
|
|
if errors.Is(publishErr, circuitbreaker.ErrCircuitOpen) {
|
|
if errors.Is(publishErr, circuitbreaker.ErrCircuitOpen) {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("circuit_open").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "circuit_open").Inc()
|
|
|
d.Metrics.CBState.WithLabelValues("nats").Set(circuitbreaker.StateOpen)
|
|
d.Metrics.CBState.WithLabelValues("nats").Set(circuitbreaker.StateOpen)
|
|
|
d.Logger.Warn("circuit breaker open",
|
|
d.Logger.Warn("circuit breaker open",
|
|
|
"subject", subject,
|
|
"subject", subject,
|
|
@@ -263,7 +264,7 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
recordHit()
|
|
recordHit()
|
|
|
return Reject("circuit_open", 503, "broker circuit breaker open")
|
|
return Reject("circuit_open", 503, "broker circuit breaker open")
|
|
|
}
|
|
}
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("broker_unavailable").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "broker_unavailable").Inc()
|
|
|
d.Logger.Error("nats publish", "err", publishErr, "subject", subject)
|
|
d.Logger.Error("nats publish", "err", publishErr, "subject", subject)
|
|
|
recordHit()
|
|
recordHit()
|
|
|
return Reject("broker_unavailable", 503, publishErr.Error())
|
|
return Reject("broker_unavailable", 503, publishErr.Error())
|
|
@@ -272,9 +273,9 @@ func (d *Deps) Process(ctx context.Context, body []byte, sig string) Result {
|
|
|
d.Metrics.PayloadBytes.Observe(float64(len(payload)))
|
|
d.Metrics.PayloadBytes.Observe(float64(len(payload)))
|
|
|
|
|
|
|
|
if isNew {
|
|
if isNew {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("accepted").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "accepted").Inc()
|
|
|
} else {
|
|
} else {
|
|
|
- d.Metrics.AlertsReceived.WithLabelValues("deduped").Inc()
|
|
|
|
|
|
|
+ d.Metrics.AlertsReceived.WithLabelValues(d.Transport, "deduped").Inc()
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
d.Logger.Info("alert accepted",
|
|
d.Logger.Info("alert accepted",
|