broad-announce/docker-compose.yml

# docker-compose.yml — single-host M0 stack.
# Run: docker compose up -d
# Then: see M0_VERIFICATION.md for the loadgen smoke test.
#
# Port conventions (project rule):
#   - app HTTP services:  8800–8899  (ingestd/routerd/deliverd-fcm/deliverd-telegram/telegramd/admind/loadgen/faketgmd)
#   - canonical ports stay (5432 postgres, 4222 nats, 6379 redis,
#     1883 mqtt, 9090 prometheus, 3000 grafana, etc.)

services:
  # ── Data tier ────────────────────────────────────────────────────
  postgres:
    image: timescale/timescaledb:latest-pg16
    environment:
      POSTGRES_USER: ba
      POSTGRES_PASSWORD: ba
      POSTGRES_DB: ba
    # No host port mapping: a host-local postgres is already on
    # :5432. The app services reach this one via the docker network
    # DNS name 'postgres'. Use `docker compose exec postgres psql ...`
    # to talk to it from the host.
    expose: ["5432"]
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ba"]
      interval: 5s
      timeout: 3s
      retries: 10

  redis:
    image: redis:7-alpine
    # No host port mapping: a host-local redis is on :6379. App
    # services reach this one via docker DNS 'redis'.
    expose: ["6379"]
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 10

  nats:
    image: nats:2.10-alpine
    # -js                       : JetStream enabled
    # -m 8222                   : HTTP monitoring port
    # -c /etc/nats/nats.conf    : load the JetStream config from
    #                             a file. 2.10 does not expose
    #                             max_file_store / max_mem_store as
    #                             CLI flags; they have to come from
    #                             a .conf. The config also sets
    #                             store_dir. See deploy/nats/nats.conf
    #                             and M11_NATS_INVESTIGATION.md.
    command: ["-js", "-m", "8222", "-c", "/etc/nats/nats.conf"]
    ports: ["4222:4222", "8222:8222"]   # 8222 is the monitoring HTTP
    volumes:
      - natsdata:/data
      - ./deploy/nats/nats.conf:/etc/nats/nats.conf:ro
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]
      interval: 5s
      timeout: 3s
      retries: 20

  emqx:
    image: emqx/emqx:5.10.4
    ports: ["1883:1883", "18083:18083"]   # MQTT + admin UI
    volumes:
      # M4: per-company ACL + auth bootstrap. acl.conf is read on
      # SIGHUP; the auth CSV is read on first boot. M11 promotes
      # this to a Postgres-backed authentication chain.
      - ./deploy/emqx/acl.conf:/opt/emqx/etc/acl.conf:ro
      - ./deploy/emqx/auth-built-in-db-bootstrap.csv:/opt/emqx/etc/auth-built-in-db-bootstrap.csv:ro
    # EMQX 5.x prefers env-var config over emqx.conf. The HOCON
    # path is emqx.conf → base.hocon → cluster.hocon → env vars
    # (highest precedence). The double-underscore separator in
    # env-var names maps to nested HOCON keys.
    environment:
      # Built-in-db authentication (one chain, password_based, plain)
      EMQX_AUTHENTICATION__1__BACKEND: "built_in_database"
      EMQX_AUTHENTICATION__1__MECHANISM: "password_based"
      EMQX_AUTHENTICATION__1__USER_ID_TYPE: "username"
      EMQX_AUTHENTICATION__1__PASSWORD_HASH_ALGORITHM__NAME: "plain"
      EMQX_AUTHENTICATION__1__PASSWORD_HASH_ALGORITHM__SALT_POSITION: "disable"
      # File-based authorization, default-deny
      EMQX_AUTHORIZATION__NO_MATCH: "deny"
      EMQX_AUTHORIZATION__DENY_ACTION: "disconnect"
      EMQX_AUTHORIZATION__SOURCES__1__TYPE: "file"
      EMQX_AUTHORIZATION__SOURCES__1__ENABLE: "true"
      EMQX_AUTHORIZATION__SOURCES__1__PATH: "/opt/emqx/etc/acl.conf"
    healthcheck:
      # The original `echo > /dev/tcp/...` ran under sh on
      # Debian-based EMQX and reported unhealthy even when the
      # broker was fine. Switch to `bash -c` and a TCP probe.
      test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/1883"]
      interval: 10s
      timeout: 5s
      retries: 20

  clickhouse:
    image: clickhouse/clickhouse-server:24-alpine
    ports: ["8123:8123", "9000:9000"]
    volumes:
      - chdata:/var/lib/clickhouse
    ulimits:
      nofile: { soft: 262144, hard: 262144 }

  # ── App tier ─────────────────────────────────────────────────────
  ingestd:
    build: .
    command: ["/app/ingestd"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8800"
      BA_NATS_URL: nats://nats:4222
      BA_REDIS_URL: redis://redis:6379/0
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      BA_INGESTD_SOURCES: "acme-001:acme-001-prom:s3cret-acme-001,acme-002:acme-002-prom:s3cret-acme-002,globex-002:grafana:s3cret-globex"
      BA_INGESTD_RATE_LIMIT_PER_SOURCE: "20000"
      BA_INGESTD_RATE_LIMIT_PER_COMPANY: "10000"
      # M4: MQTT subscriber. ingestd subscribes to ba/+/+/incoming
      # with the dedicated `ingestd` user. The auth file lists
      # this user (password "ingestd-broker-only") so EMQX's
      # built-in-db authenticates the connection.
      BA_INGESTD_MQTT_BROKER: "tcp://emqx:1883"
      BA_INGESTD_MQTT_USERNAME: "ingestd"
      BA_INGESTD_MQTT_PASSWORD: "ingestd-broker-only"
      BA_INGESTD_MQTT_SUBSCRIBE: "ba/+/+/incoming"
      # M5: WebSocket ingest + live tail. TAIL_TOKEN gates the
      # /v1/tail/ws endpoint; a static token is fine for the
      # dev path; M11 swaps for JWT.
      BA_INGESTD_TAIL_TOKEN: "tail-dev-token-please-change-in-prod"
      BA_INGESTD_MAX_CONCURRENT_PER_IP: "32"
      BA_INGESTD_DEDUPE_TTL_SECONDS: "300"
      # M9 layer 6: circuit breaker (trips after 5 failures in 10s, 30s open)
      BA_INGESTD_CB_FAILURE_THRESHOLD: "5"
      BA_INGESTD_CB_FAILURE_WINDOW_SECS: "10"
      BA_INGESTD_CB_OPEN_DURATION_SECS: "30"
      BA_INGESTD_CB_MAX_HALF_OPEN: "1"
      # M9 layer 7: quarantine (bans source at 100 hits/5min for 10min)
      BA_INGESTD_QUARANTINE_HITS_THRESHOLD: "100"
      BA_INGESTD_QUARANTINE_WINDOW_SECONDS: "300"
      BA_INGESTD_QUARANTINE_DURATION_SECONDS: "600"
      # M13a W4: JWT gate for the /v1/admin/ingest route.
      # Shares the secret with authd. Set to empty string to
      # disable the admin route (back to M11 behavior).
      BA_INGESTD_AUTHD_JWT_SECRET: "${BA_AUTHD_JWT_SECRET:-}"
      BA_AUTHD_ISSUER: "broad-announce"
    ports: ["8800:8800"]
    depends_on:
      nats:    { condition: service_healthy }
      redis:   { condition: service_healthy }
      postgres: { condition: service_healthy }
      emqx:    { condition: service_healthy }

  routerd:
    build: .
    command: ["/app/routerd"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8801"
      BA_NATS_URL: nats://nats:4222
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      # M6.5: router-level dedupe collapse window. A burst of
      # identical alerts is held for up to this many ms, then
      # a single delivery is fanned out with the final
      # dedupe_count. A continuous stream re-flushes every
      # DedupeFlushMs.
      BA_ROUTERD_DEDUPE_FLUSH_MS: "2000"
    ports: ["8801:8801"]
    depends_on:
      nats:     { condition: service_healthy }
      postgres: { condition: service_healthy }

  deliverd-fcm:
    build: .
    command: ["/app/deliverd-fcm"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8802"
      BA_NATS_URL: nats://nats:4222
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      BA_FAKECMD_URL: "http://fakefcmd:8820"
      # M8: in-process retry. Defaults match
      # internal/retry.Default() — 10 attempts, base
      # 100ms, cap 2s, total budget 30s. Tuned to
      # terminate in ~12s for a fully failing target
      # so a single misbehaving source can't tie up a
      # consumer.
      BA_DELIVERD_MAX_ATTEMPTS: "10"
      BA_DELIVERD_RETRY_BASE_MS: "100"
      BA_DELIVERD_RETRY_MAX_MS: "2000"
      BA_DELIVERD_RETRY_BUDGET_MS: "30000"
    ports: ["8802:8802"]
    depends_on:
      nats:      { condition: service_healthy }
      postgres:  { condition: service_healthy }
      fakefcmd:  { condition: service_started }

  deliverd-telegram:
    build: .
    command: ["/app/deliverd-telegram"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8821"
      BA_NATS_URL: nats://nats:4222
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      BA_TELEGRAM_BOT_TOKEN: "fake-tg-bot-token-acme-001"
      BA_TELEGRAM_FAKE_URL: "http://faketgmd:8830"
      # M8: same retry knobs as deliverd-fcm.
      BA_DELIVERD_MAX_ATTEMPTS: "10"
      BA_DELIVERD_RETRY_BASE_MS: "100"
      BA_DELIVERD_RETRY_MAX_MS: "2000"
      BA_DELIVERD_RETRY_BUDGET_MS: "30000"
    ports: ["8821:8821"]
    depends_on:
      nats:      { condition: service_healthy }
      postgres:  { condition: service_healthy }
      faketgmd:  { condition: service_started }

  telegramd:
    build: .
    command: ["/app/telegramd"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8822"
      BA_NATS_URL: nats://nats:4222
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      BA_TELEGRAM_FAKE_URL: "http://faketgmd:8830"
    ports: ["8822:8822"]
    depends_on:
      nats:      { condition: service_healthy }
      postgres:  { condition: service_healthy }
      faketgmd:  { condition: service_started }

  faketgmd:
    build: .
    command: ["/app/faketgmd", "--addr", ":8830", "--timeout", "5"]
    ports: ["8830:8830"]

  fakefcmd:
    build: .
    command: ["/app/fakefcmd", "--addr", ":8820"]
    ports: ["8820:8820"]
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8820/health"]
      interval: 5s
      timeout: 3s
      retries: 10

  seed:
    build: .
    command: ["/app/seed"]
    environment:
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      BA_MIGRATIONS_DIR: /migrations
    volumes:
      - ./migrations:/migrations:ro
    depends_on:
      postgres: { condition: service_healthy }

  admind:
    build: .
    command: ["/app/admind"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8803"
      BA_NATS_URL: nats://nats:4222
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      # M8: admind connects to NATS to publish DLQ
      # replays back onto the deliveries.<chan>.<co>
      # subject. The HTTPAddr serves /v1/ping, /v1/dlq*,
      # /dlq, /health, /metrics.
      # M13a W4: enable the JWT gate by sharing the authd secret.
      BA_AUTHD_JWT_SECRET: "${BA_AUTHD_JWT_SECRET:?BA_AUTHD_JWT_SECRET must be set (see scripts/generate-jwt-secret.sh)}"
      BA_AUTHD_ISSUER: "broad-announce"
    ports: ["8803:8803"]
    depends_on:
      nats:     { condition: service_healthy }
      postgres: { condition: service_healthy }
      authd:    { condition: service_healthy }

  # M13a: multi-tenant auth IdP. Issues JWTs and stores
  # refresh tokens in Postgres. Other services share
  # BA_AUTHD_JWT_SECRET with this one to verify tokens.
  authd:
    build: .
    command: ["/app/authd"]
    environment:
      BA_ENV: dev
      BA_AUTHD_HTTP_ADDR: ":8804"
      BA_AUTHD_ISSUER: "broad-announce"
      BA_AUTHD_JWT_SECRET: "${BA_AUTHD_JWT_SECRET:?BA_AUTHD_JWT_SECRET must be set (see scripts/generate-jwt-secret.sh)}"
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      # M13a W1: dev-only flag that lets authd generate a
      # secret on first run if BA_AUTHD_JWT_SECRET is unset.
      # In production this MUST be unset; the secret comes
      # from a sealed-secret or KMS.
      BA_AUTHD_ALLOW_GENERATED_SECRET: "1"
      BA_AUTHD_SECRET_FILE: /var/run/broad-announce/authd.jwt
    ports: ["8804:8804"]
    volumes:
      - authd-data:/var/run/broad-announce
    depends_on:
      postgres: { condition: service_healthy }

  archiverd:
    build: .
    command: ["/app/archiverd"]
    environment:
      BA_ENV: dev
      BA_HTTP_ADDR: ":8805"
      BA_POSTGRES_DSN: postgres://ba:ba@postgres:5432/ba?sslmode=disable
      # M7: archiver cadence + retention cutoff. The
      # Timescale retention policy does the same at 7d;
      # the archiver just runs ahead so ClickHouse has
      # the data before TS drops it.
      BA_ARCHIVERD_RUN_EVERY_SECONDS: "3600"
      BA_ARCHIVERD_OLDER_THAN_HOURS: "168"
      BA_ARCHIVERD_BATCH_SIZE: "10000"
      BA_ARCHIVERD_CLICKHOUSE_URL: "http://clickhouse:8123"
    ports: ["8805:8805"]
    depends_on:
      postgres:    { condition: service_healthy }
      clickhouse:  { condition: service_started }

  # ── Observability ────────────────────────────────────────────────
  prometheus:
    image: prom/prometheus:latest
    command:
      - --config.file=/etc/prometheus/prometheus.yml
    volumes:
      - ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      # F2: alert rules (NATS resource limits, ingestd publish-path health).
      - ./deploy/prometheus/rules:/etc/prometheus/rules:ro
    ports: ["9090:9090"]
    depends_on: [ingestd, routerd, deliverd-fcm, deliverd-telegram, telegramd, admind]

  grafana:
    image: grafana/grafana:latest
    # :3000 is held by gogs (git3) on this host. Map to :3001
    # on the host, keep :3000 internal. M0 ports convention
    # says canonical ports stay; the host-port override is a
    # one-line exception, fully isolated to grafana.
    ports: ["3001:3000"]
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_SECURITY_DISABLE_LOGIN_FORM: "false"
      # M9: provisioning paths for dashboards + datasources.
      GF_PATHS_PROVISIONING: /etc/grafana/provisioning
      GF_DASHBOARDS_ENABLED: "true"
    volumes:
      - ./deploy/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
      - ./deploy/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
      - ./deploy/grafana/provisioning/dashboards/broad-announce-overview.json:/var/lib/grafana/dashboards/broad-announce-overview.json:ro
    depends_on: [prometheus]

  # ── Loadgen (one-shot smoke profile) ────────────────────
  loadgen-http:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-001:prom-prod:s3cret-acme
      - --mode=normal
      - --rate=50
      - --duration=30s
      - --metrics=:8891
      - --instance=loadgen-http
      - --cluster-id=smoke
    profiles: ["loadgen"]
    depends_on:
      ingestd: { condition: service_started }

  # M10: 3-instance cluster hitting 5k/s. Each instance does ~1.7k/s.
  # Per-company cap is 10k/s, so 3 × 1.7k = 5.1k is safe.
  # Use `docker compose --profile loadgen-m10 up -d` to bring up all 3.
  loadgen-http-1:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-001:prom-prod:s3cret-acme-001
      - --mode=normal
      - --rate=1700
      - --duration=10m
      - --ramp-up=30s
      - --metrics=:8891
      - --instance=loadgen-http-1
      - --cluster-id=m10
    profiles: ["loadgen-m10"]
    depends_on:
      ingestd: { condition: service_started }

  loadgen-http-2:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-002:prom-prod:s3cret-acme-002
      - --mode=normal
      - --rate=1700
      - --duration=10m
      - --ramp-up=30s
      - --metrics=:8891
      - --instance=loadgen-http-2
      - --cluster-id=m10
    profiles: ["loadgen-m10"]
    depends_on:
      ingestd: { condition: service_started }

  loadgen-http-3:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-003:prom-prod:s3cret-acme-003
      - --mode=normal
      - --rate=1700
      - --duration=10m
      - --ramp-up=30s
      - --metrics=:8891
      - --instance=loadgen-http-3
      - --cluster-id=m10
    profiles: ["loadgen-m10"]
    depends_on:
      ingestd: { condition: service_started }

  # M10 W4: Rogue loadgen — same company+source as loadgen-http-1 but
  # firing at 10× the per-source cap (100/s). Simulates a compromised source
  # at 1000/s. Per-source limiter drops 900/s (429); other companies stay clean.
  # NOT started by default with --profile loadgen-m10; spawned by step3_runaway_test().
  loadgen-http-4:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-001:prom-prod:s3cret-acme-001
      - --mode=normal
      - --rate=1000
      - --duration=10m
      - --ramp-up=10s
      - --metrics=:8891
      - --instance=loadgen-http-4
      - --cluster-id=m10
    profiles: ["loadgen-m10"]
    depends_on:
      ingestd: { condition: service_started }

  # M11 W4: gRPC loadgen — 2-instance cluster at 10k/s total.
  # Each instance opens 8 parallel streams (workers) at 5k/s each.
  # Use: docker compose --profile loadgen-grpc up -d
  loadgen-grpc-1:
    build: .
    command:
      - /app/loadgen-grpc
      - --target=ingestd:9090
      - --api-key=acme-001:acme-001-prom:s3cret-acme-001
      - --rate=8000
      - --workers=16
      - --dedupe-pct=0
      - --duration=15m
      - --metrics=:8892
      - --instance=loadgen-grpc-1
      - --cluster-id=m11
    profiles: ["loadgen-grpc"]
    depends_on:
      ingestd: { condition: service_started }

  loadgen-grpc-2:
    build: .
    command:
      - /app/loadgen-grpc
      - --target=ingestd:9090
      - --api-key=acme-002:acme-002-prom:s3cret-acme-002
      - --rate=8000
      - --workers=16
      - --dedupe-pct=0
      - --duration=15m
      - --metrics=:8892
      - --instance=loadgen-grpc-2
      - --cluster-id=m11
    profiles: ["loadgen-grpc"]
    depends_on:
      ingestd: { condition: service_started }

  # M10-bench: delivery tier stubbed with deliverd-bench (no-op).
  # No FCM, no Telegram, no Postgres writes — just consume and ACK.
  # Allows broker+router ceiling testing at 50k/s without burning FCM credits.
  deliverd-bench:
    build: .
    command: ["/app/deliverd-bench"]
    environment:
      BA_ENV: dev
      BA_NATS_URL: nats://nats:4222
    profiles: ["bench"]
    depends_on:
      nats: { condition: service_healthy }

  # 10-instance loadgen cluster for 50k/s bench (5 × 5k instances × 2 = 50k).
  # Split across 2 host machines in production; on docker-compose single-host
  # we run 10 instances at 5k/s each for a total of 50k/s.
  loadgen-bench-1:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-bench:prom-bench:s3cret-bench
      - --mode=normal
      - --rate=5000
      - --duration=5m
      - --ramp-up=15s
      - --metrics=:8891
      - --instance=loadgen-bench-1
      - --cluster-id=m10-bench
    profiles: ["bench"]
    depends_on:
      ingestd: { condition: service_started }

  loadgen-bench-2:
    build: .
    command:
      - /app/loadgen-http
      - --target=http://ingestd:8800
      - --api-key=acme-bench:prom-bench:s3cret-bench
      - --mode=normal
      - --rate=5000
      - --duration=5m
      - --ramp-up=15s
      - --metrics=:8891
      - --instance=loadgen-bench-2
      - --cluster-id=m10-bench
    profiles: ["bench"]
    depends_on:
      ingestd: { condition: service_started }

volumes:
  pgdata: {}
  natsdata: {}
  chdata: {}
  authd-data: {}