// Command archiverd is the M7 periodic data-tier job. It // moves deliveries older than 7 days from TimescaleDB to // ClickHouse, then drops the source rows. The hot-path // services are unaffected — archiverd only reads and // deletes, never publishes or queues. // // Cadence: BA_ARCHIVERD_RUN_EVERY_SECONDS (default 3600). // Health: /health returns 200 if the last successful run // was less than 2x the cadence ago. /metrics exposes // ba_archiverd_rows_archived_total and // ba_archiverd_last_run_timestamp_seconds. package main import ( "context" "encoding/json" "fmt" "log/slog" "net/http" "os" "os/signal" "strconv" "sync/atomic" "syscall" "time" "git3.techno-world.net/lrosales/broad-announce/internal/archiver" "git3.techno-world.net/lrosales/broad-announce/internal/config" "git3.techno-world.net/lrosales/broad-announce/internal/httpserver" "git3.techno-world.net/lrosales/broad-announce/internal/observability" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) func main() { cfg, err := config.LoadArchiverd() if err != nil { os.Stderr.WriteString("config: " + err.Error() + "\n") os.Exit(1) } logger := observability.Init(cfg.Env, cfg.LogLevel, "archiverd") logger.Info("starting", "env", cfg.Env, "addr", cfg.HTTPAddr, "run_every_seconds", cfg.RunEverySeconds, "older_than_hours", cfg.OlderThanHours, "batch_size", cfg.BatchSize, ) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() // Metrics. We use a tiny custom registry here rather // than the shared IngestdMetrics struct because the // archiver has different counters (rows_archived by // table) and doesn't need the full ingestd suite. reg := prometheus.NewRegistry() reg.MustRegister(prometheus.NewGoCollector(), prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) rowsArchived := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "ba", Subsystem: "archiverd", Name: "rows_archived_total", Help: "M7: number of rows moved from Timescale to ClickHouse by the archiver.", }, []string{"table"}) lastRunTs := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "ba", Subsystem: "archiverd", Name: "last_run_timestamp_seconds", Help: "M7: unix timestamp of the last successful archiverd run.", }) runDuration := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "ba", Subsystem: "archiverd", Name: "last_run_duration_seconds", Help: "M7: duration of the last archiverd run, in seconds.", }) reg.MustRegister(rowsArchived, lastRunTs, runDuration) // Shared atomic for the /health check. var lastRunUnix atomic.Int64 lastRunUnix.Store(0) // Periodic loop. RunOnce is called in a goroutine so // the /health endpoint stays responsive between runs. go runLoop(ctx, logger, archiver.RunOptions{ PostgresDSN: cfg.PostgresDSN, ClickHouseURL: cfg.ClickHouseURL, OlderThan: time.Duration(cfg.OlderThanHours) * time.Hour, BatchSize: cfg.BatchSize, Logger: logger, }, rowsArchived, &lastRunUnix, runDuration) srv := httpserver.New(httpserver.Config{ Addr: cfg.HTTPAddr, ServiceName: "archiverd", ShutdownGrace: cfg.ShutdownGrace, }, logger, metricsHandler(reg, &lastRunUnix, cfg.RunEverySeconds*2)) errCh := make(chan error, 1) go func() { errCh <- srv.Start() }() select { case <-ctx.Done(): logger.Info("shutdown signal received") case err := <-errCh: if err != nil { logger.Error("http server", "err", err) os.Exit(1) } } if err := srv.Shutdown(ctx); err != nil { logger.Warn("graceful shutdown", "err", err) } logger.Info("bye") } // runLoop drives the periodic execution. The first run // fires immediately on startup (so a fresh deploy catches // up on backlog), then every `RunEvery` seconds. func runLoop( ctx context.Context, logger *slog.Logger, opts archiver.RunOptions, rowsArchived *prometheus.CounterVec, lastRunUnix *atomic.Int64, runDuration prometheus.Gauge, ) { tick := time.NewTicker(time.Duration(optsRunEverySeconds()) * time.Second) defer tick.Stop() for { // Fire immediately on first iteration. oneRun(ctx, logger, opts, rowsArchived, lastRunUnix, runDuration) select { case <-ctx.Done(): return case <-tick.C: } } } func oneRun( ctx context.Context, logger *slog.Logger, opts archiver.RunOptions, rowsArchived *prometheus.CounterVec, lastRunUnix *atomic.Int64, runDuration prometheus.Gauge, ) { runCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() rep, err := archiver.RunOnce(runCtx, opts) if err != nil { logger.Error("archiver run failed", "err", err) return } lastRunUnix.Store(rep.FinishedAt.Unix()) for _, t := range rep.Tables { rowsArchived.WithLabelValues(t.Name).Add(float64(t.Rows)) } runDuration.Set(rep.Duration.Seconds()) logger.Info("archiver run ok", "duration_seconds", rep.Duration.Seconds(), "tables", rep.Tables, ) } // metricsHandler is a small wrapper that exposes /health // (returns 200 if the last run was within 2x the cadence) // alongside the /metrics endpoint. func metricsHandler(reg *prometheus.Registry, lastRun *atomic.Int64, healthyAfterSeconds int) *http.ServeMux { mux := http.NewServeMux() mux.Handle("GET /metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) mux.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) { last := lastRun.Load() if last == 0 { // No run yet — still healthy if we're // within the first 2x cadence of // startup. w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(map[string]any{ "status": "starting", "note": "no run has completed yet", }) return } age := time.Now().Unix() - last if age > int64(healthyAfterSeconds) { w.WriteHeader(http.StatusServiceUnavailable) _ = json.NewEncoder(w).Encode(map[string]any{ "status": "stale", "age_secs": age, }) return } w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "age_secs": age, }) }) return mux } // optsRunEverySeconds is read from the env at startup. // We keep it in a func rather than the config struct // because the loop closure captures it. func optsRunEverySeconds() int { v := os.Getenv("BA_ARCHIVERD_RUN_EVERY_SECONDS") if v == "" { return 3600 } n, err := strconv.Atoi(v) if err != nil || n <= 0 { return 3600 } return n } // keep fmt import var _ = fmt.Sprintf