diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 2ec0446..98b8e3d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -111,11 +111,18 @@ func run(confFile string) int { go adminSrv.Start() } + watchdogStopped := make(chan struct{}) if cfg.Watchdog.Enabled { wlog := log.With(slog.String("component", "watchdog")) - go watchdog.New(db, cfg, notify.NewStderr(wlog), wlog, + w := watchdog.New(db, cfg, notify.NewStderr(wlog), wlog, time.Duration(cfg.Watchdog.IntervalSeconds)*time.Second, - ).Run(ctx) + ) + go func() { + defer close(watchdogStopped) + w.Run(ctx) + }() + } else { + close(watchdogStopped) } <-ctx.Done() @@ -125,7 +132,7 @@ func run(confFile string) int { if adminSrv != nil { servers["admin"] = adminSrv } - closeDB = shutdown(log, q, servers, shutdownTimeout) + closeDB = shutdown(log, q, servers, watchdogStopped, shutdownTimeout) return 0 } @@ -149,7 +156,7 @@ type httpShutdowner interface { Shutdown(context.Context) error } -func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, timeout time.Duration) bool { +func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, watchdogStopped <-chan struct{}, timeout time.Duration) bool { log.Info("shutting down", "timeout", timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -170,9 +177,16 @@ func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdown select { case <-drained: log.Info("workers drained") - return true case <-ctx.Done(): log.Error("workers still running after timeout, leaving database untouched") return false } + + select { + case <-watchdogStopped: + return true + case <-ctx.Done(): + log.Error("watchdog still running after timeout, leaving database untouched") + return false + } } diff --git a/cmd/agent/main_test.go b/cmd/agent/main_test.go index 9f3209c..36100f1 100644 --- a/cmd/agent/main_test.go +++ b/cmd/agent/main_test.go @@ -46,7 +46,7 @@ func TestShutdown_DrainReussi(t *testing.T) { api, prom := &fakeServer{}, &fakeServer{} servers := map[string]httpShutdowner{"api": api, "prometheus": prom} - if !shutdown(discardLogger(), q, servers, 5*time.Second) { + if !shutdown(discardLogger(), q, servers, stopped(), 5*time.Second) { t.Fatal("un drainage réussi doit autoriser la fermeture de la base") } if !api.called.Load() || !prom.called.Load() { @@ -65,7 +65,7 @@ func TestShutdown_TimeoutLaisseLaBaseIntacte(t *testing.T) { var buf strings.Builder log := slog.New(slog.NewTextHandler(&buf, nil)) - if shutdown(log, q, map[string]httpShutdowner{}, 50*time.Millisecond) { + if shutdown(log, q, map[string]httpShutdowner{}, stopped(), 50*time.Millisecond) { t.Fatal("un drainage incomplet ne doit pas autoriser la fermeture de la base") } if !strings.Contains(buf.String(), "leaving database untouched") { @@ -84,7 +84,7 @@ func TestShutdown_ErreurServeurNEmpechePasLeDrainage(t *testing.T) { "prometheus": &fakeServer{}, } - if !shutdown(log, q, servers, 5*time.Second) { + if !shutdown(log, q, servers, stopped(), 5*time.Second) { t.Fatal("une erreur d'arrêt HTTP ne doit pas empêcher le drainage") } if !strings.Contains(buf.String(), "http server shutdown") { @@ -96,7 +96,7 @@ func TestShutdown_SansServeur(t *testing.T) { q := worker.New(10) q.Start(1) - if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, 5*time.Second) { + if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, stopped(), 5*time.Second) { t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre") } } @@ -108,6 +108,47 @@ func (f *fakeBase) ApplyBase(frr.Base) error { return nil } +func stopped() <-chan struct{} { + c := make(chan struct{}) + close(c) + return c +} + +func TestShutdown_AttendLaFinDuWatchdogAvantLaBase(t *testing.T) { + q := worker.New(10) + q.Start(1) + + watchdogStopped := make(chan struct{}) + var finished atomic.Bool + go func() { + time.Sleep(100 * time.Millisecond) + finished.Store(true) + close(watchdogStopped) + }() + + if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, watchdogStopped, 5*time.Second) { + t.Fatal("un watchdog arrêté dans le délai doit autoriser la fermeture de la base") + } + if !finished.Load() { + t.Error("la fermeture de la base a été autorisée avant la fin du passage du watchdog") + } +} + +func TestShutdown_WatchdogEncoreEnCoursLaisseLaBaseIntacte(t *testing.T) { + q := worker.New(10) + q.Start(1) + + var buf strings.Builder + log := slog.New(slog.NewTextHandler(&buf, nil)) + + if shutdown(log, q, map[string]httpShutdowner{}, make(chan struct{}), 50*time.Millisecond) { + t.Fatal("un watchdog encore en cours ne doit pas autoriser la fermeture de la base") + } + if !strings.Contains(buf.String(), "watchdog still running after timeout") { + t.Errorf("le dépassement devrait être logué, obtenu %q", buf.String()) + } +} + func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) { cfg := &configuration.Config{} cfg.BGP.RouteReflectors = []string{"10.255.255.1"}