f-78: agent: attendre la fin du watchdog avant de fermer la base à l'arrêt #78
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
a47af03eef
commit
54e0faac55
2 changed files with 64 additions and 9 deletions
|
|
@ -111,11 +111,18 @@ func run(confFile string) int {
|
|||
go adminSrv.Start()
|
||||
}
|
||||
|
||||
watchdogStopped := make(chan struct{})
|
||||
if cfg.Watchdog.Enabled {
|
||||
wlog := log.With(slog.String("component", "watchdog"))
|
||||
go watchdog.New(db, cfg, notify.NewStderr(wlog), wlog,
|
||||
w := watchdog.New(db, cfg, notify.NewStderr(wlog), wlog,
|
||||
time.Duration(cfg.Watchdog.IntervalSeconds)*time.Second,
|
||||
).Run(ctx)
|
||||
)
|
||||
go func() {
|
||||
defer close(watchdogStopped)
|
||||
w.Run(ctx)
|
||||
}()
|
||||
} else {
|
||||
close(watchdogStopped)
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
|
|
@ -125,7 +132,7 @@ func run(confFile string) int {
|
|||
if adminSrv != nil {
|
||||
servers["admin"] = adminSrv
|
||||
}
|
||||
closeDB = shutdown(log, q, servers, shutdownTimeout)
|
||||
closeDB = shutdown(log, q, servers, watchdogStopped, shutdownTimeout)
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
@ -149,7 +156,7 @@ type httpShutdowner interface {
|
|||
Shutdown(context.Context) error
|
||||
}
|
||||
|
||||
func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, timeout time.Duration) bool {
|
||||
func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, watchdogStopped <-chan struct{}, timeout time.Duration) bool {
|
||||
log.Info("shutting down", "timeout", timeout)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
|
|
@ -170,9 +177,16 @@ func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdown
|
|||
select {
|
||||
case <-drained:
|
||||
log.Info("workers drained")
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
log.Error("workers still running after timeout, leaving database untouched")
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-watchdogStopped:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
log.Error("watchdog still running after timeout, leaving database untouched")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func TestShutdown_DrainReussi(t *testing.T) {
|
|||
api, prom := &fakeServer{}, &fakeServer{}
|
||||
servers := map[string]httpShutdowner{"api": api, "prometheus": prom}
|
||||
|
||||
if !shutdown(discardLogger(), q, servers, 5*time.Second) {
|
||||
if !shutdown(discardLogger(), q, servers, stopped(), 5*time.Second) {
|
||||
t.Fatal("un drainage réussi doit autoriser la fermeture de la base")
|
||||
}
|
||||
if !api.called.Load() || !prom.called.Load() {
|
||||
|
|
@ -65,7 +65,7 @@ func TestShutdown_TimeoutLaisseLaBaseIntacte(t *testing.T) {
|
|||
var buf strings.Builder
|
||||
log := slog.New(slog.NewTextHandler(&buf, nil))
|
||||
|
||||
if shutdown(log, q, map[string]httpShutdowner{}, 50*time.Millisecond) {
|
||||
if shutdown(log, q, map[string]httpShutdowner{}, stopped(), 50*time.Millisecond) {
|
||||
t.Fatal("un drainage incomplet ne doit pas autoriser la fermeture de la base")
|
||||
}
|
||||
if !strings.Contains(buf.String(), "leaving database untouched") {
|
||||
|
|
@ -84,7 +84,7 @@ func TestShutdown_ErreurServeurNEmpechePasLeDrainage(t *testing.T) {
|
|||
"prometheus": &fakeServer{},
|
||||
}
|
||||
|
||||
if !shutdown(log, q, servers, 5*time.Second) {
|
||||
if !shutdown(log, q, servers, stopped(), 5*time.Second) {
|
||||
t.Fatal("une erreur d'arrêt HTTP ne doit pas empêcher le drainage")
|
||||
}
|
||||
if !strings.Contains(buf.String(), "http server shutdown") {
|
||||
|
|
@ -96,7 +96,7 @@ func TestShutdown_SansServeur(t *testing.T) {
|
|||
q := worker.New(10)
|
||||
q.Start(1)
|
||||
|
||||
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, 5*time.Second) {
|
||||
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, stopped(), 5*time.Second) {
|
||||
t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre")
|
||||
}
|
||||
}
|
||||
|
|
@ -108,6 +108,47 @@ func (f *fakeBase) ApplyBase(frr.Base) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func stopped() <-chan struct{} {
|
||||
c := make(chan struct{})
|
||||
close(c)
|
||||
return c
|
||||
}
|
||||
|
||||
func TestShutdown_AttendLaFinDuWatchdogAvantLaBase(t *testing.T) {
|
||||
q := worker.New(10)
|
||||
q.Start(1)
|
||||
|
||||
watchdogStopped := make(chan struct{})
|
||||
var finished atomic.Bool
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
finished.Store(true)
|
||||
close(watchdogStopped)
|
||||
}()
|
||||
|
||||
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, watchdogStopped, 5*time.Second) {
|
||||
t.Fatal("un watchdog arrêté dans le délai doit autoriser la fermeture de la base")
|
||||
}
|
||||
if !finished.Load() {
|
||||
t.Error("la fermeture de la base a été autorisée avant la fin du passage du watchdog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdown_WatchdogEncoreEnCoursLaisseLaBaseIntacte(t *testing.T) {
|
||||
q := worker.New(10)
|
||||
q.Start(1)
|
||||
|
||||
var buf strings.Builder
|
||||
log := slog.New(slog.NewTextHandler(&buf, nil))
|
||||
|
||||
if shutdown(log, q, map[string]httpShutdowner{}, make(chan struct{}), 50*time.Millisecond) {
|
||||
t.Fatal("un watchdog encore en cours ne doit pas autoriser la fermeture de la base")
|
||||
}
|
||||
if !strings.Contains(buf.String(), "watchdog still running after timeout") {
|
||||
t.Errorf("le dépassement devrait être logué, obtenu %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) {
|
||||
cfg := &configuration.Config{}
|
||||
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue