f-78: agent: attendre la fin du watchdog avant de fermer la base à l'arrêt #78
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
a47af03eef
commit
54e0faac55
2 changed files with 64 additions and 9 deletions
|
|
@ -111,11 +111,18 @@ func run(confFile string) int {
|
||||||
go adminSrv.Start()
|
go adminSrv.Start()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
watchdogStopped := make(chan struct{})
|
||||||
if cfg.Watchdog.Enabled {
|
if cfg.Watchdog.Enabled {
|
||||||
wlog := log.With(slog.String("component", "watchdog"))
|
wlog := log.With(slog.String("component", "watchdog"))
|
||||||
go watchdog.New(db, cfg, notify.NewStderr(wlog), wlog,
|
w := watchdog.New(db, cfg, notify.NewStderr(wlog), wlog,
|
||||||
time.Duration(cfg.Watchdog.IntervalSeconds)*time.Second,
|
time.Duration(cfg.Watchdog.IntervalSeconds)*time.Second,
|
||||||
).Run(ctx)
|
)
|
||||||
|
go func() {
|
||||||
|
defer close(watchdogStopped)
|
||||||
|
w.Run(ctx)
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
close(watchdogStopped)
|
||||||
}
|
}
|
||||||
|
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
|
|
@ -125,7 +132,7 @@ func run(confFile string) int {
|
||||||
if adminSrv != nil {
|
if adminSrv != nil {
|
||||||
servers["admin"] = adminSrv
|
servers["admin"] = adminSrv
|
||||||
}
|
}
|
||||||
closeDB = shutdown(log, q, servers, shutdownTimeout)
|
closeDB = shutdown(log, q, servers, watchdogStopped, shutdownTimeout)
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -149,7 +156,7 @@ type httpShutdowner interface {
|
||||||
Shutdown(context.Context) error
|
Shutdown(context.Context) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, timeout time.Duration) bool {
|
func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdowner, watchdogStopped <-chan struct{}, timeout time.Duration) bool {
|
||||||
log.Info("shutting down", "timeout", timeout)
|
log.Info("shutting down", "timeout", timeout)
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
|
@ -170,9 +177,16 @@ func shutdown(log *slog.Logger, q *worker.Queue, servers map[string]httpShutdown
|
||||||
select {
|
select {
|
||||||
case <-drained:
|
case <-drained:
|
||||||
log.Info("workers drained")
|
log.Info("workers drained")
|
||||||
return true
|
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Error("workers still running after timeout, leaving database untouched")
|
log.Error("workers still running after timeout, leaving database untouched")
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-watchdogStopped:
|
||||||
|
return true
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.Error("watchdog still running after timeout, leaving database untouched")
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ func TestShutdown_DrainReussi(t *testing.T) {
|
||||||
api, prom := &fakeServer{}, &fakeServer{}
|
api, prom := &fakeServer{}, &fakeServer{}
|
||||||
servers := map[string]httpShutdowner{"api": api, "prometheus": prom}
|
servers := map[string]httpShutdowner{"api": api, "prometheus": prom}
|
||||||
|
|
||||||
if !shutdown(discardLogger(), q, servers, 5*time.Second) {
|
if !shutdown(discardLogger(), q, servers, stopped(), 5*time.Second) {
|
||||||
t.Fatal("un drainage réussi doit autoriser la fermeture de la base")
|
t.Fatal("un drainage réussi doit autoriser la fermeture de la base")
|
||||||
}
|
}
|
||||||
if !api.called.Load() || !prom.called.Load() {
|
if !api.called.Load() || !prom.called.Load() {
|
||||||
|
|
@ -65,7 +65,7 @@ func TestShutdown_TimeoutLaisseLaBaseIntacte(t *testing.T) {
|
||||||
var buf strings.Builder
|
var buf strings.Builder
|
||||||
log := slog.New(slog.NewTextHandler(&buf, nil))
|
log := slog.New(slog.NewTextHandler(&buf, nil))
|
||||||
|
|
||||||
if shutdown(log, q, map[string]httpShutdowner{}, 50*time.Millisecond) {
|
if shutdown(log, q, map[string]httpShutdowner{}, stopped(), 50*time.Millisecond) {
|
||||||
t.Fatal("un drainage incomplet ne doit pas autoriser la fermeture de la base")
|
t.Fatal("un drainage incomplet ne doit pas autoriser la fermeture de la base")
|
||||||
}
|
}
|
||||||
if !strings.Contains(buf.String(), "leaving database untouched") {
|
if !strings.Contains(buf.String(), "leaving database untouched") {
|
||||||
|
|
@ -84,7 +84,7 @@ func TestShutdown_ErreurServeurNEmpechePasLeDrainage(t *testing.T) {
|
||||||
"prometheus": &fakeServer{},
|
"prometheus": &fakeServer{},
|
||||||
}
|
}
|
||||||
|
|
||||||
if !shutdown(log, q, servers, 5*time.Second) {
|
if !shutdown(log, q, servers, stopped(), 5*time.Second) {
|
||||||
t.Fatal("une erreur d'arrêt HTTP ne doit pas empêcher le drainage")
|
t.Fatal("une erreur d'arrêt HTTP ne doit pas empêcher le drainage")
|
||||||
}
|
}
|
||||||
if !strings.Contains(buf.String(), "http server shutdown") {
|
if !strings.Contains(buf.String(), "http server shutdown") {
|
||||||
|
|
@ -96,7 +96,7 @@ func TestShutdown_SansServeur(t *testing.T) {
|
||||||
q := worker.New(10)
|
q := worker.New(10)
|
||||||
q.Start(1)
|
q.Start(1)
|
||||||
|
|
||||||
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, 5*time.Second) {
|
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, stopped(), 5*time.Second) {
|
||||||
t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre")
|
t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -108,6 +108,47 @@ func (f *fakeBase) ApplyBase(frr.Base) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stopped() <-chan struct{} {
|
||||||
|
c := make(chan struct{})
|
||||||
|
close(c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShutdown_AttendLaFinDuWatchdogAvantLaBase(t *testing.T) {
|
||||||
|
q := worker.New(10)
|
||||||
|
q.Start(1)
|
||||||
|
|
||||||
|
watchdogStopped := make(chan struct{})
|
||||||
|
var finished atomic.Bool
|
||||||
|
go func() {
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
finished.Store(true)
|
||||||
|
close(watchdogStopped)
|
||||||
|
}()
|
||||||
|
|
||||||
|
if !shutdown(discardLogger(), q, map[string]httpShutdowner{}, watchdogStopped, 5*time.Second) {
|
||||||
|
t.Fatal("un watchdog arrêté dans le délai doit autoriser la fermeture de la base")
|
||||||
|
}
|
||||||
|
if !finished.Load() {
|
||||||
|
t.Error("la fermeture de la base a été autorisée avant la fin du passage du watchdog")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShutdown_WatchdogEncoreEnCoursLaisseLaBaseIntacte(t *testing.T) {
|
||||||
|
q := worker.New(10)
|
||||||
|
q.Start(1)
|
||||||
|
|
||||||
|
var buf strings.Builder
|
||||||
|
log := slog.New(slog.NewTextHandler(&buf, nil))
|
||||||
|
|
||||||
|
if shutdown(log, q, map[string]httpShutdowner{}, make(chan struct{}), 50*time.Millisecond) {
|
||||||
|
t.Fatal("un watchdog encore en cours ne doit pas autoriser la fermeture de la base")
|
||||||
|
}
|
||||||
|
if !strings.Contains(buf.String(), "watchdog still running after timeout") {
|
||||||
|
t.Errorf("le dépassement devrait être logué, obtenu %q", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) {
|
func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) {
|
||||||
cfg := &configuration.Config{}
|
cfg := &configuration.Config{}
|
||||||
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
|
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue