From ee1737fedc4e2baf4173ba7be461e0893c3ca4cc Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Mon, 19 Oct 2026 16:50:35 +0200 Subject: [PATCH] =?UTF-8?q?f-78:=20watchdog:=20r=C3=A9servations=20DHCP=20?= =?UTF-8?q?des=20VM=20en=20creating,=20deleting=20ou=20error=20non=20compa?= =?UTF-8?q?r=C3=A9es=20#78?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- docs/exploitation/diagnostic.rst | 4 +- internal/watchdog/check_dhcp.go | 49 +++++++++++--- internal/watchdog/check_dhcp_test.go | 90 ++++++++++++++++++++++++-- release_notes/0.2.0.md | 4 +- test/e2e/suite/01-lab.sh | 4 +- test/e2e/suite/61-redemarrage-agent.sh | 4 +- 6 files changed, 135 insertions(+), 20 deletions(-) diff --git a/docs/exploitation/diagnostic.rst b/docs/exploitation/diagnostic.rst index 6c2a29f..8ed51fd 100644 --- a/docs/exploitation/diagnostic.rst +++ b/docs/exploitation/diagnostic.rst @@ -58,7 +58,9 @@ la MAC n'est pas réservée — l'ordre ``set-host`` n'a jamais atteint le serve Le watchdog signale ces écarts de lui-même, à chaque tick, en comparant l'état servi à la base : ``dhcp reservation missing on the server``, ``stale dhcp reservation``, ``dhcp reservation -diverges``. Regarder ses notifications avant de sonder à la main. +diverges``. Regarder ses notifications avant de sonder à la main. Seules les VM ``running`` sont +comparées : pendant une création ou une suppression, ou pour une VM en ``error``, la réservation +peut être présente ou absente sans alerte. Si le serveur ne voit passer aucune requête, le problème est en amont : tap absent, bridge non raccordé, VM dans le mauvais netns. diff --git a/internal/watchdog/check_dhcp.go b/internal/watchdog/check_dhcp.go index 5e03e57..a3f5bf7 100644 --- a/internal/watchdog/check_dhcp.go +++ b/internal/watchdog/check_dhcp.go @@ -25,28 +25,58 @@ type stateReporter interface { State(s dhcpbackend.Subnet) (dhcpapi.State, string, error) } -func expectedHosts(db *badger.DB, subnetName string) ([]dhcpapi.Host, error) { +func expectedHosts(db *badger.DB, subnetName string) (hosts []dhcpapi.Host, transient map[string]bool, err error) { pairs, err := kv.ListByPrefix(db, prefixVM) if err != nil { - return nil, fmt.Errorf("listing vms: %w", err) + return nil, nil, fmt.Errorf("listing vms: %w", err) } - hosts := make([]dhcpapi.Host, 0) + hosts, transient = make([]dhcpapi.Host, 0), map[string]bool{} for _, vmName := range resourceNames(pairs, prefixVM) { st, err := state.Get(db, prefixVM+vmName) - if err != nil || st != state.Running { + if err != nil { + continue + } + if st == state.Creating || st == state.Deleting || st == state.Error { + transient[vmName] = true + continue + } + if st != state.Running { continue } vmHosts, err := expectedVMHosts(db, vmName, subnetName) if err != nil { - return nil, err + return nil, nil, err } hosts = append(hosts, vmHosts...) } dhcpapi.SortHosts(hosts) - return hosts, nil + return hosts, transient, nil +} + +func withoutTransient(served, expected []dhcpapi.Host, transient map[string]bool) []dhcpapi.Host { + wanted := make(map[string]bool, len(expected)) + for _, h := range expected { + wanted[canonicalMAC(h)] = true + } + + kept := make([]dhcpapi.Host, 0, len(served)) + for _, h := range served { + if transient[h.VM] && !wanted[canonicalMAC(h)] { + continue + } + kept = append(kept, h) + } + return kept +} + +func canonicalMAC(h dhcpapi.Host) string { + if c, err := dhcpapi.CanonicalHost(h); err == nil { + return c.MAC + } + return h.MAC } func expectedVMHosts(db *badger.DB, vmName, subnetName string) ([]dhcpapi.Host, error) { @@ -125,18 +155,19 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep n.Notify(kindSubnet, name, "dhcp server has no subnet configuration: it serves nothing") } - expected, err := expectedHosts(db, name) + expected, transient, err := expectedHosts(db, name) if err != nil { n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations unreadable in database: %v", err)) return } + hosts := withoutTransient(served.Hosts, expected, transient) expectedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: expected}) if err != nil { n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations inconsistent in database: %v", err)) return } - servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: served.Hosts}) + servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: hosts}) if err != nil { n.Notify(kindSubnet, name, fmt.Sprintf("dhcp reservations reported by the server are inconsistent: %v", err)) return @@ -145,7 +176,7 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep return } - for _, gap := range reservationGaps(expected, served.Hosts) { + for _, gap := range reservationGaps(expected, hosts) { n.Notify(kindSubnet, name, gap) } } diff --git a/internal/watchdog/check_dhcp_test.go b/internal/watchdog/check_dhcp_test.go index a9c4801..a39e4c3 100644 --- a/internal/watchdog/check_dhcp_test.go +++ b/internal/watchdog/check_dhcp_test.go @@ -83,7 +83,7 @@ func TestExpectedHosts_ReadsRunningVMsOnThatSubnet(t *testing.T) { db := newTestDB(t) seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true) - hosts, err := expectedHosts(db, testSubnetName) + hosts, _, err := expectedHosts(db, testSubnetName) if err != nil { t.Fatalf("expectedHosts: %v", err) } @@ -105,13 +105,16 @@ func TestExpectedHosts_IgnoresVMsThatAreNotRunning(t *testing.T) { t.Fatalf("state.Set: %v", err) } - hosts, err := expectedHosts(db, testSubnetName) + hosts, transient, err := expectedHosts(db, testSubnetName) if err != nil { t.Fatalf("expectedHosts: %v", err) } if len(hosts) != 0 { t.Errorf("hosts = %+v, want none: a vm being deleted is not expected to be served", hosts) } + if !transient["vm-web"] { + t.Errorf("transient = %v, want vm-web: a vm being deleted may still be served", transient) + } } func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) { @@ -121,7 +124,7 @@ func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) { seedKV(t, db, "vm/vm-web/nic/1/ip", "10.0.6.10") seedKV(t, db, "vm/vm-web/nic/1/primary", "false") - hosts, err := expectedHosts(db, testSubnetName) + hosts, _, err := expectedHosts(db, testSubnetName) if err != nil { t.Fatalf("expectedHosts: %v", err) } @@ -135,7 +138,7 @@ func TestExpectedHosts_SortsByMAC(t *testing.T) { seedSubnetWithVM(t, db, "vm-b", "10.0.5.11", "00:22:33:00:00:0B", true) seedSubnetWithVM(t, db, "vm-a", "10.0.5.10", "00:22:33:00:00:0A", true) - hosts, err := expectedHosts(db, testSubnetName) + hosts, _, err := expectedHosts(db, testSubnetName) if err != nil { t.Fatalf("expectedHosts: %v", err) } @@ -149,7 +152,7 @@ func TestExpectedHosts_ReportsAnIPWithoutAMAC(t *testing.T) { seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true) seedKV(t, db, "vm/vm-web/nic/0/ip", "10.0.5.99") - if _, err := expectedHosts(db, testSubnetName); err == nil { + if _, _, err := expectedHosts(db, testSubnetName); err == nil { t.Fatal("an ip absent from the address plan must be reported, not skipped") } } @@ -250,6 +253,83 @@ func TestCheckDHCPState_ReportsAStaleReservation(t *testing.T) { } } +func TestCheckDHCPState_ToleratesAVMInATransientState(t *testing.T) { + for _, st := range []state.State{state.Creating, state.Deleting, state.Error} { + t.Run(string(st), func(t *testing.T) { + for _, served := range []bool{true, false} { + db := newTestDB(t) + seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true) + if err := state.Set(db, "vm/vm-web", st); err != nil { + t.Fatalf("state.Set: %v", err) + } + + b, _ := servedBy(t) + if served { + if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{ + {Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true}, + }); err != nil { + t.Fatalf("SetVM: %v", err) + } + } + + r := &recorder{} + checkDHCPState(db, testSubnetName, twoSubnet(), b, r) + + if len(r.calls) != 0 { + t.Errorf("served=%t: notifications = %+v, want none while the vm is %s", served, r.calls, st) + } + } + }) + } +} + +func TestCheckDHCPState_ToleranceIsLimitedToTheTransientVM(t *testing.T) { + db := newTestDB(t) + seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true) + if err := state.Set(db, "vm/vm-web", state.Deleting); err != nil { + t.Fatalf("state.Set: %v", err) + } + + b, _ := servedBy(t) + if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{ + {Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true}, + }); err != nil { + t.Fatalf("SetVM: %v", err) + } + if err := b.SetVM(twoSubnet(), "vm-gone", []dhcpbackend.Reservation{ + {Index: 0, MAC: "00:22:33:00:00:0B", IP: "10.0.5.11", DefaultRoute: true}, + }); err != nil { + t.Fatalf("SetVM: %v", err) + } + + r := &recorder{} + checkDHCPState(db, testSubnetName, twoSubnet(), b, r) + + if len(r.calls) != 1 || !r.hasProblemContaining("stale dhcp reservation on the server: 00:22:33:00:00:0b") { + t.Errorf("notifications = %+v, want only the reservation of the vm that no longer exists", r.calls) + } +} + +func TestCheckDHCPState_AReservationOfATransientVMOnAnExpectedMACIsStillCompared(t *testing.T) { + db := newTestDB(t) + seedSubnetWithVM(t, db, "vm-new", "10.0.5.10", "00:22:33:00:00:0A", true) + seedResource(t, db, "vm/", "vm-old", state.Deleting) + + b, _ := servedBy(t) + if err := b.SetVM(twoSubnet(), "vm-old", []dhcpbackend.Reservation{ + {Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.99", DefaultRoute: true}, + }); err != nil { + t.Fatalf("SetVM: %v", err) + } + + r := &recorder{} + checkDHCPState(db, testSubnetName, twoSubnet(), b, r) + + if len(r.calls) != 1 || !r.hasProblemContaining("dhcp reservation diverges") { + t.Errorf("notifications = %+v, want the divergence on the mac a running vm expects", r.calls) + } +} + func TestCheckDHCPState_ReportsADivergingIP(t *testing.T) { db := newTestDB(t) seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true) diff --git a/release_notes/0.2.0.md b/release_notes/0.2.0.md index 0e99d4d..e2d0580 100644 --- a/release_notes/0.2.0.md +++ b/release_notes/0.2.0.md @@ -86,7 +86,9 @@ L3VNI, avec une configuration BGP pilotée par l'agent ; documentation du projet sans propriétaire est signalée comme **trou noir**, une route attendue absente comme manquante - Le watchdog interroge le serveur DHCP intégré et compare les réservations servies à celles que la base implique. Il nomme ce qui diverge — ordre perdu à la création, ordre perdu à la - suppression, adresse ou route par défaut divergente — et reste en lecture seule + suppression, adresse ou route par défaut divergente — et reste en lecture seule. Une VM en + `creating`, `deleting` ou `error` n'est pas comparée : sa réservation peut être présente ou + absente sans alerte, le temps que la création ou la suppression aboutisse (#78) - La socket de contrôle répond à `get-state` et à `probe`, ce dernier montrant sans effet de bord ce qui serait envoyé à une MAC donnée : adresse, masque, routeur, DNS et routes - Documentation Sphinx du projet : concepts, architecture, déploiement et exploitation diff --git a/test/e2e/suite/01-lab.sh b/test/e2e/suite/01-lab.sh index aad104a..d66a7c7 100644 --- a/test/e2e/suite/01-lab.sh +++ b/test/e2e/suite/01-lab.sh @@ -33,8 +33,8 @@ for rr in 10.255.255.1 10.255.255.2; do check "${HOST} : session EVPN vers ${rr} établie" sh -c "vtysh -c 'show bgp neighbors ${rr}' | grep -q 'BGP state = Established'" check "${HOST} : loopback de ${rr} joignable" ping -c 2 -W 2 "${rr}" done -check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min (hors réservation DHCP périmée, #78)" watchdog_quiet 300 'stale dhcp reservation' -known_flaky "#78" "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation' +check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min" watchdog_quiet 300 +check "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation' check "${HOST} : image de VM compatible two prête" two_image check "${HOST} : clé SSH des VM prête" vm_key NODE diff --git a/test/e2e/suite/61-redemarrage-agent.sh b/test/e2e/suite/61-redemarrage-agent.sh index 9450fc8..69ea5af 100644 --- a/test/e2e/suite/61-redemarrage-agent.sh +++ b/test/e2e/suite/61-redemarrage-agent.sh @@ -57,8 +57,8 @@ check "après redémarrage : vrf-t61 exporte toujours t61-a 10.61.1.10/32" vrf_r check "après redémarrage : vrf-t61 apprend toujours 10.61.2.10/32 via hv2" vrf_route_via vp-t61 10.61.2.10 192.168.14.12 check "après redémarrage : bail de t61-a renouvelé (DHCP servi)" lease_renewed vp-t61 10.61.1.10 check "après redémarrage : métadonnées de t61-a servies (instance-id t61-a)" vm_grep vp-t61 10.61.1.10 'curl -fsS http://169.254.169.254/meta-data' 't61-a' -check "après redémarrage : aucune incohérence signalée par le watchdog (hors réservation DHCP périmée, #78)" watchdog_quiet "$(( $(date +%s) - START ))" 'stale dhcp reservation' -known_flaky "#78" "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation' +check "après redémarrage : aucune incohérence signalée par le watchdog" watchdog_quiet "$(( $(date +%s) - START ))" +check "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation' vm_ssh vp-t61 10.61.1.10 'pkill -x ping' || true sleep 1 stats=$(vm_ssh vp-t61 10.61.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n+0, max+0}' /tmp/t61.log")