f-78: watchdog: réservations DHCP des VM en creating, deleting ou error non comparées #78
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
ccd1213593
commit
ee1737fedc
6 changed files with 135 additions and 20 deletions
|
|
@ -58,7 +58,9 @@ la MAC n'est pas réservée — l'ordre ``set-host`` n'a jamais atteint le serve
|
|||
|
||||
Le watchdog signale ces écarts de lui-même, à chaque tick, en comparant l'état servi à la base :
|
||||
``dhcp reservation missing on the server``, ``stale dhcp reservation``, ``dhcp reservation
|
||||
diverges``. Regarder ses notifications avant de sonder à la main.
|
||||
diverges``. Regarder ses notifications avant de sonder à la main. Seules les VM ``running`` sont
|
||||
comparées : pendant une création ou une suppression, ou pour une VM en ``error``, la réservation
|
||||
peut être présente ou absente sans alerte.
|
||||
|
||||
Si le serveur ne voit passer aucune requête, le problème est en amont : tap absent, bridge non
|
||||
raccordé, VM dans le mauvais netns.
|
||||
|
|
|
|||
|
|
@ -25,28 +25,58 @@ type stateReporter interface {
|
|||
State(s dhcpbackend.Subnet) (dhcpapi.State, string, error)
|
||||
}
|
||||
|
||||
func expectedHosts(db *badger.DB, subnetName string) ([]dhcpapi.Host, error) {
|
||||
func expectedHosts(db *badger.DB, subnetName string) (hosts []dhcpapi.Host, transient map[string]bool, err error) {
|
||||
pairs, err := kv.ListByPrefix(db, prefixVM)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing vms: %w", err)
|
||||
return nil, nil, fmt.Errorf("listing vms: %w", err)
|
||||
}
|
||||
|
||||
hosts := make([]dhcpapi.Host, 0)
|
||||
hosts, transient = make([]dhcpapi.Host, 0), map[string]bool{}
|
||||
for _, vmName := range resourceNames(pairs, prefixVM) {
|
||||
st, err := state.Get(db, prefixVM+vmName)
|
||||
if err != nil || st != state.Running {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if st == state.Creating || st == state.Deleting || st == state.Error {
|
||||
transient[vmName] = true
|
||||
continue
|
||||
}
|
||||
if st != state.Running {
|
||||
continue
|
||||
}
|
||||
|
||||
vmHosts, err := expectedVMHosts(db, vmName, subnetName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
hosts = append(hosts, vmHosts...)
|
||||
}
|
||||
|
||||
dhcpapi.SortHosts(hosts)
|
||||
return hosts, nil
|
||||
return hosts, transient, nil
|
||||
}
|
||||
|
||||
func withoutTransient(served, expected []dhcpapi.Host, transient map[string]bool) []dhcpapi.Host {
|
||||
wanted := make(map[string]bool, len(expected))
|
||||
for _, h := range expected {
|
||||
wanted[canonicalMAC(h)] = true
|
||||
}
|
||||
|
||||
kept := make([]dhcpapi.Host, 0, len(served))
|
||||
for _, h := range served {
|
||||
if transient[h.VM] && !wanted[canonicalMAC(h)] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, h)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
func canonicalMAC(h dhcpapi.Host) string {
|
||||
if c, err := dhcpapi.CanonicalHost(h); err == nil {
|
||||
return c.MAC
|
||||
}
|
||||
return h.MAC
|
||||
}
|
||||
|
||||
func expectedVMHosts(db *badger.DB, vmName, subnetName string) ([]dhcpapi.Host, error) {
|
||||
|
|
@ -125,18 +155,19 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep
|
|||
n.Notify(kindSubnet, name, "dhcp server has no subnet configuration: it serves nothing")
|
||||
}
|
||||
|
||||
expected, err := expectedHosts(db, name)
|
||||
expected, transient, err := expectedHosts(db, name)
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations unreadable in database: %v", err))
|
||||
return
|
||||
}
|
||||
hosts := withoutTransient(served.Hosts, expected, transient)
|
||||
|
||||
expectedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: expected})
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations inconsistent in database: %v", err))
|
||||
return
|
||||
}
|
||||
servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: served.Hosts})
|
||||
servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: hosts})
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp reservations reported by the server are inconsistent: %v", err))
|
||||
return
|
||||
|
|
@ -145,7 +176,7 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep
|
|||
return
|
||||
}
|
||||
|
||||
for _, gap := range reservationGaps(expected, served.Hosts) {
|
||||
for _, gap := range reservationGaps(expected, hosts) {
|
||||
n.Notify(kindSubnet, name, gap)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ func TestExpectedHosts_ReadsRunningVMsOnThatSubnet(t *testing.T) {
|
|||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
|
|
@ -105,13 +105,16 @@ func TestExpectedHosts_IgnoresVMsThatAreNotRunning(t *testing.T) {
|
|||
t.Fatalf("state.Set: %v", err)
|
||||
}
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
hosts, transient, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
if len(hosts) != 0 {
|
||||
t.Errorf("hosts = %+v, want none: a vm being deleted is not expected to be served", hosts)
|
||||
}
|
||||
if !transient["vm-web"] {
|
||||
t.Errorf("transient = %v, want vm-web: a vm being deleted may still be served", transient)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
||||
|
|
@ -121,7 +124,7 @@ func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
|||
seedKV(t, db, "vm/vm-web/nic/1/ip", "10.0.6.10")
|
||||
seedKV(t, db, "vm/vm-web/nic/1/primary", "false")
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
|
|
@ -135,7 +138,7 @@ func TestExpectedHosts_SortsByMAC(t *testing.T) {
|
|||
seedSubnetWithVM(t, db, "vm-b", "10.0.5.11", "00:22:33:00:00:0B", true)
|
||||
seedSubnetWithVM(t, db, "vm-a", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
|
|
@ -149,7 +152,7 @@ func TestExpectedHosts_ReportsAnIPWithoutAMAC(t *testing.T) {
|
|||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
seedKV(t, db, "vm/vm-web/nic/0/ip", "10.0.5.99")
|
||||
|
||||
if _, err := expectedHosts(db, testSubnetName); err == nil {
|
||||
if _, _, err := expectedHosts(db, testSubnetName); err == nil {
|
||||
t.Fatal("an ip absent from the address plan must be reported, not skipped")
|
||||
}
|
||||
}
|
||||
|
|
@ -250,6 +253,83 @@ func TestCheckDHCPState_ReportsAStaleReservation(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ToleratesAVMInATransientState(t *testing.T) {
|
||||
for _, st := range []state.State{state.Creating, state.Deleting, state.Error} {
|
||||
t.Run(string(st), func(t *testing.T) {
|
||||
for _, served := range []bool{true, false} {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
if err := state.Set(db, "vm/vm-web", st); err != nil {
|
||||
t.Fatalf("state.Set: %v", err)
|
||||
}
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if served {
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("served=%t: notifications = %+v, want none while the vm is %s", served, r.calls, st)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ToleranceIsLimitedToTheTransientVM(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
if err := state.Set(db, "vm/vm-web", state.Deleting); err != nil {
|
||||
t.Fatalf("state.Set: %v", err)
|
||||
}
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
if err := b.SetVM(twoSubnet(), "vm-gone", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0B", IP: "10.0.5.11", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 1 || !r.hasProblemContaining("stale dhcp reservation on the server: 00:22:33:00:00:0b") {
|
||||
t.Errorf("notifications = %+v, want only the reservation of the vm that no longer exists", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_AReservationOfATransientVMOnAnExpectedMACIsStillCompared(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-new", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
seedResource(t, db, "vm/", "vm-old", state.Deleting)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-old", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.99", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 1 || !r.hasProblemContaining("dhcp reservation diverges") {
|
||||
t.Errorf("notifications = %+v, want the divergence on the mac a running vm expects", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsADivergingIP(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
|
|
|||
|
|
@ -86,7 +86,9 @@ L3VNI, avec une configuration BGP pilotée par l'agent ; documentation du projet
|
|||
sans propriétaire est signalée comme **trou noir**, une route attendue absente comme manquante
|
||||
- Le watchdog interroge le serveur DHCP intégré et compare les réservations servies à celles que
|
||||
la base implique. Il nomme ce qui diverge — ordre perdu à la création, ordre perdu à la
|
||||
suppression, adresse ou route par défaut divergente — et reste en lecture seule
|
||||
suppression, adresse ou route par défaut divergente — et reste en lecture seule. Une VM en
|
||||
`creating`, `deleting` ou `error` n'est pas comparée : sa réservation peut être présente ou
|
||||
absente sans alerte, le temps que la création ou la suppression aboutisse (#78)
|
||||
- La socket de contrôle répond à `get-state` et à `probe`, ce dernier montrant sans effet de bord
|
||||
ce qui serait envoyé à une MAC donnée : adresse, masque, routeur, DNS et routes
|
||||
- Documentation Sphinx du projet : concepts, architecture, déploiement et exploitation
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ for rr in 10.255.255.1 10.255.255.2; do
|
|||
check "${HOST} : session EVPN vers ${rr} établie" sh -c "vtysh -c 'show bgp neighbors ${rr}' | grep -q 'BGP state = Established'"
|
||||
check "${HOST} : loopback de ${rr} joignable" ping -c 2 -W 2 "${rr}"
|
||||
done
|
||||
check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min (hors réservation DHCP périmée, #78)" watchdog_quiet 300 'stale dhcp reservation'
|
||||
known_flaky "#78" "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation'
|
||||
check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min" watchdog_quiet 300
|
||||
check "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation'
|
||||
check "${HOST} : image de VM compatible two prête" two_image
|
||||
check "${HOST} : clé SSH des VM prête" vm_key
|
||||
NODE
|
||||
|
|
|
|||
|
|
@ -57,8 +57,8 @@ check "après redémarrage : vrf-t61 exporte toujours t61-a 10.61.1.10/32" vrf_r
|
|||
check "après redémarrage : vrf-t61 apprend toujours 10.61.2.10/32 via hv2" vrf_route_via vp-t61 10.61.2.10 192.168.14.12
|
||||
check "après redémarrage : bail de t61-a renouvelé (DHCP servi)" lease_renewed vp-t61 10.61.1.10
|
||||
check "après redémarrage : métadonnées de t61-a servies (instance-id t61-a)" vm_grep vp-t61 10.61.1.10 'curl -fsS http://169.254.169.254/meta-data' 't61-a'
|
||||
check "après redémarrage : aucune incohérence signalée par le watchdog (hors réservation DHCP périmée, #78)" watchdog_quiet "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
||||
known_flaky "#78" "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
||||
check "après redémarrage : aucune incohérence signalée par le watchdog" watchdog_quiet "$(( $(date +%s) - START ))"
|
||||
check "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
||||
vm_ssh vp-t61 10.61.1.10 'pkill -x ping' || true
|
||||
sleep 1
|
||||
stats=$(vm_ssh vp-t61 10.61.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n+0, max+0}' /tmp/t61.log")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue