f-78: watchdog: réservations DHCP des VM en creating, deleting ou error non comparées #78
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
ccd1213593
commit
ee1737fedc
6 changed files with 135 additions and 20 deletions
|
|
@ -58,7 +58,9 @@ la MAC n'est pas réservée — l'ordre ``set-host`` n'a jamais atteint le serve
|
||||||
|
|
||||||
Le watchdog signale ces écarts de lui-même, à chaque tick, en comparant l'état servi à la base :
|
Le watchdog signale ces écarts de lui-même, à chaque tick, en comparant l'état servi à la base :
|
||||||
``dhcp reservation missing on the server``, ``stale dhcp reservation``, ``dhcp reservation
|
``dhcp reservation missing on the server``, ``stale dhcp reservation``, ``dhcp reservation
|
||||||
diverges``. Regarder ses notifications avant de sonder à la main.
|
diverges``. Regarder ses notifications avant de sonder à la main. Seules les VM ``running`` sont
|
||||||
|
comparées : pendant une création ou une suppression, ou pour une VM en ``error``, la réservation
|
||||||
|
peut être présente ou absente sans alerte.
|
||||||
|
|
||||||
Si le serveur ne voit passer aucune requête, le problème est en amont : tap absent, bridge non
|
Si le serveur ne voit passer aucune requête, le problème est en amont : tap absent, bridge non
|
||||||
raccordé, VM dans le mauvais netns.
|
raccordé, VM dans le mauvais netns.
|
||||||
|
|
|
||||||
|
|
@ -25,28 +25,58 @@ type stateReporter interface {
|
||||||
State(s dhcpbackend.Subnet) (dhcpapi.State, string, error)
|
State(s dhcpbackend.Subnet) (dhcpapi.State, string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func expectedHosts(db *badger.DB, subnetName string) ([]dhcpapi.Host, error) {
|
func expectedHosts(db *badger.DB, subnetName string) (hosts []dhcpapi.Host, transient map[string]bool, err error) {
|
||||||
pairs, err := kv.ListByPrefix(db, prefixVM)
|
pairs, err := kv.ListByPrefix(db, prefixVM)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("listing vms: %w", err)
|
return nil, nil, fmt.Errorf("listing vms: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
hosts := make([]dhcpapi.Host, 0)
|
hosts, transient = make([]dhcpapi.Host, 0), map[string]bool{}
|
||||||
for _, vmName := range resourceNames(pairs, prefixVM) {
|
for _, vmName := range resourceNames(pairs, prefixVM) {
|
||||||
st, err := state.Get(db, prefixVM+vmName)
|
st, err := state.Get(db, prefixVM+vmName)
|
||||||
if err != nil || st != state.Running {
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if st == state.Creating || st == state.Deleting || st == state.Error {
|
||||||
|
transient[vmName] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if st != state.Running {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
vmHosts, err := expectedVMHosts(db, vmName, subnetName)
|
vmHosts, err := expectedVMHosts(db, vmName, subnetName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
hosts = append(hosts, vmHosts...)
|
hosts = append(hosts, vmHosts...)
|
||||||
}
|
}
|
||||||
|
|
||||||
dhcpapi.SortHosts(hosts)
|
dhcpapi.SortHosts(hosts)
|
||||||
return hosts, nil
|
return hosts, transient, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func withoutTransient(served, expected []dhcpapi.Host, transient map[string]bool) []dhcpapi.Host {
|
||||||
|
wanted := make(map[string]bool, len(expected))
|
||||||
|
for _, h := range expected {
|
||||||
|
wanted[canonicalMAC(h)] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
kept := make([]dhcpapi.Host, 0, len(served))
|
||||||
|
for _, h := range served {
|
||||||
|
if transient[h.VM] && !wanted[canonicalMAC(h)] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kept = append(kept, h)
|
||||||
|
}
|
||||||
|
return kept
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalMAC(h dhcpapi.Host) string {
|
||||||
|
if c, err := dhcpapi.CanonicalHost(h); err == nil {
|
||||||
|
return c.MAC
|
||||||
|
}
|
||||||
|
return h.MAC
|
||||||
}
|
}
|
||||||
|
|
||||||
func expectedVMHosts(db *badger.DB, vmName, subnetName string) ([]dhcpapi.Host, error) {
|
func expectedVMHosts(db *badger.DB, vmName, subnetName string) ([]dhcpapi.Host, error) {
|
||||||
|
|
@ -125,18 +155,19 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep
|
||||||
n.Notify(kindSubnet, name, "dhcp server has no subnet configuration: it serves nothing")
|
n.Notify(kindSubnet, name, "dhcp server has no subnet configuration: it serves nothing")
|
||||||
}
|
}
|
||||||
|
|
||||||
expected, err := expectedHosts(db, name)
|
expected, transient, err := expectedHosts(db, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations unreadable in database: %v", err))
|
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations unreadable in database: %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
hosts := withoutTransient(served.Hosts, expected, transient)
|
||||||
|
|
||||||
expectedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: expected})
|
expectedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: expected})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations inconsistent in database: %v", err))
|
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations inconsistent in database: %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: served.Hosts})
|
servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: hosts})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp reservations reported by the server are inconsistent: %v", err))
|
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp reservations reported by the server are inconsistent: %v", err))
|
||||||
return
|
return
|
||||||
|
|
@ -145,7 +176,7 @@ func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateRep
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, gap := range reservationGaps(expected, served.Hosts) {
|
for _, gap := range reservationGaps(expected, hosts) {
|
||||||
n.Notify(kindSubnet, name, gap)
|
n.Notify(kindSubnet, name, gap)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -83,7 +83,7 @@ func TestExpectedHosts_ReadsRunningVMsOnThatSubnet(t *testing.T) {
|
||||||
db := newTestDB(t)
|
db := newTestDB(t)
|
||||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
|
||||||
hosts, err := expectedHosts(db, testSubnetName)
|
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expectedHosts: %v", err)
|
t.Fatalf("expectedHosts: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -105,13 +105,16 @@ func TestExpectedHosts_IgnoresVMsThatAreNotRunning(t *testing.T) {
|
||||||
t.Fatalf("state.Set: %v", err)
|
t.Fatalf("state.Set: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
hosts, err := expectedHosts(db, testSubnetName)
|
hosts, transient, err := expectedHosts(db, testSubnetName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expectedHosts: %v", err)
|
t.Fatalf("expectedHosts: %v", err)
|
||||||
}
|
}
|
||||||
if len(hosts) != 0 {
|
if len(hosts) != 0 {
|
||||||
t.Errorf("hosts = %+v, want none: a vm being deleted is not expected to be served", hosts)
|
t.Errorf("hosts = %+v, want none: a vm being deleted is not expected to be served", hosts)
|
||||||
}
|
}
|
||||||
|
if !transient["vm-web"] {
|
||||||
|
t.Errorf("transient = %v, want vm-web: a vm being deleted may still be served", transient)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
||||||
|
|
@ -121,7 +124,7 @@ func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
||||||
seedKV(t, db, "vm/vm-web/nic/1/ip", "10.0.6.10")
|
seedKV(t, db, "vm/vm-web/nic/1/ip", "10.0.6.10")
|
||||||
seedKV(t, db, "vm/vm-web/nic/1/primary", "false")
|
seedKV(t, db, "vm/vm-web/nic/1/primary", "false")
|
||||||
|
|
||||||
hosts, err := expectedHosts(db, testSubnetName)
|
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expectedHosts: %v", err)
|
t.Fatalf("expectedHosts: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -135,7 +138,7 @@ func TestExpectedHosts_SortsByMAC(t *testing.T) {
|
||||||
seedSubnetWithVM(t, db, "vm-b", "10.0.5.11", "00:22:33:00:00:0B", true)
|
seedSubnetWithVM(t, db, "vm-b", "10.0.5.11", "00:22:33:00:00:0B", true)
|
||||||
seedSubnetWithVM(t, db, "vm-a", "10.0.5.10", "00:22:33:00:00:0A", true)
|
seedSubnetWithVM(t, db, "vm-a", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
|
||||||
hosts, err := expectedHosts(db, testSubnetName)
|
hosts, _, err := expectedHosts(db, testSubnetName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expectedHosts: %v", err)
|
t.Fatalf("expectedHosts: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -149,7 +152,7 @@ func TestExpectedHosts_ReportsAnIPWithoutAMAC(t *testing.T) {
|
||||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
seedKV(t, db, "vm/vm-web/nic/0/ip", "10.0.5.99")
|
seedKV(t, db, "vm/vm-web/nic/0/ip", "10.0.5.99")
|
||||||
|
|
||||||
if _, err := expectedHosts(db, testSubnetName); err == nil {
|
if _, _, err := expectedHosts(db, testSubnetName); err == nil {
|
||||||
t.Fatal("an ip absent from the address plan must be reported, not skipped")
|
t.Fatal("an ip absent from the address plan must be reported, not skipped")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -250,6 +253,83 @@ func TestCheckDHCPState_ReportsAStaleReservation(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCheckDHCPState_ToleratesAVMInATransientState(t *testing.T) {
|
||||||
|
for _, st := range []state.State{state.Creating, state.Deleting, state.Error} {
|
||||||
|
t.Run(string(st), func(t *testing.T) {
|
||||||
|
for _, served := range []bool{true, false} {
|
||||||
|
db := newTestDB(t)
|
||||||
|
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
if err := state.Set(db, "vm/vm-web", st); err != nil {
|
||||||
|
t.Fatalf("state.Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b, _ := servedBy(t)
|
||||||
|
if served {
|
||||||
|
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||||
|
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("SetVM: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
r := &recorder{}
|
||||||
|
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||||
|
|
||||||
|
if len(r.calls) != 0 {
|
||||||
|
t.Errorf("served=%t: notifications = %+v, want none while the vm is %s", served, r.calls, st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckDHCPState_ToleranceIsLimitedToTheTransientVM(t *testing.T) {
|
||||||
|
db := newTestDB(t)
|
||||||
|
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
if err := state.Set(db, "vm/vm-web", state.Deleting); err != nil {
|
||||||
|
t.Fatalf("state.Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b, _ := servedBy(t)
|
||||||
|
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||||
|
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("SetVM: %v", err)
|
||||||
|
}
|
||||||
|
if err := b.SetVM(twoSubnet(), "vm-gone", []dhcpbackend.Reservation{
|
||||||
|
{Index: 0, MAC: "00:22:33:00:00:0B", IP: "10.0.5.11", DefaultRoute: true},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("SetVM: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r := &recorder{}
|
||||||
|
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||||
|
|
||||||
|
if len(r.calls) != 1 || !r.hasProblemContaining("stale dhcp reservation on the server: 00:22:33:00:00:0b") {
|
||||||
|
t.Errorf("notifications = %+v, want only the reservation of the vm that no longer exists", r.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckDHCPState_AReservationOfATransientVMOnAnExpectedMACIsStillCompared(t *testing.T) {
|
||||||
|
db := newTestDB(t)
|
||||||
|
seedSubnetWithVM(t, db, "vm-new", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
seedResource(t, db, "vm/", "vm-old", state.Deleting)
|
||||||
|
|
||||||
|
b, _ := servedBy(t)
|
||||||
|
if err := b.SetVM(twoSubnet(), "vm-old", []dhcpbackend.Reservation{
|
||||||
|
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.99", DefaultRoute: true},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("SetVM: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r := &recorder{}
|
||||||
|
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||||
|
|
||||||
|
if len(r.calls) != 1 || !r.hasProblemContaining("dhcp reservation diverges") {
|
||||||
|
t.Errorf("notifications = %+v, want the divergence on the mac a running vm expects", r.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCheckDHCPState_ReportsADivergingIP(t *testing.T) {
|
func TestCheckDHCPState_ReportsADivergingIP(t *testing.T) {
|
||||||
db := newTestDB(t)
|
db := newTestDB(t)
|
||||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,9 @@ L3VNI, avec une configuration BGP pilotée par l'agent ; documentation du projet
|
||||||
sans propriétaire est signalée comme **trou noir**, une route attendue absente comme manquante
|
sans propriétaire est signalée comme **trou noir**, une route attendue absente comme manquante
|
||||||
- Le watchdog interroge le serveur DHCP intégré et compare les réservations servies à celles que
|
- Le watchdog interroge le serveur DHCP intégré et compare les réservations servies à celles que
|
||||||
la base implique. Il nomme ce qui diverge — ordre perdu à la création, ordre perdu à la
|
la base implique. Il nomme ce qui diverge — ordre perdu à la création, ordre perdu à la
|
||||||
suppression, adresse ou route par défaut divergente — et reste en lecture seule
|
suppression, adresse ou route par défaut divergente — et reste en lecture seule. Une VM en
|
||||||
|
`creating`, `deleting` ou `error` n'est pas comparée : sa réservation peut être présente ou
|
||||||
|
absente sans alerte, le temps que la création ou la suppression aboutisse (#78)
|
||||||
- La socket de contrôle répond à `get-state` et à `probe`, ce dernier montrant sans effet de bord
|
- La socket de contrôle répond à `get-state` et à `probe`, ce dernier montrant sans effet de bord
|
||||||
ce qui serait envoyé à une MAC donnée : adresse, masque, routeur, DNS et routes
|
ce qui serait envoyé à une MAC donnée : adresse, masque, routeur, DNS et routes
|
||||||
- Documentation Sphinx du projet : concepts, architecture, déploiement et exploitation
|
- Documentation Sphinx du projet : concepts, architecture, déploiement et exploitation
|
||||||
|
|
|
||||||
|
|
@ -33,8 +33,8 @@ for rr in 10.255.255.1 10.255.255.2; do
|
||||||
check "${HOST} : session EVPN vers ${rr} établie" sh -c "vtysh -c 'show bgp neighbors ${rr}' | grep -q 'BGP state = Established'"
|
check "${HOST} : session EVPN vers ${rr} établie" sh -c "vtysh -c 'show bgp neighbors ${rr}' | grep -q 'BGP state = Established'"
|
||||||
check "${HOST} : loopback de ${rr} joignable" ping -c 2 -W 2 "${rr}"
|
check "${HOST} : loopback de ${rr} joignable" ping -c 2 -W 2 "${rr}"
|
||||||
done
|
done
|
||||||
check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min (hors réservation DHCP périmée, #78)" watchdog_quiet 300 'stale dhcp reservation'
|
check "${HOST} : aucune incohérence signalée par le watchdog depuis 5 min" watchdog_quiet 300
|
||||||
known_flaky "#78" "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation'
|
check "${HOST} : aucune réservation DHCP signalée périmée par le watchdog depuis 5 min" watchdog_quiet_about 300 'stale dhcp reservation'
|
||||||
check "${HOST} : image de VM compatible two prête" two_image
|
check "${HOST} : image de VM compatible two prête" two_image
|
||||||
check "${HOST} : clé SSH des VM prête" vm_key
|
check "${HOST} : clé SSH des VM prête" vm_key
|
||||||
NODE
|
NODE
|
||||||
|
|
|
||||||
|
|
@ -57,8 +57,8 @@ check "après redémarrage : vrf-t61 exporte toujours t61-a 10.61.1.10/32" vrf_r
|
||||||
check "après redémarrage : vrf-t61 apprend toujours 10.61.2.10/32 via hv2" vrf_route_via vp-t61 10.61.2.10 192.168.14.12
|
check "après redémarrage : vrf-t61 apprend toujours 10.61.2.10/32 via hv2" vrf_route_via vp-t61 10.61.2.10 192.168.14.12
|
||||||
check "après redémarrage : bail de t61-a renouvelé (DHCP servi)" lease_renewed vp-t61 10.61.1.10
|
check "après redémarrage : bail de t61-a renouvelé (DHCP servi)" lease_renewed vp-t61 10.61.1.10
|
||||||
check "après redémarrage : métadonnées de t61-a servies (instance-id t61-a)" vm_grep vp-t61 10.61.1.10 'curl -fsS http://169.254.169.254/meta-data' 't61-a'
|
check "après redémarrage : métadonnées de t61-a servies (instance-id t61-a)" vm_grep vp-t61 10.61.1.10 'curl -fsS http://169.254.169.254/meta-data' 't61-a'
|
||||||
check "après redémarrage : aucune incohérence signalée par le watchdog (hors réservation DHCP périmée, #78)" watchdog_quiet "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
check "après redémarrage : aucune incohérence signalée par le watchdog" watchdog_quiet "$(( $(date +%s) - START ))"
|
||||||
known_flaky "#78" "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
check "après redémarrage : aucune réservation DHCP signalée périmée par le watchdog" watchdog_quiet_about "$(( $(date +%s) - START ))" 'stale dhcp reservation'
|
||||||
vm_ssh vp-t61 10.61.1.10 'pkill -x ping' || true
|
vm_ssh vp-t61 10.61.1.10 'pkill -x ping' || true
|
||||||
sleep 1
|
sleep 1
|
||||||
stats=$(vm_ssh vp-t61 10.61.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n+0, max+0}' /tmp/t61.log")
|
stats=$(vm_ssh vp-t61 10.61.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n+0, max+0}' /tmp/t61.log")
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue