f-46: watchdog: reconcile the served dhcp state against the database #46
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
a220cce77e
commit
7d4d856c95
8 changed files with 584 additions and 39 deletions
|
|
@ -2,6 +2,7 @@ package dhcpbackend
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/dhcp"
|
||||
"git.g3e.fr/syonad/two/pkg/systemd"
|
||||
|
|
@ -125,3 +126,7 @@ func (b Dnsmasq) DelVM(s Subnet, vmName string, _ []Reservation) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b Dnsmasq) ConfigPath(s Subnet) string {
|
||||
return filepath.Join(b.confDir(), s.Instance()+".conf")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,3 +137,11 @@ func (b Two) DelVM(s Subnet, vmName string, res []Reservation) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b Two) StatePath(s Subnet) string {
|
||||
return b.statePath(s)
|
||||
}
|
||||
|
||||
func (b Two) State(s Subnet) (dhcpapi.State, string, error) {
|
||||
return b.client(s).GetState()
|
||||
}
|
||||
|
|
|
|||
189
internal/watchdog/check_dhcp.go
Normal file
189
internal/watchdog/check_dhcp.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
dhcpapi "git.g3e.fr/syonad/two/internal/api/dhcp"
|
||||
"git.g3e.fr/syonad/two/internal/dhcp"
|
||||
"git.g3e.fr/syonad/two/internal/dhcpbackend"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
"git.g3e.fr/syonad/two/pkg/db/kv"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
type configFileReporter interface {
|
||||
ConfigPath(s dhcpbackend.Subnet) string
|
||||
}
|
||||
|
||||
type stateReporter interface {
|
||||
State(s dhcpbackend.Subnet) (dhcpapi.State, string, error)
|
||||
}
|
||||
|
||||
func expectedHosts(db *badger.DB, subnetName string) ([]dhcpapi.Host, error) {
|
||||
pairs, err := kv.ListByPrefix(db, prefixVM)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing vms: %w", err)
|
||||
}
|
||||
|
||||
hosts := make([]dhcpapi.Host, 0)
|
||||
for _, vmName := range resourceNames(pairs, prefixVM) {
|
||||
st, err := state.Get(db, prefixVM+vmName)
|
||||
if err != nil || st != state.Running {
|
||||
continue
|
||||
}
|
||||
|
||||
vmHosts, err := expectedVMHosts(db, vmName, subnetName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hosts = append(hosts, vmHosts...)
|
||||
}
|
||||
|
||||
dhcpapi.SortHosts(hosts)
|
||||
return hosts, nil
|
||||
}
|
||||
|
||||
func expectedVMHosts(db *badger.DB, vmName, subnetName string) ([]dhcpapi.Host, error) {
|
||||
prefix := prefixVM + vmName + "/nic/"
|
||||
entries, err := kv.ListByPrefix(db, prefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing nics of vm %s: %w", vmName, err)
|
||||
}
|
||||
|
||||
indexes := make([]int, 0)
|
||||
for key := range entries {
|
||||
parts := strings.Split(strings.TrimPrefix(key, prefix), "/")
|
||||
if len(parts) != 2 || parts[1] != "subnet" {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid nic index %q for vm %s", parts[0], vmName)
|
||||
}
|
||||
indexes = append(indexes, idx)
|
||||
}
|
||||
sort.Ints(indexes)
|
||||
|
||||
hosts := make([]dhcpapi.Host, 0, len(indexes))
|
||||
for _, idx := range indexes {
|
||||
nic := fmt.Sprintf("%s%d/", prefix, idx)
|
||||
if entries[nic+"subnet"] != subnetName {
|
||||
continue
|
||||
}
|
||||
|
||||
ip := entries[nic+"ip"]
|
||||
if ip == "" {
|
||||
return nil, fmt.Errorf("nic %d of vm %s has no ip", idx, vmName)
|
||||
}
|
||||
mac, err := dhcp.GetMACForIP(db, subnetName, ip)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get mac for ip %s: %w", ip, err)
|
||||
}
|
||||
|
||||
hosts = append(hosts, dhcpapi.Host{
|
||||
MAC: mac,
|
||||
IP: ip,
|
||||
VM: vmName,
|
||||
DefaultRoute: entries[nic+"primary"] == "true",
|
||||
})
|
||||
}
|
||||
return hosts, nil
|
||||
}
|
||||
|
||||
func checkDHCP(db *badger.DB, name string, s dhcpbackend.Subnet, backend dhcpbackend.Backend, u unitChecker, n notify.Notifier) {
|
||||
if backend == nil {
|
||||
return
|
||||
}
|
||||
if r, ok := backend.(configFileReporter); ok {
|
||||
checkDHCPConfigFile(name, r.ConfigPath(s), n)
|
||||
}
|
||||
if r, ok := backend.(stateReporter); ok {
|
||||
checkDHCPState(db, name, s, r, n)
|
||||
}
|
||||
checkUnit(kindSubnet, name, backend.Unit(s), u, n)
|
||||
}
|
||||
|
||||
func checkDHCPConfigFile(name, path string, n notify.Notifier) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dnsmasq config missing (%s): %v", path, err))
|
||||
}
|
||||
}
|
||||
|
||||
func checkDHCPState(db *badger.DB, name string, s dhcpbackend.Subnet, r stateReporter, n notify.Notifier) {
|
||||
served, _, err := r.State(s)
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp server unreachable: %v", err))
|
||||
return
|
||||
}
|
||||
if served.Subnet == nil {
|
||||
n.Notify(kindSubnet, name, "dhcp server has no subnet configuration: it serves nothing")
|
||||
}
|
||||
|
||||
expected, err := expectedHosts(db, name)
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations unreadable in database: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
expectedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: expected})
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("expected dhcp reservations inconsistent in database: %v", err))
|
||||
return
|
||||
}
|
||||
servedDigest, err := dhcpapi.Digest(dhcpapi.State{Hosts: served.Hosts})
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp reservations reported by the server are inconsistent: %v", err))
|
||||
return
|
||||
}
|
||||
if expectedDigest == servedDigest {
|
||||
return
|
||||
}
|
||||
|
||||
for _, gap := range reservationGaps(expected, served.Hosts) {
|
||||
n.Notify(kindSubnet, name, gap)
|
||||
}
|
||||
}
|
||||
|
||||
func reservationGaps(expected, served []dhcpapi.Host) []string {
|
||||
index := func(hosts []dhcpapi.Host) map[string]dhcpapi.Host {
|
||||
byMAC := make(map[string]dhcpapi.Host, len(hosts))
|
||||
for _, h := range hosts {
|
||||
if c, err := dhcpapi.CanonicalHost(h); err == nil {
|
||||
byMAC[c.MAC] = c
|
||||
} else {
|
||||
byMAC[h.MAC] = h
|
||||
}
|
||||
}
|
||||
return byMAC
|
||||
}
|
||||
|
||||
want, got := index(expected), index(served)
|
||||
|
||||
gaps := make([]string, 0)
|
||||
for mac, h := range want {
|
||||
s, ok := got[mac]
|
||||
if !ok {
|
||||
gaps = append(gaps, fmt.Sprintf("dhcp reservation missing on the server: %s → %s (vm %s)", mac, h.IP, h.VM))
|
||||
continue
|
||||
}
|
||||
if s.IP != h.IP {
|
||||
gaps = append(gaps, fmt.Sprintf("dhcp reservation diverges for %s: server serves %s, database says %s (vm %s)", mac, s.IP, h.IP, h.VM))
|
||||
}
|
||||
if s.DefaultRoute != h.DefaultRoute {
|
||||
gaps = append(gaps, fmt.Sprintf("dhcp default route diverges for %s: server says %t, database says %t (vm %s)", mac, s.DefaultRoute, h.DefaultRoute, h.VM))
|
||||
}
|
||||
}
|
||||
for mac, h := range got {
|
||||
if _, ok := want[mac]; !ok {
|
||||
gaps = append(gaps, fmt.Sprintf("stale dhcp reservation on the server: %s → %s (vm %s)", mac, h.IP, h.VM))
|
||||
}
|
||||
}
|
||||
sort.Strings(gaps)
|
||||
return gaps
|
||||
}
|
||||
348
internal/watchdog/check_dhcp_test.go
Normal file
348
internal/watchdog/check_dhcp_test.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
dhcpapi "git.g3e.fr/syonad/two/internal/api/dhcp"
|
||||
"git.g3e.fr/syonad/two/internal/dhcpbackend"
|
||||
"git.g3e.fr/syonad/two/internal/dhcpd"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
testSubnetName = "sn-000001"
|
||||
testVPC = "vp-admin"
|
||||
testBridge = "br-000001"
|
||||
)
|
||||
|
||||
func twoSubnet() dhcpbackend.Subnet {
|
||||
return dhcpbackend.Subnet{Name: testSubnetName, VPC: testVPC, Bridge: testBridge}
|
||||
}
|
||||
|
||||
func shortTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp("", "dhcpd")
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirTemp: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
return dir
|
||||
}
|
||||
|
||||
func servedBy(t *testing.T) (dhcpbackend.Two, *dhcpd.Store) {
|
||||
t.Helper()
|
||||
|
||||
b := dhcpbackend.Two{RunDir: shortTempDir(t)}
|
||||
s := twoSubnet()
|
||||
|
||||
store := dhcpd.NewStore(dhcpapi.StatePath(b.RunDir, s.Instance()))
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
server, err := dhcpapi.Listen(store, dhcpapi.SocketPath(b.RunDir, s.Instance()), slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
if err != nil {
|
||||
t.Fatalf("Listen: %v", err)
|
||||
}
|
||||
go server.Serve()
|
||||
t.Cleanup(func() { server.Close() })
|
||||
|
||||
_, network, err := net.ParseCIDR("10.0.5.0/24")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCIDR: %v", err)
|
||||
}
|
||||
if err := store.SetSubnet(dhcpd.SubnetConfig{Network: network, InterfaceIP: net.ParseIP("10.0.5.1")}); err != nil {
|
||||
t.Fatalf("SetSubnet: %v", err)
|
||||
}
|
||||
return b, store
|
||||
}
|
||||
|
||||
func seedSubnetWithVM(t *testing.T, db *badger.DB, vmName, ip, mac string, primary bool) {
|
||||
t.Helper()
|
||||
|
||||
seedKV(t, db, "subnet/"+testSubnetName+"/vpc", testVPC)
|
||||
seedKV(t, db, "subnet/"+testSubnetName+"/dhcp/"+ip, mac)
|
||||
|
||||
if err := state.Set(db, "vm/"+vmName, state.Running); err != nil {
|
||||
t.Fatalf("state.Set: %v", err)
|
||||
}
|
||||
seedKV(t, db, "vm/"+vmName+"/nic/0/subnet", testSubnetName)
|
||||
seedKV(t, db, "vm/"+vmName+"/nic/0/ip", ip)
|
||||
seedKV(t, db, "vm/"+vmName+"/nic/0/primary", fmt.Sprintf("%t", primary))
|
||||
}
|
||||
|
||||
func TestExpectedHosts_ReadsRunningVMsOnThatSubnet(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
if len(hosts) != 1 {
|
||||
t.Fatalf("hosts = %+v, want one", hosts)
|
||||
}
|
||||
if hosts[0].MAC != "00:22:33:00:00:0A" || hosts[0].IP != "10.0.5.10" {
|
||||
t.Errorf("host = %+v, want the mac derived from the address plan", hosts[0])
|
||||
}
|
||||
if hosts[0].VM != "vm-web" || !hosts[0].DefaultRoute {
|
||||
t.Errorf("host = %+v, want vm-web carrying the default route", hosts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpectedHosts_IgnoresVMsThatAreNotRunning(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
if err := state.Set(db, "vm/vm-web", state.Deleting); err != nil {
|
||||
t.Fatalf("state.Set: %v", err)
|
||||
}
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
if len(hosts) != 0 {
|
||||
t.Errorf("hosts = %+v, want none: a vm being deleted is not expected to be served", hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpectedHosts_IgnoresInterfacesOnOtherSubnets(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
seedKV(t, db, "vm/vm-web/nic/1/subnet", "sn-000002")
|
||||
seedKV(t, db, "vm/vm-web/nic/1/ip", "10.0.6.10")
|
||||
seedKV(t, db, "vm/vm-web/nic/1/primary", "false")
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
if len(hosts) != 1 {
|
||||
t.Errorf("hosts = %+v, want only the interface on this subnet", hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpectedHosts_SortsByMAC(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-b", "10.0.5.11", "00:22:33:00:00:0B", true)
|
||||
seedSubnetWithVM(t, db, "vm-a", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
hosts, err := expectedHosts(db, testSubnetName)
|
||||
if err != nil {
|
||||
t.Fatalf("expectedHosts: %v", err)
|
||||
}
|
||||
if len(hosts) != 2 || hosts[0].MAC != "00:22:33:00:00:0A" {
|
||||
t.Errorf("hosts = %+v, want sorted by mac", hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpectedHosts_ReportsAnIPWithoutAMAC(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
seedKV(t, db, "vm/vm-web/nic/0/ip", "10.0.5.99")
|
||||
|
||||
if _, err := expectedHosts(db, testSubnetName); err == nil {
|
||||
t.Fatal("an ip absent from the address plan must be reported, not skipped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_MACCaseAloneIsNotADivergence(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("dhcp.Entries stores uppercase macs and the server normalizes to lowercase: that alone must not read as drift, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ADivergenceIsReportedOnceNotAsBothMissingAndStale(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.99", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 1 {
|
||||
t.Errorf("notifications = %+v, want a single diverging-reservation report", r.calls)
|
||||
}
|
||||
if r.hasProblemContaining("missing on the server") || r.hasProblemContaining("stale dhcp") {
|
||||
t.Errorf("without mac normalization the same host reads as both missing and stale: %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_SilentWhenServerMatchesDatabase(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("notifications = %+v, want none when the server agrees with the database", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsAReservationTheServerNeverGot(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if !r.hasProblemContaining("dhcp reservation missing on the server") {
|
||||
t.Errorf("a lost set-host order must be reported, got %+v", r.calls)
|
||||
}
|
||||
if !r.hasProblemContaining("vm vm-web") {
|
||||
t.Errorf("the report must name the vm, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsAStaleReservation(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedKV(t, db, "subnet/"+testSubnetName+"/vpc", testVPC)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-gone", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if !r.hasProblemContaining("stale dhcp reservation on the server") {
|
||||
t.Errorf("a lost del-host order must be reported, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsADivergingIP(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.99", DefaultRoute: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if !r.hasProblemContaining("dhcp reservation diverges") {
|
||||
t.Errorf("a mac served with the wrong address must be reported, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsADivergingDefaultRoute(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedSubnetWithVM(t, db, "vm-web", "10.0.5.10", "00:22:33:00:00:0A", true)
|
||||
|
||||
b, _ := servedBy(t)
|
||||
if err := b.SetVM(twoSubnet(), "vm-web", []dhcpbackend.Reservation{
|
||||
{Index: 0, MAC: "00:22:33:00:00:0A", IP: "10.0.5.10", DefaultRoute: false},
|
||||
}); err != nil {
|
||||
t.Fatalf("SetVM: %v", err)
|
||||
}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if !r.hasProblemContaining("dhcp default route diverges") {
|
||||
t.Errorf("a wrong default route would break multi-subnet routing, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsAnUnreachableServer(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedKV(t, db, "subnet/"+testSubnetName+"/vpc", testVPC)
|
||||
|
||||
b := dhcpbackend.Two{RunDir: shortTempDir(t)}
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, twoSubnet(), b, r)
|
||||
|
||||
if !r.hasProblemContaining("dhcp server unreachable") {
|
||||
t.Errorf("a dead server must be reported, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPState_ReportsAServerWithNoSubnetConfiguration(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedKV(t, db, "subnet/"+testSubnetName+"/vpc", testVPC)
|
||||
|
||||
b := dhcpbackend.Two{RunDir: shortTempDir(t)}
|
||||
s := twoSubnet()
|
||||
store := dhcpd.NewStore(dhcpapi.StatePath(b.RunDir, s.Instance()))
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
server, err := dhcpapi.Listen(store, dhcpapi.SocketPath(b.RunDir, s.Instance()), slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
if err != nil {
|
||||
t.Fatalf("Listen: %v", err)
|
||||
}
|
||||
go server.Serve()
|
||||
t.Cleanup(func() { server.Close() })
|
||||
|
||||
r := &recorder{}
|
||||
checkDHCPState(db, testSubnetName, s, b, r)
|
||||
|
||||
if !r.hasProblemContaining("no subnet configuration") {
|
||||
t.Errorf("a server that was never configured serves nothing, got %+v", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCPConfigFile_ReportsAMissingDnsmasqConfig(t *testing.T) {
|
||||
r := &recorder{}
|
||||
checkDHCPConfigFile(testSubnetName, filepath.Join(t.TempDir(), "absent.conf"), r)
|
||||
|
||||
if !r.hasProblemContaining("dnsmasq config missing") {
|
||||
t.Errorf("notifications = %+v, want the missing config reported", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDHCP_WithoutABackendChecksNothing(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
r := &recorder{}
|
||||
|
||||
checkDHCP(db, testSubnetName, twoSubnet(), nil, nil, r)
|
||||
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("notifications = %+v, want none: the caller already reported the unusable backend", r.calls)
|
||||
}
|
||||
}
|
||||
|
|
@ -2,12 +2,11 @@ package watchdog
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/dhcp"
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/dhcpbackend"
|
||||
"git.g3e.fr/syonad/two/internal/netns"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
|
|
@ -30,16 +29,14 @@ func subnetIfaceNames(subnetName string) (hostVeth, nsVeth, bridge string, err e
|
|||
return "v-" + id + "-e", "v-" + id + "-i", "br-" + id, nil
|
||||
}
|
||||
|
||||
func dnsmasqName(vpc, bridge string) string {
|
||||
return vpc + "_" + bridge
|
||||
}
|
||||
|
||||
func CheckSubnets(db *badger.DB, u unitChecker, n notify.Notifier) error {
|
||||
func CheckSubnets(db *badger.DB, cfg *configuration.Config, u unitChecker, n notify.Notifier) error {
|
||||
pairs, err := kv.ListByPrefix(db, prefixSubnet)
|
||||
if err != nil {
|
||||
return fmt.Errorf("watchdog: listing subnets: %w", err)
|
||||
}
|
||||
|
||||
backend, backendErr := dhcpbackend.New(cfg)
|
||||
|
||||
for _, name := range resourceNames(pairs, prefixSubnet) {
|
||||
st, err := state.Get(db, prefixSubnet+name)
|
||||
if err != nil {
|
||||
|
|
@ -49,12 +46,15 @@ func CheckSubnets(db *badger.DB, u unitChecker, n notify.Notifier) error {
|
|||
if st != state.Running {
|
||||
continue
|
||||
}
|
||||
checkSubnet(db, name, u, n)
|
||||
checkSubnet(db, name, backend, u, n)
|
||||
if backendErr != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dhcp checks skipped, backend unusable: %v", backendErr))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkSubnet(db *badger.DB, name string, u unitChecker, n notify.Notifier) {
|
||||
func checkSubnet(db *badger.DB, name string, backend dhcpbackend.Backend, u unitChecker, n notify.Notifier) {
|
||||
hostVeth, nsVeth, bridge, err := subnetIfaceNames(name)
|
||||
if err != nil {
|
||||
n.Notify(kindSubnet, name, err.Error())
|
||||
|
|
@ -90,13 +90,7 @@ func checkSubnet(db *badger.DB, name string, u unitChecker, n notify.Notifier) {
|
|||
|
||||
checkSubnetNetns(name, vpc, nsVeth, bridge, n)
|
||||
|
||||
dnsName := dnsmasqName(vpc, bridge)
|
||||
conf := filepath.Join(dhcp.DefaultConfDir, dnsName+".conf")
|
||||
if _, err := os.Stat(conf); err != nil {
|
||||
n.Notify(kindSubnet, name, fmt.Sprintf("dnsmasq config missing (%s): %v", conf, err))
|
||||
}
|
||||
|
||||
checkUnit(kindSubnet, name, "dnsmasq@"+dnsName+".service", u, n)
|
||||
checkDHCP(db, name, dhcpbackend.Subnet{Name: name, VPC: vpc, Bridge: bridge}, backend, u, n)
|
||||
}
|
||||
|
||||
func checkVxlanIface(db *badger.DB, name string, n notify.Notifier) {
|
||||
|
|
|
|||
|
|
@ -39,17 +39,11 @@ func TestSubnetIfaceNames_TiretFinal(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqName(t *testing.T) {
|
||||
if got := dnsmasqName("vp-admin", "br-000000"); got != "vp-admin_br-000000" {
|
||||
t.Errorf("dnsmasqName = %q, attendu vp-admin_br-000000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSubnets_BaseVide(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
if len(r.calls) != 0 {
|
||||
|
|
@ -64,7 +58,7 @@ func TestCheckSubnets_IgnoreLesEtatsNonRunning(t *testing.T) {
|
|||
}
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
if len(r.calls) != 0 {
|
||||
|
|
@ -77,7 +71,7 @@ func TestCheckSubnets_VPCManquantEnBase(t *testing.T) {
|
|||
seedResource(t, db, prefixSubnet, "br-000042", state.Running)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -96,7 +90,7 @@ func TestCheckSubnets_ModeManquantEnBase(t *testing.T) {
|
|||
seedKV(t, db, prefixSubnet+"br-000042/vpc", "vp-admin")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +104,7 @@ func TestCheckSubnets_ModeInconnu(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", "macvlan")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +118,7 @@ func TestCheckSubnets_ModeBridgeNeVerifiePasDeVxlan(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +137,7 @@ func TestCheckSubnets_ModeVxlanSansVxlanID(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", modeVxlan)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -158,7 +152,7 @@ func TestCheckSubnets_ModeVxlanVxlanIDInvalide(t *testing.T) {
|
|||
seedKV(t, db, prefixSubnet+"br-000042/vxlan_id", "pas-un-nombre")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -173,7 +167,7 @@ func TestCheckSubnets_ModeVxlanVerifieLInterfaceVxlan(t *testing.T) {
|
|||
seedKV(t, db, prefixSubnet+"br-000042/vxlan_id", "42")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -187,7 +181,7 @@ func TestCheckSubnets_ConfigDnsmasqAbsente(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -205,7 +199,7 @@ func TestCheckSubnets_UnitDnsmasqInterrogee(t *testing.T) {
|
|||
u := newFakeUnits().active("dnsmasq@vp-admin_br-000042.service")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, u, r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), u, r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -223,7 +217,7 @@ func TestCheckSubnets_UnitDnsmasqInactive(t *testing.T) {
|
|||
u := newFakeUnits().inactive("dnsmasq@vp-admin_br-000042.service", "failed")
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, u, r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), u, r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -238,7 +232,7 @@ func TestCheckSubnets_UnitIllisible(t *testing.T) {
|
|||
u := newFakeUnits().failing("dnsmasq@vp-admin_br-000042.service", errors.New("dbus indisponible"))
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, u, r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), u, r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -252,7 +246,7 @@ func TestCheckSubnets_SansUnitCheckerPasDeVerificationDUnit(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, nil, r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), nil, r); err != nil {
|
||||
t.Fatalf("erreur inattendue: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -267,7 +261,7 @@ func TestCheckSubnets_EtatCorrompuNInterrompPasLaBoucle(t *testing.T) {
|
|||
seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
|
||||
r := &recorder{}
|
||||
|
||||
if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
|
||||
if err := CheckSubnets(db, dnsmasqConfig(), newFakeUnits(), r); err != nil {
|
||||
t.Fatalf("un état corrompu ne doit pas faire échouer CheckSubnets: %v", err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
"git.g3e.fr/syonad/two/pkg/db/kv"
|
||||
|
|
@ -108,3 +109,9 @@ func seedResource(t *testing.T, db *badger.DB, prefix, name string, s state.Stat
|
|||
t.Fatalf("seedResource %s%s: %v", prefix, name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func dnsmasqConfig() *configuration.Config {
|
||||
cfg := &configuration.Config{}
|
||||
cfg.DHCP.Backend = configuration.BackendDnsmasq
|
||||
return cfg
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ func (w *Watchdog) tick() {
|
|||
if err := CheckVPCs(w.db, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: vpc check failed", "err", err)
|
||||
}
|
||||
if err := CheckSubnets(w.db, u, w.notifier); err != nil {
|
||||
if err := CheckSubnets(w.db, w.cfg, u, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: subnet check failed", "err", err)
|
||||
}
|
||||
if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue