diff --git a/docs/exploitation/observabilite.rst b/docs/exploitation/observabilite.rst index b8f7a4c..d0b97e1 100644 --- a/docs/exploitation/observabilite.rst +++ b/docs/exploitation/observabilite.rst @@ -38,6 +38,20 @@ toujours sur le système, et **notifie les écarts sans jamais réparer**. Il contrôle notamment l'existence des network namespaces, des liens réseau des subnets, des taps de VM et la réponse des units systemd associées. +Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de contrôle : + +* **Sessions BGP** : une notification par route reflector dont la session n'est pas + ``Established`` (``show bgp neighbors ``). +* **VPC routés** (``l3vni``, état ``running``) : présence et état des interfaces ``vrf-``, + ``brl3-``, ``vxl3-``, ``l3--o`` et, dans le netns, ``l3--i`` ; L3VNI de type + ``L3``, ``Up``, avec la MAC de routeur attendue (``show evpn vni ``). +* **Réconciliation des routes exportées** dans la VRF : chaque ``/24`` de subnet ``vxlan`` et chaque + ``/32`` d'interface de VM ``running`` doit être exporté ; à l'inverse, une route exportée dont + le propriétaire n'existe plus — ni ``running``, ni en ``creating``, ``deleting`` ou ``error`` — + est signalée comme un **trou noir** : l'annonce attire du trafic que plus rien ne reçoit. + +Un VPC routé alors que la section ``bgp`` a disparu de la configuration est signalé lui aussi. + .. note:: Un écart persistant est signalé **à chaque tick**, sans déduplication. Le volume de diff --git a/internal/frr/status.go b/internal/frr/status.go new file mode 100644 index 0000000..031ec39 --- /dev/null +++ b/internal/frr/status.go @@ -0,0 +1,49 @@ +package frr + +import ( + "fmt" + "net" + "strings" +) + +type VNIState struct { + Type string + State string + RouterMAC string +} + +func (c *Client) NeighborEstablished(peer net.IP) (bool, error) { + out, err := c.exec("show bgp neighbors " + peer.String()) + if err != nil { + return false, err + } + return strings.Contains(out, "BGP state = Established"), nil +} + +func (c *Client) VNI(vni uint32) (VNIState, error) { + out, err := c.exec(fmt.Sprintf("show evpn vni %d", vni)) + if err != nil { + return VNIState{}, err + } + return parseVNI(out), nil +} + +func parseVNI(out string) VNIState { + var s VNIState + for _, l := range strings.Split(out, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(l), ": ") + if !ok { + continue + } + value = strings.TrimSpace(value) + switch strings.ToLower(key) { + case "type": + s.Type = value + case "state": + s.State = value + case "router mac": + s.RouterMAC = strings.ToLower(value) + } + } + return s +} diff --git a/internal/frr/status_test.go b/internal/frr/status_test.go new file mode 100644 index 0000000..9b65f7d --- /dev/null +++ b/internal/frr/status_test.go @@ -0,0 +1,50 @@ +package frr + +import ( + "net" + "strings" + "testing" +) + +const vniOutput = `VNI: 2700 + Type: L3 + Tenant VRF: vrf-s7 + Vxlan-Intf: vxl3-s7 + SVI-If: brl3-s7 + State: Up + VNI Filter: none + System MAC: 02:02:c0:a8:0e:aa + Router MAC: 02:02:C0:A8:0E:0B + L2 VNIs: +` + +func TestVNIReadsTypeStateAndRouterMAC(t *testing.T) { + f := &fakeVtysh{fail: map[string]string{"show evpn vni 2700": vniOutput}} + s, err := f.client().VNI(2700) + if err != nil { + t.Fatal(err) + } + want := VNIState{Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"} + if s != want { + t.Errorf("VNI = %+v, attendu %+v", s, want) + } +} + +func TestVNIReportsAnUnknownVNI(t *testing.T) { + f := &fakeVtysh{fail: map[string]string{"show evpn vni 9": "% VNI 9 does not exist\n"}} + if _, err := f.client().VNI(9); err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Errorf("erreur %v", err) + } +} + +func TestNeighborEstablished(t *testing.T) { + rr := net.ParseIP("10.255.255.1") + f := &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1, remote AS 64600, local AS 64600, internal link\n BGP version 4, remote router ID 10.255.255.1\n BGP state = Established, up for 00:10:02\n"}} + if up, err := f.client().NeighborEstablished(rr); err != nil || !up { + t.Errorf("établie : %v, %v", up, err) + } + f = &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1\n BGP state = Active\n"}} + if up, err := f.client().NeighborEstablished(rr); err != nil || up { + t.Errorf("active : %v, %v", up, err) + } +} diff --git a/internal/netif/route_linux.go b/internal/netif/route_linux.go index f68d191..fc444d1 100644 --- a/internal/netif/route_linux.go +++ b/internal/netif/route_linux.go @@ -19,3 +19,24 @@ func RouteAdd(iface string, subnet *net.IPNet) error { Scope: netlink.SCOPE_LINK, }) } + +func TableRoutesVia(table int, iface string) ([]*net.IPNet, error) { + link, err := netlink.LinkByName(iface) + if err != nil { + return nil, err + } + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{ + Table: table, + LinkIndex: link.Attrs().Index, + }, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil { + return nil, err + } + dsts := make([]*net.IPNet, 0, len(routes)) + for _, r := range routes { + if r.Dst != nil && r.Gw != nil { + dsts = append(dsts, r.Dst) + } + } + return dsts, nil +} diff --git a/internal/netif/route_other.go b/internal/netif/route_other.go index 685c5bf..71daec7 100644 --- a/internal/netif/route_other.go +++ b/internal/netif/route_other.go @@ -2,8 +2,15 @@ package netif -import "net" +import ( + "errors" + "net" +) func RouteAdd(_ string, _ *net.IPNet) error { return nil } + +func TableRoutesVia(_ int, _ string) ([]*net.IPNet, error) { + return nil, errors.New("route listing is only implemented on linux") +} diff --git a/internal/vpc/l3.go b/internal/vpc/l3.go index 225d650..79b2088 100644 --- a/internal/vpc/l3.go +++ b/internal/vpc/l3.go @@ -137,3 +137,33 @@ func deleteL3(h host, cfg *configuration.Config, v l3VPC) error { } return nil } + +type Routing struct { + VNI uint32 + Supernet *net.IPNet + VRF string + Bridge string + Vxlan string + VethHost string + VethNetns string +} + +func LoadRouting(db *badger.DB, vpcName string) (Routing, bool, error) { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return Routing{}, ok, err + } + return Routing{ + VNI: v.vni, + Supernet: v.supernet, + VRF: v.vrf(), + Bridge: v.bridge(), + Vxlan: v.vxlan(), + VethHost: v.vethHost(), + VethNetns: v.vethNetns(), + }, true, nil +} + +func RouterMAC(vtep net.IP) (net.HardwareAddr, error) { + return routerMAC(vtep) +} diff --git a/internal/watchdog/check_l3.go b/internal/watchdog/check_l3.go new file mode 100644 index 0000000..537cf29 --- /dev/null +++ b/internal/watchdog/check_l3.go @@ -0,0 +1,232 @@ +package watchdog + +import ( + "fmt" + "net" + "sort" + "strings" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/netns" + "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" + "git.g3e.fr/syonad/two/internal/watchdog/notify" + "git.g3e.fr/syonad/two/pkg/db/kv" + + "github.com/dgraph-io/badger/v4" +) + +const kindBGP = "bgp" + +type l3Probe interface { + VTEPAddress(iface string) (net.IP, error) + NeighborEstablished(peer net.IP) (bool, error) + VNI(vni uint32) (frr.VNIState, error) + ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) + LinkProblem(ns, iface string) string +} + +type systemProbe struct { + frr *frr.Client +} + +func (systemProbe) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) } + +func (p systemProbe) NeighborEstablished(peer net.IP) (bool, error) { + return p.frr.NeighborEstablished(peer) +} + +func (p systemProbe) VNI(vni uint32) (frr.VNIState, error) { return p.frr.VNI(vni) } + +func (systemProbe) ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) { + return netif.TableRoutesVia(int(table), iface) +} + +func (systemProbe) LinkProblem(ns, iface string) string { + if ns == "" { + return linkProblem(iface) + } + var p string + if err := netns.Call(ns, func() error { + p = linkProblem(iface) + return nil + }); err != nil { + return fmt.Sprintf("cannot enter netns %s: %v", ns, err) + } + if p != "" { + return p + " (in netns)" + } + return "" +} + +func CheckBGP(cfg *configuration.Config, p l3Probe, n notify.Notifier) { + if !cfg.BGP.Enabled() { + return + } + reflectors, err := cfg.BGP.Reflectors() + if err != nil { + n.Notify(kindBGP, "config", err.Error()) + return + } + for _, rr := range reflectors { + up, err := p.NeighborEstablished(rr) + switch { + case err != nil: + n.Notify(kindBGP, rr.String(), fmt.Sprintf("session unreadable: %v", err)) + case !up: + n.Notify(kindBGP, rr.String(), "session to the route reflector is not established") + } + } +} + +func CheckL3VPCs(db *badger.DB, cfg *configuration.Config, p l3Probe, n notify.Notifier) error { + pairs, err := kv.ListByPrefix(db, prefixVPC) + if err != nil { + return fmt.Errorf("watchdog: listing vpcs: %w", err) + } + var rmac string + for _, name := range resourceNames(pairs, prefixVPC) { + if st, err := state.Get(db, prefixVPC+name); err != nil || st != state.Running { + continue + } + r, ok, err := vpc.LoadRouting(db, name) + if err != nil { + n.Notify(kindVPC, name, err.Error()) + continue + } + if !ok { + continue + } + if !cfg.BGP.Enabled() { + n.Notify(kindVPC, name, "vpc has an l3vni but the agent configuration has no bgp section") + continue + } + if rmac == "" { + rmac = expectedRouterMAC(cfg, p, n) + } + checkL3VPC(db, name, r, rmac, p, n) + } + return nil +} + +func expectedRouterMAC(cfg *configuration.Config, p l3Probe, n notify.Notifier) string { + vtep, err := p.VTEPAddress(cfg.DefaultInterface) + if err != nil { + n.Notify(kindBGP, "vtep", fmt.Sprintf("vtep address unreadable on %s: %v", cfg.DefaultInterface, err)) + return "unknown" + } + mac, err := vpc.RouterMAC(vtep) + if err != nil { + n.Notify(kindBGP, "vtep", err.Error()) + return "unknown" + } + return mac.String() +} + +func checkL3VPC(db *badger.DB, name string, r vpc.Routing, rmac string, p l3Probe, n notify.Notifier) { + for _, iface := range []string{r.VRF, r.Bridge, r.Vxlan, r.VethHost} { + if problem := p.LinkProblem("", iface); problem != "" { + n.Notify(kindVPC, name, problem) + } + } + if problem := p.LinkProblem(name, r.VethNetns); problem != "" { + n.Notify(kindVPC, name, problem) + } + + s, err := p.VNI(r.VNI) + switch { + case err != nil: + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d unreadable in frr: %v", r.VNI, err)) + case s.Type != "L3" || s.State != "Up": + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d is type %q state %q in frr, expected L3 Up", r.VNI, s.Type, s.State)) + case rmac != "unknown" && s.RouterMAC != rmac: + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d has router mac %q in frr, expected %s", r.VNI, s.RouterMAC, rmac)) + } + + actual, err := p.ExportedRoutes(r.VNI, r.VethHost) + if err != nil { + n.Notify(kindVPC, name, fmt.Sprintf("routes of %s unreadable: %v", r.VRF, err)) + return + } + expected, tolerated, err := expectedExports(db, name) + if err != nil { + n.Notify(kindVPC, name, err.Error()) + return + } + exported := make(map[string]bool, len(actual)) + for _, dst := range actual { + exported[dst.String()] = true + } + for _, dst := range sortedKeys(exported) { + if !expected[dst] && !tolerated[dst] { + n.Notify(kindVPC, name, fmt.Sprintf("route %s exported in %s without a running owner: its traffic is black-holed", dst, r.VRF)) + } + } + for _, dst := range sortedKeys(expected) { + if !exported[dst] { + n.Notify(kindVPC, name, fmt.Sprintf("route %s not exported in %s", dst, r.VRF)) + } + } +} + +func expectedExports(db *badger.DB, vpcName string) (expected, tolerated map[string]bool, err error) { + expected, tolerated = map[string]bool{}, map[string]bool{} + classify := func(st state.State, dst string) { + switch st { + case state.Running: + expected[dst] = true + case state.Creating, state.Deleting, state.Error: + tolerated[dst] = true + } + } + + subnets, err := kv.ListByPrefix(db, prefixSubnet) + if err != nil { + return nil, nil, fmt.Errorf("watchdog: listing subnets: %w", err) + } + routed := map[string]bool{} + for key, value := range subnets { + subnet, ok := strings.CutSuffix(strings.TrimPrefix(key, prefixSubnet), "/vpc") + if !ok || strings.Contains(subnet, "/") || value != vpcName || subnets[prefixSubnet+subnet+"/mode"] != "vxlan" { + continue + } + routed[subnet] = true + _, cidr, err := net.ParseCIDR(subnets[prefixSubnet+subnet+"/cidr"]) + if err != nil { + continue + } + if st, err := state.Get(db, prefixSubnet+subnet); err == nil { + classify(st, cidr.String()) + } + } + + vms, err := kv.ListByPrefix(db, prefixVM) + if err != nil { + return nil, nil, fmt.Errorf("watchdog: listing vms: %w", err) + } + for key, value := range vms { + parts := strings.Split(key, "/") + if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || !routed[value] { + continue + } + ip := net.ParseIP(vms[strings.TrimSuffix(key, "subnet")+"ip"]).To4() + if ip == nil { + continue + } + if st, err := state.Get(db, prefixVM+parts[1]); err == nil { + classify(st, ip.String()+"/32") + } + } + return expected, tolerated, nil +} + +func sortedKeys(m map[string]bool) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/watchdog/check_l3_test.go b/internal/watchdog/check_l3_test.go new file mode 100644 index 0000000..c307b30 --- /dev/null +++ b/internal/watchdog/check_l3_test.go @@ -0,0 +1,242 @@ +package watchdog + +import ( + "errors" + "net" + "reflect" + "testing" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/internal/state" +) + +type fakeProbe struct { + established map[string]bool + neighborErr error + vni map[uint32]frr.VNIState + routes map[uint32][]string + routesErr error + links map[string]string + vtepErr error +} + +func (f *fakeProbe) VTEPAddress(string) (net.IP, error) { + if f.vtepErr != nil { + return nil, f.vtepErr + } + return net.ParseIP("192.168.14.11"), nil +} + +func (f *fakeProbe) NeighborEstablished(peer net.IP) (bool, error) { + return f.established[peer.String()], f.neighborErr +} + +func (f *fakeProbe) VNI(vni uint32) (frr.VNIState, error) { + s, ok := f.vni[vni] + if !ok { + return s, errors.New("% VNI does not exist") + } + return s, nil +} + +func (f *fakeProbe) ExportedRoutes(table uint32, _ string) ([]*net.IPNet, error) { + var out []*net.IPNet + for _, r := range f.routes[table] { + _, n, _ := net.ParseCIDR(r) + out = append(out, n) + } + return out, f.routesErr +} + +func (f *fakeProbe) LinkProblem(ns, iface string) string { + return f.links[ns+"/"+iface] +} + +func bgpCfg() *configuration.Config { + cfg := &configuration.Config{DefaultInterface: "br-000000"} + cfg.BGP.ASN = 64600 + cfg.BGP.RouteReflectors = []string{"10.255.255.1", "10.255.255.2"} + return cfg +} + +func notified(r *recorder) []string { + var out []string + for _, c := range r.calls { + out = append(out, c.kind+" "+c.name+": "+c.problem) + } + return out +} + +func TestCheckBGPReportsEachReflectorWithoutSession(t *testing.T) { + r := &recorder{} + CheckBGP(bgpCfg(), &fakeProbe{established: map[string]bool{"10.255.255.1": true}}, r) + want := []string{"bgp 10.255.255.2: session to the route reflector is not established"} + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications %q, attendu %q", got, want) + } +} + +func TestCheckBGPReportsAnUnreadableSession(t *testing.T) { + r := &recorder{} + CheckBGP(bgpCfg(), &fakeProbe{neighborErr: errors.New("vtysh absent")}, r) + if len(r.calls) != 2 || !r.hasProblemContaining("session unreadable: vtysh absent") { + t.Errorf("notifications %q", notified(r)) + } +} + +func TestCheckBGPIsSilentWithoutBGPSection(t *testing.T) { + r := &recorder{} + CheckBGP(&configuration.Config{}, &fakeProbe{}, r) + if len(r.calls) != 0 { + t.Errorf("notifications %q", notified(r)) + } +} + +func seedRoutedVPC(t *testing.T) *fakeProbe { + return &fakeProbe{ + vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}}, + } +} + +func TestCheckL3VPCsReconcilesExportedRoutesWithTheirOwners(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-flat", state.Running) + seedKV(t, db, "vpc/vp-flat/cidr", "10.242.0.0/16") + + seedResource(t, db, prefixSubnet, "sn-a", state.Running) + seedKV(t, db, "subnet/sn-a/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-a/mode", "vxlan") + seedKV(t, db, "subnet/sn-a/cidr", "10.241.1.0/24") + seedResource(t, db, prefixSubnet, "sn-b", state.Creating) + seedKV(t, db, "subnet/sn-b/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-b/mode", "vxlan") + seedKV(t, db, "subnet/sn-b/cidr", "10.241.2.0/24") + seedResource(t, db, prefixSubnet, "sn-br", state.Running) + seedKV(t, db, "subnet/sn-br/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-br/mode", "bridge") + seedKV(t, db, "subnet/sn-br/cidr", "192.168.50.0/24") + + vm := func(name string, st state.State, subnet, ip string) { + seedResource(t, db, prefixVM, name, st) + seedKV(t, db, "vm/"+name+"/nic/0/subnet", subnet) + seedKV(t, db, "vm/"+name+"/nic/0/ip", ip) + } + vm("i-run", state.Running, "sn-a", "10.241.1.10") + vm("i-missing", state.Running, "sn-a", "10.241.1.11") + vm("i-starting", state.Creating, "sn-a", "10.241.1.12") + vm("i-gone", state.Deleted, "sn-a", "10.241.1.13") + vm("i-bridge", state.Running, "sn-br", "192.168.50.10") + seedResource(t, db, prefixSubnet, "sn-flat", state.Running) + seedKV(t, db, "subnet/sn-flat/vpc", "vp-flat") + seedKV(t, db, "subnet/sn-flat/mode", "vxlan") + seedKV(t, db, "subnet/sn-flat/cidr", "10.242.1.0/24") + vm("i-flat", state.Running, "sn-flat", "10.242.1.10") + + p := seedRoutedVPC(t) + p.routes = map[uint32][]string{2700: { + "10.241.1.0/24", "10.241.2.0/24", + "10.241.1.10/32", "10.241.1.12/32", "10.241.1.13/32", "10.241.1.50/32", + }} + r := &recorder{} + if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil { + t.Fatal(err) + } + want := []string{ + "vpc vp-s7: route 10.241.1.13/32 exported in vrf-s7 without a running owner: its traffic is black-holed", + "vpc vp-s7: route 10.241.1.50/32 exported in vrf-s7 without a running owner: its traffic is black-holed", + "vpc vp-s7: route 10.241.1.11/32 not exported in vrf-s7", + } + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications :\n%q\nattendu :\n%q", got, want) + } +} + +func TestCheckL3VPCsChecksInterfacesAndTheL3VNI(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-down", state.Error) + seedKV(t, db, "vpc/vp-down/cidr", "10.243.0.0/16") + seedKV(t, db, "vpc/vp-down/l3vni", "2800") + + p := &fakeProbe{ + vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:99"}}, + links: map[string]string{"/vxl3-s7": "interface vxl3-s7 down", "vp-s7/l3-s7-i": "interface l3-s7-i not found (in netns)"}, + } + r := &recorder{} + if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil { + t.Fatal(err) + } + want := []string{ + "vpc vp-s7: interface vxl3-s7 down", + "vpc vp-s7: interface l3-s7-i not found (in netns)", + `vpc vp-s7: l3vni 2700 has router mac "02:02:c0:a8:0e:99" in frr, expected 02:02:c0:a8:0e:0b`, + } + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications :\n%q\nattendu :\n%q", got, want) + } + + p.vni[2700] = frr.VNIState{Type: "L2", State: "Up"} + p.links = nil + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining(`l3vni 2700 is type "L2" state "Up" in frr, expected L3 Up`) { + t.Errorf("notifications %q", notified(r)) + } + + p.vni[2700] = frr.VNIState{Type: "L3", State: "Down", RouterMAC: "02:02:c0:a8:0e:0b"} + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining(`l3vni 2700 is type "L3" state "Down" in frr, expected L3 Up`) { + t.Errorf("notifications %q", notified(r)) + } + + delete(p.vni, 2700) + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining("l3vni 2700 unreadable in frr") { + t.Errorf("notifications %q", notified(r)) + } +} + +func TestCheckL3VPCsReportsAnL3VNIWithoutBGP(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + r := &recorder{} + CheckL3VPCs(db, &configuration.Config{}, &fakeProbe{}, r) + want := []string{"vpc vp-s7: vpc has an l3vni but the agent configuration has no bgp section"} + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications %q, attendu %q", got, want) + } +} + +func TestCheckL3VPCsReportsUnreadableRoutesAndVTEP(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-s8", state.Running) + seedKV(t, db, "vpc/vp-s8/cidr", "10.242.0.0/16") + seedKV(t, db, "vpc/vp-s8/l3vni", "2800") + p := seedRoutedVPC(t) + p.routesErr = errors.New("netlink refused") + p.vtepErr = errors.New("no address") + r := &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if n := len(r.forName("vtep")); n != 1 { + t.Errorf("VTEP illisible signalé %d fois, attendu une fois par passage : %q", n, notified(r)) + } + if !r.hasProblemContaining("routes of vrf-s7 unreadable: netlink refused") || !r.hasProblemContaining("vtep address unreadable on br-000000") { + t.Errorf("notifications %q", notified(r)) + } + if r.hasProblemContaining("router mac") { + t.Errorf("RMAC comparée sans VTEP : %q", notified(r)) + } +} diff --git a/internal/watchdog/watchdog.go b/internal/watchdog/watchdog.go index 6da33b8..7a0101f 100644 --- a/internal/watchdog/watchdog.go +++ b/internal/watchdog/watchdog.go @@ -6,6 +6,7 @@ import ( "time" configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" "git.g3e.fr/syonad/two/internal/watchdog/notify" "git.g3e.fr/syonad/two/pkg/systemd" @@ -21,6 +22,7 @@ type Watchdog struct { logger *slog.Logger interval time.Duration dbusDown bool + l3 l3Probe } func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog { @@ -38,6 +40,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl notifier: n, logger: logger, interval: interval, + l3: systemProbe{frr: frr.NewClient()}, } } @@ -71,6 +74,10 @@ func (w *Watchdog) tick() { if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil { w.logger.Error("watchdog: vm check failed", "err", err) } + CheckBGP(w.cfg, w.l3, w.notifier) + if err := CheckL3VPCs(w.db, w.cfg, w.l3, w.notifier); err != nil { + w.logger.Error("watchdog: l3 vpc check failed", "err", err) + } } func (w *Watchdog) units() (unitChecker, func()) {