f-41: watchdog: sessions BGP, L3VNI et routes exportées des VPC routés #41
All checks were successful
Documentation / publish (push) Successful in 1m27s
Release Pipeline / set-release-target (push) Successful in 2s
Release Pipeline / upload-assets (agent.service, systemd/agent.service) (push) Successful in 8s
Release Pipeline / upload-assets (dhcp@.service, systemd/dhcp@.service) (push) Successful in 7s
Release Pipeline / upload-assets (dnsmasq@.service, systemd/dnsmasq@.service) (push) Successful in 7s
Release Pipeline / upload-assets (metadata@.service, systemd/metadata@.service) (push) Successful in 7s
Release Pipeline / upload-assets (run-dhcp-in-netns.sh, scripts/run-dhcp-in-netns.sh) (push) Successful in 7s
Release Pipeline / build (dhcp, amd64, linux) (push) Successful in 0s
Release Pipeline / build (agent, amd64, linux) (push) Successful in 0s
Release Pipeline / build (metadata, amd64, linux) (push) Successful in 0s
Release Pipeline / upload-assets (run-dnsmasq-in-netns.sh, scripts/run-dnsmasq-in-netns.sh) (push) Successful in 7s
Release Pipeline / checksums (push) Successful in 8s
Release Pipeline / release (push) Successful in 28s
Release Pipeline / publish (push) Successful in 0s
Release Pipeline / build (push) Successful in 2m9s
All checks were successful
Documentation / publish (push) Successful in 1m27s
Release Pipeline / set-release-target (push) Successful in 2s
Release Pipeline / upload-assets (agent.service, systemd/agent.service) (push) Successful in 8s
Release Pipeline / upload-assets (dhcp@.service, systemd/dhcp@.service) (push) Successful in 7s
Release Pipeline / upload-assets (dnsmasq@.service, systemd/dnsmasq@.service) (push) Successful in 7s
Release Pipeline / upload-assets (metadata@.service, systemd/metadata@.service) (push) Successful in 7s
Release Pipeline / upload-assets (run-dhcp-in-netns.sh, scripts/run-dhcp-in-netns.sh) (push) Successful in 7s
Release Pipeline / build (dhcp, amd64, linux) (push) Successful in 0s
Release Pipeline / build (agent, amd64, linux) (push) Successful in 0s
Release Pipeline / build (metadata, amd64, linux) (push) Successful in 0s
Release Pipeline / upload-assets (run-dnsmasq-in-netns.sh, scripts/run-dnsmasq-in-netns.sh) (push) Successful in 7s
Release Pipeline / checksums (push) Successful in 8s
Release Pipeline / release (push) Successful in 28s
Release Pipeline / publish (push) Successful in 0s
Release Pipeline / build (push) Successful in 2m9s
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
03029eabe8
commit
9477dc91d2
9 changed files with 653 additions and 1 deletions
|
|
@ -38,6 +38,20 @@ toujours sur le système, et **notifie les écarts sans jamais réparer**.
|
|||
Il contrôle notamment l'existence des network namespaces, des liens réseau des subnets, des taps
|
||||
de VM et la réponse des units systemd associées.
|
||||
|
||||
Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de contrôle :
|
||||
|
||||
* **Sessions BGP** : une notification par route reflector dont la session n'est pas
|
||||
``Established`` (``show bgp neighbors <ip>``).
|
||||
* **VPC routés** (``l3vni``, état ``running``) : présence et état des interfaces ``vrf-<id>``,
|
||||
``brl3-<id>``, ``vxl3-<id>``, ``l3-<id>-o`` et, dans le netns, ``l3-<id>-i`` ; L3VNI de type
|
||||
``L3``, ``Up``, avec la MAC de routeur attendue (``show evpn vni <l3vni>``).
|
||||
* **Réconciliation des routes exportées** dans la VRF : chaque ``/24`` de subnet ``vxlan`` et chaque
|
||||
``/32`` d'interface de VM ``running`` doit être exporté ; à l'inverse, une route exportée dont
|
||||
le propriétaire n'existe plus — ni ``running``, ni en ``creating``, ``deleting`` ou ``error`` —
|
||||
est signalée comme un **trou noir** : l'annonce attire du trafic que plus rien ne reçoit.
|
||||
|
||||
Un VPC routé alors que la section ``bgp`` a disparu de la configuration est signalé lui aussi.
|
||||
|
||||
.. note::
|
||||
|
||||
Un écart persistant est signalé **à chaque tick**, sans déduplication. Le volume de
|
||||
|
|
|
|||
49
internal/frr/status.go
Normal file
49
internal/frr/status.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package frr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type VNIState struct {
|
||||
Type string
|
||||
State string
|
||||
RouterMAC string
|
||||
}
|
||||
|
||||
func (c *Client) NeighborEstablished(peer net.IP) (bool, error) {
|
||||
out, err := c.exec("show bgp neighbors " + peer.String())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return strings.Contains(out, "BGP state = Established"), nil
|
||||
}
|
||||
|
||||
func (c *Client) VNI(vni uint32) (VNIState, error) {
|
||||
out, err := c.exec(fmt.Sprintf("show evpn vni %d", vni))
|
||||
if err != nil {
|
||||
return VNIState{}, err
|
||||
}
|
||||
return parseVNI(out), nil
|
||||
}
|
||||
|
||||
func parseVNI(out string) VNIState {
|
||||
var s VNIState
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
key, value, ok := strings.Cut(strings.TrimSpace(l), ": ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
value = strings.TrimSpace(value)
|
||||
switch strings.ToLower(key) {
|
||||
case "type":
|
||||
s.Type = value
|
||||
case "state":
|
||||
s.State = value
|
||||
case "router mac":
|
||||
s.RouterMAC = strings.ToLower(value)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
50
internal/frr/status_test.go
Normal file
50
internal/frr/status_test.go
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
package frr
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const vniOutput = `VNI: 2700
|
||||
Type: L3
|
||||
Tenant VRF: vrf-s7
|
||||
Vxlan-Intf: vxl3-s7
|
||||
SVI-If: brl3-s7
|
||||
State: Up
|
||||
VNI Filter: none
|
||||
System MAC: 02:02:c0:a8:0e:aa
|
||||
Router MAC: 02:02:C0:A8:0E:0B
|
||||
L2 VNIs:
|
||||
`
|
||||
|
||||
func TestVNIReadsTypeStateAndRouterMAC(t *testing.T) {
|
||||
f := &fakeVtysh{fail: map[string]string{"show evpn vni 2700": vniOutput}}
|
||||
s, err := f.client().VNI(2700)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := VNIState{Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}
|
||||
if s != want {
|
||||
t.Errorf("VNI = %+v, attendu %+v", s, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVNIReportsAnUnknownVNI(t *testing.T) {
|
||||
f := &fakeVtysh{fail: map[string]string{"show evpn vni 9": "% VNI 9 does not exist\n"}}
|
||||
if _, err := f.client().VNI(9); err == nil || !strings.Contains(err.Error(), "does not exist") {
|
||||
t.Errorf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeighborEstablished(t *testing.T) {
|
||||
rr := net.ParseIP("10.255.255.1")
|
||||
f := &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1, remote AS 64600, local AS 64600, internal link\n BGP version 4, remote router ID 10.255.255.1\n BGP state = Established, up for 00:10:02\n"}}
|
||||
if up, err := f.client().NeighborEstablished(rr); err != nil || !up {
|
||||
t.Errorf("établie : %v, %v", up, err)
|
||||
}
|
||||
f = &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1\n BGP state = Active\n"}}
|
||||
if up, err := f.client().NeighborEstablished(rr); err != nil || up {
|
||||
t.Errorf("active : %v, %v", up, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -19,3 +19,24 @@ func RouteAdd(iface string, subnet *net.IPNet) error {
|
|||
Scope: netlink.SCOPE_LINK,
|
||||
})
|
||||
}
|
||||
|
||||
func TableRoutesVia(table int, iface string) ([]*net.IPNet, error) {
|
||||
link, err := netlink.LinkByName(iface)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{
|
||||
Table: table,
|
||||
LinkIndex: link.Attrs().Index,
|
||||
}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsts := make([]*net.IPNet, 0, len(routes))
|
||||
for _, r := range routes {
|
||||
if r.Dst != nil && r.Gw != nil {
|
||||
dsts = append(dsts, r.Dst)
|
||||
}
|
||||
}
|
||||
return dsts, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,15 @@
|
|||
|
||||
package netif
|
||||
|
||||
import "net"
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
)
|
||||
|
||||
func RouteAdd(_ string, _ *net.IPNet) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TableRoutesVia(_ int, _ string) ([]*net.IPNet, error) {
|
||||
return nil, errors.New("route listing is only implemented on linux")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,3 +137,33 @@ func deleteL3(h host, cfg *configuration.Config, v l3VPC) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Routing struct {
|
||||
VNI uint32
|
||||
Supernet *net.IPNet
|
||||
VRF string
|
||||
Bridge string
|
||||
Vxlan string
|
||||
VethHost string
|
||||
VethNetns string
|
||||
}
|
||||
|
||||
func LoadRouting(db *badger.DB, vpcName string) (Routing, bool, error) {
|
||||
v, ok, err := loadRouted(db, vpcName)
|
||||
if err != nil || !ok {
|
||||
return Routing{}, ok, err
|
||||
}
|
||||
return Routing{
|
||||
VNI: v.vni,
|
||||
Supernet: v.supernet,
|
||||
VRF: v.vrf(),
|
||||
Bridge: v.bridge(),
|
||||
Vxlan: v.vxlan(),
|
||||
VethHost: v.vethHost(),
|
||||
VethNetns: v.vethNetns(),
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func RouterMAC(vtep net.IP) (net.HardwareAddr, error) {
|
||||
return routerMAC(vtep)
|
||||
}
|
||||
|
|
|
|||
232
internal/watchdog/check_l3.go
Normal file
232
internal/watchdog/check_l3.go
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/frr"
|
||||
"git.g3e.fr/syonad/two/internal/netif"
|
||||
"git.g3e.fr/syonad/two/internal/netns"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
"git.g3e.fr/syonad/two/internal/vpc"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
"git.g3e.fr/syonad/two/pkg/db/kv"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
const kindBGP = "bgp"
|
||||
|
||||
type l3Probe interface {
|
||||
VTEPAddress(iface string) (net.IP, error)
|
||||
NeighborEstablished(peer net.IP) (bool, error)
|
||||
VNI(vni uint32) (frr.VNIState, error)
|
||||
ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error)
|
||||
LinkProblem(ns, iface string) string
|
||||
}
|
||||
|
||||
type systemProbe struct {
|
||||
frr *frr.Client
|
||||
}
|
||||
|
||||
func (systemProbe) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) }
|
||||
|
||||
func (p systemProbe) NeighborEstablished(peer net.IP) (bool, error) {
|
||||
return p.frr.NeighborEstablished(peer)
|
||||
}
|
||||
|
||||
func (p systemProbe) VNI(vni uint32) (frr.VNIState, error) { return p.frr.VNI(vni) }
|
||||
|
||||
func (systemProbe) ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) {
|
||||
return netif.TableRoutesVia(int(table), iface)
|
||||
}
|
||||
|
||||
func (systemProbe) LinkProblem(ns, iface string) string {
|
||||
if ns == "" {
|
||||
return linkProblem(iface)
|
||||
}
|
||||
var p string
|
||||
if err := netns.Call(ns, func() error {
|
||||
p = linkProblem(iface)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Sprintf("cannot enter netns %s: %v", ns, err)
|
||||
}
|
||||
if p != "" {
|
||||
return p + " (in netns)"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func CheckBGP(cfg *configuration.Config, p l3Probe, n notify.Notifier) {
|
||||
if !cfg.BGP.Enabled() {
|
||||
return
|
||||
}
|
||||
reflectors, err := cfg.BGP.Reflectors()
|
||||
if err != nil {
|
||||
n.Notify(kindBGP, "config", err.Error())
|
||||
return
|
||||
}
|
||||
for _, rr := range reflectors {
|
||||
up, err := p.NeighborEstablished(rr)
|
||||
switch {
|
||||
case err != nil:
|
||||
n.Notify(kindBGP, rr.String(), fmt.Sprintf("session unreadable: %v", err))
|
||||
case !up:
|
||||
n.Notify(kindBGP, rr.String(), "session to the route reflector is not established")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CheckL3VPCs(db *badger.DB, cfg *configuration.Config, p l3Probe, n notify.Notifier) error {
|
||||
pairs, err := kv.ListByPrefix(db, prefixVPC)
|
||||
if err != nil {
|
||||
return fmt.Errorf("watchdog: listing vpcs: %w", err)
|
||||
}
|
||||
var rmac string
|
||||
for _, name := range resourceNames(pairs, prefixVPC) {
|
||||
if st, err := state.Get(db, prefixVPC+name); err != nil || st != state.Running {
|
||||
continue
|
||||
}
|
||||
r, ok, err := vpc.LoadRouting(db, name)
|
||||
if err != nil {
|
||||
n.Notify(kindVPC, name, err.Error())
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !cfg.BGP.Enabled() {
|
||||
n.Notify(kindVPC, name, "vpc has an l3vni but the agent configuration has no bgp section")
|
||||
continue
|
||||
}
|
||||
if rmac == "" {
|
||||
rmac = expectedRouterMAC(cfg, p, n)
|
||||
}
|
||||
checkL3VPC(db, name, r, rmac, p, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func expectedRouterMAC(cfg *configuration.Config, p l3Probe, n notify.Notifier) string {
|
||||
vtep, err := p.VTEPAddress(cfg.DefaultInterface)
|
||||
if err != nil {
|
||||
n.Notify(kindBGP, "vtep", fmt.Sprintf("vtep address unreadable on %s: %v", cfg.DefaultInterface, err))
|
||||
return "unknown"
|
||||
}
|
||||
mac, err := vpc.RouterMAC(vtep)
|
||||
if err != nil {
|
||||
n.Notify(kindBGP, "vtep", err.Error())
|
||||
return "unknown"
|
||||
}
|
||||
return mac.String()
|
||||
}
|
||||
|
||||
func checkL3VPC(db *badger.DB, name string, r vpc.Routing, rmac string, p l3Probe, n notify.Notifier) {
|
||||
for _, iface := range []string{r.VRF, r.Bridge, r.Vxlan, r.VethHost} {
|
||||
if problem := p.LinkProblem("", iface); problem != "" {
|
||||
n.Notify(kindVPC, name, problem)
|
||||
}
|
||||
}
|
||||
if problem := p.LinkProblem(name, r.VethNetns); problem != "" {
|
||||
n.Notify(kindVPC, name, problem)
|
||||
}
|
||||
|
||||
s, err := p.VNI(r.VNI)
|
||||
switch {
|
||||
case err != nil:
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d unreadable in frr: %v", r.VNI, err))
|
||||
case s.Type != "L3" || s.State != "Up":
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d is type %q state %q in frr, expected L3 Up", r.VNI, s.Type, s.State))
|
||||
case rmac != "unknown" && s.RouterMAC != rmac:
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d has router mac %q in frr, expected %s", r.VNI, s.RouterMAC, rmac))
|
||||
}
|
||||
|
||||
actual, err := p.ExportedRoutes(r.VNI, r.VethHost)
|
||||
if err != nil {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("routes of %s unreadable: %v", r.VRF, err))
|
||||
return
|
||||
}
|
||||
expected, tolerated, err := expectedExports(db, name)
|
||||
if err != nil {
|
||||
n.Notify(kindVPC, name, err.Error())
|
||||
return
|
||||
}
|
||||
exported := make(map[string]bool, len(actual))
|
||||
for _, dst := range actual {
|
||||
exported[dst.String()] = true
|
||||
}
|
||||
for _, dst := range sortedKeys(exported) {
|
||||
if !expected[dst] && !tolerated[dst] {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("route %s exported in %s without a running owner: its traffic is black-holed", dst, r.VRF))
|
||||
}
|
||||
}
|
||||
for _, dst := range sortedKeys(expected) {
|
||||
if !exported[dst] {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("route %s not exported in %s", dst, r.VRF))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectedExports(db *badger.DB, vpcName string) (expected, tolerated map[string]bool, err error) {
|
||||
expected, tolerated = map[string]bool{}, map[string]bool{}
|
||||
classify := func(st state.State, dst string) {
|
||||
switch st {
|
||||
case state.Running:
|
||||
expected[dst] = true
|
||||
case state.Creating, state.Deleting, state.Error:
|
||||
tolerated[dst] = true
|
||||
}
|
||||
}
|
||||
|
||||
subnets, err := kv.ListByPrefix(db, prefixSubnet)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("watchdog: listing subnets: %w", err)
|
||||
}
|
||||
routed := map[string]bool{}
|
||||
for key, value := range subnets {
|
||||
subnet, ok := strings.CutSuffix(strings.TrimPrefix(key, prefixSubnet), "/vpc")
|
||||
if !ok || strings.Contains(subnet, "/") || value != vpcName || subnets[prefixSubnet+subnet+"/mode"] != "vxlan" {
|
||||
continue
|
||||
}
|
||||
routed[subnet] = true
|
||||
_, cidr, err := net.ParseCIDR(subnets[prefixSubnet+subnet+"/cidr"])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if st, err := state.Get(db, prefixSubnet+subnet); err == nil {
|
||||
classify(st, cidr.String())
|
||||
}
|
||||
}
|
||||
|
||||
vms, err := kv.ListByPrefix(db, prefixVM)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("watchdog: listing vms: %w", err)
|
||||
}
|
||||
for key, value := range vms {
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || !routed[value] {
|
||||
continue
|
||||
}
|
||||
ip := net.ParseIP(vms[strings.TrimSuffix(key, "subnet")+"ip"]).To4()
|
||||
if ip == nil {
|
||||
continue
|
||||
}
|
||||
if st, err := state.Get(db, prefixVM+parts[1]); err == nil {
|
||||
classify(st, ip.String()+"/32")
|
||||
}
|
||||
}
|
||||
return expected, tolerated, nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
242
internal/watchdog/check_l3_test.go
Normal file
242
internal/watchdog/check_l3_test.go
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/frr"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
)
|
||||
|
||||
type fakeProbe struct {
|
||||
established map[string]bool
|
||||
neighborErr error
|
||||
vni map[uint32]frr.VNIState
|
||||
routes map[uint32][]string
|
||||
routesErr error
|
||||
links map[string]string
|
||||
vtepErr error
|
||||
}
|
||||
|
||||
func (f *fakeProbe) VTEPAddress(string) (net.IP, error) {
|
||||
if f.vtepErr != nil {
|
||||
return nil, f.vtepErr
|
||||
}
|
||||
return net.ParseIP("192.168.14.11"), nil
|
||||
}
|
||||
|
||||
func (f *fakeProbe) NeighborEstablished(peer net.IP) (bool, error) {
|
||||
return f.established[peer.String()], f.neighborErr
|
||||
}
|
||||
|
||||
func (f *fakeProbe) VNI(vni uint32) (frr.VNIState, error) {
|
||||
s, ok := f.vni[vni]
|
||||
if !ok {
|
||||
return s, errors.New("% VNI does not exist")
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (f *fakeProbe) ExportedRoutes(table uint32, _ string) ([]*net.IPNet, error) {
|
||||
var out []*net.IPNet
|
||||
for _, r := range f.routes[table] {
|
||||
_, n, _ := net.ParseCIDR(r)
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, f.routesErr
|
||||
}
|
||||
|
||||
func (f *fakeProbe) LinkProblem(ns, iface string) string {
|
||||
return f.links[ns+"/"+iface]
|
||||
}
|
||||
|
||||
func bgpCfg() *configuration.Config {
|
||||
cfg := &configuration.Config{DefaultInterface: "br-000000"}
|
||||
cfg.BGP.ASN = 64600
|
||||
cfg.BGP.RouteReflectors = []string{"10.255.255.1", "10.255.255.2"}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func notified(r *recorder) []string {
|
||||
var out []string
|
||||
for _, c := range r.calls {
|
||||
out = append(out, c.kind+" "+c.name+": "+c.problem)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCheckBGPReportsEachReflectorWithoutSession(t *testing.T) {
|
||||
r := &recorder{}
|
||||
CheckBGP(bgpCfg(), &fakeProbe{established: map[string]bool{"10.255.255.1": true}}, r)
|
||||
want := []string{"bgp 10.255.255.2: session to the route reflector is not established"}
|
||||
if got := notified(r); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("notifications %q, attendu %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBGPReportsAnUnreadableSession(t *testing.T) {
|
||||
r := &recorder{}
|
||||
CheckBGP(bgpCfg(), &fakeProbe{neighborErr: errors.New("vtysh absent")}, r)
|
||||
if len(r.calls) != 2 || !r.hasProblemContaining("session unreadable: vtysh absent") {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBGPIsSilentWithoutBGPSection(t *testing.T) {
|
||||
r := &recorder{}
|
||||
CheckBGP(&configuration.Config{}, &fakeProbe{}, r)
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
}
|
||||
|
||||
func seedRoutedVPC(t *testing.T) *fakeProbe {
|
||||
return &fakeProbe{
|
||||
vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckL3VPCsReconcilesExportedRoutesWithTheirOwners(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
|
||||
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
|
||||
seedResource(t, db, prefixVPC, "vp-flat", state.Running)
|
||||
seedKV(t, db, "vpc/vp-flat/cidr", "10.242.0.0/16")
|
||||
|
||||
seedResource(t, db, prefixSubnet, "sn-a", state.Running)
|
||||
seedKV(t, db, "subnet/sn-a/vpc", "vp-s7")
|
||||
seedKV(t, db, "subnet/sn-a/mode", "vxlan")
|
||||
seedKV(t, db, "subnet/sn-a/cidr", "10.241.1.0/24")
|
||||
seedResource(t, db, prefixSubnet, "sn-b", state.Creating)
|
||||
seedKV(t, db, "subnet/sn-b/vpc", "vp-s7")
|
||||
seedKV(t, db, "subnet/sn-b/mode", "vxlan")
|
||||
seedKV(t, db, "subnet/sn-b/cidr", "10.241.2.0/24")
|
||||
seedResource(t, db, prefixSubnet, "sn-br", state.Running)
|
||||
seedKV(t, db, "subnet/sn-br/vpc", "vp-s7")
|
||||
seedKV(t, db, "subnet/sn-br/mode", "bridge")
|
||||
seedKV(t, db, "subnet/sn-br/cidr", "192.168.50.0/24")
|
||||
|
||||
vm := func(name string, st state.State, subnet, ip string) {
|
||||
seedResource(t, db, prefixVM, name, st)
|
||||
seedKV(t, db, "vm/"+name+"/nic/0/subnet", subnet)
|
||||
seedKV(t, db, "vm/"+name+"/nic/0/ip", ip)
|
||||
}
|
||||
vm("i-run", state.Running, "sn-a", "10.241.1.10")
|
||||
vm("i-missing", state.Running, "sn-a", "10.241.1.11")
|
||||
vm("i-starting", state.Creating, "sn-a", "10.241.1.12")
|
||||
vm("i-gone", state.Deleted, "sn-a", "10.241.1.13")
|
||||
vm("i-bridge", state.Running, "sn-br", "192.168.50.10")
|
||||
seedResource(t, db, prefixSubnet, "sn-flat", state.Running)
|
||||
seedKV(t, db, "subnet/sn-flat/vpc", "vp-flat")
|
||||
seedKV(t, db, "subnet/sn-flat/mode", "vxlan")
|
||||
seedKV(t, db, "subnet/sn-flat/cidr", "10.242.1.0/24")
|
||||
vm("i-flat", state.Running, "sn-flat", "10.242.1.10")
|
||||
|
||||
p := seedRoutedVPC(t)
|
||||
p.routes = map[uint32][]string{2700: {
|
||||
"10.241.1.0/24", "10.241.2.0/24",
|
||||
"10.241.1.10/32", "10.241.1.12/32", "10.241.1.13/32", "10.241.1.50/32",
|
||||
}}
|
||||
r := &recorder{}
|
||||
if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"vpc vp-s7: route 10.241.1.13/32 exported in vrf-s7 without a running owner: its traffic is black-holed",
|
||||
"vpc vp-s7: route 10.241.1.50/32 exported in vrf-s7 without a running owner: its traffic is black-holed",
|
||||
"vpc vp-s7: route 10.241.1.11/32 not exported in vrf-s7",
|
||||
}
|
||||
if got := notified(r); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("notifications :\n%q\nattendu :\n%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckL3VPCsChecksInterfacesAndTheL3VNI(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
|
||||
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
|
||||
seedResource(t, db, prefixVPC, "vp-down", state.Error)
|
||||
seedKV(t, db, "vpc/vp-down/cidr", "10.243.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-down/l3vni", "2800")
|
||||
|
||||
p := &fakeProbe{
|
||||
vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:99"}},
|
||||
links: map[string]string{"/vxl3-s7": "interface vxl3-s7 down", "vp-s7/l3-s7-i": "interface l3-s7-i not found (in netns)"},
|
||||
}
|
||||
r := &recorder{}
|
||||
if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"vpc vp-s7: interface vxl3-s7 down",
|
||||
"vpc vp-s7: interface l3-s7-i not found (in netns)",
|
||||
`vpc vp-s7: l3vni 2700 has router mac "02:02:c0:a8:0e:99" in frr, expected 02:02:c0:a8:0e:0b`,
|
||||
}
|
||||
if got := notified(r); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("notifications :\n%q\nattendu :\n%q", got, want)
|
||||
}
|
||||
|
||||
p.vni[2700] = frr.VNIState{Type: "L2", State: "Up"}
|
||||
p.links = nil
|
||||
r = &recorder{}
|
||||
CheckL3VPCs(db, bgpCfg(), p, r)
|
||||
if !r.hasProblemContaining(`l3vni 2700 is type "L2" state "Up" in frr, expected L3 Up`) {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
|
||||
p.vni[2700] = frr.VNIState{Type: "L3", State: "Down", RouterMAC: "02:02:c0:a8:0e:0b"}
|
||||
r = &recorder{}
|
||||
CheckL3VPCs(db, bgpCfg(), p, r)
|
||||
if !r.hasProblemContaining(`l3vni 2700 is type "L3" state "Down" in frr, expected L3 Up`) {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
|
||||
delete(p.vni, 2700)
|
||||
r = &recorder{}
|
||||
CheckL3VPCs(db, bgpCfg(), p, r)
|
||||
if !r.hasProblemContaining("l3vni 2700 unreadable in frr") {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckL3VPCsReportsAnL3VNIWithoutBGP(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
|
||||
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
|
||||
r := &recorder{}
|
||||
CheckL3VPCs(db, &configuration.Config{}, &fakeProbe{}, r)
|
||||
want := []string{"vpc vp-s7: vpc has an l3vni but the agent configuration has no bgp section"}
|
||||
if got := notified(r); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("notifications %q, attendu %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckL3VPCsReportsUnreadableRoutesAndVTEP(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
|
||||
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
|
||||
seedResource(t, db, prefixVPC, "vp-s8", state.Running)
|
||||
seedKV(t, db, "vpc/vp-s8/cidr", "10.242.0.0/16")
|
||||
seedKV(t, db, "vpc/vp-s8/l3vni", "2800")
|
||||
p := seedRoutedVPC(t)
|
||||
p.routesErr = errors.New("netlink refused")
|
||||
p.vtepErr = errors.New("no address")
|
||||
r := &recorder{}
|
||||
CheckL3VPCs(db, bgpCfg(), p, r)
|
||||
if n := len(r.forName("vtep")); n != 1 {
|
||||
t.Errorf("VTEP illisible signalé %d fois, attendu une fois par passage : %q", n, notified(r))
|
||||
}
|
||||
if !r.hasProblemContaining("routes of vrf-s7 unreadable: netlink refused") || !r.hasProblemContaining("vtep address unreadable on br-000000") {
|
||||
t.Errorf("notifications %q", notified(r))
|
||||
}
|
||||
if r.hasProblemContaining("router mac") {
|
||||
t.Errorf("RMAC comparée sans VTEP : %q", notified(r))
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/frr"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
"git.g3e.fr/syonad/two/pkg/systemd"
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ type Watchdog struct {
|
|||
logger *slog.Logger
|
||||
interval time.Duration
|
||||
dbusDown bool
|
||||
l3 l3Probe
|
||||
}
|
||||
|
||||
func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog {
|
||||
|
|
@ -38,6 +40,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl
|
|||
notifier: n,
|
||||
logger: logger,
|
||||
interval: interval,
|
||||
l3: systemProbe{frr: frr.NewClient()},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -71,6 +74,10 @@ func (w *Watchdog) tick() {
|
|||
if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: vm check failed", "err", err)
|
||||
}
|
||||
CheckBGP(w.cfg, w.l3, w.notifier)
|
||||
if err := CheckL3VPCs(w.db, w.cfg, w.l3, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: l3 vpc check failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watchdog) units() (unitChecker, func()) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue