From 0f31b2faf49451c6ab1f2be438631de00462e17a Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Tue, 6 Oct 2026 00:50:06 +0200 Subject: [PATCH 1/9] =?UTF-8?q?f-41:=20subnet:=20MAC=20de=20gateway=20anyc?= =?UTF-8?q?ast=20d=C3=A9riv=C3=A9e=20de=20la=20VNI=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- docs/concepts/modes-reseau.rst | 6 ++++ internal/netif/upDown.go | 10 ++++++ internal/subnet/create.go | 9 +++++ internal/subnet/gateway.go | 19 +++++++++++ internal/subnet/gateway_test.go | 58 +++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 internal/subnet/gateway.go create mode 100644 internal/subnet/gateway_test.go diff --git a/docs/concepts/modes-reseau.rst b/docs/concepts/modes-reseau.rst index 270cb17..74031e9 100644 --- a/docs/concepts/modes-reseau.rst +++ b/docs/concepts/modes-reseau.rst @@ -42,6 +42,12 @@ vxlan Le subnet vit dans le netns du VPC. La VM n'est donc **pas joignable depuis l'host** sans route explicite — point à connaître avant de câbler un outil externe dessus. +Le bridge du subnet, qui porte ``interface_ip``, a une **MAC de gateway anycast** calculée depuis +le ``vxlan_id`` : ``02:01:00`` suivi des 24 bits de la VNI (``vxlan_id`` 2811 → +``02:01:00:00:0a:fb``). Elle est identique sur tous les hôtes qui portent le subnet : une VM qui +change d'hôte garde une entrée ARP valide pour sa gateway. Elle n'est posée qu'à la création du +subnet — un subnet créé avant garde la MAC choisie par le noyau jusqu'à sa recréation. + bridge ------ diff --git a/internal/netif/upDown.go b/internal/netif/upDown.go index 6d27cc2..4e9cc61 100644 --- a/internal/netif/upDown.go +++ b/internal/netif/upDown.go @@ -1,6 +1,8 @@ package netif import ( + "net" + "github.com/vishvananda/netlink" ) @@ -19,3 +21,11 @@ func LinkSetDown(name string) error { } return netlink.LinkSetDown(link) } + +func LinkSetHardwareAddr(name string, mac net.HardwareAddr) error { + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + return netlink.LinkSetHardwareAddr(link, mac) +} diff --git a/internal/subnet/create.go b/internal/subnet/create.go index 1164219..568b64e 100644 --- a/internal/subnet/create.go +++ b/internal/subnet/create.go @@ -70,6 +70,15 @@ func createSubnet(db *badger.DB, subnetName string, d subnetData, backend dhcpba if err := netif.CreateBridge(d.bridge, 1500); err != nil { return fmt.Errorf("create bridge: %w", err) } + if d.mode == ModeVxlan { + mac, err := anycastGatewayMAC(d.vxlanID) + if err != nil { + return err + } + if err := netif.LinkSetHardwareAddr(d.bridge, mac); err != nil { + return fmt.Errorf("set anycast gateway mac %s: %w", mac, err) + } + } return netif.BridgeSetMaster(vethI, d.bridge) }); err != nil { return fmt.Errorf("setup bridge in netns: %w", err) diff --git a/internal/subnet/gateway.go b/internal/subnet/gateway.go new file mode 100644 index 0000000..a61eb11 --- /dev/null +++ b/internal/subnet/gateway.go @@ -0,0 +1,19 @@ +package subnet + +import ( + "fmt" + "net" +) + +const ( + maxVNI = 1<<24 - 1 + roleAnycastGateway = 0x01 + locallyAdministered = 0x02 +) + +func anycastGatewayMAC(vni int) (net.HardwareAddr, error) { + if vni < 0 || vni > maxVNI { + return nil, fmt.Errorf("vxlan_id %d does not fit in 24 bits", vni) + } + return net.HardwareAddr{locallyAdministered, roleAnycastGateway, 0x00, byte(vni >> 16), byte(vni >> 8), byte(vni)}, nil +} diff --git a/internal/subnet/gateway_test.go b/internal/subnet/gateway_test.go new file mode 100644 index 0000000..2352996 --- /dev/null +++ b/internal/subnet/gateway_test.go @@ -0,0 +1,58 @@ +package subnet + +import "testing" + +func TestAnycastGatewayMAC(t *testing.T) { + cases := []struct { + vni int + want string + }{ + {2811, "02:01:00:00:0a:fb"}, + {0, "02:01:00:00:00:00"}, + {1, "02:01:00:00:00:01"}, + {0x123456, "02:01:00:12:34:56"}, + {maxVNI, "02:01:00:ff:ff:ff"}, + } + for _, c := range cases { + got, err := anycastGatewayMAC(c.vni) + if err != nil { + t.Fatalf("VNI %d : erreur inattendue %v", c.vni, err) + } + if got.String() != c.want { + t.Errorf("VNI %d : %s, %s attendu", c.vni, got, c.want) + } + } +} + +func TestAnycastGatewayMACIsLocalUnicast(t *testing.T) { + got, err := anycastGatewayMAC(2811) + if err != nil { + t.Fatal(err) + } + if got[0]&0x02 == 0 { + t.Errorf("%s : bit « administrée localement » absent", got) + } + if got[0]&0x01 != 0 { + t.Errorf("%s : MAC de groupe (multicast)", got) + } +} + +func TestAnycastGatewayMACIsIdenticalForTheSameVNIAndDistinctOtherwise(t *testing.T) { + a, _ := anycastGatewayMAC(2811) + b, _ := anycastGatewayMAC(2811) + c, _ := anycastGatewayMAC(2812) + if a.String() != b.String() { + t.Errorf("même VNI, MAC différentes : %s et %s", a, b) + } + if a.String() == c.String() { + t.Errorf("VNI différentes, même MAC : %s", a) + } +} + +func TestAnycastGatewayMACRejectsVNIOutside24Bits(t *testing.T) { + for _, vni := range []int{-1, maxVNI + 1} { + if mac, err := anycastGatewayMAC(vni); err == nil { + t.Errorf("VNI %d accepté : %s", vni, mac) + } + } +} From 94c2f82867582a20c5d543eb85563737f0284647 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Tue, 6 Oct 2026 14:13:08 +0200 Subject: [PATCH 2/9] f-41: frr, sysctl, netif: primitives du L3VNI #41 Signed-off-by: GnomeZworc --- internal/frr/client.go | 107 +++++++++++++++++++ internal/frr/client_test.go | 184 +++++++++++++++++++++++++++++++++ internal/frr/config.go | 129 +++++++++++++++++++++++ internal/frr/config_test.go | 142 +++++++++++++++++++++++++ internal/frr/teardown.go | 84 +++++++++++++++ internal/frr/teardown_test.go | 142 +++++++++++++++++++++++++ internal/netif/addr.go | 8 ++ internal/netif/route.go | 27 +++++ internal/netif/vrf.go | 16 +++ internal/sysctl/sysctl.go | 23 +++++ internal/sysctl/sysctl_test.go | 48 +++++++++ 11 files changed, 910 insertions(+) create mode 100644 internal/frr/client.go create mode 100644 internal/frr/client_test.go create mode 100644 internal/frr/config.go create mode 100644 internal/frr/config_test.go create mode 100644 internal/frr/teardown.go create mode 100644 internal/frr/teardown_test.go create mode 100644 internal/netif/route.go create mode 100644 internal/netif/vrf.go create mode 100644 internal/sysctl/sysctl.go create mode 100644 internal/sysctl/sysctl_test.go diff --git a/internal/frr/client.go b/internal/frr/client.go new file mode 100644 index 0000000..1f76d4e --- /dev/null +++ b/internal/frr/client.go @@ -0,0 +1,107 @@ +package frr + +import ( + "fmt" + "os/exec" + "strings" +) + +type runner func(args ...string) ([]byte, error) + +type Client struct { + run runner +} + +func NewClient() *Client { + return &Client{run: func(args ...string) ([]byte, error) { + return exec.Command("vtysh", args...).CombinedOutput() + }} +} + +func (c *Client) ApplyBase(b Base) error { + if err := b.validate(); err != nil { + return err + } + if err := c.configure(b.commands()...); err != nil { + return err + } + return c.save() +} + +func (c *Client) ApplyVRF(v VRF) error { + if err := v.validate(); err != nil { + return err + } + if err := c.configure(v.commands()...); err != nil { + return err + } + return c.save() +} + +func (c *Client) WithdrawVRF(v VRF) error { + return c.teardown(v, routingTeardown) +} + +func (c *Client) RemoveVRF(v VRF) error { + if err := c.teardown(v, vrfTeardown); err != nil { + return err + } + running, err := c.runningConfig() + if err != nil { + return err + } + return remaining(routingTeardown(v, running)) +} + +func (c *Client) teardown(v VRF, plan func(VRF, string) []step) error { + if err := v.validateName(); err != nil { + return err + } + running, err := c.runningConfig() + if err != nil { + return err + } + for _, s := range plan(v, running) { + if err := c.configure(s...); err != nil { + return err + } + } + if err := c.save(); err != nil { + return err + } + if running, err = c.runningConfig(); err != nil { + return err + } + return remaining(plan(v, running)) +} + +func (c *Client) runningConfig() (string, error) { + return c.exec("show running-config") +} + +func (c *Client) configure(cmds ...string) error { + _, err := c.exec(append([]string{"configure terminal"}, cmds...)...) + return err +} + +func (c *Client) save() error { + _, err := c.exec("write memory") + return err +} + +func (c *Client) exec(cmds ...string) (string, error) { + args := make([]string, 0, 2*len(cmds)) + for _, cmd := range cmds { + args = append(args, "-c", cmd) + } + out, err := c.run(args...) + if err != nil { + return "", fmt.Errorf("vtysh %q: %w: %s", cmds, err, strings.TrimSpace(string(out))) + } + for _, l := range strings.Split(string(out), "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "%") { + return "", fmt.Errorf("vtysh %q: %s", cmds, strings.TrimSpace(l)) + } + } + return string(out), nil +} diff --git a/internal/frr/client_test.go b/internal/frr/client_test.go new file mode 100644 index 0000000..d48ba4d --- /dev/null +++ b/internal/frr/client_test.go @@ -0,0 +1,184 @@ +package frr + +import ( + "errors" + "net" + "reflect" + "strings" + "testing" +) + +type fakeVtysh struct { + calls [][]string + running []string + fail map[string]string + status map[string]error +} + +func (f *fakeVtysh) run(args ...string) ([]byte, error) { + var cmds []string + for i := 1; i < len(args); i += 2 { + cmds = append(cmds, args[i]) + } + f.calls = append(f.calls, cmds) + last := cmds[len(cmds)-1] + if last == "show running-config" { + if len(f.running) == 0 { + return nil, errors.New("show running-config inattendu") + } + out := f.running[0] + f.running = f.running[1:] + return []byte(out), nil + } + for _, c := range cmds { + if out, ok := f.fail[c]; ok { + return []byte(out), f.status[c] + } + } + return nil, nil +} + +func (f *fakeVtysh) client() *Client { + return &Client{run: f.run} +} + +func (f *fakeVtysh) sent(cmd string) bool { + for _, c := range f.calls { + for _, x := range c { + if x == cmd { + return true + } + } + } + return false +} + +func TestApplyVRFConfiguresThenSaves(t *testing.T) { + f := &fakeVtysh{} + if err := f.client().ApplyVRF(labVRF(t)); err != nil { + t.Fatal(err) + } + want := [][]string{ + append([]string{"configure terminal"}, labVRF(t).commands()...), + {"write memory"}, + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want) + } +} + +func TestApplyVRFRefusesAnInvalidVRFWithoutCallingVtysh(t *testing.T) { + f := &fakeVtysh{} + v := labVRF(t) + v.L3VNI = 0 + if err := f.client().ApplyVRF(v); err == nil || !strings.Contains(err.Error(), "l3vni 0") { + t.Errorf("erreur %v", err) + } + if len(f.calls) != 0 { + t.Errorf("vtysh appelé : %q", f.calls) + } +} + +func TestApplyBaseConfiguresThenSaves(t *testing.T) { + f := &fakeVtysh{} + b := Base{ASN: 64600, RouterID: labVRF(t).VTEP, RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}} + if err := f.client().ApplyBase(b); err != nil { + t.Fatal(err) + } + if len(f.calls) != 2 || f.calls[0][0] != "configure terminal" || f.calls[0][1] != "router bgp 64600" || f.calls[1][0] != "write memory" { + t.Errorf("appels : %q", f.calls) + } +} + +func TestExecReportsAnErrorPrintedByVtysh(t *testing.T) { + f := &fakeVtysh{fail: map[string]string{"vni 4001": "% Please unconfigure l3vni 4001\n"}} + err := f.client().ApplyVRF(labVRF(t)) + if err == nil || !strings.Contains(err.Error(), "% Please unconfigure l3vni 4001") { + t.Errorf("erreur %v", err) + } + if f.sent("write memory") { + t.Error("configuration sauvegardée malgré l'erreur") + } +} + +func TestExecReportsANonZeroExit(t *testing.T) { + f := &fakeVtysh{ + fail: map[string]string{"write memory": "Building Configuration...\n"}, + status: map[string]error{"write memory": errors.New("exit status 1")}, + } + err := f.client().ApplyVRF(labVRF(t)) + if err == nil || !strings.Contains(err.Error(), "exit status 1") || !strings.Contains(err.Error(), "Building Configuration...") { + t.Errorf("erreur %v", err) + } +} + +func TestWithdrawVRFRunsEachStepInOrderThenSaves(t *testing.T) { + f := &fakeVtysh{running: []string{fullRunning, otherOnlyRunning}} + if err := f.client().WithdrawVRF(labVRF(t)); err != nil { + t.Fatal(err) + } + want := [][]string{ + {"show running-config"}, + {"configure terminal", "router bgp 64600 vrf vrf-a1b2", "address-family ipv4 unicast", "no redistribute kernel"}, + {"configure terminal", "vrf vrf-a1b2", "no vni 4001"}, + {"configure terminal", "no router bgp 64600 vrf vrf-a1b2"}, + {"configure terminal", "no route-map EXPORT-a1b2"}, + {"configure terminal", "no ip prefix-list PL-a1b2"}, + {"write memory"}, + {"show running-config"}, + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want) + } +} + +func TestWithdrawVRFStopsAtTheFirstRefusedStep(t *testing.T) { + f := &fakeVtysh{ + running: []string{fullRunning}, + fail: map[string]string{"no vni 4001": "% Please unconfigure l3vni 4001\n"}, + } + if err := f.client().WithdrawVRF(labVRF(t)); err == nil || !strings.Contains(err.Error(), "Please unconfigure") { + t.Errorf("erreur %v", err) + } + if f.sent("no router bgp 64600 vrf vrf-a1b2") { + t.Error("instance BGP supprimée après un refus de l'étape précédente") + } +} + +func TestWithdrawVRFFailsWhenFRRKeptTheConfiguration(t *testing.T) { + f := &fakeVtysh{running: []string{fullRunning, fullRunning}} + err := f.client().WithdrawVRF(labVRF(t)) + if err == nil || !strings.Contains(err.Error(), "configuration still present after teardown") || !strings.Contains(err.Error(), "no router bgp 64600 vrf vrf-a1b2") { + t.Errorf("erreur %v", err) + } +} + +func TestRemoveVRFDeletesTheVRFAndChecksNothingIsLeft(t *testing.T) { + gone := "vrf vrf-a1b2c\n vni 4002\nexit-vrf\n" + f := &fakeVtysh{running: []string{otherOnlyRunning, gone, gone}} + if err := f.client().RemoveVRF(labVRF(t)); err != nil { + t.Fatal(err) + } + if !f.sent("no vrf vrf-a1b2") { + t.Errorf("appels : %q", f.calls) + } +} + +func TestRemoveVRFFailsWhenRoutingConfigurationIsStillThere(t *testing.T) { + f := &fakeVtysh{running: []string{otherOnlyRunning, "", fullRunning}} + err := f.client().RemoveVRF(labVRF(t)) + if err == nil || !strings.Contains(err.Error(), "configuration still present after teardown") { + t.Errorf("erreur %v", err) + } +} + +func TestTeardownNeedsOnlyTheASNAndTheID(t *testing.T) { + f := &fakeVtysh{running: []string{"", ""}} + if err := f.client().WithdrawVRF(VRF{ASN: 64600, ID: "a1b2"}); err != nil { + t.Errorf("démontage refusé : %v", err) + } + f = &fakeVtysh{} + if err := f.client().WithdrawVRF(VRF{ASN: 64600}); err == nil || len(f.calls) != 0 { + t.Errorf("démontage sans id : erreur %v, appels %q", err, f.calls) + } +} diff --git a/internal/frr/config.go b/internal/frr/config.go new file mode 100644 index 0000000..10cdd4d --- /dev/null +++ b/internal/frr/config.go @@ -0,0 +1,129 @@ +package frr + +import ( + "fmt" + "net" +) + +const maxVNI = 1<<24 - 1 + +type Base struct { + ASN uint32 + RouterID net.IP + RouteReflectors []net.IP +} + +type VRF struct { + ASN uint32 + ID string + L3VNI uint32 + VTEP net.IP + Supernet *net.IPNet +} + +func (b Base) validate() error { + if b.ASN == 0 { + return fmt.Errorf("frr: asn must not be 0") + } + if b.RouterID.To4() == nil { + return fmt.Errorf("frr: router-id %v is not an IPv4 address", b.RouterID) + } + if len(b.RouteReflectors) == 0 { + return fmt.Errorf("frr: at least one route reflector is required") + } + for _, rr := range b.RouteReflectors { + if rr.To4() == nil { + return fmt.Errorf("frr: route reflector %v is not an IPv4 address", rr) + } + } + return nil +} + +func (b Base) commands() []string { + cmds := []string{ + fmt.Sprintf("router bgp %d", b.ASN), + fmt.Sprintf("bgp router-id %s", b.RouterID), + "no bgp default ipv4-unicast", + "neighbor fabric peer-group", + fmt.Sprintf("neighbor fabric remote-as %d", b.ASN), + "neighbor fabric capability extended-nexthop", + } + for _, rr := range b.RouteReflectors { + cmds = append(cmds, fmt.Sprintf("neighbor %s peer-group fabric", rr)) + } + return append(cmds, + "address-family l2vpn evpn", + "neighbor fabric activate", + "advertise-all-vni", + "exit-address-family", + "exit", + ) +} + +func (v VRF) Name() string { + return "vrf-" + v.ID +} + +func (v VRF) prefixList() string { + return "PL-" + v.ID +} + +func (v VRF) routeMap() string { + return "EXPORT-" + v.ID +} + +func (v VRF) bgpInstance() string { + return fmt.Sprintf("router bgp %d vrf %s", v.ASN, v.Name()) +} + +func (v VRF) validateName() error { + if v.ASN == 0 { + return fmt.Errorf("frr: asn must not be 0") + } + if v.ID == "" { + return fmt.Errorf("frr: vrf id must not be empty") + } + return nil +} + +func (v VRF) validate() error { + if err := v.validateName(); err != nil { + return err + } + if v.L3VNI == 0 || v.L3VNI > maxVNI { + return fmt.Errorf("frr: l3vni %d does not fit in 24 bits", v.L3VNI) + } + if v.VTEP.To4() == nil { + return fmt.Errorf("frr: vtep %v is not an IPv4 address", v.VTEP) + } + if v.Supernet == nil || v.Supernet.IP.To4() == nil { + return fmt.Errorf("frr: supernet %v is not an IPv4 prefix", v.Supernet) + } + return nil +} + +func (v VRF) commands() []string { + return []string{ + fmt.Sprintf("ip prefix-list %s seq 10 permit %s le 32", v.prefixList(), v.Supernet), + fmt.Sprintf("route-map %s permit 10", v.routeMap()), + fmt.Sprintf("match ip address prefix-list %s", v.prefixList()), + "exit", + fmt.Sprintf("route-map %s deny 20", v.routeMap()), + "exit", + fmt.Sprintf("vrf %s", v.Name()), + fmt.Sprintf("vni %d", v.L3VNI), + "exit-vrf", + v.bgpInstance(), + fmt.Sprintf("bgp router-id %s", v.VTEP), + "no bgp default ipv4-unicast", + "address-family ipv4 unicast", + fmt.Sprintf("redistribute kernel route-map %s", v.routeMap()), + "exit-address-family", + "address-family l2vpn evpn", + fmt.Sprintf("rd %s:%d", v.VTEP, v.L3VNI), + fmt.Sprintf("route-target both %d:%d", v.ASN, v.L3VNI), + "advertise ipv4 unicast", + "exit-address-family", + "exit", + } +} diff --git a/internal/frr/config_test.go b/internal/frr/config_test.go new file mode 100644 index 0000000..ac30a87 --- /dev/null +++ b/internal/frr/config_test.go @@ -0,0 +1,142 @@ +package frr + +import ( + "net" + "reflect" + "strings" + "testing" +) + +func mustCIDR(t *testing.T, s string) *net.IPNet { + t.Helper() + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n +} + +func labVRF(t *testing.T) VRF { + return VRF{ + ASN: 64600, + ID: "a1b2", + L3VNI: 4001, + VTEP: net.ParseIP("192.168.14.11"), + Supernet: mustCIDR(t, "10.10.0.0/16"), + } +} + +func TestBaseCommands(t *testing.T) { + b := Base{ + ASN: 64600, + RouterID: net.ParseIP("192.168.14.13"), + RouteReflectors: []net.IP{net.ParseIP("10.255.255.1"), net.ParseIP("10.255.255.2")}, + } + want := []string{ + "router bgp 64600", + "bgp router-id 192.168.14.13", + "no bgp default ipv4-unicast", + "neighbor fabric peer-group", + "neighbor fabric remote-as 64600", + "neighbor fabric capability extended-nexthop", + "neighbor 10.255.255.1 peer-group fabric", + "neighbor 10.255.255.2 peer-group fabric", + "address-family l2vpn evpn", + "neighbor fabric activate", + "advertise-all-vni", + "exit-address-family", + "exit", + } + if got := b.commands(); !reflect.DeepEqual(got, want) { + t.Errorf("commandes de base :\n%s\nattendu :\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +func TestVRFCommands(t *testing.T) { + want := []string{ + "ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32", + "route-map EXPORT-a1b2 permit 10", + "match ip address prefix-list PL-a1b2", + "exit", + "route-map EXPORT-a1b2 deny 20", + "exit", + "vrf vrf-a1b2", + "vni 4001", + "exit-vrf", + "router bgp 64600 vrf vrf-a1b2", + "bgp router-id 192.168.14.11", + "no bgp default ipv4-unicast", + "address-family ipv4 unicast", + "redistribute kernel route-map EXPORT-a1b2", + "exit-address-family", + "address-family l2vpn evpn", + "rd 192.168.14.11:4001", + "route-target both 64600:4001", + "advertise ipv4 unicast", + "exit-address-family", + "exit", + } + if got := labVRF(t).commands(); !reflect.DeepEqual(got, want) { + t.Errorf("commandes de VRF :\n%s\nattendu :\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +func TestBaseValidateRejects(t *testing.T) { + valid := func() Base { + return Base{ASN: 64600, RouterID: net.ParseIP("192.168.14.13"), RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}} + } + if err := valid().validate(); err != nil { + t.Fatalf("base valide refusée : %v", err) + } + cases := map[string]struct { + mutate func(*Base) + msg string + }{ + "asn nul": {func(b *Base) { b.ASN = 0 }, "asn must not be 0"}, + "router-id absent": {func(b *Base) { b.RouterID = nil }, "router-id is not an IPv4 address"}, + "router-id IPv6": {func(b *Base) { b.RouterID = net.ParseIP("fd00::1") }, "router-id fd00::1 is not an IPv4 address"}, + "aucun RR": {func(b *Base) { b.RouteReflectors = nil }, "at least one route reflector is required"}, + "RR IPv6": {func(b *Base) { b.RouteReflectors = append(b.RouteReflectors, net.ParseIP("fd00::2")) }, "route reflector fd00::2 is not an IPv4 address"}, + } + for name, c := range cases { + b := valid() + c.mutate(&b) + if err := b.validate(); err == nil || !strings.Contains(err.Error(), c.msg) { + t.Errorf("%s : erreur %v, « %s » attendu", name, err, c.msg) + } + } +} + +func TestVRFValidateRejects(t *testing.T) { + if err := labVRF(t).validate(); err != nil { + t.Fatalf("VRF valide refusée : %v", err) + } + cases := map[string]struct { + mutate func(*VRF) + msg string + }{ + "asn nul": {func(v *VRF) { v.ASN = 0 }, "asn must not be 0"}, + "id vide": {func(v *VRF) { v.ID = "" }, "vrf id must not be empty"}, + "l3vni nul": {func(v *VRF) { v.L3VNI = 0 }, "l3vni 0 does not fit in 24 bits"}, + "l3vni sur 25 bits": {func(v *VRF) { v.L3VNI = 16777216 }, "l3vni 16777216 does not fit in 24 bits"}, + "vtep absent": {func(v *VRF) { v.VTEP = nil }, "vtep is not an IPv4 address"}, + "vtep IPv6": {func(v *VRF) { v.VTEP = net.ParseIP("fd00::1") }, "vtep fd00::1 is not an IPv4 address"}, + "supernet absent": {func(v *VRF) { v.Supernet = nil }, "supernet is not an IPv4 prefix"}, + "supernet IPv6": {func(v *VRF) { v.Supernet = mustCIDR(t, "fd00::/48") }, "supernet fd00::/48 is not an IPv4 prefix"}, + } + for name, c := range cases { + v := labVRF(t) + c.mutate(&v) + if err := v.validate(); err == nil || !strings.Contains(err.Error(), c.msg) { + t.Errorf("%s : erreur %v, « %s » attendu", name, err, c.msg) + } + } +} + +func TestVRFAcceptsTheLargest24BitVNI(t *testing.T) { + v := labVRF(t) + v.L3VNI = 16777215 + if err := v.validate(); err != nil { + t.Errorf("l3vni 16777215 refusée : %v", err) + } +} diff --git a/internal/frr/teardown.go b/internal/frr/teardown.go new file mode 100644 index 0000000..bcdd18d --- /dev/null +++ b/internal/frr/teardown.go @@ -0,0 +1,84 @@ +package frr + +import ( + "fmt" + "strings" +) + +type step []string + +func routingTeardown(v VRF, running string) []step { + lines := strings.Split(running, "\n") + bgp := hasLine(lines, v.bgpInstance()) + var steps []step + if bgp { + steps = append(steps, step{v.bgpInstance(), "address-family ipv4 unicast", "no redistribute kernel"}) + } + if vni := vrfVNI(lines, v.Name()); vni != "" { + steps = append(steps, step{"vrf " + v.Name(), "no vni " + vni}) + } + if bgp { + steps = append(steps, step{"no " + v.bgpInstance()}) + } + if hasPrefix(lines, "route-map "+v.routeMap()+" ") { + steps = append(steps, step{"no route-map " + v.routeMap()}) + } + if hasPrefix(lines, "ip prefix-list "+v.prefixList()+" ") { + steps = append(steps, step{"no ip prefix-list " + v.prefixList()}) + } + return steps +} + +func vrfTeardown(v VRF, running string) []step { + if hasLine(strings.Split(running, "\n"), "vrf "+v.Name()) { + return []step{{"no vrf " + v.Name()}} + } + return nil +} + +func hasLine(lines []string, want string) bool { + for _, l := range lines { + if strings.TrimRight(l, " \r") == want { + return true + } + } + return false +} + +func hasPrefix(lines []string, prefix string) bool { + for _, l := range lines { + if strings.HasPrefix(l, prefix) { + return true + } + } + return false +} + +func vrfVNI(lines []string, name string) string { + in := false + for _, l := range lines { + l = strings.TrimRight(l, " \r") + switch { + case l == "vrf "+name: + in = true + case in && l == "exit-vrf": + return "" + case in: + if f := strings.Fields(l); len(f) >= 2 && f[0] == "vni" { + return f[1] + } + } + } + return "" +} + +func remaining(steps []step) error { + if len(steps) == 0 { + return nil + } + var parts []string + for _, s := range steps { + parts = append(parts, strings.Join(s, " / ")) + } + return fmt.Errorf("configuration still present after teardown: %s", strings.Join(parts, "; ")) +} diff --git a/internal/frr/teardown_test.go b/internal/frr/teardown_test.go new file mode 100644 index 0000000..cd27049 --- /dev/null +++ b/internal/frr/teardown_test.go @@ -0,0 +1,142 @@ +package frr + +import ( + "reflect" + "testing" +) + +const fullRunning = `frr version 10.7.1 +frr defaults traditional +hostname hv1 +! +vrf vrf-a1b2 + vni 4001 +exit-vrf +! +vrf vrf-a1b2c + vni 4002 +exit-vrf +! +router bgp 64600 + bgp router-id 192.168.14.11 + no bgp default ipv4-unicast + neighbor fabric peer-group + neighbor fabric remote-as 64600 + neighbor 10.255.255.1 peer-group fabric + ! + address-family l2vpn evpn + neighbor fabric activate + advertise-all-vni + exit-address-family +exit +! +router bgp 64600 vrf vrf-a1b2 + bgp router-id 192.168.14.11 + no bgp default ipv4-unicast + ! + address-family ipv4 unicast + redistribute kernel route-map EXPORT-a1b2 + exit-address-family + ! + address-family l2vpn evpn + advertise ipv4 unicast + rd 192.168.14.11:4001 + route-target import 64600:4001 + route-target export 64600:4001 + exit-address-family +exit +! +router bgp 64600 vrf vrf-a1b2c + bgp router-id 192.168.14.11 +exit +! +ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32 +ip prefix-list PL-a1b2c seq 10 permit 10.20.0.0/16 le 32 +! +route-map EXPORT-a1b2 permit 10 + match ip address prefix-list PL-a1b2 +exit +! +route-map EXPORT-a1b2 deny 20 +exit +! +route-map EXPORT-a1b2c permit 10 +exit +! +end +` + +const otherOnlyRunning = `vrf vrf-a1b2 +exit-vrf +! +vrf vrf-a1b2c + vni 4002 +exit-vrf +! +router bgp 64600 vrf vrf-a1b2c +exit +! +ip prefix-list PL-a1b2c seq 10 permit 10.20.0.0/16 le 32 +! +route-map EXPORT-a1b2c permit 10 +exit +! +end +` + +func TestRoutingTeardownFollowsTheOrderFRRRequires(t *testing.T) { + want := []step{ + {"router bgp 64600 vrf vrf-a1b2", "address-family ipv4 unicast", "no redistribute kernel"}, + {"vrf vrf-a1b2", "no vni 4001"}, + {"no router bgp 64600 vrf vrf-a1b2"}, + {"no route-map EXPORT-a1b2"}, + {"no ip prefix-list PL-a1b2"}, + } + if got := routingTeardown(labVRF(t), fullRunning); !reflect.DeepEqual(got, want) { + t.Errorf("démontage :\n%q\nattendu :\n%q", got, want) + } +} + +func TestRoutingTeardownIgnoresAVRFWhoseNameExtendsOurs(t *testing.T) { + if got := routingTeardown(labVRF(t), otherOnlyRunning); got != nil { + t.Errorf("étapes visant vrf-a1b2c : %q", got) + } +} + +func TestRoutingTeardownUsesTheVNIFoundInFRR(t *testing.T) { + v := labVRF(t) + v.L3VNI = 9999 + running := "vrf vrf-a1b2\n vni 4001\nexit-vrf\n" + want := []step{{"vrf vrf-a1b2", "no vni 4001"}} + if got := routingTeardown(v, running); !reflect.DeepEqual(got, want) { + t.Errorf("démontage : %q, %q attendu", got, want) + } +} + +func TestRoutingTeardownSkipsWhatIsAbsent(t *testing.T) { + running := "vrf vrf-a1b2\n vni 4001\nexit-vrf\n!\nip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32\n" + want := []step{ + {"vrf vrf-a1b2", "no vni 4001"}, + {"no ip prefix-list PL-a1b2"}, + } + if got := routingTeardown(labVRF(t), running); !reflect.DeepEqual(got, want) { + t.Errorf("démontage :\n%q\nattendu :\n%q", got, want) + } +} + +func TestRoutingTeardownDoesNotReadAVNIOutsideTheVRFBlock(t *testing.T) { + running := "vrf vrf-a1b2\nexit-vrf\n!\nvrf vrf-a1b2c\n vni 4002\nexit-vrf\n" + if got := routingTeardown(labVRF(t), running); got != nil { + t.Errorf("étapes : %q", got) + } +} + +func TestVRFTeardown(t *testing.T) { + want := []step{{"no vrf vrf-a1b2"}} + if got := vrfTeardown(labVRF(t), otherOnlyRunning); !reflect.DeepEqual(got, want) { + t.Errorf("démontage : %q, %q attendu", got, want) + } + if got := vrfTeardown(labVRF(t), "vrf vrf-a1b2c\nexit-vrf\n"); got != nil { + t.Errorf("étapes visant vrf-a1b2c : %q", got) + } +} diff --git a/internal/netif/addr.go b/internal/netif/addr.go index b45b6aa..cd5dd74 100644 --- a/internal/netif/addr.go +++ b/internal/netif/addr.go @@ -19,3 +19,11 @@ func AddrAdd(iface string, ip net.IP) error { } return netlink.AddrAdd(link, addr) } + +func AddrAddPrefix(iface string, prefix *net.IPNet) error { + link, err := netlink.LinkByName(iface) + if err != nil { + return err + } + return netlink.AddrAdd(link, &netlink.Addr{IPNet: prefix}) +} diff --git a/internal/netif/route.go b/internal/netif/route.go new file mode 100644 index 0000000..f10b159 --- /dev/null +++ b/internal/netif/route.go @@ -0,0 +1,27 @@ +package netif + +import ( + "net" + + "github.com/vishvananda/netlink" +) + +func RouteReplaceVia(table int, dst *net.IPNet, gw net.IP, iface string) error { + link, err := netlink.LinkByName(iface) + if err != nil { + return err + } + return netlink.RouteReplace(&netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: dst, + Gw: gw, + Table: table, + }) +} + +func RouteDel(table int, dst *net.IPNet) error { + return netlink.RouteDel(&netlink.Route{ + Dst: dst, + Table: table, + }) +} diff --git a/internal/netif/vrf.go b/internal/netif/vrf.go new file mode 100644 index 0000000..8d8e47e --- /dev/null +++ b/internal/netif/vrf.go @@ -0,0 +1,16 @@ +package netif + +import ( + "github.com/vishvananda/netlink" +) + +func CreateVRF(name string, table uint32) error { + vrf := &netlink.Vrf{ + LinkAttrs: netlink.LinkAttrs{Name: name}, + Table: table, + } + if err := netlink.LinkAdd(vrf); err != nil { + return err + } + return netlink.LinkSetUp(vrf) +} diff --git a/internal/sysctl/sysctl.go b/internal/sysctl/sysctl.go new file mode 100644 index 0000000..50f0a1a --- /dev/null +++ b/internal/sysctl/sysctl.go @@ -0,0 +1,23 @@ +package sysctl + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +var root = "/proc/sys" + +func Set(key, value string) error { + path := filepath.Join(root, strings.ReplaceAll(key, ".", "/")) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) + if err != nil { + return fmt.Errorf("sysctl %s: %w", key, err) + } + if _, err := f.WriteString(value); err != nil { + f.Close() + return fmt.Errorf("sysctl %s=%s: %w", key, value, err) + } + return f.Close() +} diff --git a/internal/sysctl/sysctl_test.go b/internal/sysctl/sysctl_test.go new file mode 100644 index 0000000..e07674f --- /dev/null +++ b/internal/sysctl/sysctl_test.go @@ -0,0 +1,48 @@ +package sysctl + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func fakeRoot(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "net", "ipv4"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "net", "ipv4", "ip_forward"), []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + old := root + root = dir + t.Cleanup(func() { root = old }) + return dir +} + +func TestSetWritesTheValueAtTheKeyPath(t *testing.T) { + dir := fakeRoot(t) + if err := Set("net.ipv4.ip_forward", "1"); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dir, "net", "ipv4", "ip_forward")) + if err != nil { + t.Fatal(err) + } + if string(got) != "1" { + t.Errorf("contenu %q, \"1\" attendu", got) + } +} + +func TestSetRefusesAnUnknownKeyWithoutCreatingIt(t *testing.T) { + dir := fakeRoot(t) + err := Set("net.ipv4.ip_forwarding", "1") + if err == nil || !strings.Contains(err.Error(), "sysctl net.ipv4.ip_forwarding") { + t.Errorf("erreur %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "net", "ipv4", "ip_forwarding")); !os.IsNotExist(err) { + t.Errorf("clé inconnue créée : %v", err) + } +} From bf0e83862fb7123aa272f9e49583f250edcb3649 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 19:52:46 +0200 Subject: [PATCH 3/9] =?UTF-8?q?f-41:=20config,=20frr:=20base=20BGP=20pilot?= =?UTF-8?q?=C3=A9e=20par=20l'agent=20au=20d=C3=A9marrage=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- cmd/agent/main.go | 26 +++++++ cmd/agent/main_test.go | 21 ++++++ conf/agent/config.exemple.yml | 9 +++ docs/deploiement/premier-hyperviseur.rst | 14 +++- docs/exploitation/configuration.rst | 32 +++++++++ internal/config/agent/bgp.go | 38 ++++++++++ internal/config/agent/bgp_test.go | 54 ++++++++++++++ internal/config/agent/struct.go | 1 + internal/frr/client.go | 17 +++++ internal/frr/client_test.go | 89 +++++++++++++++++++++--- internal/frr/config.go | 26 +++++++ internal/netif/vxlan.go | 22 ++++-- 12 files changed, 331 insertions(+), 18 deletions(-) create mode 100644 internal/config/agent/bgp.go create mode 100644 internal/config/agent/bgp_test.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index b7ecc32..e824dbc 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -13,7 +13,9 @@ import ( agentapi "git.g3e.fr/syonad/two/internal/api/agent" configuration "git.g3e.fr/syonad/two/internal/config/agent" dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent" + "git.g3e.fr/syonad/two/internal/frr" "git.g3e.fr/syonad/two/internal/migration" + "git.g3e.fr/syonad/two/internal/netif" agentmetrics "git.g3e.fr/syonad/two/internal/prometheus/agent" "git.g3e.fr/syonad/two/internal/watchdog" "git.g3e.fr/syonad/two/internal/watchdog/notify" @@ -58,6 +60,14 @@ func main() { return } + if cfg.BGP.Enabled() { + if err := applyBGPBase(cfg, frr.NewClient()); err != nil { + log.Error("failed to apply the bgp base configuration", "error", err) + return + } + log.Info("bgp base configuration applied", "asn", cfg.BGP.ASN, "route_reflectors", cfg.BGP.RouteReflectors) + } + q := worker.New(cfg.Worker.BufferSize) q.Start(cfg.Worker.Count) @@ -110,6 +120,22 @@ func main() { closeDB = shutdown(log, q, servers, shutdownTimeout) } +type baseApplier interface { + ApplyBase(frr.Base) error +} + +func applyBGPBase(cfg *configuration.Config, frrClient baseApplier) error { + reflectors, err := cfg.BGP.Reflectors() + if err != nil { + return err + } + vtep, err := netif.VTEPAddress(cfg.DefaultInterface) + if err != nil { + return err + } + return frrClient.ApplyBase(frr.Base{ASN: cfg.BGP.ASN, RouterID: vtep, RouteReflectors: reflectors}) +} + type httpShutdowner interface { Shutdown(context.Context) error } diff --git a/cmd/agent/main_test.go b/cmd/agent/main_test.go index 12291c7..302d0e7 100644 --- a/cmd/agent/main_test.go +++ b/cmd/agent/main_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" "git.g3e.fr/syonad/two/pkg/worker" ) @@ -96,3 +98,22 @@ func TestShutdown_SansServeur(t *testing.T) { t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre") } } + +type fakeBase struct{ called bool } + +func (f *fakeBase) ApplyBase(frr.Base) error { + f.called = true + return nil +} + +func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) { + cfg := &configuration.Config{} + cfg.BGP.RouteReflectors = []string{"10.255.255.1"} + f := &fakeBase{} + if err := applyBGPBase(cfg, f); err == nil || !strings.Contains(err.Error(), "bgp.asn must be set") { + t.Errorf("erreur %v", err) + } + if f.called { + t.Error("ApplyBase appelé avec une section bgp invalide") + } +} diff --git a/conf/agent/config.exemple.yml b/conf/agent/config.exemple.yml index de0f6ae..a31a58d 100644 --- a/conf/agent/config.exemple.yml +++ b/conf/agent/config.exemple.yml @@ -35,6 +35,15 @@ interfaces: internet: br-000000 admin: br-000000 +# BGP control plane of the hypervisor (optional). +# Absent: the agent leaves FRR alone. Present: at every start, before serving its +# API, the agent applies the base BGP configuration through vtysh (router-id = the +# primary IPv4 of default_interface) and saves it with "write memory". +# Changing the asn requires removing the previous instance by hand. +# bgp: +# asn: 64600 +# route_reflectors: [10.255.255.1] + # Metadata server runtime directory (cloud-init files per VM) metadata: run_dir: "/run/two/metadata" diff --git a/docs/deploiement/premier-hyperviseur.rst b/docs/deploiement/premier-hyperviseur.rst index dc480fe..f44260e 100644 --- a/docs/deploiement/premier-hyperviseur.rst +++ b/docs/deploiement/premier-hyperviseur.rst @@ -27,8 +27,18 @@ l'agent désactive l'apprentissage et ne configure aucun voisin — cf. Configuration de FRR ~~~~~~~~~~~~~~~~~~~~ -Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. D'un hyperviseur à l'autre ne changent que -``hostname`` et ``router-id``, l'adresse de l'hyperviseur sur son segment. +Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. Le reste est posé par l'agent, à chaque +démarrage, à partir de la section ``bgp`` de ``agent.yml`` (:doc:`/exploitation/configuration`) : + +.. code-block:: yaml + + bgp: + asn: 64600 + route_reflectors: [10.255.255.1] + +L'agent en déduit la configuration ci-dessous ; le ``router-id`` est l'adresse IPv4 primaire de +``default_interface``, la seule valeur qui change d'un hyperviseur à l'autre. ``hostname`` et +``log`` ne sont pas gérés par l'agent. .. literalinclude:: ../../test/e2e/topologies/frr/hv1.conf :language: text diff --git a/docs/exploitation/configuration.rst b/docs/exploitation/configuration.rst index 7024527..8c364b0 100644 --- a/docs/exploitation/configuration.rst +++ b/docs/exploitation/configuration.rst @@ -104,6 +104,38 @@ Traduit les clés logiques ``iface_type`` de l'API vers les bridges physiques de inconnue ou omise retombe silencieusement sur ``default_interface`` — ce n'est pas une erreur, mais c'est une source de subnets branchés au mauvais endroit sans le dire. +Plan de contrôle BGP +-------------------- + +.. code-block:: yaml + + bgp: + asn: 64600 + route_reflectors: [10.255.255.1] + +Section **facultative**. Absente, l'agent ne touche pas à FRR : sa configuration reste à poser +à la main, comme avant. Présente, l'agent **pilote la configuration BGP de l'hyperviseur** : +à chaque démarrage, avant d'ouvrir son API, il applique par ``vtysh`` la base décrite dans +:doc:`/deploiement/premier-hyperviseur`, puis l'enregistre par ``write memory``. + +* ``asn`` — l'AS de l'hyperviseur, commun à tous les hyperviseurs du cluster (iBGP vers le + route reflector). Obligatoire dès que la section existe. +* ``route_reflectors`` — au moins une adresse IPv4, sans doublon. Un route reflector retiré de + la liste est retiré de FRR au démarrage suivant. + +Le ``router-id`` n'est pas configurable : c'est l'adresse IPv4 primaire de +``default_interface``, celle qui sert de VTEP. + +Le démarrage **échoue** si la section est incomplète, si ``default_interface`` n'a pas +d'adresse IPv4 primaire, si ``vtysh`` refuse une commande, ou si FRR tourne déjà sous un autre +AS : changer d'``asn`` suppose de retirer l'ancienne instance à la main. + +.. warning:: + + ``/etc/frr/frr.conf`` devient un fichier **généré** : ``write memory`` le réécrit à partir de + la configuration en mémoire de FRR. Une modification faite dans le fichier sans être + rechargée est perdue au prochain démarrage de l'agent. + Metadata et QEMU ---------------- diff --git a/internal/config/agent/bgp.go b/internal/config/agent/bgp.go new file mode 100644 index 0000000..e79cf54 --- /dev/null +++ b/internal/config/agent/bgp.go @@ -0,0 +1,38 @@ +package configuration + +import ( + "fmt" + "net" +) + +type BGP struct { + ASN uint32 `mapstructure:"asn"` + RouteReflectors []string `mapstructure:"route_reflectors"` +} + +func (b BGP) Enabled() bool { + return b.ASN != 0 || len(b.RouteReflectors) > 0 +} + +func (b BGP) Reflectors() ([]net.IP, error) { + if b.ASN == 0 { + return nil, fmt.Errorf("bgp.asn must be set") + } + if len(b.RouteReflectors) == 0 { + return nil, fmt.Errorf("bgp.route_reflectors must list at least one address") + } + seen := make(map[string]bool, len(b.RouteReflectors)) + ips := make([]net.IP, 0, len(b.RouteReflectors)) + for _, rr := range b.RouteReflectors { + ip := net.ParseIP(rr).To4() + if ip == nil { + return nil, fmt.Errorf("bgp.route_reflectors: %q is not an IPv4 address", rr) + } + if seen[ip.String()] { + return nil, fmt.Errorf("bgp.route_reflectors: %s is listed twice", ip) + } + seen[ip.String()] = true + ips = append(ips, ip) + } + return ips, nil +} diff --git a/internal/config/agent/bgp_test.go b/internal/config/agent/bgp_test.go new file mode 100644 index 0000000..4004d0a --- /dev/null +++ b/internal/config/agent/bgp_test.go @@ -0,0 +1,54 @@ +package configuration + +import ( + "strings" + "testing" +) + +func TestLoadConfig_BGPAbsentIsDisabled(t *testing.T) { + cfg, err := LoadConfig(writeConfig(t, "database:\n path: /tmp/two\n")) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if cfg.BGP.Enabled() { + t.Errorf("bgp enabled without a bgp section: %+v", cfg.BGP) + } +} + +func TestLoadConfig_ReadsTheBGPSection(t *testing.T) { + cfg, err := LoadConfig(writeConfig(t, "bgp:\n asn: 4200000000\n route_reflectors: [10.255.255.1, 10.255.255.2]\n")) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if !cfg.BGP.Enabled() || cfg.BGP.ASN != 4200000000 { + t.Fatalf("bgp = %+v", cfg.BGP) + } + rrs, err := cfg.BGP.Reflectors() + if err != nil { + t.Fatalf("Reflectors: %v", err) + } + if len(rrs) != 2 || rrs[0].String() != "10.255.255.1" || rrs[1].String() != "10.255.255.2" { + t.Errorf("reflectors = %v", rrs) + } +} + +func TestBGPReflectors_RejectsAnIncompleteOrInvalidSection(t *testing.T) { + cases := []struct { + bgp BGP + want string + }{ + {BGP{RouteReflectors: []string{"10.255.255.1"}}, "bgp.asn must be set"}, + {BGP{ASN: 64600}, "at least one address"}, + {BGP{ASN: 64600, RouteReflectors: []string{"rr1"}}, `"rr1" is not an IPv4 address`}, + {BGP{ASN: 64600, RouteReflectors: []string{"fd00::1"}}, `"fd00::1" is not an IPv4 address`}, + {BGP{ASN: 64600, RouteReflectors: []string{"10.255.255.1", "10.255.255.1"}}, "10.255.255.1 is listed twice"}, + } + for _, c := range cases { + if !c.bgp.Enabled() { + t.Errorf("%+v: Enabled() = false", c.bgp) + } + if _, err := c.bgp.Reflectors(); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%+v: erreur %v, attendu %q", c.bgp, err, c.want) + } + } +} diff --git a/internal/config/agent/struct.go b/internal/config/agent/struct.go index 0f84a09..17ddd43 100644 --- a/internal/config/agent/struct.go +++ b/internal/config/agent/struct.go @@ -55,6 +55,7 @@ type Config struct { Enabled bool `mapstructure:"enabled"` IntervalSeconds int `mapstructure:"interval_seconds"` } `mapstructure:"watchdog"` + BGP BGP `mapstructure:"bgp"` DefaultInterface string `mapstructure:"default_interface"` Interfaces map[string]string `mapstructure:"interfaces"` } diff --git a/internal/frr/client.go b/internal/frr/client.go index 1f76d4e..04b2ea8 100644 --- a/internal/frr/client.go +++ b/internal/frr/client.go @@ -22,9 +22,26 @@ func (c *Client) ApplyBase(b Base) error { if err := b.validate(); err != nil { return err } + running, err := c.runningConfig() + if err != nil { + return err + } + stale, err := b.staleReflectors(running) + if err != nil { + return err + } if err := c.configure(b.commands()...); err != nil { return err } + if len(stale) > 0 { + cmds := []string{fmt.Sprintf("router bgp %d", b.ASN)} + for _, rr := range stale { + cmds = append(cmds, "no neighbor "+rr) + } + if err := c.configure(append(cmds, "exit")...); err != nil { + return err + } + } return c.save() } diff --git a/internal/frr/client_test.go b/internal/frr/client_test.go index d48ba4d..96050be 100644 --- a/internal/frr/client_test.go +++ b/internal/frr/client_test.go @@ -79,17 +79,6 @@ func TestApplyVRFRefusesAnInvalidVRFWithoutCallingVtysh(t *testing.T) { } } -func TestApplyBaseConfiguresThenSaves(t *testing.T) { - f := &fakeVtysh{} - b := Base{ASN: 64600, RouterID: labVRF(t).VTEP, RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}} - if err := f.client().ApplyBase(b); err != nil { - t.Fatal(err) - } - if len(f.calls) != 2 || f.calls[0][0] != "configure terminal" || f.calls[0][1] != "router bgp 64600" || f.calls[1][0] != "write memory" { - t.Errorf("appels : %q", f.calls) - } -} - func TestExecReportsAnErrorPrintedByVtysh(t *testing.T) { f := &fakeVtysh{fail: map[string]string{"vni 4001": "% Please unconfigure l3vni 4001\n"}} err := f.client().ApplyVRF(labVRF(t)) @@ -182,3 +171,81 @@ func TestTeardownNeedsOnlyTheASNAndTheID(t *testing.T) { t.Errorf("démontage sans id : erreur %v, appels %q", err, f.calls) } } + +const baseRunning = `frr version 10.7.1 +router bgp 64600 + bgp router-id 192.168.14.13 + neighbor fabric peer-group + neighbor fabric remote-as 64600 + neighbor 10.255.255.1 peer-group fabric + neighbor 10.255.255.9 peer-group fabric + neighbor 10.0.0.5 peer-group monitoring + ! + address-family l2vpn evpn + neighbor fabric activate + advertise-all-vni + exit-address-family +exit +! +router bgp 64600 vrf vrf-a1b2 + neighbor 10.255.255.7 peer-group fabric +exit +` + +func labBase() Base { + return Base{ASN: 64600, RouterID: net.ParseIP("192.168.14.13"), RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}} +} + +func TestApplyBaseOnAFreshFRRConfiguresThenSaves(t *testing.T) { + f := &fakeVtysh{running: []string{""}} + if err := f.client().ApplyBase(labBase()); err != nil { + t.Fatal(err) + } + want := [][]string{ + {"show running-config"}, + append([]string{"configure terminal"}, labBase().commands()...), + {"write memory"}, + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want) + } +} + +func TestApplyBaseRemovesReflectorsNoLongerConfiguredAfterAddingTheNewOnes(t *testing.T) { + f := &fakeVtysh{running: []string{baseRunning}} + if err := f.client().ApplyBase(labBase()); err != nil { + t.Fatal(err) + } + want := [][]string{ + {"show running-config"}, + append([]string{"configure terminal"}, labBase().commands()...), + {"configure terminal", "router bgp 64600", "no neighbor 10.255.255.9", "exit"}, + {"write memory"}, + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want) + } +} + +func TestApplyBaseRefusesToChangeTheASN(t *testing.T) { + f := &fakeVtysh{running: []string{baseRunning}} + b := labBase() + b.ASN = 64601 + err := f.client().ApplyBase(b) + if err == nil || !strings.Contains(err.Error(), "bgp already runs as AS 64600, not 64601") { + t.Errorf("erreur %v", err) + } + if len(f.calls) != 1 { + t.Errorf("appels après le refus : %q", f.calls) + } +} + +func TestApplyBaseStopsWhenTheRunningConfigIsUnreadable(t *testing.T) { + f := &fakeVtysh{} + if err := f.client().ApplyBase(labBase()); err == nil { + t.Fatal("aucune erreur") + } + if f.sent("write memory") || f.sent("router bgp 64600") { + t.Errorf("appels : %q", f.calls) + } +} diff --git a/internal/frr/config.go b/internal/frr/config.go index 10cdd4d..2ebad5e 100644 --- a/internal/frr/config.go +++ b/internal/frr/config.go @@ -3,6 +3,7 @@ package frr import ( "fmt" "net" + "strings" ) const maxVNI = 1<<24 - 1 @@ -60,6 +61,31 @@ func (b Base) commands() []string { ) } +func (b Base) staleReflectors(running string) ([]string, error) { + want := make(map[string]bool, len(b.RouteReflectors)) + for _, rr := range b.RouteReflectors { + want[rr.String()] = true + } + var stale []string + in := false + for _, l := range strings.Split(running, "\n") { + l = strings.TrimRight(l, " \r") + f := strings.Fields(l) + switch { + case len(f) == 3 && f[0] == "router" && f[1] == "bgp": + if f[2] != fmt.Sprint(b.ASN) { + return nil, fmt.Errorf("frr: bgp already runs as AS %s, not %d: changing the asn requires removing that instance by hand", f[2], b.ASN) + } + in = true + case in && l == "exit": + in = false + case in && len(f) == 4 && f[0] == "neighbor" && f[2] == "peer-group" && f[3] == "fabric" && !want[f[1]]: + stale = append(stale, f[1]) + } + } + return stale, nil +} + func (v VRF) Name() string { return "vrf-" + v.ID } diff --git a/internal/netif/vxlan.go b/internal/netif/vxlan.go index 0930cb3..81ec3a2 100644 --- a/internal/netif/vxlan.go +++ b/internal/netif/vxlan.go @@ -18,11 +18,7 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error { if err != nil { return err } - addrs, err := netlink.AddrList(link, syscall.AF_INET) - if err != nil { - return err - } - local, err := vtepAddress(localIface, addrs) + local, err := linkVTEPAddress(link) if err != nil { return err } @@ -40,6 +36,22 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error { return netlink.LinkAdd(vxlan) } +func VTEPAddress(iface string) (net.IP, error) { + link, err := netlink.LinkByName(iface) + if err != nil { + return nil, err + } + return linkVTEPAddress(link) +} + +func linkVTEPAddress(link netlink.Link) (net.IP, error) { + addrs, err := netlink.AddrList(link, syscall.AF_INET) + if err != nil { + return nil, err + } + return vtepAddress(link.Attrs().Name, addrs) +} + func vtepAddress(iface string, addrs []netlink.Addr) (net.IP, error) { for _, a := range addrs { if a.IPNet == nil || a.IP.To4() == nil || a.Scope != scopeUniverse || a.Flags&addressSecondary != 0 { From 3dd94f35dd07c20f4fd8bfc11a989d720c83b131 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:05:20 +0200 Subject: [PATCH 4/9] f-41: api, vpc: champ l3vni des VPC #41 Signed-off-by: GnomeZworc --- api/agent.yaml | 19 ++++- docs/architecture/stockage.rst | 1 + internal/api/agent/helpers_test.go | 6 +- internal/api/agent/models.go | 6 +- internal/api/agent/vpc.go | 3 +- internal/api/agent/vpc_test.go | 84 +++++++++++++++++++ internal/api/agent/vpcs.go | 19 ++++- internal/dispatcher/agent/vpc_commands.go | 43 +++++++++- .../dispatcher/agent/vpc_commands_test.go | 80 ++++++++++++++++++ internal/vpc/l3vni.go | 15 ++++ internal/vpc/l3vni_test.go | 29 +++++++ 11 files changed, 295 insertions(+), 10 deletions(-) create mode 100644 internal/vpc/l3vni.go create mode 100644 internal/vpc/l3vni_test.go diff --git a/api/agent.yaml b/api/agent.yaml index fb78148..97d8ef6 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -45,13 +45,13 @@ paths: schema: $ref: "#/components/schemas/VPC" "400": - description: Missing required field or invalid request body + description: Missing required field, invalid request body, or l3vni out of range content: application/json: schema: $ref: "#/components/schemas/Error" "409": - description: VPC already exists + description: VPC already exists, l3vni already used by another VPC, or l3vni requested on an agent without a bgp section content: application/json: schema: @@ -317,6 +317,17 @@ components: type: string description: CIDR block for the entire VPC address space example: "10.0.0.0/16" + l3vni: + type: integer + format: int32 + minimum: 1 + maximum: 16777215 + description: >- + Optional. L3VNI of the VPC, used for inter-subnet routing between hypervisors + (symmetric IRB). Also the id of the routing table of the VPC VRF, so 253-255 are + rejected. Requires the bgp section in the agent configuration, and must not be used + by another VPC of the agent. Absent: the VPC has no L3VNI. + example: 4001 VPC: type: object @@ -331,6 +342,10 @@ components: cidr: type: string example: "10.0.0.0/16" + l3vni: + type: integer + description: L3VNI of the VPC, absent when the VPC has none + example: 4001 SubnetCreateRequest: type: object diff --git a/docs/architecture/stockage.rst b/docs/architecture/stockage.rst index fd404a2..02f03a3 100644 --- a/docs/architecture/stockage.rst +++ b/docs/architecture/stockage.rst @@ -8,6 +8,7 @@ de sérialisation structurée. vpc//state → creating | running | error | deleting | deleted vpc//cidr → + vpc//l3vni → (optionnel : VPC routé par L3VNI) subnet//state → creating | running | error | deleting | deleted subnet//vpc → diff --git a/internal/api/agent/helpers_test.go b/internal/api/agent/helpers_test.go index c7162c7..69adfb2 100644 --- a/internal/api/agent/helpers_test.go +++ b/internal/api/agent/helpers_test.go @@ -16,11 +16,15 @@ import ( // The worker queue is buffered but has no running goroutines: Dispatch enqueues // without blocking and Execute never runs, so DB state reflects only Prepare writes. func newTestServer(t *testing.T) (*Server, *badger.DB) { + t.Helper() + return newTestServerWith(t, &configuration.Config{DefaultInterface: "br-test"}) +} + +func newTestServerWith(t *testing.T, cfg *configuration.Config) (*Server, *badger.DB) { t.Helper() db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) t.Cleanup(func() { db.Close() }) q := worker.New(100) - cfg := &configuration.Config{DefaultInterface: "br-test"} logger := slog.New(slog.NewTextHandler(io.Discard, nil)) d := dispatcher.New(q, db, cfg, logger) return New(d, db, logger, "127.0.0.1:0"), db diff --git a/internal/api/agent/models.go b/internal/api/agent/models.go index f30ddf0..de01684 100644 --- a/internal/api/agent/models.go +++ b/internal/api/agent/models.go @@ -1,14 +1,16 @@ package agentapi type VPCCreateRequest struct { - Name string `json:"name"` - CIDR string `json:"cidr"` + Name string `json:"name"` + CIDR string `json:"cidr"` + L3VNI *uint32 `json:"l3vni,omitempty"` } type VPC struct { Name string `json:"name"` State string `json:"state"` CIDR string `json:"cidr"` + L3VNI uint32 `json:"l3vni,omitempty"` } type SubnetCreateRequest struct { diff --git a/internal/api/agent/vpc.go b/internal/api/agent/vpc.go index bb3f5d7..33e4002 100644 --- a/internal/api/agent/vpc.go +++ b/internal/api/agent/vpc.go @@ -36,8 +36,9 @@ func (s *Server) getVpc(w http.ResponseWriter, _ *http.Request, name string) { return } cidr, _ := kv.GetFromDB(s.db, "vpc/"+name+"/cidr") + l3vni, _ := kv.GetFromDB(s.db, "vpc/"+name+"/l3vni") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(VPC{Name: name, State: state, CIDR: cidr}) + json.NewEncoder(w).Encode(VPC{Name: name, State: state, CIDR: cidr, L3VNI: parseL3VNI(l3vni)}) } func (s *Server) deleteVpc(w http.ResponseWriter, _ *http.Request, name string) { diff --git a/internal/api/agent/vpc_test.go b/internal/api/agent/vpc_test.go index b5ba517..626667a 100644 --- a/internal/api/agent/vpc_test.go +++ b/internal/api/agent/vpc_test.go @@ -5,9 +5,12 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" + configuration "git.g3e.fr/syonad/two/internal/config/agent" "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" ) // --- VpcsHandler --- @@ -236,3 +239,84 @@ func TestVpcByName_InvalidMethod(t *testing.T) { t.Errorf("attendu 405, obtenu %d", w.Code) } } + +func newBGPTestServer(t *testing.T) (*Server, *badger.DB) { + cfg := &configuration.Config{DefaultInterface: "br-test"} + cfg.BGP.ASN = 64600 + cfg.BGP.RouteReflectors = []string{"10.255.255.1"} + return newTestServerWith(t, cfg) +} + +func postVpcJSON(s *Server, body string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + s.VpcsHandler(w, httptest.NewRequest(http.MethodPost, "/vpcs", strings.NewReader(body))) + return w +} + +func TestPostVpc_WithL3VNI(t *testing.T) { + s, db := newBGPTestServer(t) + w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":4001}`) + if w.Code != http.StatusAccepted { + t.Fatalf("attendu 202, obtenu %d: %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), `"l3vni":4001`) { + t.Errorf("réponse sans l3vni : %s", w.Body.String()) + } + if got, _ := kv.GetFromDB(db, "vpc/vp-1/l3vni"); got != "4001" { + t.Errorf("l3vni en DB = %q", got) + } +} + +func TestPostVpc_WithoutL3VNIOmitsTheField(t *testing.T) { + s, _ := newBGPTestServer(t) + w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16"}`) + if w.Code != http.StatusAccepted || strings.Contains(w.Body.String(), "l3vni") { + t.Errorf("%d : %s", w.Code, w.Body.String()) + } +} + +func TestPostVpc_RejectsAnInvalidL3VNIAsABadRequest(t *testing.T) { + s, db := newBGPTestServer(t) + for _, body := range []string{ + `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":0}`, + `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":255}`, + `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":16777216}`, + `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":-1}`, + } { + if w := postVpcJSON(s, body); w.Code != http.StatusBadRequest { + t.Errorf("%s : attendu 400, obtenu %d: %s", body, w.Code, w.Body.String()) + } + } + if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 { + t.Errorf("écritures malgré le refus : %v", entries) + } +} + +func TestPostVpc_L3VNIWithoutBGPIsAConflict(t *testing.T) { + s, _ := newTestServer(t) + w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":4001}`) + if w.Code != http.StatusConflict || !strings.Contains(w.Body.String(), "requires the bgp section") { + t.Errorf("%d : %s", w.Code, w.Body.String()) + } +} + +func TestGetAndListVpc_ReturnTheL3VNI(t *testing.T) { + s, db := newTestServer(t) + kv.AddInDB(db, "vpc/vp-1/state", "running") + kv.AddInDB(db, "vpc/vp-1/cidr", "10.0.0.0/16") + kv.AddInDB(db, "vpc/vp-1/l3vni", "4001") + w := httptest.NewRecorder() + s.VpcByNameHandler(w, httptest.NewRequest(http.MethodGet, "/vpcs/vp-1", nil)) + var got VPC + json.NewDecoder(w.Body).Decode(&got) + if got.L3VNI != 4001 { + t.Errorf("GET : l3vni = %d", got.L3VNI) + } + w = httptest.NewRecorder() + s.VpcsHandler(w, httptest.NewRequest(http.MethodGet, "/vpcs", nil)) + var list []VPC + json.NewDecoder(w.Body).Decode(&list) + if len(list) != 1 || list[0].L3VNI != 4001 { + t.Errorf("liste : %+v", list) + } +} diff --git a/internal/api/agent/vpcs.go b/internal/api/agent/vpcs.go index 905cdee..1e0ed68 100644 --- a/internal/api/agent/vpcs.go +++ b/internal/api/agent/vpcs.go @@ -4,9 +4,11 @@ import ( "encoding/json" "net" "net/http" + "strconv" "strings" dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent" + "git.g3e.fr/syonad/two/internal/vpc" "git.g3e.fr/syonad/two/pkg/db/kv" ) @@ -44,6 +46,8 @@ func (s *Server) listVpcs(w http.ResponseWriter, _ *http.Request) { vpcs[name].State = value case "cidr": vpcs[name].CIDR = value + case "l3vni": + vpcs[name].L3VNI = parseL3VNI(value) } } result := make([]VPC, 0, len(vpcs)) @@ -77,6 +81,14 @@ func (s *Server) postVpc(w http.ResponseWriter, r *http.Request) { return } cmd := dispatcher.CreateVPCCommand{Name: req.Name, CIDR: req.CIDR} + if req.L3VNI != nil { + if err := vpc.ValidL3VNI(*req.L3VNI); err != nil { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()}) + return + } + cmd.L3VNI = *req.L3VNI + } if err := s.dispatcher.Prepare(cmd); err != nil { w.WriteHeader(http.StatusConflict) json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()}) @@ -90,5 +102,10 @@ func (s *Server) postVpc(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusAccepted) - json.NewEncoder(w).Encode(VPC{Name: req.Name, State: state, CIDR: req.CIDR}) + json.NewEncoder(w).Encode(VPC{Name: req.Name, State: state, CIDR: req.CIDR, L3VNI: cmd.L3VNI}) +} + +func parseL3VNI(value string) uint32 { + id, _ := strconv.ParseUint(value, 10, 32) + return uint32(id) } diff --git a/internal/dispatcher/agent/vpc_commands.go b/internal/dispatcher/agent/vpc_commands.go index 9e26819..2b6cc35 100644 --- a/internal/dispatcher/agent/vpc_commands.go +++ b/internal/dispatcher/agent/vpc_commands.go @@ -3,6 +3,7 @@ package dispatcher import ( "fmt" "net" + "strconv" "strings" "time" @@ -14,25 +15,61 @@ import ( ) type CreateVPCCommand struct { - Name string - CIDR string + Name string + CIDR string + L3VNI uint32 } func (c CreateVPCCommand) Key() string { return "vpc/" + c.Name } -func (c CreateVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error { +func (c CreateVPCCommand) Prepare(db *badger.DB, cfg *configuration.Config) error { if _, err := kv.GetFromDB(db, "vpc/"+c.Name+"/state"); err == nil { return fmt.Errorf("vpc %q already exists", c.Name) } if _, _, err := net.ParseCIDR(c.CIDR); err != nil { return fmt.Errorf("invalid cidr %q: %w", c.CIDR, err) } + if c.L3VNI != 0 { + if err := c.checkL3VNI(db, cfg); err != nil { + return err + } + } if err := kv.AddInDB(db, "vpc/"+c.Name+"/cidr", c.CIDR); err != nil { return err } + if c.L3VNI != 0 { + if err := kv.AddInDB(db, "vpc/"+c.Name+"/l3vni", strconv.FormatUint(uint64(c.L3VNI), 10)); err != nil { + return err + } + } return state.Set(db, c.Key(), state.Creating) } +func (c CreateVPCCommand) checkL3VNI(db *badger.DB, cfg *configuration.Config) error { + if err := vpc.ValidL3VNI(c.L3VNI); err != nil { + return err + } + if cfg == nil || !cfg.BGP.Enabled() { + return fmt.Errorf("l3vni requires the bgp section in the agent configuration") + } + entries, err := kv.ListByPrefix(db, "vpc/") + if err != nil { + return fmt.Errorf("failed to list vpcs: %w", err) + } + want := strconv.FormatUint(uint64(c.L3VNI), 10) + for key, value := range entries { + parts := strings.Split(key, "/") + if len(parts) != 3 || parts[2] != "l3vni" || value != want { + continue + } + if s, err := state.Get(db, "vpc/"+parts[1]); err == nil && s == state.Deleted { + continue + } + return fmt.Errorf("l3vni %d is already used by vpc %q", c.L3VNI, parts[1]) + } + return nil +} + func (c CreateVPCCommand) Execute(db *badger.DB, _ *configuration.Config) error { return vpc.CreateVPC(db, c.Name) } diff --git a/internal/dispatcher/agent/vpc_commands_test.go b/internal/dispatcher/agent/vpc_commands_test.go index 78b3d98..862f9c2 100644 --- a/internal/dispatcher/agent/vpc_commands_test.go +++ b/internal/dispatcher/agent/vpc_commands_test.go @@ -1,8 +1,10 @@ package dispatcher import ( + "strings" "testing" + configuration "git.g3e.fr/syonad/two/internal/config/agent" "git.g3e.fr/syonad/two/pkg/db/kv" ) @@ -137,3 +139,81 @@ func TestDeleteVPCCommand_Prepare_AllowedWhenSubnetDeleting(t *testing.T) { t.Fatalf("Prepare devrait réussir si le subnet est deleting : %v", err) } } + +func bgpConfig() *configuration.Config { + cfg := &configuration.Config{} + cfg.BGP.ASN = 64600 + cfg.BGP.RouteReflectors = []string{"10.255.255.1"} + return cfg +} + +func TestCreateVPCCommand_Prepare_StoresTheL3VNI(t *testing.T) { + _, db := newTestDispatcher(t) + cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 4001} + if err := cmd.Prepare(db, bgpConfig()); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } + if got, err := kv.GetFromDB(db, "vpc/vpc-1/l3vni"); err != nil || got != "4001" { + t.Errorf("l3vni en DB = %q, %v ; attendu 4001", got, err) + } +} + +func TestCreateVPCCommand_Prepare_WithoutL3VNIStoresNothing(t *testing.T) { + _, db := newTestDispatcher(t) + cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16"} + if err := cmd.Prepare(db, nil); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } + if got, err := kv.GetFromDB(db, "vpc/vpc-1/l3vni"); err == nil { + t.Errorf("clé l3vni écrite : %q", got) + } +} + +func TestCreateVPCCommand_Prepare_RefusesAnL3VNIWithoutBGP(t *testing.T) { + _, db := newTestDispatcher(t) + cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 4001} + err := cmd.Prepare(db, &configuration.Config{}) + if err == nil || !strings.Contains(err.Error(), "requires the bgp section") { + t.Fatalf("erreur %v", err) + } + if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 { + t.Errorf("écritures malgré le refus : %v", entries) + } +} + +func TestCreateVPCCommand_Prepare_RefusesAReservedL3VNI(t *testing.T) { + _, db := newTestDispatcher(t) + cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 254} + if err := cmd.Prepare(db, bgpConfig()); err == nil || !strings.Contains(err.Error(), "l3vni 254 is reserved") { + t.Fatalf("erreur %v", err) + } + if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 { + t.Errorf("écritures malgré le refus : %v", entries) + } +} + +func TestCreateVPCCommand_Prepare_RefusesAnL3VNIAlreadyUsed(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vpc/vpc-a/state", "deleting") + kv.AddInDB(db, "vpc/vpc-a/l3vni", "4001") + cmd := CreateVPCCommand{Name: "vpc-b", CIDR: "10.0.0.0/16", L3VNI: 4001} + err := cmd.Prepare(db, bgpConfig()) + if err == nil || !strings.Contains(err.Error(), `l3vni 4001 is already used by vpc "vpc-a"`) { + t.Fatalf("erreur %v", err) + } + if _, err := kv.GetFromDB(db, "vpc/vpc-b/state"); err == nil { + t.Error("vpc-b créé malgré le doublon") + } +} + +func TestCreateVPCCommand_Prepare_ReusesTheL3VNIOfADeletedVPC(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vpc/vpc-a/state", "deleted") + kv.AddInDB(db, "vpc/vpc-a/l3vni", "4001") + kv.AddInDB(db, "vpc/vpc-c/state", "running") + kv.AddInDB(db, "vpc/vpc-c/l3vni", "4002") + cmd := CreateVPCCommand{Name: "vpc-b", CIDR: "10.0.0.0/16", L3VNI: 4001} + if err := cmd.Prepare(db, bgpConfig()); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } +} diff --git a/internal/vpc/l3vni.go b/internal/vpc/l3vni.go new file mode 100644 index 0000000..eddf26d --- /dev/null +++ b/internal/vpc/l3vni.go @@ -0,0 +1,15 @@ +package vpc + +import "fmt" + +const maxL3VNI = 1<<24 - 1 + +func ValidL3VNI(id uint32) error { + if id == 0 || id > maxL3VNI { + return fmt.Errorf("l3vni %d does not fit in 24 bits (1-%d)", id, maxL3VNI) + } + if id >= 253 && id <= 255 { + return fmt.Errorf("l3vni %d is reserved: it is also the routing table id, and tables 253-255 belong to the kernel", id) + } + return nil +} diff --git a/internal/vpc/l3vni_test.go b/internal/vpc/l3vni_test.go new file mode 100644 index 0000000..c93c86c --- /dev/null +++ b/internal/vpc/l3vni_test.go @@ -0,0 +1,29 @@ +package vpc + +import ( + "strings" + "testing" +) + +func TestValidL3VNI_AcceptsEvery24BitValueOutsideTheKernelTables(t *testing.T) { + for _, id := range []uint32{1, 252, 256, 4001, 16777215} { + if err := ValidL3VNI(id); err != nil { + t.Errorf("ValidL3VNI(%d) = %v", id, err) + } + } +} + +func TestValidL3VNI_RejectsOutOfRangeAndReservedTables(t *testing.T) { + cases := map[uint32]string{ + 0: "does not fit in 24 bits", + 16777216: "does not fit in 24 bits", + 253: "is reserved", + 254: "is reserved", + 255: "is reserved", + } + for id, want := range cases { + if err := ValidL3VNI(id); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("ValidL3VNI(%d) = %v, attendu %q", id, err, want) + } + } +} From e2faf85e962479ca7e7c597713c7394cd11c56db Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:21:14 +0200 Subject: [PATCH 5/9] =?UTF-8?q?f-41:=20dispatcher:=20arr=C3=AAt=20des=20wo?= =?UTF-8?q?rkers=20avant=20la=20fermeture=20de=20la=20base=20dans=20les=20?= =?UTF-8?q?tests=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- internal/dispatcher/agent/helpers_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/dispatcher/agent/helpers_test.go b/internal/dispatcher/agent/helpers_test.go index c14f6a3..2e6cbcc 100644 --- a/internal/dispatcher/agent/helpers_test.go +++ b/internal/dispatcher/agent/helpers_test.go @@ -17,6 +17,7 @@ func newTestDispatcher(t *testing.T) (*Dispatcher, *badger.DB) { t.Cleanup(func() { db.Close() }) q := worker.New(100) q.Start(2) + t.Cleanup(q.Stop) cfg := &configuration.Config{DefaultInterface: "br-default"} cfg.Interfaces = map[string]string{"vms": "br-vms"} logger := slog.New(slog.NewTextHandler(io.Discard, nil)) From 3c6852dc187af4fdcd0d72f326c3c93ad9dbe879 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:21:19 +0200 Subject: [PATCH 6/9] =?UTF-8?q?f-41:=20vpc:=20VRF,=20L3VNI=20et=20transit?= =?UTF-8?q?=20=C3=A0=20la=20cr=C3=A9ation=20des=20VPC=20rout=C3=A9es=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- docs/concepts/vpc-subnet-vm.rst | 29 +++ internal/dispatcher/agent/vpc_commands.go | 6 +- internal/netif/delete.go | 14 ++ internal/vpc/create.go | 12 +- internal/vpc/delete.go | 13 +- internal/vpc/host.go | 87 ++++++++ internal/vpc/l3.go | 139 ++++++++++++ internal/vpc/l3_test.go | 246 ++++++++++++++++++++++ 8 files changed, 541 insertions(+), 5 deletions(-) create mode 100644 internal/vpc/host.go create mode 100644 internal/vpc/l3.go create mode 100644 internal/vpc/l3_test.go diff --git a/docs/concepts/vpc-subnet-vm.rst b/docs/concepts/vpc-subnet-vm.rst index 6337ea9..1c55ec0 100644 --- a/docs/concepts/vpc-subnet-vm.rst +++ b/docs/concepts/vpc-subnet-vm.rst @@ -20,6 +20,35 @@ d'isolation : deux VPC ne se voient pas, et peuvent réutiliser les mêmes plage Un VPC ne peut être supprimé que si tous ses subnets le sont déjà — sinon 409. +Routage entre hyperviseurs : ``l3vni`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Un VPC créé avec ``l3vni`` route ses subnets entre hyperviseurs par un **VNI de transit** propre +au VPC (IRB symétrique, `#41 `_) : un hyperviseur n'a +plus besoin de porter un subnet pour joindre ses VM. Sans ``l3vni``, le VPC se comporte comme +avant. Le champ exige la section ``bgp`` de l'agent (:doc:`/exploitation/configuration`) et ne +se change pas après création. + +À la création, l'agent pose sur l'hyperviseur, pour un VPC ``vp-`` : + +.. code-block:: text + + vrf- VRF, table de routage = l3vni + ├── brl3- bridge sans adresse, MAC de routeur 02:02: + │ └── vxl3- VXLAN du l3vni, VTEP = IPv4 primaire de default_interface + └── l3--o 169.254.0.1/30 ── l3--i 169.254.0.2/30, dans le netns du VPC + +Dans le netns, le ``cidr`` du VPC est routé vers ``169.254.0.1`` ; le transit ``/30`` est le même +sur tous les hyperviseurs et pour tous les VPC. Le forwarding IPv4 est activé dans le netns, et +sur l'hôte **seulement** sur ``brl3-`` et ``l3--o`` : l'hyperviseur ne devient pas un +routeur entre ses autres interfaces. FRR reçoit la VRF, son ``vni``, une instance +``router bgp vrf vrf-`` avec RD ``:`` et RT ``:``, et +l'export des routes du ``cidr`` présentes dans la VRF. + +La suppression retire d'abord la configuration de routage de FRR, puis les interfaces, puis la +VRF de FRR. Cet ordre est imposé : supprimer la VRF du noyau sous une instance BGP encore +configurée fait planter ``bgpd`` (FRR 10.7.1). + Subnet ------ diff --git a/internal/dispatcher/agent/vpc_commands.go b/internal/dispatcher/agent/vpc_commands.go index 2b6cc35..d37c31b 100644 --- a/internal/dispatcher/agent/vpc_commands.go +++ b/internal/dispatcher/agent/vpc_commands.go @@ -70,8 +70,8 @@ func (c CreateVPCCommand) checkL3VNI(db *badger.DB, cfg *configuration.Config) e return nil } -func (c CreateVPCCommand) Execute(db *badger.DB, _ *configuration.Config) error { - return vpc.CreateVPC(db, c.Name) +func (c CreateVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error { + return vpc.CreateVPC(db, c.Name, cfg) } type DeleteVPCCommand struct { @@ -132,7 +132,7 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): } } - if err := vpc.DeleteVPC(db, c.Name); err != nil { + if err := vpc.DeleteVPC(db, c.Name, cfg); err != nil { return err } current, err := state.Get(db, c.Key()) diff --git a/internal/netif/delete.go b/internal/netif/delete.go index 9b06c55..8fc18f6 100644 --- a/internal/netif/delete.go +++ b/internal/netif/delete.go @@ -1,6 +1,8 @@ package netif import ( + "errors" + "github.com/vishvananda/netlink" ) @@ -11,3 +13,15 @@ func DeleteLink(name string) error { } return netlink.LinkDel(link) } + +func DeleteLinkIfExists(name string) error { + link, err := netlink.LinkByName(name) + if err != nil { + var notFound netlink.LinkNotFoundError + if errors.As(err, ¬Found) { + return nil + } + return err + } + return netlink.LinkDel(link) +} diff --git a/internal/vpc/create.go b/internal/vpc/create.go index 52e2567..a7f5b24 100644 --- a/internal/vpc/create.go +++ b/internal/vpc/create.go @@ -3,6 +3,7 @@ package vpc import ( "strings" + configuration "git.g3e.fr/syonad/two/internal/config/agent" "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" @@ -10,7 +11,7 @@ import ( "github.com/dgraph-io/badger/v4" ) -func CreateVPC(db *badger.DB, name string) error { +func CreateVPC(db *badger.DB, name string, cfg *configuration.Config) error { if current, err := state.Get(db, "vpc/"+name); err != nil { return err } else if current == state.Creating { @@ -48,6 +49,15 @@ func CreateVPC(db *badger.DB, name string) error { }); err != nil { return err } + l3, ok, err := loadL3(db, name, vpcID) + if err != nil { + return err + } + if ok { + if err := createL3(newSystem(), cfg, l3); err != nil { + return err + } + } return state.Set(db, "vpc/"+name, state.Running) } return nil diff --git a/internal/vpc/delete.go b/internal/vpc/delete.go index ff4d83f..66d6d7a 100644 --- a/internal/vpc/delete.go +++ b/internal/vpc/delete.go @@ -3,6 +3,7 @@ package vpc import ( "strings" + configuration "git.g3e.fr/syonad/two/internal/config/agent" "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" @@ -10,12 +11,22 @@ import ( "github.com/dgraph-io/badger/v4" ) -func DeleteVPC(db *badger.DB, name string) error { +func DeleteVPC(db *badger.DB, name string, cfg *configuration.Config) error { if current, err := state.Get(db, "vpc/"+name); err != nil { return err } else if current == state.Deleting { vpcID := strings.SplitN(name, "-", 2)[1] + l3, ok, err := loadL3(db, name, vpcID) + if err != nil { + return err + } + if ok { + if err := deleteL3(newSystem(), cfg, l3); err != nil { + return err + } + } + if err := netif.DeleteLink("vp-" + vpcID + "-e"); err != nil { return err } diff --git a/internal/vpc/host.go b/internal/vpc/host.go new file mode 100644 index 0000000..8f2351b --- /dev/null +++ b/internal/vpc/host.go @@ -0,0 +1,87 @@ +package vpc + +import ( + "net" + + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/netns" + "git.g3e.fr/syonad/two/internal/sysctl" +) + +type host interface { + VTEPAddress(iface string) (net.IP, error) + Sysctl(ns, key, value string) error + CreateVRF(name string, table uint32) error + CreateBridge(name string, mtu int) error + SetHardwareAddr(name string, mac net.HardwareAddr) error + SetMaster(iface, master string) error + CreateVxlan(name string, vni int, iface string, mtu int) error + CreateVeth(hostIf, nsIf, ns string, mtu int) error + AddrAdd(ns, iface string, prefix *net.IPNet) error + LinkUp(ns, iface string) error + RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error + DeleteLink(name string) error + ApplyVRF(v frr.VRF) error + WithdrawVRF(v frr.VRF) error + RemoveVRF(v frr.VRF) error +} + +type system struct { + frr *frr.Client +} + +func newSystem() system { + return system{frr: frr.NewClient()} +} + +func in(ns string, fn func() error) error { + if ns == "" { + return fn() + } + return netns.Call(ns, fn) +} + +func (system) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) } + +func (system) Sysctl(ns, key, value string) error { + return in(ns, func() error { return sysctl.Set(key, value) }) +} + +func (system) CreateVRF(name string, table uint32) error { return netif.CreateVRF(name, table) } + +func (system) CreateBridge(name string, mtu int) error { return netif.CreateBridge(name, mtu) } + +func (system) SetHardwareAddr(name string, mac net.HardwareAddr) error { + return netif.LinkSetHardwareAddr(name, mac) +} + +func (system) SetMaster(iface, master string) error { return netif.BridgeSetMaster(iface, master) } + +func (system) CreateVxlan(name string, vni int, iface string, mtu int) error { + return netif.CreateVxlan(name, vni, iface, mtu) +} + +func (system) CreateVeth(hostIf, nsIf, ns string, mtu int) error { + return netif.CreateVethToNetns(hostIf, nsIf, "/var/run/netns/"+ns, mtu) +} + +func (system) AddrAdd(ns, iface string, prefix *net.IPNet) error { + return in(ns, func() error { return netif.AddrAddPrefix(iface, prefix) }) +} + +func (system) LinkUp(ns, iface string) error { + return in(ns, func() error { return netif.LinkSetUp(iface) }) +} + +func (system) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error { + return in(ns, func() error { return netif.RouteReplaceVia(0, dst, gw, iface) }) +} + +func (system) DeleteLink(name string) error { return netif.DeleteLinkIfExists(name) } + +func (s system) ApplyVRF(v frr.VRF) error { return s.frr.ApplyVRF(v) } + +func (s system) WithdrawVRF(v frr.VRF) error { return s.frr.WithdrawVRF(v) } + +func (s system) RemoveVRF(v frr.VRF) error { return s.frr.RemoveVRF(v) } diff --git a/internal/vpc/l3.go b/internal/vpc/l3.go new file mode 100644 index 0000000..225d650 --- /dev/null +++ b/internal/vpc/l3.go @@ -0,0 +1,139 @@ +package vpc + +import ( + "fmt" + "net" + "strconv" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/pkg/db/kv" + + "github.com/dgraph-io/badger/v4" +) + +const l3MTU = 1500 + +var ( + transitHost = &net.IPNet{IP: net.IPv4(169, 254, 0, 1).To4(), Mask: net.CIDRMask(30, 32)} + transitNetns = &net.IPNet{IP: net.IPv4(169, 254, 0, 2).To4(), Mask: net.CIDRMask(30, 32)} +) + +type l3VPC struct { + netns string + id string + vni uint32 + supernet *net.IPNet +} + +func loadL3(db *badger.DB, name, id string) (l3VPC, bool, error) { + raw, err := kv.GetFromDB(db, "vpc/"+name+"/l3vni") + if err != nil { + return l3VPC{}, false, nil + } + vni, err := strconv.ParseUint(raw, 10, 32) + if err != nil { + return l3VPC{}, false, fmt.Errorf("vpc %q: invalid l3vni %q", name, raw) + } + if err := ValidL3VNI(uint32(vni)); err != nil { + return l3VPC{}, false, fmt.Errorf("vpc %q: %w", name, err) + } + cidr, err := kv.GetFromDB(db, "vpc/"+name+"/cidr") + if err != nil { + return l3VPC{}, false, fmt.Errorf("vpc %q: no cidr: %w", name, err) + } + _, supernet, err := net.ParseCIDR(cidr) + if err != nil { + return l3VPC{}, false, fmt.Errorf("vpc %q: invalid cidr %q: %w", name, cidr, err) + } + return l3VPC{netns: name, id: id, vni: uint32(vni), supernet: supernet}, true, nil +} + +func (v l3VPC) vrf() string { return "vrf-" + v.id } +func (v l3VPC) bridge() string { return "brl3-" + v.id } +func (v l3VPC) vxlan() string { return "vxl3-" + v.id } +func (v l3VPC) vethHost() string { return "l3-" + v.id + "-o" } +func (v l3VPC) vethNetns() string { return "l3-" + v.id + "-i" } + +func routerMAC(vtep net.IP) (net.HardwareAddr, error) { + ip := vtep.To4() + if ip == nil { + return nil, fmt.Errorf("vtep %v is not an IPv4 address", vtep) + } + return net.HardwareAddr{0x02, 0x02, ip[0], ip[1], ip[2], ip[3]}, nil +} + +func bgpASN(cfg *configuration.Config) (uint32, error) { + if cfg == nil || !cfg.BGP.Enabled() { + return 0, fmt.Errorf("the vpc has an l3vni but the agent configuration has no bgp section") + } + return cfg.BGP.ASN, nil +} + +func createL3(h host, cfg *configuration.Config, v l3VPC) error { + asn, err := bgpASN(cfg) + if err != nil { + return err + } + vtep, err := h.VTEPAddress(cfg.DefaultInterface) + if err != nil { + return fmt.Errorf("vtep: %w", err) + } + rmac, err := routerMAC(vtep) + if err != nil { + return err + } + steps := []struct { + what string + run func() error + }{ + {"create " + v.vrf(), func() error { return h.CreateVRF(v.vrf(), v.vni) }}, + {"create " + v.bridge(), func() error { return h.CreateBridge(v.bridge(), l3MTU) }}, + {"set router mac on " + v.bridge(), func() error { return h.SetHardwareAddr(v.bridge(), rmac) }}, + {"attach " + v.bridge(), func() error { return h.SetMaster(v.bridge(), v.vrf()) }}, + {"create " + v.vxlan(), func() error { return h.CreateVxlan(v.vxlan(), int(v.vni), cfg.DefaultInterface, l3MTU) }}, + {"attach " + v.vxlan(), func() error { return h.SetMaster(v.vxlan(), v.bridge()) }}, + {"set up " + v.vxlan(), func() error { return h.LinkUp("", v.vxlan()) }}, + {"create " + v.vethHost(), func() error { return h.CreateVeth(v.vethHost(), v.vethNetns(), v.netns, l3MTU) }}, + {"attach " + v.vethHost(), func() error { return h.SetMaster(v.vethHost(), v.vrf()) }}, + {"address " + v.vethHost(), func() error { return h.AddrAdd("", v.vethHost(), transitHost) }}, + {"set up " + v.vethHost(), func() error { return h.LinkUp("", v.vethHost()) }}, + {"forwarding on " + v.bridge(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.bridge()+".forwarding", "1") }}, + {"forwarding on " + v.vethHost(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.vethHost()+".forwarding", "1") }}, + {"address " + v.vethNetns(), func() error { return h.AddrAdd(v.netns, v.vethNetns(), transitNetns) }}, + {"set up " + v.vethNetns(), func() error { return h.LinkUp(v.netns, v.vethNetns()) }}, + {"forwarding in " + v.netns, func() error { return h.Sysctl(v.netns, "net.ipv4.ip_forward", "1") }}, + {"route " + v.supernet.String() + " in " + v.netns, func() error { + return h.RouteVia(v.netns, v.supernet, transitHost.IP, v.vethNetns()) + }}, + {"frr " + v.vrf(), func() error { + return h.ApplyVRF(frr.VRF{ASN: asn, ID: v.id, L3VNI: v.vni, VTEP: vtep, Supernet: v.supernet}) + }}, + } + for _, s := range steps { + if err := s.run(); err != nil { + return fmt.Errorf("l3vni %d: %s: %w", v.vni, s.what, err) + } + } + return nil +} + +func deleteL3(h host, cfg *configuration.Config, v l3VPC) error { + asn, err := bgpASN(cfg) + if err != nil { + return err + } + frrVRF := frr.VRF{ASN: asn, ID: v.id} + if err := h.WithdrawVRF(frrVRF); err != nil { + return fmt.Errorf("l3vni %d: withdraw frr: %w", v.vni, err) + } + for _, link := range []string{v.vethHost(), v.vxlan(), v.bridge(), v.vrf()} { + if err := h.DeleteLink(link); err != nil { + return fmt.Errorf("l3vni %d: delete %s: %w", v.vni, link, err) + } + } + if err := h.RemoveVRF(frrVRF); err != nil { + return fmt.Errorf("l3vni %d: remove frr vrf: %w", v.vni, err) + } + return nil +} diff --git a/internal/vpc/l3_test.go b/internal/vpc/l3_test.go new file mode 100644 index 0000000..f06a7bf --- /dev/null +++ b/internal/vpc/l3_test.go @@ -0,0 +1,246 @@ +package vpc + +import ( + "errors" + "fmt" + "net" + "reflect" + "strings" + "testing" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/pkg/db/kv" +) + +type fakeHost struct { + calls []string + fail map[string]error +} + +func (f *fakeHost) record(call string) error { + f.calls = append(f.calls, call) + for prefix, err := range f.fail { + if strings.HasPrefix(call, prefix) { + return err + } + } + return nil +} + +func (f *fakeHost) VTEPAddress(iface string) (net.IP, error) { + if err := f.record("vtep " + iface); err != nil { + return nil, err + } + return net.ParseIP("192.168.14.11"), nil +} + +func (f *fakeHost) Sysctl(ns, key, value string) error { + return f.record(fmt.Sprintf("sysctl [%s] %s=%s", ns, key, value)) +} + +func (f *fakeHost) CreateVRF(name string, table uint32) error { + return f.record(fmt.Sprintf("vrf %s table %d", name, table)) +} + +func (f *fakeHost) CreateBridge(name string, mtu int) error { + return f.record(fmt.Sprintf("bridge %s mtu %d", name, mtu)) +} + +func (f *fakeHost) SetHardwareAddr(name string, mac net.HardwareAddr) error { + return f.record(fmt.Sprintf("mac %s %s", name, mac)) +} + +func (f *fakeHost) SetMaster(iface, master string) error { + return f.record(fmt.Sprintf("master %s %s", iface, master)) +} + +func (f *fakeHost) CreateVxlan(name string, vni int, iface string, mtu int) error { + return f.record(fmt.Sprintf("vxlan %s id %d dev %s mtu %d", name, vni, iface, mtu)) +} + +func (f *fakeHost) CreateVeth(hostIf, nsIf, ns string, mtu int) error { + return f.record(fmt.Sprintf("veth %s peer %s netns %s mtu %d", hostIf, nsIf, ns, mtu)) +} + +func (f *fakeHost) AddrAdd(ns, iface string, prefix *net.IPNet) error { + return f.record(fmt.Sprintf("addr [%s] %s %s", ns, iface, prefix)) +} + +func (f *fakeHost) LinkUp(ns, iface string) error { + return f.record(fmt.Sprintf("up [%s] %s", ns, iface)) +} + +func (f *fakeHost) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error { + return f.record(fmt.Sprintf("route [%s] %s via %s dev %s", ns, dst, gw, iface)) +} + +func (f *fakeHost) DeleteLink(name string) error { + return f.record("del " + name) +} + +func (f *fakeHost) ApplyVRF(v frr.VRF) error { + return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet)) +} + +func (f *fakeHost) WithdrawVRF(v frr.VRF) error { + return f.record(fmt.Sprintf("frr withdraw asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI)) +} + +func (f *fakeHost) RemoveVRF(v frr.VRF) error { + return f.record(fmt.Sprintf("frr remove asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI)) +} + +func bgpConfig() *configuration.Config { + cfg := &configuration.Config{DefaultInterface: "br-000000"} + cfg.BGP.ASN = 64600 + cfg.BGP.RouteReflectors = []string{"10.255.255.1"} + return cfg +} + +func labL3(t *testing.T) l3VPC { + t.Helper() + _, supernet, err := net.ParseCIDR("10.241.0.0/16") + if err != nil { + t.Fatal(err) + } + return l3VPC{netns: "vp-s7", id: "s7", vni: 2700, supernet: supernet} +} + +func TestCreateL3FollowsTheLabRecipe(t *testing.T) { + f := &fakeHost{} + if err := createL3(f, bgpConfig(), labL3(t)); err != nil { + t.Fatal(err) + } + want := []string{ + "vtep br-000000", + "vrf vrf-s7 table 2700", + "bridge brl3-s7 mtu 1500", + "mac brl3-s7 02:02:c0:a8:0e:0b", + "master brl3-s7 vrf-s7", + "vxlan vxl3-s7 id 2700 dev br-000000 mtu 1500", + "master vxl3-s7 brl3-s7", + "up [] vxl3-s7", + "veth l3-s7-o peer l3-s7-i netns vp-s7 mtu 1500", + "master l3-s7-o vrf-s7", + "addr [] l3-s7-o 169.254.0.1/30", + "up [] l3-s7-o", + "sysctl [] net.ipv4.conf.brl3-s7.forwarding=1", + "sysctl [] net.ipv4.conf.l3-s7-o.forwarding=1", + "addr [vp-s7] l3-s7-i 169.254.0.2/30", + "up [vp-s7] l3-s7-i", + "sysctl [vp-s7] net.ipv4.ip_forward=1", + "route [vp-s7] 10.241.0.0/16 via 169.254.0.1 dev l3-s7-i", + "frr apply asn=64600 id=s7 vni=2700 vtep=192.168.14.11 supernet=10.241.0.0/16", + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) + } +} + +func TestCreateL3RequiresTheBGPSectionBeforeTouchingTheHost(t *testing.T) { + f := &fakeHost{} + cfg := &configuration.Config{DefaultInterface: "br-000000"} + if err := createL3(f, cfg, labL3(t)); err == nil || !strings.Contains(err.Error(), "no bgp section") { + t.Errorf("erreur %v", err) + } + if err := createL3(f, nil, labL3(t)); err == nil { + t.Error("config absente acceptée") + } + if len(f.calls) != 0 { + t.Errorf("appels : %q", f.calls) + } +} + +func TestCreateL3StopsAtTheFirstFailure(t *testing.T) { + f := &fakeHost{fail: map[string]error{"vxlan ": errors.New("file exists")}} + err := createL3(f, bgpConfig(), labL3(t)) + if err == nil || !strings.Contains(err.Error(), "l3vni 2700: create vxl3-s7: file exists") { + t.Errorf("erreur %v", err) + } + if last := f.calls[len(f.calls)-1]; !strings.HasPrefix(last, "vxlan ") { + t.Errorf("appels après l'échec : %q", f.calls) + } +} + +func TestDeleteL3WithdrawsFRRThenTheKernelThenTheVRF(t *testing.T) { + f := &fakeHost{} + if err := deleteL3(f, bgpConfig(), labL3(t)); err != nil { + t.Fatal(err) + } + want := []string{ + "frr withdraw asn=64600 id=s7 vni=0", + "del l3-s7-o", + "del vxl3-s7", + "del brl3-s7", + "del vrf-s7", + "frr remove asn=64600 id=s7 vni=0", + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) + } +} + +func TestDeleteL3KeepsTheKernelWhenFRRRefusesTheWithdrawal(t *testing.T) { + f := &fakeHost{fail: map[string]error{"frr withdraw": errors.New("% Please unconfigure l3vni 2700")}} + err := deleteL3(f, bgpConfig(), labL3(t)) + if err == nil || !strings.Contains(err.Error(), "withdraw frr") { + t.Errorf("erreur %v", err) + } + if len(f.calls) != 1 { + t.Errorf("le noyau a été touché malgré le refus de FRR : %q", f.calls) + } +} + +func TestDeleteL3KeepsTheFRRVRFWhenAKernelLinkCannotBeDeleted(t *testing.T) { + f := &fakeHost{fail: map[string]error{"del brl3-s7": errors.New("device busy")}} + if err := deleteL3(f, bgpConfig(), labL3(t)); err == nil || !strings.Contains(err.Error(), "delete brl3-s7") { + t.Errorf("erreur %v", err) + } + for _, c := range f.calls { + if strings.HasPrefix(c, "frr remove") || c == "del vrf-s7" { + t.Errorf("étape exécutée après l'échec : %q", f.calls) + } + } +} + +func TestDeleteL3RequiresTheBGPSection(t *testing.T) { + f := &fakeHost{} + if err := deleteL3(f, &configuration.Config{}, labL3(t)); err == nil || len(f.calls) != 0 { + t.Errorf("erreur %v, appels %q", err, f.calls) + } +} + +func TestRouterMAC(t *testing.T) { + mac, err := routerMAC(net.ParseIP("192.168.14.11")) + if err != nil || mac.String() != "02:02:c0:a8:0e:0b" { + t.Errorf("routerMAC = %v, %v", mac, err) + } + if _, err := routerMAC(net.ParseIP("fd00::1")); err == nil { + t.Error("VTEP IPv6 accepté") + } +} + +func TestLoadL3(t *testing.T) { + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vpc/vp-a/cidr", "10.241.0.0/16") + if _, ok, err := loadL3(db, "vp-a", "a"); ok || err != nil { + t.Errorf("VPC sans l3vni : ok=%v err=%v", ok, err) + } + kv.AddInDB(db, "vpc/vp-a/l3vni", "2700") + got, ok, err := loadL3(db, "vp-a", "a") + if !ok || err != nil || got.vni != 2700 || got.id != "a" || got.netns != "vp-a" || got.supernet.String() != "10.241.0.0/16" { + t.Errorf("loadL3 = %+v, %v, %v", got, ok, err) + } + for _, bad := range []string{"abc", "254", "0"} { + kv.AddInDB(db, "vpc/vp-a/l3vni", bad) + if _, _, err := loadL3(db, "vp-a", "a"); err == nil { + t.Errorf("l3vni %q acceptée", bad) + } + } + kv.AddInDB(db, "vpc/vp-b/l3vni", "2700") + if _, _, err := loadL3(db, "vp-b", "b"); err == nil || !strings.Contains(err.Error(), "no cidr") { + t.Errorf("VPC sans cidr : %v", err) + } +} From fe27d4526239a463143343098317cd1f0081f23b Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:28:16 +0200 Subject: [PATCH 7/9] =?UTF-8?q?f-41:=20subnet:=20export=20du=20pr=C3=A9fix?= =?UTF-8?q?e=20dans=20la=20VRF,=20suppression=20refus=C3=A9e=20tant=20qu'u?= =?UTF-8?q?ne=20VM=20l'utilise=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- api/agent.yaml | 2 +- docs/concepts/vpc-subnet-vm.rst | 5 ++ docs/exploitation/api-agent/asynchronisme.rst | 3 +- internal/dispatcher/agent/subnet_commands.go | 29 ++++++ .../dispatcher/agent/subnet_commands_test.go | 36 ++++++++ internal/netif/route.go | 9 ++ internal/subnet/create.go | 7 ++ internal/subnet/delete.go | 7 ++ internal/vpc/host.go | 10 +++ internal/vpc/l3_test.go | 8 ++ internal/vpc/route.go | 50 +++++++++++ internal/vpc/route_test.go | 89 +++++++++++++++++++ 12 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 internal/vpc/route.go create mode 100644 internal/vpc/route_test.go diff --git a/api/agent.yaml b/api/agent.yaml index 97d8ef6..e8e5b7c 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -281,7 +281,7 @@ paths: "404": $ref: "#/components/responses/NotFound" "409": - description: Subnet not deletable — only running or error states can be deleted + description: Subnet not deletable — only running or error states can be deleted, and no VM that is not deleted may still have an interface on it content: application/json: schema: diff --git a/docs/concepts/vpc-subnet-vm.rst b/docs/concepts/vpc-subnet-vm.rst index 1c55ec0..abf2333 100644 --- a/docs/concepts/vpc-subnet-vm.rst +++ b/docs/concepts/vpc-subnet-vm.rst @@ -63,6 +63,11 @@ hosts dont le nommage réseau diffère. Le comportement réseau dépend du :doc:`mode `. +Un subnet ne peut être supprimé que si aucune VM non supprimée n'a d'interface dessus — sinon +409, avec la liste des VM. Dans un VPC routé par ``l3vni``, un subnet ``vxlan`` annonce son +préfixe dans la VRF du VPC à la création et le retire en premier à la suppression : c'est le +filet qui couvre une VM tant que son ``/32`` n'est pas annoncé. + VM -- diff --git a/docs/exploitation/api-agent/asynchronisme.rst b/docs/exploitation/api-agent/asynchronisme.rst index a506d55..afcee1d 100644 --- a/docs/exploitation/api-agent/asynchronisme.rst +++ b/docs/exploitation/api-agent/asynchronisme.rst @@ -71,7 +71,8 @@ Codes de retour - ressource inexistante * - ``409`` - conflit d'existence ou d'état : ressource déjà créée, ou suppression depuis un état qui - ne l'autorise pas ; pour un VPC, subnets encore présents + ne l'autorise pas ; pour un VPC, subnets encore présents ; pour un subnet, VM non + supprimées qui l'utilisent encore * - ``422`` - dépendance absente ou pas prête : VPC parent d'un subnet, subnet d'une VM * - ``500`` diff --git a/internal/dispatcher/agent/subnet_commands.go b/internal/dispatcher/agent/subnet_commands.go index b2fb5f6..f6e7f2d 100644 --- a/internal/dispatcher/agent/subnet_commands.go +++ b/internal/dispatcher/agent/subnet_commands.go @@ -2,7 +2,9 @@ package dispatcher import ( "fmt" + "sort" "strconv" + "strings" "time" configuration "git.g3e.fr/syonad/two/internal/config/agent" @@ -101,9 +103,36 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err if !state.CanDelete(current) { return fmt.Errorf("subnet %q cannot be deleted while %s", c.Name, current) } + vms, err := vmsUsingSubnet(db, c.Name) + if err != nil { + return err + } + if len(vms) > 0 { + return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", ")) + } return state.Set(db, c.Key(), state.Deleting) } +func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) { + entries, err := kv.ListByPrefix(db, "vm/") + if err != nil { + return nil, fmt.Errorf("failed to list vms: %w", err) + } + var users []string + for key, value := range entries { + parts := strings.Split(key, "/") + if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || value != subnetName { + continue + } + if s, err := state.Get(db, "vm/"+parts[1]); err == nil && s == state.Deleted { + continue + } + users = append(users, parts[1]) + } + sort.Strings(users) + return users, nil +} + func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error { if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil { return err diff --git a/internal/dispatcher/agent/subnet_commands_test.go b/internal/dispatcher/agent/subnet_commands_test.go index d94f5e4..3474df1 100644 --- a/internal/dispatcher/agent/subnet_commands_test.go +++ b/internal/dispatcher/agent/subnet_commands_test.go @@ -1,6 +1,7 @@ package dispatcher import ( + "strings" "testing" configuration "git.g3e.fr/syonad/two/internal/config/agent" @@ -327,3 +328,38 @@ func TestCreateSubnetCommand_Prepare_RejectsUnknownMode(t *testing.T) { t.Error("un mode inconnu doit être refusé") } } + +func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-gone/state", "deleted") + kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-other/state", "running") + kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b") + kv.AddInDB(db, "vm/i-b/state", "deleting") + kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-b") + kv.AddInDB(db, "vm/i-b/nic/1/subnet", "sn-a") + kv.AddInDB(db, "vm/i-a/state", "error") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-c/state", "running") + kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a") + err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) { + t.Fatalf("erreur %v", err) + } + if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" { + t.Errorf("état modifié malgré le refus : %q", s) + } +} + +func TestDeleteSubnetCommand_Prepare_AllowedOnceItsVMsAreDeleted(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-gone/state", "deleted") + kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-other/state", "running") + kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b") + if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } +} diff --git a/internal/netif/route.go b/internal/netif/route.go index f10b159..d49ec02 100644 --- a/internal/netif/route.go +++ b/internal/netif/route.go @@ -1,7 +1,9 @@ package netif import ( + "errors" "net" + "syscall" "github.com/vishvananda/netlink" ) @@ -25,3 +27,10 @@ func RouteDel(table int, dst *net.IPNet) error { Table: table, }) } + +func RouteDelIfExists(table int, dst *net.IPNet) error { + if err := RouteDel(table, dst); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} diff --git a/internal/subnet/create.go b/internal/subnet/create.go index 568b64e..ae31b49 100644 --- a/internal/subnet/create.go +++ b/internal/subnet/create.go @@ -10,6 +10,7 @@ import ( "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "github.com/dgraph-io/badger/v4" ) @@ -37,6 +38,12 @@ func CreateSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e return err } + if d.mode == ModeVxlan { + if err := vpc.ExportRoute(db, d.vpc, d.cidr); err != nil { + return err + } + } + return state.Set(db, "subnet/"+subnetName, state.Running) } diff --git a/internal/subnet/delete.go b/internal/subnet/delete.go index a8f12d0..8ad0b3d 100644 --- a/internal/subnet/delete.go +++ b/internal/subnet/delete.go @@ -9,6 +9,7 @@ import ( "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "git.g3e.fr/syonad/two/pkg/db/kv" "github.com/dgraph-io/badger/v4" @@ -33,6 +34,12 @@ func DeleteSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e return err } + if d.mode == ModeVxlan { + if err := vpc.WithdrawRoute(db, d.vpc, d.cidr); err != nil { + return err + } + } + if err := stopDHCP(db, subnetName, d, backend); err != nil { return err } diff --git a/internal/vpc/host.go b/internal/vpc/host.go index 8f2351b..37a1720 100644 --- a/internal/vpc/host.go +++ b/internal/vpc/host.go @@ -22,6 +22,8 @@ type host interface { LinkUp(ns, iface string) error RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error DeleteLink(name string) error + TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error + TableRouteDel(table uint32, dst *net.IPNet) error ApplyVRF(v frr.VRF) error WithdrawVRF(v frr.VRF) error RemoveVRF(v frr.VRF) error @@ -80,6 +82,14 @@ func (system) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error func (system) DeleteLink(name string) error { return netif.DeleteLinkIfExists(name) } +func (system) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return netif.RouteReplaceVia(int(table), dst, gw, iface) +} + +func (system) TableRouteDel(table uint32, dst *net.IPNet) error { + return netif.RouteDelIfExists(int(table), dst) +} + func (s system) ApplyVRF(v frr.VRF) error { return s.frr.ApplyVRF(v) } func (s system) WithdrawVRF(v frr.VRF) error { return s.frr.WithdrawVRF(v) } diff --git a/internal/vpc/l3_test.go b/internal/vpc/l3_test.go index f06a7bf..a56fad9 100644 --- a/internal/vpc/l3_test.go +++ b/internal/vpc/l3_test.go @@ -79,6 +79,14 @@ func (f *fakeHost) DeleteLink(name string) error { return f.record("del " + name) } +func (f *fakeHost) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return f.record(fmt.Sprintf("table %d route %s via %s dev %s", table, dst, gw, iface)) +} + +func (f *fakeHost) TableRouteDel(table uint32, dst *net.IPNet) error { + return f.record(fmt.Sprintf("table %d del %s", table, dst)) +} + func (f *fakeHost) ApplyVRF(v frr.VRF) error { return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet)) } diff --git a/internal/vpc/route.go b/internal/vpc/route.go new file mode 100644 index 0000000..3b5da19 --- /dev/null +++ b/internal/vpc/route.go @@ -0,0 +1,50 @@ +package vpc + +import ( + "fmt" + "net" + "strings" + + "github.com/dgraph-io/badger/v4" +) + +func ExportRoute(db *badger.DB, vpcName string, dst *net.IPNet) error { + return exportRoute(newSystem(), db, vpcName, dst) +} + +func WithdrawRoute(db *badger.DB, vpcName string, dst *net.IPNet) error { + return withdrawRoute(newSystem(), db, vpcName, dst) +} + +func loadRouted(db *badger.DB, vpcName string) (l3VPC, bool, error) { + parts := strings.SplitN(vpcName, "-", 2) + if len(parts) != 2 { + return l3VPC{}, false, fmt.Errorf("invalid vpc name %q", vpcName) + } + return loadL3(db, vpcName, parts[1]) +} + +func exportRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return err + } + if !v.supernet.Contains(dst.IP) { + return fmt.Errorf("vpc %q: %s is outside %s", vpcName, dst, v.supernet) + } + if err := h.TableRouteVia(v.vni, dst, transitNetns.IP, v.vethHost()); err != nil { + return fmt.Errorf("export %s in %s: %w", dst, v.vrf(), err) + } + return nil +} + +func withdrawRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return err + } + if err := h.TableRouteDel(v.vni, dst); err != nil { + return fmt.Errorf("withdraw %s from %s: %w", dst, v.vrf(), err) + } + return nil +} diff --git a/internal/vpc/route_test.go b/internal/vpc/route_test.go new file mode 100644 index 0000000..ded011a --- /dev/null +++ b/internal/vpc/route_test.go @@ -0,0 +1,89 @@ +package vpc + +import ( + "errors" + "net" + "reflect" + "strings" + "testing" + + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +func routedDB(t *testing.T) *badger.DB { + t.Helper() + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vpc/vp-s7/cidr", "10.241.0.0/16") + kv.AddInDB(db, "vpc/vp-s7/l3vni", "2700") + kv.AddInDB(db, "vpc/vp-flat/cidr", "10.242.0.0/16") + return db +} + +func cidr(t *testing.T, s string) *net.IPNet { + t.Helper() + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n +} + +func TestExportRouteInstallsTheRouteInTheVRFTowardsTheNetns(t *testing.T) { + f := &fakeHost{} + if err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.0/24")); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 route 10.241.1.0/24 via 169.254.0.2 dev l3-s7-o"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestExportRouteRefusesAPrefixOutsideTheVPC(t *testing.T) { + f := &fakeHost{} + err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.99.1.0/24")) + if err == nil || !strings.Contains(err.Error(), "outside 10.241.0.0/16") || len(f.calls) != 0 { + t.Errorf("erreur %v, appels %q", err, f.calls) + } +} + +func TestWithdrawRouteDeletesFromTheVRFTable(t *testing.T) { + f := &fakeHost{} + if err := withdrawRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.10/32")); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 del 10.241.1.10/32"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestRoutesAreIgnoredForAVPCWithoutL3VNI(t *testing.T) { + f := &fakeHost{} + db := routedDB(t) + if err := exportRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil { + t.Error(err) + } + if err := withdrawRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil { + t.Error(err) + } + if len(f.calls) != 0 { + t.Errorf("appels %q", f.calls) + } +} + +func TestRouteErrorsArePropagated(t *testing.T) { + db := routedDB(t) + f := &fakeHost{fail: map[string]error{"table ": errors.New("network unreachable")}} + if err := exportRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "export 10.241.1.0/24 in vrf-s7") { + t.Errorf("export : %v", err) + } + if err := withdrawRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "withdraw 10.241.1.0/24 from vrf-s7") { + t.Errorf("withdraw : %v", err) + } + if err := exportRoute(f, db, "novpc", cidr(t, "10.241.1.0/24")); err == nil { + t.Error("nom de VPC sans tiret accepté") + } +} From 03029eabe8fab35f60e74b25a7a3b5e327a71ec4 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:32:16 +0200 Subject: [PATCH 8/9] =?UTF-8?q?f-41:=20vm:=20/32=20de=20chaque=20interface?= =?UTF-8?q?=20annonc=C3=A9=20dans=20la=20VRF=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- docs/concepts/vpc-subnet-vm.rst | 6 +++ internal/vm/create.go | 5 +++ internal/vm/delete.go | 5 +++ internal/vm/routes.go | 43 +++++++++++++++++++++ internal/vm/routes_test.go | 66 +++++++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+) create mode 100644 internal/vm/routes.go create mode 100644 internal/vm/routes_test.go diff --git a/docs/concepts/vpc-subnet-vm.rst b/docs/concepts/vpc-subnet-vm.rst index abf2333..4d507d0 100644 --- a/docs/concepts/vpc-subnet-vm.rst +++ b/docs/concepts/vpc-subnet-vm.rst @@ -78,6 +78,12 @@ le nom de l'interface dans le guest. Exactement une interface doit être ``prima la route par défaut et le serveur de metadata. Tous les subnets d'une VM doivent appartenir au **même VPC**. +**Routes.** Dans un VPC routé par ``l3vni``, chaque interface sur un subnet ``vxlan`` est annoncée +en ``/32`` dans la VRF du VPC une fois QEMU démarré, juste avant le passage en ``running``. À la +suppression, ces ``/32`` sont retirés **avant** l'arrêt de QEMU. Comme un subnet ne peut être +supprimé tant qu'une VM l'utilise, le ``/32`` d'une VM est toujours retiré avant le ``/24`` de son +subnet — l'ordre qui évite le trou noir de `#41 `_. + **Stockage.** Un seul disque ``vdX`` (virtio-blk) par VM ; les disques supplémentaires passent par le contrôleur SCSI (``sdX``). Cette contrainte vient de la carte PCI figée — voir :doc:`/architecture/contraintes`. diff --git a/internal/vm/create.go b/internal/vm/create.go index 692cc25..1896b81 100644 --- a/internal/vm/create.go +++ b/internal/vm/create.go @@ -15,6 +15,7 @@ import ( "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/qemu" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "github.com/dgraph-io/badger/v4" ) @@ -111,6 +112,10 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error { return fmt.Errorf("start qemu: %w", err) } + if err := applyHostRoutes(db, d, vpc.ExportRoute); err != nil { + return err + } + return state.Set(db, "vm/"+name, state.Running) } diff --git a/internal/vm/delete.go b/internal/vm/delete.go index 9b2a50c..12651fb 100644 --- a/internal/vm/delete.go +++ b/internal/vm/delete.go @@ -14,6 +14,7 @@ import ( "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/qmp" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "github.com/dgraph-io/badger/v4" ) @@ -38,6 +39,10 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { return err } + if err := applyHostRoutes(db, d, vpc.WithdrawRoute); err != nil { + return err + } + socketPath := filepath.Join(cfg.QEMU.QMPDir, name+".sock") if _, err := os.Stat(socketPath); err == nil { diff --git a/internal/vm/routes.go b/internal/vm/routes.go new file mode 100644 index 0000000..09090f7 --- /dev/null +++ b/internal/vm/routes.go @@ -0,0 +1,43 @@ +package vm + +import ( + "fmt" + "net" + + "github.com/dgraph-io/badger/v4" +) + +type hostRoute struct { + vpc string + dst *net.IPNet +} + +type routeFunc func(db *badger.DB, vpcName string, dst *net.IPNet) error + +func hostRoutes(d vmData) ([]hostRoute, error) { + var routes []hostRoute + for _, n := range d.nics { + if n.mode != "vxlan" { + continue + } + ip := net.ParseIP(n.ip).To4() + if ip == nil { + return nil, fmt.Errorf("interface %d: %q is not an IPv4 address", n.index, n.ip) + } + routes = append(routes, hostRoute{vpc: n.vpcName, dst: &net.IPNet{IP: ip, Mask: net.CIDRMask(32, 32)}}) + } + return routes, nil +} + +func applyHostRoutes(db *badger.DB, d vmData, apply routeFunc) error { + routes, err := hostRoutes(d) + if err != nil { + return err + } + for _, r := range routes { + if err := apply(db, r.vpc, r.dst); err != nil { + return err + } + } + return nil +} diff --git a/internal/vm/routes_test.go b/internal/vm/routes_test.go new file mode 100644 index 0000000..592b44d --- /dev/null +++ b/internal/vm/routes_test.go @@ -0,0 +1,66 @@ +package vm + +import ( + "errors" + "net" + "reflect" + "strings" + "testing" + + "github.com/dgraph-io/badger/v4" +) + +func routedVM() vmData { + return vmData{nics: []nicData{ + {index: 0, vpcName: "vp-s7", mode: "vxlan", ip: "10.241.1.10"}, + {index: 1, vpcName: "vp-s7", mode: "bridge", ip: "192.168.50.10"}, + {index: 2, vpcName: "vp-s7", mode: "vxlan", ip: "10.241.2.10"}, + }} +} + +type recorder struct { + calls []string + fail string +} + +func (r *recorder) apply(_ *badger.DB, vpcName string, dst *net.IPNet) error { + r.calls = append(r.calls, vpcName+" "+dst.String()) + if dst.String() == r.fail { + return errors.New("refused") + } + return nil +} + +func TestApplyHostRoutesCoversEveryVxlanInterfaceAsA32(t *testing.T) { + r := &recorder{} + if err := applyHostRoutes(nil, routedVM(), r.apply); err != nil { + t.Fatal(err) + } + want := []string{"vp-s7 10.241.1.10/32", "vp-s7 10.241.2.10/32"} + if !reflect.DeepEqual(r.calls, want) { + t.Errorf("appels %q, attendu %q", r.calls, want) + } +} + +func TestApplyHostRoutesStopsAtTheFirstError(t *testing.T) { + r := &recorder{fail: "10.241.1.10/32"} + if err := applyHostRoutes(nil, routedVM(), r.apply); err == nil || err.Error() != "refused" { + t.Errorf("erreur %v", err) + } + if len(r.calls) != 1 { + t.Errorf("appels après l'échec : %q", r.calls) + } +} + +func TestApplyHostRoutesRefusesANonIPv4AddressBeforeAnyCall(t *testing.T) { + d := routedVM() + d.nics[2].ip = "fd00::10" + r := &recorder{} + err := applyHostRoutes(nil, d, r.apply) + if err == nil || !strings.Contains(err.Error(), `interface 2: "fd00::10" is not an IPv4 address`) { + t.Errorf("erreur %v", err) + } + if len(r.calls) != 0 { + t.Errorf("appels : %q", r.calls) + } +} From 9477dc91d25215dae856c1b07fe5f2e13e5d1202 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 11 Oct 2026 20:42:01 +0200 Subject: [PATCH 9/9] =?UTF-8?q?f-41:=20watchdog:=20sessions=20BGP,=20L3VNI?= =?UTF-8?q?=20et=20routes=20export=C3=A9es=20des=20VPC=20rout=C3=A9s=20#41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- docs/exploitation/observabilite.rst | 14 ++ internal/frr/status.go | 49 ++++++ internal/frr/status_test.go | 50 ++++++ internal/netif/route_linux.go | 21 +++ internal/netif/route_other.go | 9 +- internal/vpc/l3.go | 30 ++++ internal/watchdog/check_l3.go | 232 ++++++++++++++++++++++++++ internal/watchdog/check_l3_test.go | 242 ++++++++++++++++++++++++++++ internal/watchdog/watchdog.go | 7 + 9 files changed, 653 insertions(+), 1 deletion(-) create mode 100644 internal/frr/status.go create mode 100644 internal/frr/status_test.go create mode 100644 internal/watchdog/check_l3.go create mode 100644 internal/watchdog/check_l3_test.go diff --git a/docs/exploitation/observabilite.rst b/docs/exploitation/observabilite.rst index b8f7a4c..d0b97e1 100644 --- a/docs/exploitation/observabilite.rst +++ b/docs/exploitation/observabilite.rst @@ -38,6 +38,20 @@ toujours sur le système, et **notifie les écarts sans jamais réparer**. Il contrôle notamment l'existence des network namespaces, des liens réseau des subnets, des taps de VM et la réponse des units systemd associées. +Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de contrôle : + +* **Sessions BGP** : une notification par route reflector dont la session n'est pas + ``Established`` (``show bgp neighbors ``). +* **VPC routés** (``l3vni``, état ``running``) : présence et état des interfaces ``vrf-``, + ``brl3-``, ``vxl3-``, ``l3--o`` et, dans le netns, ``l3--i`` ; L3VNI de type + ``L3``, ``Up``, avec la MAC de routeur attendue (``show evpn vni ``). +* **Réconciliation des routes exportées** dans la VRF : chaque ``/24`` de subnet ``vxlan`` et chaque + ``/32`` d'interface de VM ``running`` doit être exporté ; à l'inverse, une route exportée dont + le propriétaire n'existe plus — ni ``running``, ni en ``creating``, ``deleting`` ou ``error`` — + est signalée comme un **trou noir** : l'annonce attire du trafic que plus rien ne reçoit. + +Un VPC routé alors que la section ``bgp`` a disparu de la configuration est signalé lui aussi. + .. note:: Un écart persistant est signalé **à chaque tick**, sans déduplication. Le volume de diff --git a/internal/frr/status.go b/internal/frr/status.go new file mode 100644 index 0000000..031ec39 --- /dev/null +++ b/internal/frr/status.go @@ -0,0 +1,49 @@ +package frr + +import ( + "fmt" + "net" + "strings" +) + +type VNIState struct { + Type string + State string + RouterMAC string +} + +func (c *Client) NeighborEstablished(peer net.IP) (bool, error) { + out, err := c.exec("show bgp neighbors " + peer.String()) + if err != nil { + return false, err + } + return strings.Contains(out, "BGP state = Established"), nil +} + +func (c *Client) VNI(vni uint32) (VNIState, error) { + out, err := c.exec(fmt.Sprintf("show evpn vni %d", vni)) + if err != nil { + return VNIState{}, err + } + return parseVNI(out), nil +} + +func parseVNI(out string) VNIState { + var s VNIState + for _, l := range strings.Split(out, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(l), ": ") + if !ok { + continue + } + value = strings.TrimSpace(value) + switch strings.ToLower(key) { + case "type": + s.Type = value + case "state": + s.State = value + case "router mac": + s.RouterMAC = strings.ToLower(value) + } + } + return s +} diff --git a/internal/frr/status_test.go b/internal/frr/status_test.go new file mode 100644 index 0000000..9b65f7d --- /dev/null +++ b/internal/frr/status_test.go @@ -0,0 +1,50 @@ +package frr + +import ( + "net" + "strings" + "testing" +) + +const vniOutput = `VNI: 2700 + Type: L3 + Tenant VRF: vrf-s7 + Vxlan-Intf: vxl3-s7 + SVI-If: brl3-s7 + State: Up + VNI Filter: none + System MAC: 02:02:c0:a8:0e:aa + Router MAC: 02:02:C0:A8:0E:0B + L2 VNIs: +` + +func TestVNIReadsTypeStateAndRouterMAC(t *testing.T) { + f := &fakeVtysh{fail: map[string]string{"show evpn vni 2700": vniOutput}} + s, err := f.client().VNI(2700) + if err != nil { + t.Fatal(err) + } + want := VNIState{Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"} + if s != want { + t.Errorf("VNI = %+v, attendu %+v", s, want) + } +} + +func TestVNIReportsAnUnknownVNI(t *testing.T) { + f := &fakeVtysh{fail: map[string]string{"show evpn vni 9": "% VNI 9 does not exist\n"}} + if _, err := f.client().VNI(9); err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Errorf("erreur %v", err) + } +} + +func TestNeighborEstablished(t *testing.T) { + rr := net.ParseIP("10.255.255.1") + f := &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1, remote AS 64600, local AS 64600, internal link\n BGP version 4, remote router ID 10.255.255.1\n BGP state = Established, up for 00:10:02\n"}} + if up, err := f.client().NeighborEstablished(rr); err != nil || !up { + t.Errorf("établie : %v, %v", up, err) + } + f = &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1\n BGP state = Active\n"}} + if up, err := f.client().NeighborEstablished(rr); err != nil || up { + t.Errorf("active : %v, %v", up, err) + } +} diff --git a/internal/netif/route_linux.go b/internal/netif/route_linux.go index f68d191..fc444d1 100644 --- a/internal/netif/route_linux.go +++ b/internal/netif/route_linux.go @@ -19,3 +19,24 @@ func RouteAdd(iface string, subnet *net.IPNet) error { Scope: netlink.SCOPE_LINK, }) } + +func TableRoutesVia(table int, iface string) ([]*net.IPNet, error) { + link, err := netlink.LinkByName(iface) + if err != nil { + return nil, err + } + routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{ + Table: table, + LinkIndex: link.Attrs().Index, + }, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil { + return nil, err + } + dsts := make([]*net.IPNet, 0, len(routes)) + for _, r := range routes { + if r.Dst != nil && r.Gw != nil { + dsts = append(dsts, r.Dst) + } + } + return dsts, nil +} diff --git a/internal/netif/route_other.go b/internal/netif/route_other.go index 685c5bf..71daec7 100644 --- a/internal/netif/route_other.go +++ b/internal/netif/route_other.go @@ -2,8 +2,15 @@ package netif -import "net" +import ( + "errors" + "net" +) func RouteAdd(_ string, _ *net.IPNet) error { return nil } + +func TableRoutesVia(_ int, _ string) ([]*net.IPNet, error) { + return nil, errors.New("route listing is only implemented on linux") +} diff --git a/internal/vpc/l3.go b/internal/vpc/l3.go index 225d650..79b2088 100644 --- a/internal/vpc/l3.go +++ b/internal/vpc/l3.go @@ -137,3 +137,33 @@ func deleteL3(h host, cfg *configuration.Config, v l3VPC) error { } return nil } + +type Routing struct { + VNI uint32 + Supernet *net.IPNet + VRF string + Bridge string + Vxlan string + VethHost string + VethNetns string +} + +func LoadRouting(db *badger.DB, vpcName string) (Routing, bool, error) { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return Routing{}, ok, err + } + return Routing{ + VNI: v.vni, + Supernet: v.supernet, + VRF: v.vrf(), + Bridge: v.bridge(), + Vxlan: v.vxlan(), + VethHost: v.vethHost(), + VethNetns: v.vethNetns(), + }, true, nil +} + +func RouterMAC(vtep net.IP) (net.HardwareAddr, error) { + return routerMAC(vtep) +} diff --git a/internal/watchdog/check_l3.go b/internal/watchdog/check_l3.go new file mode 100644 index 0000000..537cf29 --- /dev/null +++ b/internal/watchdog/check_l3.go @@ -0,0 +1,232 @@ +package watchdog + +import ( + "fmt" + "net" + "sort" + "strings" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/netns" + "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" + "git.g3e.fr/syonad/two/internal/watchdog/notify" + "git.g3e.fr/syonad/two/pkg/db/kv" + + "github.com/dgraph-io/badger/v4" +) + +const kindBGP = "bgp" + +type l3Probe interface { + VTEPAddress(iface string) (net.IP, error) + NeighborEstablished(peer net.IP) (bool, error) + VNI(vni uint32) (frr.VNIState, error) + ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) + LinkProblem(ns, iface string) string +} + +type systemProbe struct { + frr *frr.Client +} + +func (systemProbe) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) } + +func (p systemProbe) NeighborEstablished(peer net.IP) (bool, error) { + return p.frr.NeighborEstablished(peer) +} + +func (p systemProbe) VNI(vni uint32) (frr.VNIState, error) { return p.frr.VNI(vni) } + +func (systemProbe) ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) { + return netif.TableRoutesVia(int(table), iface) +} + +func (systemProbe) LinkProblem(ns, iface string) string { + if ns == "" { + return linkProblem(iface) + } + var p string + if err := netns.Call(ns, func() error { + p = linkProblem(iface) + return nil + }); err != nil { + return fmt.Sprintf("cannot enter netns %s: %v", ns, err) + } + if p != "" { + return p + " (in netns)" + } + return "" +} + +func CheckBGP(cfg *configuration.Config, p l3Probe, n notify.Notifier) { + if !cfg.BGP.Enabled() { + return + } + reflectors, err := cfg.BGP.Reflectors() + if err != nil { + n.Notify(kindBGP, "config", err.Error()) + return + } + for _, rr := range reflectors { + up, err := p.NeighborEstablished(rr) + switch { + case err != nil: + n.Notify(kindBGP, rr.String(), fmt.Sprintf("session unreadable: %v", err)) + case !up: + n.Notify(kindBGP, rr.String(), "session to the route reflector is not established") + } + } +} + +func CheckL3VPCs(db *badger.DB, cfg *configuration.Config, p l3Probe, n notify.Notifier) error { + pairs, err := kv.ListByPrefix(db, prefixVPC) + if err != nil { + return fmt.Errorf("watchdog: listing vpcs: %w", err) + } + var rmac string + for _, name := range resourceNames(pairs, prefixVPC) { + if st, err := state.Get(db, prefixVPC+name); err != nil || st != state.Running { + continue + } + r, ok, err := vpc.LoadRouting(db, name) + if err != nil { + n.Notify(kindVPC, name, err.Error()) + continue + } + if !ok { + continue + } + if !cfg.BGP.Enabled() { + n.Notify(kindVPC, name, "vpc has an l3vni but the agent configuration has no bgp section") + continue + } + if rmac == "" { + rmac = expectedRouterMAC(cfg, p, n) + } + checkL3VPC(db, name, r, rmac, p, n) + } + return nil +} + +func expectedRouterMAC(cfg *configuration.Config, p l3Probe, n notify.Notifier) string { + vtep, err := p.VTEPAddress(cfg.DefaultInterface) + if err != nil { + n.Notify(kindBGP, "vtep", fmt.Sprintf("vtep address unreadable on %s: %v", cfg.DefaultInterface, err)) + return "unknown" + } + mac, err := vpc.RouterMAC(vtep) + if err != nil { + n.Notify(kindBGP, "vtep", err.Error()) + return "unknown" + } + return mac.String() +} + +func checkL3VPC(db *badger.DB, name string, r vpc.Routing, rmac string, p l3Probe, n notify.Notifier) { + for _, iface := range []string{r.VRF, r.Bridge, r.Vxlan, r.VethHost} { + if problem := p.LinkProblem("", iface); problem != "" { + n.Notify(kindVPC, name, problem) + } + } + if problem := p.LinkProblem(name, r.VethNetns); problem != "" { + n.Notify(kindVPC, name, problem) + } + + s, err := p.VNI(r.VNI) + switch { + case err != nil: + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d unreadable in frr: %v", r.VNI, err)) + case s.Type != "L3" || s.State != "Up": + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d is type %q state %q in frr, expected L3 Up", r.VNI, s.Type, s.State)) + case rmac != "unknown" && s.RouterMAC != rmac: + n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d has router mac %q in frr, expected %s", r.VNI, s.RouterMAC, rmac)) + } + + actual, err := p.ExportedRoutes(r.VNI, r.VethHost) + if err != nil { + n.Notify(kindVPC, name, fmt.Sprintf("routes of %s unreadable: %v", r.VRF, err)) + return + } + expected, tolerated, err := expectedExports(db, name) + if err != nil { + n.Notify(kindVPC, name, err.Error()) + return + } + exported := make(map[string]bool, len(actual)) + for _, dst := range actual { + exported[dst.String()] = true + } + for _, dst := range sortedKeys(exported) { + if !expected[dst] && !tolerated[dst] { + n.Notify(kindVPC, name, fmt.Sprintf("route %s exported in %s without a running owner: its traffic is black-holed", dst, r.VRF)) + } + } + for _, dst := range sortedKeys(expected) { + if !exported[dst] { + n.Notify(kindVPC, name, fmt.Sprintf("route %s not exported in %s", dst, r.VRF)) + } + } +} + +func expectedExports(db *badger.DB, vpcName string) (expected, tolerated map[string]bool, err error) { + expected, tolerated = map[string]bool{}, map[string]bool{} + classify := func(st state.State, dst string) { + switch st { + case state.Running: + expected[dst] = true + case state.Creating, state.Deleting, state.Error: + tolerated[dst] = true + } + } + + subnets, err := kv.ListByPrefix(db, prefixSubnet) + if err != nil { + return nil, nil, fmt.Errorf("watchdog: listing subnets: %w", err) + } + routed := map[string]bool{} + for key, value := range subnets { + subnet, ok := strings.CutSuffix(strings.TrimPrefix(key, prefixSubnet), "/vpc") + if !ok || strings.Contains(subnet, "/") || value != vpcName || subnets[prefixSubnet+subnet+"/mode"] != "vxlan" { + continue + } + routed[subnet] = true + _, cidr, err := net.ParseCIDR(subnets[prefixSubnet+subnet+"/cidr"]) + if err != nil { + continue + } + if st, err := state.Get(db, prefixSubnet+subnet); err == nil { + classify(st, cidr.String()) + } + } + + vms, err := kv.ListByPrefix(db, prefixVM) + if err != nil { + return nil, nil, fmt.Errorf("watchdog: listing vms: %w", err) + } + for key, value := range vms { + parts := strings.Split(key, "/") + if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || !routed[value] { + continue + } + ip := net.ParseIP(vms[strings.TrimSuffix(key, "subnet")+"ip"]).To4() + if ip == nil { + continue + } + if st, err := state.Get(db, prefixVM+parts[1]); err == nil { + classify(st, ip.String()+"/32") + } + } + return expected, tolerated, nil +} + +func sortedKeys(m map[string]bool) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/watchdog/check_l3_test.go b/internal/watchdog/check_l3_test.go new file mode 100644 index 0000000..c307b30 --- /dev/null +++ b/internal/watchdog/check_l3_test.go @@ -0,0 +1,242 @@ +package watchdog + +import ( + "errors" + "net" + "reflect" + "testing" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" + "git.g3e.fr/syonad/two/internal/state" +) + +type fakeProbe struct { + established map[string]bool + neighborErr error + vni map[uint32]frr.VNIState + routes map[uint32][]string + routesErr error + links map[string]string + vtepErr error +} + +func (f *fakeProbe) VTEPAddress(string) (net.IP, error) { + if f.vtepErr != nil { + return nil, f.vtepErr + } + return net.ParseIP("192.168.14.11"), nil +} + +func (f *fakeProbe) NeighborEstablished(peer net.IP) (bool, error) { + return f.established[peer.String()], f.neighborErr +} + +func (f *fakeProbe) VNI(vni uint32) (frr.VNIState, error) { + s, ok := f.vni[vni] + if !ok { + return s, errors.New("% VNI does not exist") + } + return s, nil +} + +func (f *fakeProbe) ExportedRoutes(table uint32, _ string) ([]*net.IPNet, error) { + var out []*net.IPNet + for _, r := range f.routes[table] { + _, n, _ := net.ParseCIDR(r) + out = append(out, n) + } + return out, f.routesErr +} + +func (f *fakeProbe) LinkProblem(ns, iface string) string { + return f.links[ns+"/"+iface] +} + +func bgpCfg() *configuration.Config { + cfg := &configuration.Config{DefaultInterface: "br-000000"} + cfg.BGP.ASN = 64600 + cfg.BGP.RouteReflectors = []string{"10.255.255.1", "10.255.255.2"} + return cfg +} + +func notified(r *recorder) []string { + var out []string + for _, c := range r.calls { + out = append(out, c.kind+" "+c.name+": "+c.problem) + } + return out +} + +func TestCheckBGPReportsEachReflectorWithoutSession(t *testing.T) { + r := &recorder{} + CheckBGP(bgpCfg(), &fakeProbe{established: map[string]bool{"10.255.255.1": true}}, r) + want := []string{"bgp 10.255.255.2: session to the route reflector is not established"} + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications %q, attendu %q", got, want) + } +} + +func TestCheckBGPReportsAnUnreadableSession(t *testing.T) { + r := &recorder{} + CheckBGP(bgpCfg(), &fakeProbe{neighborErr: errors.New("vtysh absent")}, r) + if len(r.calls) != 2 || !r.hasProblemContaining("session unreadable: vtysh absent") { + t.Errorf("notifications %q", notified(r)) + } +} + +func TestCheckBGPIsSilentWithoutBGPSection(t *testing.T) { + r := &recorder{} + CheckBGP(&configuration.Config{}, &fakeProbe{}, r) + if len(r.calls) != 0 { + t.Errorf("notifications %q", notified(r)) + } +} + +func seedRoutedVPC(t *testing.T) *fakeProbe { + return &fakeProbe{ + vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}}, + } +} + +func TestCheckL3VPCsReconcilesExportedRoutesWithTheirOwners(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-flat", state.Running) + seedKV(t, db, "vpc/vp-flat/cidr", "10.242.0.0/16") + + seedResource(t, db, prefixSubnet, "sn-a", state.Running) + seedKV(t, db, "subnet/sn-a/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-a/mode", "vxlan") + seedKV(t, db, "subnet/sn-a/cidr", "10.241.1.0/24") + seedResource(t, db, prefixSubnet, "sn-b", state.Creating) + seedKV(t, db, "subnet/sn-b/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-b/mode", "vxlan") + seedKV(t, db, "subnet/sn-b/cidr", "10.241.2.0/24") + seedResource(t, db, prefixSubnet, "sn-br", state.Running) + seedKV(t, db, "subnet/sn-br/vpc", "vp-s7") + seedKV(t, db, "subnet/sn-br/mode", "bridge") + seedKV(t, db, "subnet/sn-br/cidr", "192.168.50.0/24") + + vm := func(name string, st state.State, subnet, ip string) { + seedResource(t, db, prefixVM, name, st) + seedKV(t, db, "vm/"+name+"/nic/0/subnet", subnet) + seedKV(t, db, "vm/"+name+"/nic/0/ip", ip) + } + vm("i-run", state.Running, "sn-a", "10.241.1.10") + vm("i-missing", state.Running, "sn-a", "10.241.1.11") + vm("i-starting", state.Creating, "sn-a", "10.241.1.12") + vm("i-gone", state.Deleted, "sn-a", "10.241.1.13") + vm("i-bridge", state.Running, "sn-br", "192.168.50.10") + seedResource(t, db, prefixSubnet, "sn-flat", state.Running) + seedKV(t, db, "subnet/sn-flat/vpc", "vp-flat") + seedKV(t, db, "subnet/sn-flat/mode", "vxlan") + seedKV(t, db, "subnet/sn-flat/cidr", "10.242.1.0/24") + vm("i-flat", state.Running, "sn-flat", "10.242.1.10") + + p := seedRoutedVPC(t) + p.routes = map[uint32][]string{2700: { + "10.241.1.0/24", "10.241.2.0/24", + "10.241.1.10/32", "10.241.1.12/32", "10.241.1.13/32", "10.241.1.50/32", + }} + r := &recorder{} + if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil { + t.Fatal(err) + } + want := []string{ + "vpc vp-s7: route 10.241.1.13/32 exported in vrf-s7 without a running owner: its traffic is black-holed", + "vpc vp-s7: route 10.241.1.50/32 exported in vrf-s7 without a running owner: its traffic is black-holed", + "vpc vp-s7: route 10.241.1.11/32 not exported in vrf-s7", + } + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications :\n%q\nattendu :\n%q", got, want) + } +} + +func TestCheckL3VPCsChecksInterfacesAndTheL3VNI(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-down", state.Error) + seedKV(t, db, "vpc/vp-down/cidr", "10.243.0.0/16") + seedKV(t, db, "vpc/vp-down/l3vni", "2800") + + p := &fakeProbe{ + vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:99"}}, + links: map[string]string{"/vxl3-s7": "interface vxl3-s7 down", "vp-s7/l3-s7-i": "interface l3-s7-i not found (in netns)"}, + } + r := &recorder{} + if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil { + t.Fatal(err) + } + want := []string{ + "vpc vp-s7: interface vxl3-s7 down", + "vpc vp-s7: interface l3-s7-i not found (in netns)", + `vpc vp-s7: l3vni 2700 has router mac "02:02:c0:a8:0e:99" in frr, expected 02:02:c0:a8:0e:0b`, + } + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications :\n%q\nattendu :\n%q", got, want) + } + + p.vni[2700] = frr.VNIState{Type: "L2", State: "Up"} + p.links = nil + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining(`l3vni 2700 is type "L2" state "Up" in frr, expected L3 Up`) { + t.Errorf("notifications %q", notified(r)) + } + + p.vni[2700] = frr.VNIState{Type: "L3", State: "Down", RouterMAC: "02:02:c0:a8:0e:0b"} + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining(`l3vni 2700 is type "L3" state "Down" in frr, expected L3 Up`) { + t.Errorf("notifications %q", notified(r)) + } + + delete(p.vni, 2700) + r = &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if !r.hasProblemContaining("l3vni 2700 unreadable in frr") { + t.Errorf("notifications %q", notified(r)) + } +} + +func TestCheckL3VPCsReportsAnL3VNIWithoutBGP(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + r := &recorder{} + CheckL3VPCs(db, &configuration.Config{}, &fakeProbe{}, r) + want := []string{"vpc vp-s7: vpc has an l3vni but the agent configuration has no bgp section"} + if got := notified(r); !reflect.DeepEqual(got, want) { + t.Errorf("notifications %q, attendu %q", got, want) + } +} + +func TestCheckL3VPCsReportsUnreadableRoutesAndVTEP(t *testing.T) { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixVPC, "vp-s8", state.Running) + seedKV(t, db, "vpc/vp-s8/cidr", "10.242.0.0/16") + seedKV(t, db, "vpc/vp-s8/l3vni", "2800") + p := seedRoutedVPC(t) + p.routesErr = errors.New("netlink refused") + p.vtepErr = errors.New("no address") + r := &recorder{} + CheckL3VPCs(db, bgpCfg(), p, r) + if n := len(r.forName("vtep")); n != 1 { + t.Errorf("VTEP illisible signalé %d fois, attendu une fois par passage : %q", n, notified(r)) + } + if !r.hasProblemContaining("routes of vrf-s7 unreadable: netlink refused") || !r.hasProblemContaining("vtep address unreadable on br-000000") { + t.Errorf("notifications %q", notified(r)) + } + if r.hasProblemContaining("router mac") { + t.Errorf("RMAC comparée sans VTEP : %q", notified(r)) + } +} diff --git a/internal/watchdog/watchdog.go b/internal/watchdog/watchdog.go index 6da33b8..7a0101f 100644 --- a/internal/watchdog/watchdog.go +++ b/internal/watchdog/watchdog.go @@ -6,6 +6,7 @@ import ( "time" configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/frr" "git.g3e.fr/syonad/two/internal/watchdog/notify" "git.g3e.fr/syonad/two/pkg/systemd" @@ -21,6 +22,7 @@ type Watchdog struct { logger *slog.Logger interval time.Duration dbusDown bool + l3 l3Probe } func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog { @@ -38,6 +40,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl notifier: n, logger: logger, interval: interval, + l3: systemProbe{frr: frr.NewClient()}, } } @@ -71,6 +74,10 @@ func (w *Watchdog) tick() { if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil { w.logger.Error("watchdog: vm check failed", "err", err) } + CheckBGP(w.cfg, w.l3, w.notifier) + if err := CheckL3VPCs(w.db, w.cfg, w.l3, w.notifier); err != nil { + w.logger.Error("watchdog: l3 vpc check failed", "err", err) + } } func (w *Watchdog) units() (unitChecker, func()) {