f-41: vpc: VRF, L3VNI et transit à la création des VPC routées #41

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-11 20:21:19 +02:00
commit 3c6852dc18
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
8 changed files with 541 additions and 5 deletions

View file

@ -70,8 +70,8 @@ func (c CreateVPCCommand) checkL3VNI(db *badger.DB, cfg *configuration.Config) e
return nil
}
func (c CreateVPCCommand) Execute(db *badger.DB, _ *configuration.Config) error {
return vpc.CreateVPC(db, c.Name)
func (c CreateVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
return vpc.CreateVPC(db, c.Name, cfg)
}
type DeleteVPCCommand struct {
@ -132,7 +132,7 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
}
}
if err := vpc.DeleteVPC(db, c.Name); err != nil {
if err := vpc.DeleteVPC(db, c.Name, cfg); err != nil {
return err
}
current, err := state.Get(db, c.Key())

View file

@ -1,6 +1,8 @@
package netif
import (
"errors"
"github.com/vishvananda/netlink"
)
@ -11,3 +13,15 @@ func DeleteLink(name string) error {
}
return netlink.LinkDel(link)
}
func DeleteLinkIfExists(name string) error {
link, err := netlink.LinkByName(name)
if err != nil {
var notFound netlink.LinkNotFoundError
if errors.As(err, &notFound) {
return nil
}
return err
}
return netlink.LinkDel(link)
}

View file

@ -3,6 +3,7 @@ package vpc
import (
"strings"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
@ -10,7 +11,7 @@ import (
"github.com/dgraph-io/badger/v4"
)
func CreateVPC(db *badger.DB, name string) error {
func CreateVPC(db *badger.DB, name string, cfg *configuration.Config) error {
if current, err := state.Get(db, "vpc/"+name); err != nil {
return err
} else if current == state.Creating {
@ -48,6 +49,15 @@ func CreateVPC(db *badger.DB, name string) error {
}); err != nil {
return err
}
l3, ok, err := loadL3(db, name, vpcID)
if err != nil {
return err
}
if ok {
if err := createL3(newSystem(), cfg, l3); err != nil {
return err
}
}
return state.Set(db, "vpc/"+name, state.Running)
}
return nil

View file

@ -3,6 +3,7 @@ package vpc
import (
"strings"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
@ -10,12 +11,22 @@ import (
"github.com/dgraph-io/badger/v4"
)
func DeleteVPC(db *badger.DB, name string) error {
func DeleteVPC(db *badger.DB, name string, cfg *configuration.Config) error {
if current, err := state.Get(db, "vpc/"+name); err != nil {
return err
} else if current == state.Deleting {
vpcID := strings.SplitN(name, "-", 2)[1]
l3, ok, err := loadL3(db, name, vpcID)
if err != nil {
return err
}
if ok {
if err := deleteL3(newSystem(), cfg, l3); err != nil {
return err
}
}
if err := netif.DeleteLink("vp-" + vpcID + "-e"); err != nil {
return err
}

87
internal/vpc/host.go Normal file
View file

@ -0,0 +1,87 @@
package vpc
import (
"net"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/sysctl"
)
type host interface {
VTEPAddress(iface string) (net.IP, error)
Sysctl(ns, key, value string) error
CreateVRF(name string, table uint32) error
CreateBridge(name string, mtu int) error
SetHardwareAddr(name string, mac net.HardwareAddr) error
SetMaster(iface, master string) error
CreateVxlan(name string, vni int, iface string, mtu int) error
CreateVeth(hostIf, nsIf, ns string, mtu int) error
AddrAdd(ns, iface string, prefix *net.IPNet) error
LinkUp(ns, iface string) error
RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error
DeleteLink(name string) error
ApplyVRF(v frr.VRF) error
WithdrawVRF(v frr.VRF) error
RemoveVRF(v frr.VRF) error
}
type system struct {
frr *frr.Client
}
func newSystem() system {
return system{frr: frr.NewClient()}
}
func in(ns string, fn func() error) error {
if ns == "" {
return fn()
}
return netns.Call(ns, fn)
}
func (system) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) }
func (system) Sysctl(ns, key, value string) error {
return in(ns, func() error { return sysctl.Set(key, value) })
}
func (system) CreateVRF(name string, table uint32) error { return netif.CreateVRF(name, table) }
func (system) CreateBridge(name string, mtu int) error { return netif.CreateBridge(name, mtu) }
func (system) SetHardwareAddr(name string, mac net.HardwareAddr) error {
return netif.LinkSetHardwareAddr(name, mac)
}
func (system) SetMaster(iface, master string) error { return netif.BridgeSetMaster(iface, master) }
func (system) CreateVxlan(name string, vni int, iface string, mtu int) error {
return netif.CreateVxlan(name, vni, iface, mtu)
}
func (system) CreateVeth(hostIf, nsIf, ns string, mtu int) error {
return netif.CreateVethToNetns(hostIf, nsIf, "/var/run/netns/"+ns, mtu)
}
func (system) AddrAdd(ns, iface string, prefix *net.IPNet) error {
return in(ns, func() error { return netif.AddrAddPrefix(iface, prefix) })
}
func (system) LinkUp(ns, iface string) error {
return in(ns, func() error { return netif.LinkSetUp(iface) })
}
func (system) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error {
return in(ns, func() error { return netif.RouteReplaceVia(0, dst, gw, iface) })
}
func (system) DeleteLink(name string) error { return netif.DeleteLinkIfExists(name) }
func (s system) ApplyVRF(v frr.VRF) error { return s.frr.ApplyVRF(v) }
func (s system) WithdrawVRF(v frr.VRF) error { return s.frr.WithdrawVRF(v) }
func (s system) RemoveVRF(v frr.VRF) error { return s.frr.RemoveVRF(v) }

139
internal/vpc/l3.go Normal file
View file

@ -0,0 +1,139 @@
package vpc
import (
"fmt"
"net"
"strconv"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
const l3MTU = 1500
var (
transitHost = &net.IPNet{IP: net.IPv4(169, 254, 0, 1).To4(), Mask: net.CIDRMask(30, 32)}
transitNetns = &net.IPNet{IP: net.IPv4(169, 254, 0, 2).To4(), Mask: net.CIDRMask(30, 32)}
)
type l3VPC struct {
netns string
id string
vni uint32
supernet *net.IPNet
}
func loadL3(db *badger.DB, name, id string) (l3VPC, bool, error) {
raw, err := kv.GetFromDB(db, "vpc/"+name+"/l3vni")
if err != nil {
return l3VPC{}, false, nil
}
vni, err := strconv.ParseUint(raw, 10, 32)
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: invalid l3vni %q", name, raw)
}
if err := ValidL3VNI(uint32(vni)); err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: %w", name, err)
}
cidr, err := kv.GetFromDB(db, "vpc/"+name+"/cidr")
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: no cidr: %w", name, err)
}
_, supernet, err := net.ParseCIDR(cidr)
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: invalid cidr %q: %w", name, cidr, err)
}
return l3VPC{netns: name, id: id, vni: uint32(vni), supernet: supernet}, true, nil
}
func (v l3VPC) vrf() string { return "vrf-" + v.id }
func (v l3VPC) bridge() string { return "brl3-" + v.id }
func (v l3VPC) vxlan() string { return "vxl3-" + v.id }
func (v l3VPC) vethHost() string { return "l3-" + v.id + "-o" }
func (v l3VPC) vethNetns() string { return "l3-" + v.id + "-i" }
func routerMAC(vtep net.IP) (net.HardwareAddr, error) {
ip := vtep.To4()
if ip == nil {
return nil, fmt.Errorf("vtep %v is not an IPv4 address", vtep)
}
return net.HardwareAddr{0x02, 0x02, ip[0], ip[1], ip[2], ip[3]}, nil
}
func bgpASN(cfg *configuration.Config) (uint32, error) {
if cfg == nil || !cfg.BGP.Enabled() {
return 0, fmt.Errorf("the vpc has an l3vni but the agent configuration has no bgp section")
}
return cfg.BGP.ASN, nil
}
func createL3(h host, cfg *configuration.Config, v l3VPC) error {
asn, err := bgpASN(cfg)
if err != nil {
return err
}
vtep, err := h.VTEPAddress(cfg.DefaultInterface)
if err != nil {
return fmt.Errorf("vtep: %w", err)
}
rmac, err := routerMAC(vtep)
if err != nil {
return err
}
steps := []struct {
what string
run func() error
}{
{"create " + v.vrf(), func() error { return h.CreateVRF(v.vrf(), v.vni) }},
{"create " + v.bridge(), func() error { return h.CreateBridge(v.bridge(), l3MTU) }},
{"set router mac on " + v.bridge(), func() error { return h.SetHardwareAddr(v.bridge(), rmac) }},
{"attach " + v.bridge(), func() error { return h.SetMaster(v.bridge(), v.vrf()) }},
{"create " + v.vxlan(), func() error { return h.CreateVxlan(v.vxlan(), int(v.vni), cfg.DefaultInterface, l3MTU) }},
{"attach " + v.vxlan(), func() error { return h.SetMaster(v.vxlan(), v.bridge()) }},
{"set up " + v.vxlan(), func() error { return h.LinkUp("", v.vxlan()) }},
{"create " + v.vethHost(), func() error { return h.CreateVeth(v.vethHost(), v.vethNetns(), v.netns, l3MTU) }},
{"attach " + v.vethHost(), func() error { return h.SetMaster(v.vethHost(), v.vrf()) }},
{"address " + v.vethHost(), func() error { return h.AddrAdd("", v.vethHost(), transitHost) }},
{"set up " + v.vethHost(), func() error { return h.LinkUp("", v.vethHost()) }},
{"forwarding on " + v.bridge(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.bridge()+".forwarding", "1") }},
{"forwarding on " + v.vethHost(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.vethHost()+".forwarding", "1") }},
{"address " + v.vethNetns(), func() error { return h.AddrAdd(v.netns, v.vethNetns(), transitNetns) }},
{"set up " + v.vethNetns(), func() error { return h.LinkUp(v.netns, v.vethNetns()) }},
{"forwarding in " + v.netns, func() error { return h.Sysctl(v.netns, "net.ipv4.ip_forward", "1") }},
{"route " + v.supernet.String() + " in " + v.netns, func() error {
return h.RouteVia(v.netns, v.supernet, transitHost.IP, v.vethNetns())
}},
{"frr " + v.vrf(), func() error {
return h.ApplyVRF(frr.VRF{ASN: asn, ID: v.id, L3VNI: v.vni, VTEP: vtep, Supernet: v.supernet})
}},
}
for _, s := range steps {
if err := s.run(); err != nil {
return fmt.Errorf("l3vni %d: %s: %w", v.vni, s.what, err)
}
}
return nil
}
func deleteL3(h host, cfg *configuration.Config, v l3VPC) error {
asn, err := bgpASN(cfg)
if err != nil {
return err
}
frrVRF := frr.VRF{ASN: asn, ID: v.id}
if err := h.WithdrawVRF(frrVRF); err != nil {
return fmt.Errorf("l3vni %d: withdraw frr: %w", v.vni, err)
}
for _, link := range []string{v.vethHost(), v.vxlan(), v.bridge(), v.vrf()} {
if err := h.DeleteLink(link); err != nil {
return fmt.Errorf("l3vni %d: delete %s: %w", v.vni, link, err)
}
}
if err := h.RemoveVRF(frrVRF); err != nil {
return fmt.Errorf("l3vni %d: remove frr vrf: %w", v.vni, err)
}
return nil
}

246
internal/vpc/l3_test.go Normal file
View file

@ -0,0 +1,246 @@
package vpc
import (
"errors"
"fmt"
"net"
"reflect"
"strings"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/pkg/db/kv"
)
type fakeHost struct {
calls []string
fail map[string]error
}
func (f *fakeHost) record(call string) error {
f.calls = append(f.calls, call)
for prefix, err := range f.fail {
if strings.HasPrefix(call, prefix) {
return err
}
}
return nil
}
func (f *fakeHost) VTEPAddress(iface string) (net.IP, error) {
if err := f.record("vtep " + iface); err != nil {
return nil, err
}
return net.ParseIP("192.168.14.11"), nil
}
func (f *fakeHost) Sysctl(ns, key, value string) error {
return f.record(fmt.Sprintf("sysctl [%s] %s=%s", ns, key, value))
}
func (f *fakeHost) CreateVRF(name string, table uint32) error {
return f.record(fmt.Sprintf("vrf %s table %d", name, table))
}
func (f *fakeHost) CreateBridge(name string, mtu int) error {
return f.record(fmt.Sprintf("bridge %s mtu %d", name, mtu))
}
func (f *fakeHost) SetHardwareAddr(name string, mac net.HardwareAddr) error {
return f.record(fmt.Sprintf("mac %s %s", name, mac))
}
func (f *fakeHost) SetMaster(iface, master string) error {
return f.record(fmt.Sprintf("master %s %s", iface, master))
}
func (f *fakeHost) CreateVxlan(name string, vni int, iface string, mtu int) error {
return f.record(fmt.Sprintf("vxlan %s id %d dev %s mtu %d", name, vni, iface, mtu))
}
func (f *fakeHost) CreateVeth(hostIf, nsIf, ns string, mtu int) error {
return f.record(fmt.Sprintf("veth %s peer %s netns %s mtu %d", hostIf, nsIf, ns, mtu))
}
func (f *fakeHost) AddrAdd(ns, iface string, prefix *net.IPNet) error {
return f.record(fmt.Sprintf("addr [%s] %s %s", ns, iface, prefix))
}
func (f *fakeHost) LinkUp(ns, iface string) error {
return f.record(fmt.Sprintf("up [%s] %s", ns, iface))
}
func (f *fakeHost) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error {
return f.record(fmt.Sprintf("route [%s] %s via %s dev %s", ns, dst, gw, iface))
}
func (f *fakeHost) DeleteLink(name string) error {
return f.record("del " + name)
}
func (f *fakeHost) ApplyVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet))
}
func (f *fakeHost) WithdrawVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr withdraw asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI))
}
func (f *fakeHost) RemoveVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr remove asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI))
}
func bgpConfig() *configuration.Config {
cfg := &configuration.Config{DefaultInterface: "br-000000"}
cfg.BGP.ASN = 64600
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
return cfg
}
func labL3(t *testing.T) l3VPC {
t.Helper()
_, supernet, err := net.ParseCIDR("10.241.0.0/16")
if err != nil {
t.Fatal(err)
}
return l3VPC{netns: "vp-s7", id: "s7", vni: 2700, supernet: supernet}
}
func TestCreateL3FollowsTheLabRecipe(t *testing.T) {
f := &fakeHost{}
if err := createL3(f, bgpConfig(), labL3(t)); err != nil {
t.Fatal(err)
}
want := []string{
"vtep br-000000",
"vrf vrf-s7 table 2700",
"bridge brl3-s7 mtu 1500",
"mac brl3-s7 02:02:c0:a8:0e:0b",
"master brl3-s7 vrf-s7",
"vxlan vxl3-s7 id 2700 dev br-000000 mtu 1500",
"master vxl3-s7 brl3-s7",
"up [] vxl3-s7",
"veth l3-s7-o peer l3-s7-i netns vp-s7 mtu 1500",
"master l3-s7-o vrf-s7",
"addr [] l3-s7-o 169.254.0.1/30",
"up [] l3-s7-o",
"sysctl [] net.ipv4.conf.brl3-s7.forwarding=1",
"sysctl [] net.ipv4.conf.l3-s7-o.forwarding=1",
"addr [vp-s7] l3-s7-i 169.254.0.2/30",
"up [vp-s7] l3-s7-i",
"sysctl [vp-s7] net.ipv4.ip_forward=1",
"route [vp-s7] 10.241.0.0/16 via 169.254.0.1 dev l3-s7-i",
"frr apply asn=64600 id=s7 vni=2700 vtep=192.168.14.11 supernet=10.241.0.0/16",
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
}
}
func TestCreateL3RequiresTheBGPSectionBeforeTouchingTheHost(t *testing.T) {
f := &fakeHost{}
cfg := &configuration.Config{DefaultInterface: "br-000000"}
if err := createL3(f, cfg, labL3(t)); err == nil || !strings.Contains(err.Error(), "no bgp section") {
t.Errorf("erreur %v", err)
}
if err := createL3(f, nil, labL3(t)); err == nil {
t.Error("config absente acceptée")
}
if len(f.calls) != 0 {
t.Errorf("appels : %q", f.calls)
}
}
func TestCreateL3StopsAtTheFirstFailure(t *testing.T) {
f := &fakeHost{fail: map[string]error{"vxlan ": errors.New("file exists")}}
err := createL3(f, bgpConfig(), labL3(t))
if err == nil || !strings.Contains(err.Error(), "l3vni 2700: create vxl3-s7: file exists") {
t.Errorf("erreur %v", err)
}
if last := f.calls[len(f.calls)-1]; !strings.HasPrefix(last, "vxlan ") {
t.Errorf("appels après l'échec : %q", f.calls)
}
}
func TestDeleteL3WithdrawsFRRThenTheKernelThenTheVRF(t *testing.T) {
f := &fakeHost{}
if err := deleteL3(f, bgpConfig(), labL3(t)); err != nil {
t.Fatal(err)
}
want := []string{
"frr withdraw asn=64600 id=s7 vni=0",
"del l3-s7-o",
"del vxl3-s7",
"del brl3-s7",
"del vrf-s7",
"frr remove asn=64600 id=s7 vni=0",
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
}
}
func TestDeleteL3KeepsTheKernelWhenFRRRefusesTheWithdrawal(t *testing.T) {
f := &fakeHost{fail: map[string]error{"frr withdraw": errors.New("% Please unconfigure l3vni 2700")}}
err := deleteL3(f, bgpConfig(), labL3(t))
if err == nil || !strings.Contains(err.Error(), "withdraw frr") {
t.Errorf("erreur %v", err)
}
if len(f.calls) != 1 {
t.Errorf("le noyau a été touché malgré le refus de FRR : %q", f.calls)
}
}
func TestDeleteL3KeepsTheFRRVRFWhenAKernelLinkCannotBeDeleted(t *testing.T) {
f := &fakeHost{fail: map[string]error{"del brl3-s7": errors.New("device busy")}}
if err := deleteL3(f, bgpConfig(), labL3(t)); err == nil || !strings.Contains(err.Error(), "delete brl3-s7") {
t.Errorf("erreur %v", err)
}
for _, c := range f.calls {
if strings.HasPrefix(c, "frr remove") || c == "del vrf-s7" {
t.Errorf("étape exécutée après l'échec : %q", f.calls)
}
}
}
func TestDeleteL3RequiresTheBGPSection(t *testing.T) {
f := &fakeHost{}
if err := deleteL3(f, &configuration.Config{}, labL3(t)); err == nil || len(f.calls) != 0 {
t.Errorf("erreur %v, appels %q", err, f.calls)
}
}
func TestRouterMAC(t *testing.T) {
mac, err := routerMAC(net.ParseIP("192.168.14.11"))
if err != nil || mac.String() != "02:02:c0:a8:0e:0b" {
t.Errorf("routerMAC = %v, %v", mac, err)
}
if _, err := routerMAC(net.ParseIP("fd00::1")); err == nil {
t.Error("VTEP IPv6 accepté")
}
}
func TestLoadL3(t *testing.T) {
db := kv.InitDB(kv.Config{Path: t.TempDir()}, false)
t.Cleanup(func() { db.Close() })
kv.AddInDB(db, "vpc/vp-a/cidr", "10.241.0.0/16")
if _, ok, err := loadL3(db, "vp-a", "a"); ok || err != nil {
t.Errorf("VPC sans l3vni : ok=%v err=%v", ok, err)
}
kv.AddInDB(db, "vpc/vp-a/l3vni", "2700")
got, ok, err := loadL3(db, "vp-a", "a")
if !ok || err != nil || got.vni != 2700 || got.id != "a" || got.netns != "vp-a" || got.supernet.String() != "10.241.0.0/16" {
t.Errorf("loadL3 = %+v, %v, %v", got, ok, err)
}
for _, bad := range []string{"abc", "254", "0"} {
kv.AddInDB(db, "vpc/vp-a/l3vni", bad)
if _, _, err := loadL3(db, "vp-a", "a"); err == nil {
t.Errorf("l3vni %q acceptée", bad)
}
}
kv.AddInDB(db, "vpc/vp-b/l3vni", "2700")
if _, _, err := loadL3(db, "vp-b", "b"); err == nil || !strings.Contains(err.Error(), "no cidr") {
t.Errorf("VPC sans cidr : %v", err)
}
}