labctl: segments sur des bridges de l'hôte, routeurs et passerelle VRRP #2
All checks were successful
Tests / test (push) Successful in 3m46s

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-13 09:57:09 +02:00
commit e9a0c19b72
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
24 changed files with 1766 additions and 578 deletions

View file

@ -31,7 +31,7 @@ func qemuArgs(p *topology.Plan, n topology.NodePlan, index int, dir string) []st
}
restrict := "on"
if n.Role == topology.RoleSwitch {
if n.Role == topology.RoleRouter {
restrict = "off"
}
args = append(args,
@ -39,19 +39,11 @@ func qemuArgs(p *topology.Plan, n topology.NodePlan, index int, dir string) []st
"-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,romfile=", AdminInterface, adminMAC(index)),
)
for _, c := range nodeCables(p, n.Name) {
args = append(args, cable(c.NodeInterface, c.NodePort, c.SwitchPort, c.NodeMAC.String(), c.MTU)...)
}
for _, c := range switchCables(p, n.Name) {
args = append(args, cable(c.SwitchInterface, c.SwitchPort, c.NodePort, c.SwitchMAC.String(), c.MTU)...)
for _, port := range nodePorts(p, n.Name) {
args = append(args,
"-netdev", fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", port.Interface, port.Tap),
"-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,host_mtu=%d,romfile=", port.Interface, port.MAC, port.MTU),
)
}
return args
}
func cable(id string, local, remote int, mac string, mtu int) []string {
return []string{
"-netdev", fmt.Sprintf("dgram,id=%s,local.type=inet,local.host=%s,local.port=%d,remote.type=inet,remote.host=%s,remote.port=%d",
id, loopback, local, loopback, remote),
"-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,host_mtu=%d,romfile=", id, mac, mtu),
}
}

View file

@ -119,32 +119,46 @@ func adminMAC(index int) net.HardwareAddr {
return net.HardwareAddr{0x02, 0x4d, byte(index >> 8), byte(index), 0x00, 0x00}
}
func nodeCables(p *topology.Plan, name string) []topology.Cable {
var out []topology.Cable
for _, c := range p.Cables {
if c.Node == name {
out = append(out, c)
func nodePorts(p *topology.Plan, name string) []topology.Port {
var out []topology.Port
for _, port := range p.Ports {
if port.Node == name {
out = append(out, port)
}
}
return out
}
func switchCables(p *topology.Plan, name string) []topology.Cable {
var out []topology.Cable
for _, c := range p.Cables {
if c.Switch == name {
out = append(out, c)
}
}
return out
}
func switchSegments(p *topology.Plan, name string) []topology.SegmentPlan {
func routedSegments(p *topology.Plan, name string) []topology.SegmentPlan {
var out []topology.SegmentPlan
for _, s := range p.Segments {
if s.Switch == name {
out = append(out, s)
for _, r := range s.Routers {
if r == name {
out = append(out, s)
}
}
}
return out
}
func segmentNamed(p *topology.Plan, name string) topology.SegmentPlan {
for _, s := range p.Segments {
if s.Name == name {
return s
}
}
return topology.SegmentPlan{}
}
func uplink(p *topology.Plan, name string) (topology.Port, bool) {
ports := nodePorts(p, name)
for _, port := range ports {
if segmentNamed(p, port.Segment).Gateway.IsValid() {
return port, true
}
}
if len(ports) > 0 {
return ports[0], false
}
return topology.Port{}, false
}

View file

@ -2,6 +2,7 @@ package render
import (
"reflect"
"strconv"
"strings"
"testing"
@ -18,12 +19,12 @@ images:
url: https://example.invalid/deb.qcow2
sums: https://example.invalid/SHA512SUMS
segments:
underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 }
underlay: { routers: [sw1], cidr: 10.250.0.0/24, mtu: 9000 }
nodes:
sw1: { role: switch, image: deb, cpus: 2, memory: 1024 }
rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] }
hv1: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
hv2: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
sw1: { role: router, image: deb, cpus: 2, memory: 1024, segments: [underlay] }
rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] }
hv1: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
hv2: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
`
const twoSegments = `name: two-seg
@ -32,13 +33,30 @@ images:
url: https://example.invalid/deb.qcow2
sums: https://example.invalid/SHA512SUMS
segments:
red: { switch: sw, cidr: 10.1.0.0/24 }
blue: { switch: sw, cidr: 10.2.0.0/24, mtu: 1500 }
red: { routers: [sw], cidr: 10.1.0.0/24 }
blue: { routers: [sw], cidr: 10.2.0.0/24, mtu: 1500 }
nodes:
sw: { role: switch, image: deb, cpus: 1, memory: 512 }
sw: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, blue] }
hv: { role: host, image: deb, cpus: 1, memory: 512, segments: [blue, red] }
`
const withVRRP = `name: vrrp
images:
deb:
url: https://example.invalid/deb.qcow2
sums: https://example.invalid/SHA512SUMS
segments:
red: { routers: [r1, r2], cidr: 10.1.0.0/24 }
blue: { routers: [r1], cidr: 10.2.0.0/24 }
pure: { cidr: 10.3.0.0/24 }
nodes:
r1: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, blue] }
r2: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, pure] }
r3: { role: router, image: deb, cpus: 1, memory: 512, segments: [pure] }
hv: { role: host, image: deb, cpus: 1, memory: 512, segments: [pure, red] }
l2: { role: host, image: deb, cpus: 1, memory: 512, segments: [pure] }
`
func plan(t *testing.T, doc string) *topology.Plan {
t.Helper()
topo, err := topology.Parse([]byte(doc))
@ -182,7 +200,7 @@ func TestQEMU_HypervisorNetwork(t *testing.T) {
hv := nodeNamed(t, renderAll(t, twoHypervisors), "hv1")
wantNetdevs := []string{
"user,id=mgmt0,restrict=on,ipv6=off,hostfwd=tcp:127.0.0.1:2202-:22",
"dgram,id=underlay,local.type=inet,local.host=127.0.0.1,local.port=20002,remote.type=inet,remote.host=127.0.0.1,remote.port=20003",
"tap,id=underlay,ifname=lt2,script=no,downscript=no",
}
wantDevices := []string{
"virtio-net-pci,netdev=mgmt0,mac=02:4d:00:02:00:00,romfile=",
@ -196,27 +214,31 @@ func TestQEMU_HypervisorNetwork(t *testing.T) {
}
}
func TestQEMU_SwitchNetwork(t *testing.T) {
sw := nodeNamed(t, renderAll(t, twoHypervisors), "sw1")
func TestQEMU_RouterNetwork(t *testing.T) {
sw := nodeNamed(t, renderAll(t, twoSegments), "sw")
wantNetdevs := []string{
"user,id=mgmt0,restrict=off,ipv6=off,hostfwd=tcp:127.0.0.1:2200-:22",
"dgram,id=p0,local.type=inet,local.host=127.0.0.1,local.port=20001,remote.type=inet,remote.host=127.0.0.1,remote.port=20000",
"dgram,id=p1,local.type=inet,local.host=127.0.0.1,local.port=20003,remote.type=inet,remote.host=127.0.0.1,remote.port=20002",
"dgram,id=p2,local.type=inet,local.host=127.0.0.1,local.port=20005,remote.type=inet,remote.host=127.0.0.1,remote.port=20004",
"tap,id=red,ifname=lt0,script=no,downscript=no",
"tap,id=blue,ifname=lt2,script=no,downscript=no",
}
wantDevices := []string{
"virtio-net-pci,netdev=mgmt0,mac=02:4d:00:00:00:00,romfile=",
"virtio-net-pci,netdev=red,mac=02:4c:00:00:00:00,host_mtu=9000,romfile=",
"virtio-net-pci,netdev=blue,mac=02:4c:00:00:01:00,host_mtu=1500,romfile=",
}
if got := value(t, sw.QEMU, "-netdev"); !reflect.DeepEqual(got, wantNetdevs) {
t.Errorf("netdevs = %v\nwant %v", got, wantNetdevs)
}
if !hasValue(sw.QEMU, "-device", "virtio-net-pci,netdev=p2,mac=02:4c:00:03:00:01,host_mtu=9000,romfile=") {
t.Errorf("p2 device missing: %v", sw.QEMU)
if got := value(t, sw.QEMU, "-device"); !reflect.DeepEqual(got, wantDevices) {
t.Errorf("devices = %v\nwant %v", got, wantDevices)
}
}
func TestQEMU_OnlyTheSwitchReachesTheOutsideThroughAdministration(t *testing.T) {
for name, n := range renderAll(t, twoHypervisors) {
func TestQEMU_OnlyRoutersReachTheOutsideThroughAdministration(t *testing.T) {
for name, n := range renderAll(t, withVRRP) {
admin := value(t, n.QEMU, "-netdev")[0]
wantRestrict := "restrict=on"
if name == "sw1" {
if name == "r1" || name == "r2" || name == "r3" {
wantRestrict = "restrict=off"
}
if !strings.Contains(admin, ","+wantRestrict+",") {
@ -228,33 +250,28 @@ func TestQEMU_OnlyTheSwitchReachesTheOutsideThroughAdministration(t *testing.T)
}
}
func TestQEMU_EveryCableEndsMatch(t *testing.T) {
ends := map[string]int{}
for _, n := range renderAll(t, twoHypervisors) {
func TestQEMU_EveryTapOfThePlanIsOpenedOnce(t *testing.T) {
p := plan(t, withVRRP)
opened := map[string]int{}
for _, n := range renderAll(t, withVRRP) {
for _, nd := range value(t, n.QEMU, "-netdev") {
if !strings.HasPrefix(nd, "dgram,") {
continue
if strings.HasPrefix(nd, "dgram") || strings.HasPrefix(nd, "socket") {
t.Errorf("%s uses a %s netdev", n.Name, nd)
}
var local, remote string
for _, kv := range strings.Split(nd, ",") {
if v, ok := strings.CutPrefix(kv, "local.port="); ok {
local = v
}
if v, ok := strings.CutPrefix(kv, "remote.port="); ok {
remote = v
if tap, ok := strings.CutPrefix(kv, "ifname="); ok {
opened[tap]++
}
}
ends[local+">"+remote]++
}
}
for pair, count := range ends {
local, remote, _ := strings.Cut(pair, ">")
if count != 1 || ends[remote+">"+local] != 1 {
t.Errorf("cable end %s has no single matching end", pair)
}
if len(opened) != len(p.Ports) {
t.Errorf("%d taps opened, plan has %d ports", len(opened), len(p.Ports))
}
if len(ends) != 6 {
t.Errorf("%d cable ends, want 6", len(ends))
for _, port := range p.Ports {
if opened[port.Tap] != 1 {
t.Errorf("tap %s opened %d times", port.Tap, opened[port.Tap])
}
}
}
@ -270,7 +287,7 @@ func TestNetworkConfig_Hypervisor(t *testing.T) {
}
under := iface(t, doc, "underlay")
if under.Match.MACAddress != "02:4c:00:02:00:00" || under.SetName != "underlay" || under.MTU != 9000 ||
!reflect.DeepEqual(under.Addresses, []string{"10.250.0.3/24"}) {
!reflect.DeepEqual(under.Addresses, []string{"10.250.0.4/24"}) {
t.Errorf("underlay = %+v", under)
}
if !reflect.DeepEqual(under.Routes, []route{{To: "0.0.0.0/0", Via: "10.250.0.1"}}) {
@ -289,17 +306,27 @@ func TestNetworkConfig_DHCPIsExplicitlyOffEverywhere(t *testing.T) {
}
}
func TestNetworkConfig_Switch(t *testing.T) {
doc := network(t, nodeNamed(t, renderAll(t, twoHypervisors), "sw1"))
func TestNetworkConfig_Router(t *testing.T) {
doc := network(t, nodeNamed(t, renderAll(t, twoSegments), "sw"))
if len(doc.Ethernets) != 3 {
t.Fatalf("network-config = %+v", doc)
}
admin := iface(t, doc, "mgmt0")
if !reflect.DeepEqual(admin.Routes, []route{{To: "0.0.0.0/0", Via: "10.0.2.2"}}) ||
admin.Nameservers == nil || !reflect.DeepEqual(admin.Nameservers.Addresses, []string{"10.0.2.3"}) {
t.Errorf("switch mgmt0 = %+v", admin)
t.Errorf("router mgmt0 = %+v", admin)
}
for port, mac := range map[string]string{"p0": "02:4c:00:01:00:01", "p1": "02:4c:00:02:00:01", "p2": "02:4c:00:03:00:01"} {
e := iface(t, doc, port)
if e.Match.MACAddress != mac || e.SetName != port || e.MTU != 9000 || len(e.Addresses) != 0 || len(e.Routes) != 0 {
t.Errorf("%s = %+v", port, e)
for name, want := range map[string]struct {
mac, address string
mtu int
}{
"red": {"02:4c:00:00:00:00", "10.1.0.2/24", 9000},
"blue": {"02:4c:00:00:01:00", "10.2.0.2/24", 1500},
} {
e := iface(t, doc, name)
if e.Match.MACAddress != want.mac || e.SetName != name || e.MTU != want.mtu ||
!reflect.DeepEqual(e.Addresses, []string{want.address}) || len(e.Routes) != 0 || e.Nameservers != nil {
t.Errorf("%s = %+v", name, e)
}
}
}
@ -333,45 +360,71 @@ func TestUserData_PasswordLoginIsExplicitlyOff(t *testing.T) {
}
}
func TestUserData_SwitchBuildsBridgeGatewayAndNAT(t *testing.T) {
func TestNetworkConfig_DefaultRouteOnTheFirstSegmentWithAGateway(t *testing.T) {
doc := network(t, nodeNamed(t, renderAll(t, withVRRP), "hv"))
pure, red := iface(t, doc, "pure"), iface(t, doc, "red")
if len(pure.Routes) != 0 || pure.Nameservers != nil || !reflect.DeepEqual(pure.Addresses, []string{"10.3.0.3/24"}) {
t.Errorf("pure = %+v", pure)
}
if !reflect.DeepEqual(red.Routes, []route{{To: "0.0.0.0/0", Via: "10.1.0.1"}}) || red.Nameservers == nil ||
!reflect.DeepEqual(red.Addresses, []string{"10.1.0.4/24"}) {
t.Errorf("red = %+v", red)
}
}
func TestNetworkConfig_NoDefaultRouteWithoutGateway(t *testing.T) {
for name, e := range network(t, nodeNamed(t, renderAll(t, withVRRP), "l2")).Ethernets {
if len(e.Routes) != 0 || e.Nameservers != nil {
t.Errorf("%s = %+v, want no route", name, e)
}
}
}
func TestNetworkConfig_RouterHasNoDefaultRouteOnItsSegments(t *testing.T) {
doc := network(t, nodeNamed(t, renderAll(t, withVRRP), "r1"))
for _, name := range []string{"red", "blue"} {
if e := iface(t, doc, name); len(e.Routes) != 0 || e.Nameservers != nil {
t.Errorf("%s = %+v", name, e)
}
}
if admin := iface(t, doc, "mgmt0"); len(admin.Routes) != 1 {
t.Errorf("mgmt0 = %+v", admin)
}
}
func TestUserData_RouterBuildsGatewayAndNAT(t *testing.T) {
cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "sw1"))
if !reflect.DeepEqual(cfg.Packages, []string{"nftables"}) {
t.Errorf("packages = %v", cfg.Packages)
}
script := fileAt(t, cfg, "/usr/local/sbin/lab-switch")
script := fileAt(t, cfg, "/usr/local/sbin/lab-router")
if script.Permissions != "0755" {
t.Errorf("script permissions = %s", script.Permissions)
}
wantScript := `#!/bin/sh
set -eu
sysctl -qw net.ipv4.ip_forward=1
ip link add br-underlay type bridge stp_state 0 2>/dev/null || true
ip link set dev p0 master br-underlay
ip link set dev p0 up
ip link set dev p1 master br-underlay
ip link set dev p1 up
ip link set dev p2 master br-underlay
ip link set dev p2 up
ip link set dev br-underlay mtu 9000
ip addr replace 10.250.0.1/24 dev br-underlay
ip link set dev br-underlay up
nft -f /etc/lab-switch.nft
ip addr replace 10.250.0.1/24 dev underlay
nft -f /etc/lab-router.nft
`
if script.Content != wantScript {
t.Errorf("script:\n%s\nwant:\n%s", script.Content, wantScript)
}
nft := fileAt(t, cfg, "/etc/lab-switch.nft").Content
nftFile := fileAt(t, cfg, "/etc/lab-router.nft")
if nftFile.Permissions != "0644" {
t.Errorf("nft permissions = %s", nftFile.Permissions)
}
for _, want := range []string{
"add table ip lab_nat\ndelete table ip lab_nat\n",
"type nat hook postrouting priority srcnat;",
`ip saddr { 10.250.0.0/24 } oifname "mgmt0" masquerade`,
} {
if !strings.Contains(nft, want) {
t.Errorf("nft rules miss %q:\n%s", want, nft)
if !strings.Contains(nftFile.Content, want) {
t.Errorf("nft rules miss %q:\n%s", want, nftFile.Content)
}
}
unit := fileAt(t, cfg, "/etc/systemd/system/lab-switch.service").Content
for _, want := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=/usr/local/sbin/lab-switch", "After=network-online.target", "WantedBy=multi-user.target"} {
unit := fileAt(t, cfg, "/etc/systemd/system/lab-router.service").Content
for _, want := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=/usr/local/sbin/lab-router", "After=network-online.target", "WantedBy=multi-user.target"} {
if !strings.Contains(unit, want) {
t.Errorf("unit misses %q", want)
}
@ -379,29 +432,82 @@ nft -f /etc/lab-switch.nft
if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"/usr/local/sbin/lab-provision"}}) {
t.Errorf("runcmd = %v", cfg.Runcmd)
}
if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-switch.service\n" {
if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-router.service\n" {
t.Errorf("lab-provision:\n%s", got)
}
}
func TestUserData_SwitchWithTwoSegments(t *testing.T) {
func TestUserData_RouterWithTwoSegments(t *testing.T) {
cfg := user(t, nodeNamed(t, renderAll(t, twoSegments), "sw"))
script := fileAt(t, cfg, "/usr/local/sbin/lab-switch").Content
for _, want := range []string{
"ip link set dev p0 master br-red\n",
"ip link set dev p1 master br-blue\n",
"ip link set dev br-blue mtu 1500\n",
"ip addr replace 10.2.0.1/24 dev br-blue\n",
script := fileAt(t, cfg, "/usr/local/sbin/lab-router").Content
want := "sysctl -qw net.ipv4.ip_forward=1\nip addr replace 10.1.0.1/24 dev red\nip addr replace 10.2.0.1/24 dev blue\nnft -f /etc/lab-router.nft\n"
if !strings.HasSuffix(script, want) {
t.Errorf("script:\n%s\ndoes not end with:\n%s", script, want)
}
if nft := fileAt(t, cfg, "/etc/lab-router.nft").Content; !strings.Contains(nft, "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }") {
t.Errorf("nft:\n%s", nft)
}
}
func TestUserData_RouterScriptWithVRRPAndSingleSegments(t *testing.T) {
script := fileAt(t, user(t, nodeNamed(t, renderAll(t, withVRRP), "r1")), "/usr/local/sbin/lab-router").Content
want := `#!/bin/sh
set -eu
sysctl -qw net.ipv4.ip_forward=1
sysctl -qw net.ipv4.conf.red.arp_ignore=1 net.ipv4.conf.red.arp_announce=2
ip link show dev vrred >/dev/null 2>&1 || { ip link add vrred link red type macvlan mode bridge && ip link set dev vrred protodown on; }
ip link set dev vrred addrgenmode random address 00:00:5e:00:01:01
ip addr replace 10.1.0.1/24 dev vrred
ip link set dev vrred up
ip addr replace 10.2.0.1/24 dev blue
nft -f /etc/lab-router.nft
`
if script != want {
t.Errorf("script:\n%s\nwant:\n%s", script, want)
}
r2 := fileAt(t, user(t, nodeNamed(t, renderAll(t, withVRRP), "r2")), "/usr/local/sbin/lab-router").Content
if !strings.Contains(r2, "ip addr replace 10.1.0.1/24 dev vrred\n") || strings.Contains(r2, "dev pure") {
t.Errorf("r2 script:\n%s", r2)
}
}
func TestUserData_RouterNATCoversTheSegmentsItRoutes(t *testing.T) {
nodes := renderAll(t, withVRRP)
for name, want := range map[string]string{
"r1": "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }",
"r2": "ip saddr { 10.1.0.0/24 }",
} {
if !strings.Contains(script, want) {
t.Errorf("script misses %q:\n%s", want, script)
if nft := fileAt(t, user(t, nodeNamed(t, nodes, name)), "/etc/lab-router.nft").Content; !strings.Contains(nft, want) {
t.Errorf("%s nft:\n%s\nmisses %s", name, nft, want)
}
}
if strings.Contains(script, "p1 master br-red") || strings.Contains(script, "p0 master br-blue") {
t.Errorf("port bridged on the wrong segment:\n%s", script)
cfg := user(t, nodeNamed(t, nodes, "r3"))
if nft := fileAt(t, cfg, "/etc/lab-router.nft").Content; nft != "add table ip lab_nat\ndelete table ip lab_nat\n" {
t.Errorf("r3 nft:\n%s", nft)
}
if nft := fileAt(t, cfg, "/etc/lab-switch.nft").Content; !strings.Contains(nft, "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }") {
t.Errorf("nft:\n%s", nft)
if script := fileAt(t, cfg, "/usr/local/sbin/lab-router").Content; script != "#!/bin/sh\nset -eu\nsysctl -qw net.ipv4.ip_forward=1\nnft -f /etc/lab-router.nft\n" {
t.Errorf("r3 script:\n%s", script)
}
}
func TestVRRPMAC(t *testing.T) {
for vrid, want := range map[int]string{1: "00:00:5e:00:01:01", 16: "00:00:5e:00:01:10", 255: "00:00:5e:00:01:ff"} {
if got := vrrpMAC(vrid); got != want {
t.Errorf("vrrpMAC(%d) = %s, want %s", vrid, got, want)
}
}
}
func TestVRRPPriorityFollowsTheRoutersOrder(t *testing.T) {
var routers []string
for i := 0; i < 22; i++ {
routers = append(routers, "r"+strconv.Itoa(i))
}
s := topology.SegmentPlan{Routers: routers}
for name, want := range map[string]int{"r0": 200, "r1": 190, "r19": 10, "r20": 1, "r21": 1} {
if got := vrrpPriority(s, name); got != want {
t.Errorf("priority of %s = %d, want %d", name, got, want)
}
}
}
@ -415,7 +521,7 @@ func TestMetaData(t *testing.T) {
}
}
func TestRender_AdminMacsAreUniqueAndApartFromCableMacs(t *testing.T) {
func TestRender_AdminMacsAreUniqueAndApartFromPortMacs(t *testing.T) {
seen := map[string]string{}
for name, n := range renderAll(t, twoHypervisors) {
for _, d := range value(t, n.QEMU, "-device") {
@ -429,8 +535,8 @@ func TestRender_AdminMacsAreUniqueAndApartFromCableMacs(t *testing.T) {
}
}
}
if len(seen) != 10 {
t.Errorf("%d distinct macs, want 10", len(seen))
if len(seen) != 8 {
t.Errorf("%d distinct macs, want 8", len(seen))
}
}

View file

@ -15,9 +15,9 @@ images:
url: https://example.invalid/deb.qcow2
sums: https://example.invalid/SHA512SUMS
segments:
underlay: { switch: sw1, cidr: 192.168.14.0/24 }
underlay: { routers: [sw1], cidr: 192.168.14.0/24 }
nodes:
sw1: { role: switch, image: deb, cpus: 2, memory: 1024, secondary: { underlay: [169.254.0.1/28] }, frr: sw1.conf }
sw1: { role: router, image: deb, cpus: 2, memory: 1024, segments: [underlay], secondary: { underlay: [169.254.0.1/28] }, frr: sw1.conf }
rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay], secondary: { underlay: [169.254.0.3/28] }, loopback: 10.255.255.1/32, frr: rr1.conf }
hv1: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay], frr: hv1.conf, provision: hv.sh, post_provision: hv-post.sh, files: { agent.yml: two.yml }, services: [agent.service] }
hv2: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
@ -79,16 +79,20 @@ sed -i 's/^bgpd=no/bgpd=yes/' /etc/frr/daemons
func TestRoles_SecondaryAddressFollowsThePrimaryOnTheNode(t *testing.T) {
doc := network(t, nodeNamed(t, renderRoles(t), "rr1"))
got := iface(t, doc, "underlay").Addresses
if !reflect.DeepEqual(got, []string{"192.168.14.2/24", "169.254.0.3/28"}) {
if !reflect.DeepEqual(got, []string{"192.168.14.3/24", "169.254.0.3/28"}) {
t.Errorf("rr1 underlay addresses = %v", got)
}
}
func TestRoles_SwitchCarriesItsSecondaryOnTheBridge(t *testing.T) {
script := fileAt(t, user(t, nodeNamed(t, renderRoles(t), "sw1")), "/usr/local/sbin/lab-switch").Content
want := "ip addr replace 192.168.14.1/24 dev br-underlay\nip addr replace 169.254.0.1/28 dev br-underlay\nip link set dev br-underlay up\n"
if !strings.Contains(script, want) {
t.Errorf("lab-switch:\n%s\ndoes not contain:\n%s", script, want)
func TestRoles_RouterCarriesItsSecondaryOnItsSegmentInterface(t *testing.T) {
sw1 := nodeNamed(t, renderRoles(t), "sw1")
under := iface(t, network(t, sw1), "underlay")
if !reflect.DeepEqual(under.Addresses, []string{"192.168.14.2/24", "169.254.0.1/28"}) || len(under.Routes) != 0 {
t.Errorf("sw1 underlay = %+v", under)
}
script := fileAt(t, user(t, sw1), "/usr/local/sbin/lab-router").Content
if !strings.Contains(script, "ip addr replace 192.168.14.1/24 dev underlay\n") || strings.Contains(script, "169.254.0.1") {
t.Errorf("lab-router:\n%s", script)
}
}
@ -156,7 +160,7 @@ func TestRoles_ProvisioningIsOneScriptThatStopsAtTheFirstFailure(t *testing.T) {
env := "LAB_NODE=hv1 LAB_UPLINK=underlay LAB_FILES=/etc/lab/files "
files := "mkdir -p /etc/lab/files /run/lab-files\nmount -o ro LABEL=LABFILES /run/lab-files\ncp -R /run/lab-files/. /etc/lab/files/\numount /run/lab-files\n"
for name, steps := range map[string]string{
"sw1": "systemctl daemon-reload\nsystemctl enable --now lab-switch.service\n/usr/local/sbin/lab-frr\n",
"sw1": "systemctl daemon-reload\nsystemctl enable --now lab-router.service\n/usr/local/sbin/lab-frr\n",
"hv1": files + env + "/usr/local/sbin/lab-node-provision\n/usr/local/sbin/lab-frr\n" + env + "/usr/local/sbin/lab-node-post-provision\n",
} {
cfg := user(t, nodeNamed(t, nodes, name))
@ -238,6 +242,40 @@ func TestRoles_UplinkIsTheInterfaceCarryingTheDefaultRoute(t *testing.T) {
}
}
func TestRoles_UplinkSkipsSegmentsWithoutGateway(t *testing.T) {
p := plan(t, withVRRP)
want := map[string]string{
"hv": "LAB_NODE=hv LAB_UPLINK=red LAB_FILES=/etc/lab/files ",
"l2": "LAB_NODE=l2 LAB_UPLINK=pure LAB_FILES=/etc/lab/files ",
"r1": "LAB_NODE=r1 LAB_UPLINK=mgmt0 LAB_FILES=/etc/lab/files ",
"r3": "LAB_NODE=r3 LAB_UPLINK=mgmt0 LAB_FILES=/etc/lab/files ",
}
for _, n := range p.Nodes {
if w, ok := want[n.Name]; ok {
if got := provisionEnv(p, n); got != w {
t.Errorf("%s provision environment = %q, want %q", n.Name, got, w)
}
}
}
}
func TestRoles_UplinkIsGivenToTheProvisionScript(t *testing.T) {
doc := strings.Replace(withVRRP, "segments: [pure, red] }", "segments: [pure, red], provision: hv.sh }", 1)
nodes, err := Render(plan(t, doc), Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey}, Provision: map[string]string{"hv": "#!/bin/sh\n"}})
if err != nil {
t.Fatalf("Render: %v", err)
}
for _, n := range nodes {
if n.Name != "hv" {
continue
}
got := fileAt(t, user(t, n), "/usr/local/sbin/lab-provision").Content
if got != "#!/bin/sh\nset -eu\nLAB_NODE=hv LAB_UPLINK=red LAB_FILES=/etc/lab/files /usr/local/sbin/lab-node-provision\n" {
t.Errorf("lab-provision:\n%s", got)
}
}
}
func TestRoles_NodesWithoutProvisionReceiveNoProvisionFiles(t *testing.T) {
nodes := renderRoles(t)
for _, name := range []string{"sw1", "rr1", "hv2"} {
@ -253,8 +291,67 @@ func TestRoles_FRRDaemons(t *testing.T) {
p := plan(t, withRoles)
want := map[string][]string{"sw1": {"bgpd", "bfdd"}, "rr1": {"bgpd", "bfdd"}, "hv1": {"bgpd"}}
for _, n := range p.Nodes {
if w, ok := want[n.Name]; ok && !reflect.DeepEqual(frrDaemons(n), w) {
t.Errorf("%s daemons = %v, want %v", n.Name, frrDaemons(n), w)
if w, ok := want[n.Name]; ok && !reflect.DeepEqual(frrDaemons(p, n), w) {
t.Errorf("%s daemons = %v, want %v", n.Name, frrDaemons(p, n), w)
}
}
}
func TestRoles_VRRPAddsVrrpdOnlyToTheRoutersThatRunIt(t *testing.T) {
p := plan(t, withVRRP)
want := map[string][]string{"r1": {"bgpd", "bfdd", "vrrpd"}, "r2": {"bgpd", "bfdd", "vrrpd"}, "r3": {"bgpd", "bfdd"}, "hv": {"bgpd"}}
needs := map[string]bool{"r1": true, "r2": true, "r3": false, "hv": false, "l2": false}
for _, n := range p.Nodes {
if w, ok := want[n.Name]; ok && !reflect.DeepEqual(frrDaemons(p, n), w) {
t.Errorf("%s daemons = %v, want %v", n.Name, frrDaemons(p, n), w)
}
if got := NeedsFRR(p, n); got != needs[n.Name] {
t.Errorf("NeedsFRR(%s) = %v, want %v", n.Name, got, needs[n.Name])
}
}
}
func TestRoles_VRRPRouterWithoutFRRFileGetsAGeneratedConfig(t *testing.T) {
nodes := renderAll(t, withVRRP)
cfg := user(t, nodeNamed(t, nodes, "r1"))
want := "frr defaults traditional\nhostname r1\n!\ninterface red\n vrrp 1 version 3\n vrrp 1 priority 200\n no vrrp 1 preempt\n vrrp 1 ip 10.1.0.1\nexit\n!\n"
if f := fileAt(t, cfg, "/etc/lab/frr.conf"); f.Content != want || f.Permissions != "0640" {
t.Errorf("r1 frr.conf (%s):\n%s\nwant:\n%s", f.Permissions, f.Content, want)
}
vrrpd := "sed -i 's/^vrrpd=no/vrrpd=yes/' /etc/frr/daemons\n"
bfd := "sed -i 's/^bfdd=no/bfdd=yes/' /etc/frr/daemons\n"
tail := "install -o frr -g frr -m 0640 /etc/lab/frr.conf /etc/frr/frr.conf\nsystemctl restart frr\n"
if f := fileAt(t, cfg, "/usr/local/sbin/lab-frr"); f.Content != frrScriptHead+bfd+vrrpd+tail {
t.Errorf("r1 lab-frr:\n%s", f.Content)
}
fileAt(t, cfg, "/usr/share/keyrings/frrouting.gpg")
want = "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-router.service\n/usr/local/sbin/lab-frr\n"
if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != want {
t.Errorf("r1 lab-provision:\n%s\nwant:\n%s", got, want)
}
for _, name := range []string{"r3", "hv", "l2"} {
for _, f := range user(t, nodeNamed(t, nodes, name)).WriteFiles {
if strings.Contains(f.Path, "frr") {
t.Errorf("%s receives %s", name, f.Path)
}
}
}
}
func TestRoles_VRRPStanzaFollowsTheUserFRRConfig(t *testing.T) {
doc := strings.Replace(withVRRP, "segments: [red, pure] }", "segments: [red, pure], frr: r2.conf }", 1)
o := Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey}, FRR: map[string]string{"r2": "hostname r2\nrouter bgp 65001\n\n\n"}}
nodes, err := Render(plan(t, doc), o)
if err != nil {
t.Fatalf("Render: %v", err)
}
for _, n := range nodes {
if n.Name != "r2" {
continue
}
want := "hostname r2\nrouter bgp 65001\ninterface red\n vrrp 1 version 3\n vrrp 1 priority 190\n no vrrp 1 preempt\n vrrp 1 ip 10.1.0.1\nexit\n!\n"
if got := fileAt(t, user(t, n), "/etc/lab/frr.conf").Content; got != want {
t.Errorf("r2 frr.conf:\n%s\nwant:\n%s", got, want)
}
}
}
@ -275,7 +372,7 @@ func TestRoles_RefusesAnUnreadFRRConfig(t *testing.T) {
}
}
func TestRoles_SwitchLoopbackIsCreatedByTheSwitchScript(t *testing.T) {
func TestRoles_RouterLoopbackIsCreatedByTheRouterScript(t *testing.T) {
doc := strings.Replace(withRoles, "secondary: { underlay: [169.254.0.1/28] }, frr: sw1.conf", "secondary: { underlay: [169.254.0.1/28] }, loopback: 10.255.254.1/32, frr: sw1.conf", 1)
nodes, err := Render(plan(t, doc), roleOptions())
if err != nil {
@ -287,10 +384,16 @@ func TestRoles_SwitchLoopbackIsCreatedByTheSwitchScript(t *testing.T) {
sw1 = n
}
}
script := fileAt(t, user(t, sw1), "/usr/local/sbin/lab-switch").Content
want := "ip link set dev br-underlay up\nip link add lo1 type dummy 2>/dev/null || true\nip addr replace 10.255.254.1/32 dev lo1\nip link set dev lo1 up\nnft -f /etc/lab-switch.nft\n"
cfg := user(t, sw1)
script := fileAt(t, cfg, "/usr/local/sbin/lab-router").Content
want := "ip addr replace 192.168.14.1/24 dev underlay\nip link add lo1 type dummy 2>/dev/null || true\nip addr replace 10.255.254.1/32 dev lo1\nip link set dev lo1 up\nnft -f /etc/lab-router.nft\n"
if !strings.HasSuffix(script, want) {
t.Errorf("lab-switch:\n%s\ndoes not end with:\n%s", script, want)
t.Errorf("lab-router:\n%s\ndoes not end with:\n%s", script, want)
}
for _, f := range cfg.WriteFiles {
if strings.HasPrefix(f.Path, "/usr/local/sbin/lab-node") || strings.Contains(f.Path, "lab-node.service") {
t.Errorf("sw1 receives %s", f.Path)
}
}
}

View file

@ -12,9 +12,10 @@ import (
)
const (
SwitchScript = "/usr/local/sbin/lab-switch"
SwitchNFT = "/etc/lab-switch.nft"
SwitchUnit = "/etc/systemd/system/lab-switch.service"
RouterScript = "/usr/local/sbin/lab-router"
RouterNFT = "/etc/lab-router.nft"
RouterUnit = "/etc/systemd/system/lab-router.service"
VRRPPrefix = "vr"
NodeScript = "/usr/local/sbin/lab-node"
NodeUnit = "/etc/systemd/system/lab-node.service"
@ -99,14 +100,14 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
}
var steps []string
switch {
case n.Role == topology.RoleSwitch:
case n.Role == topology.RoleRouter:
cfg.Packages = []string{"nftables"}
cfg.WriteFiles = []writeFile{
{Path: SwitchScript, Permissions: "0755", Content: switchScript(p, n)},
{Path: SwitchNFT, Permissions: "0644", Content: switchNFT(p, n.Name)},
{Path: SwitchUnit, Permissions: "0644", Content: unit("Lab switch: bridges, gateways and NAT", SwitchScript)},
{Path: RouterScript, Permissions: "0755", Content: routerScript(p, n)},
{Path: RouterNFT, Permissions: "0644", Content: routerNFT(p, n.Name)},
{Path: RouterUnit, Permissions: "0644", Content: unit("Lab router: gateways and NAT", RouterScript)},
}
steps = append(steps, "systemctl daemon-reload", "systemctl enable --now lab-switch.service")
steps = append(steps, "systemctl daemon-reload", "systemctl enable --now lab-router.service")
case n.Loopback.IsValid():
cfg.WriteFiles = []writeFile{
{Path: NodeScript, Permissions: "0755", Content: "#!/bin/sh\nset -eu\n" + loopbackLines(n)},
@ -126,11 +127,11 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
cfg.WriteFiles = append(cfg.WriteFiles, writeFile{Path: NodeProvision, Permissions: "0755", Encoding: "b64", Content: base64.StdEncoding.EncodeToString([]byte(o.Provision[n.Name]))})
steps = append(steps, provisionEnv(p, n)+NodeProvision)
}
if n.FRR != "" {
if NeedsFRR(p, n) {
cfg.WriteFiles = append(cfg.WriteFiles,
writeFile{Path: FRRKey, Permissions: "0644", Encoding: "b64", Content: base64.StdEncoding.EncodeToString(frrKey)},
writeFile{Path: FRRConfig, Permissions: "0640", Content: o.FRR[n.Name]},
writeFile{Path: FRRScript, Permissions: "0755", Content: frrScript(n)},
writeFile{Path: FRRConfig, Permissions: "0640", Content: frrConfig(p, n, o.FRR[n.Name])},
writeFile{Path: FRRScript, Permissions: "0755", Content: frrScript(p, n)},
)
steps = append(steps, FRRScript)
}
@ -156,27 +157,77 @@ func retry(command string) string {
}
func provisionEnv(p *topology.Plan, n topology.NodePlan) string {
uplink := ""
if cables := nodeCables(p, n.Name); len(cables) > 0 {
uplink = cables[0].NodeInterface
port, _ := uplink(p, n.Name)
up := port.Interface
if n.Role == topology.RoleRouter {
up = AdminInterface
}
return fmt.Sprintf("LAB_NODE=%s LAB_UPLINK=%s LAB_FILES=%s ", n.Name, uplink, FilesDir)
return fmt.Sprintf("LAB_NODE=%s LAB_UPLINK=%s LAB_FILES=%s ", n.Name, up, FilesDir)
}
func frrDaemons(n topology.NodePlan) []string {
if n.Role == topology.RoleHost {
return []string{"bgpd"}
func vrrpSegments(p *topology.Plan, name string) []topology.SegmentPlan {
var out []topology.SegmentPlan
for _, s := range routedSegments(p, name) {
if s.VRID != 0 {
out = append(out, s)
}
}
return []string{"bgpd", "bfdd"}
return out
}
func frrScript(n topology.NodePlan) string {
func NeedsFRR(p *topology.Plan, n topology.NodePlan) bool {
return n.FRR != "" || len(vrrpSegments(p, n.Name)) > 0
}
func frrDaemons(p *topology.Plan, n topology.NodePlan) []string {
daemons := []string{"bgpd"}
if n.Role != topology.RoleHost {
daemons = append(daemons, "bfdd")
}
if len(vrrpSegments(p, n.Name)) > 0 {
daemons = append(daemons, "vrrpd")
}
return daemons
}
func frrConfig(p *topology.Plan, n topology.NodePlan, given string) string {
segments := vrrpSegments(p, n.Name)
if len(segments) == 0 {
return given
}
var b strings.Builder
if given == "" {
fmt.Fprintf(&b, "frr defaults traditional\nhostname %s\n!\n", n.Name)
} else {
b.WriteString(strings.TrimRight(given, "\n") + "\n")
}
for _, s := range segments {
fmt.Fprintf(&b, "interface %s\n vrrp %d version 3\n vrrp %d priority %d\n no vrrp %d preempt\n vrrp %d ip %s\nexit\n!\n",
s.Name, s.VRID, s.VRID, vrrpPriority(s, n.Name), s.VRID, s.VRID, s.Gateway)
}
return b.String()
}
func vrrpPriority(s topology.SegmentPlan, name string) int {
for i, r := range s.Routers {
if r == name {
return max(200-10*i, 1)
}
}
return 1
}
func vrrpMAC(vrid int) string {
return fmt.Sprintf("00:00:5e:00:01:%02x", vrid)
}
func frrScript(p *topology.Plan, n topology.NodePlan) string {
var b strings.Builder
b.WriteString("#!/bin/sh\nset -eu\nexport DEBIAN_FRONTEND=noninteractive\n. /etc/os-release\n")
fmt.Fprintf(&b, "echo \"deb [signed-by=%s] %s ${VERSION_CODENAME} %s\" > /etc/apt/sources.list.d/frr.list\n", FRRKey, FRRRepo, FRRSuite)
b.WriteString(retry("apt-get update -qq --error-on=any"))
fmt.Fprintf(&b, "apt-get install -y -qq --no-install-recommends %s\n", FRRPackages)
for _, d := range frrDaemons(n) {
for _, d := range frrDaemons(p, n) {
fmt.Fprintf(&b, "sed -i 's/^%s=no/%s=yes/' /etc/frr/daemons\n", d, d)
}
fmt.Fprintf(&b, "install -o frr -g frr -m 0640 %s /etc/frr/frr.conf\n", FRRConfig)
@ -199,77 +250,62 @@ func networkConfig(p *topology.Plan, n topology.NodePlan, index int) ([]byte, er
SetName: AdminInterface,
Addresses: []string{AdminAddress},
}
if n.Role == topology.RoleSwitch {
router := n.Role == topology.RoleRouter
if router {
admin.Routes = []route{{To: "0.0.0.0/0", Via: AdminGateway}}
admin.Nameservers = &nameservers{Addresses: []string{AdminDNS}}
for _, c := range switchCables(p, n.Name) {
doc.Ethernets[c.SwitchInterface] = ethernet{
Match: match{MACAddress: c.SwitchMAC.String()},
SetName: c.SwitchInterface,
MTU: c.MTU,
}
}
}
doc.Ethernets[AdminInterface] = admin
for i, c := range nodeCables(p, n.Name) {
up, routed := uplink(p, n.Name)
for _, port := range nodePorts(p, n.Name) {
e := ethernet{
Match: match{MACAddress: c.NodeMAC.String()},
SetName: c.NodeInterface,
MTU: c.MTU,
Addresses: []string{c.NodeAddress.String()},
Match: match{MACAddress: port.MAC.String()},
SetName: port.Interface,
MTU: port.MTU,
Addresses: []string{port.Address.String()},
}
for _, prefix := range n.Secondary[c.Segment] {
for _, prefix := range n.Secondary[port.Segment] {
e.Addresses = append(e.Addresses, prefix.String())
}
if i == 0 {
e.Routes = []route{{To: "0.0.0.0/0", Via: gatewayOf(p, c.Segment)}}
if !router && routed && port.Segment == up.Segment {
e.Routes = []route{{To: "0.0.0.0/0", Via: segmentNamed(p, port.Segment).Gateway.String()}}
e.Nameservers = &nameservers{Addresses: Nameservers}
}
doc.Ethernets[c.NodeInterface] = e
doc.Ethernets[port.Interface] = e
}
return yaml.Marshal(doc)
}
func gatewayOf(p *topology.Plan, segment string) string {
for _, s := range p.Segments {
if s.Name == segment {
return s.Gateway.String()
}
}
return ""
}
func switchScript(p *topology.Plan, n topology.NodePlan) string {
name := n.Name
func routerScript(p *topology.Plan, n topology.NodePlan) string {
var b strings.Builder
b.WriteString("#!/bin/sh\nset -eu\nsysctl -qw net.ipv4.ip_forward=1\n")
for _, s := range switchSegments(p, name) {
fmt.Fprintf(&b, "ip link add %s type bridge stp_state 0 2>/dev/null || true\n", s.Bridge)
for _, c := range switchCables(p, name) {
if c.Segment != s.Name {
continue
}
fmt.Fprintf(&b, "ip link set dev %s master %s\n", c.SwitchInterface, s.Bridge)
fmt.Fprintf(&b, "ip link set dev %s up\n", c.SwitchInterface)
for _, s := range routedSegments(p, n.Name) {
gateway := fmt.Sprintf("%s/%d", s.Gateway, s.Network.Bits())
if s.VRID == 0 {
fmt.Fprintf(&b, "ip addr replace %s dev %s\n", gateway, s.Name)
continue
}
fmt.Fprintf(&b, "ip link set dev %s mtu %d\n", s.Bridge, s.MTU)
fmt.Fprintf(&b, "ip addr replace %s/%d dev %s\n", s.Gateway, s.Network.Bits(), s.Bridge)
for _, prefix := range n.Secondary[s.Name] {
fmt.Fprintf(&b, "ip addr replace %s dev %s\n", prefix, s.Bridge)
}
fmt.Fprintf(&b, "ip link set dev %s up\n", s.Bridge)
vrrp := VRRPPrefix + s.Name
fmt.Fprintf(&b, "sysctl -qw net.ipv4.conf.%s.arp_ignore=1 net.ipv4.conf.%s.arp_announce=2\n", s.Name, s.Name)
fmt.Fprintf(&b, "ip link show dev %s >/dev/null 2>&1 || { ip link add %s link %s type macvlan mode bridge && ip link set dev %s protodown on; }\n", vrrp, vrrp, s.Name, vrrp)
fmt.Fprintf(&b, "ip link set dev %s addrgenmode random address %s\n", vrrp, vrrpMAC(s.VRID))
fmt.Fprintf(&b, "ip addr replace %s dev %s\n", gateway, vrrp)
fmt.Fprintf(&b, "ip link set dev %s up\n", vrrp)
}
b.WriteString(loopbackLines(n))
fmt.Fprintf(&b, "nft -f %s\n", SwitchNFT)
fmt.Fprintf(&b, "nft -f %s\n", RouterNFT)
return b.String()
}
func switchNFT(p *topology.Plan, name string) string {
func routerNFT(p *topology.Plan, name string) string {
var networks []string
for _, s := range switchSegments(p, name) {
for _, s := range routedSegments(p, name) {
networks = append(networks, s.Network.String())
}
if len(networks) == 0 {
return "add table ip lab_nat\ndelete table ip lab_nat\n"
}
return fmt.Sprintf(`add table ip lab_nat
delete table ip lab_nat
table ip lab_nat {