package render import ( "reflect" "strconv" "strings" "testing" "go.yaml.in/yaml/v3" "git.g3e.fr/syonad/labctl/internal/topology" ) const labKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O lab-automation" const twoHypervisors = `name: evpn-2hv images: deb: url: https://example.invalid/deb.qcow2 sums: https://example.invalid/SHA512SUMS segments: underlay: { routers: [sw1], cidr: 10.250.0.0/24, mtu: 9000 } nodes: sw1: { role: router, image: deb, cpus: 2, memory: 1024, segments: [underlay] } rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] } hv1: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] } hv2: { role: host, image: deb, cpus: 4, memory: 16384, segments: [underlay] } ` const twoSegments = `name: two-seg images: deb: url: https://example.invalid/deb.qcow2 sums: https://example.invalid/SHA512SUMS segments: red: { routers: [sw], cidr: 10.1.0.0/24 } blue: { routers: [sw], cidr: 10.2.0.0/24, mtu: 1500 } nodes: sw: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, blue] } hv: { role: host, image: deb, cpus: 1, memory: 512, segments: [blue, red] } ` const withVRRP = `name: vrrp images: deb: url: https://example.invalid/deb.qcow2 sums: https://example.invalid/SHA512SUMS segments: red: { routers: [r1, r2], cidr: 10.1.0.0/24 } blue: { routers: [r1], cidr: 10.2.0.0/24 } pure: { cidr: 10.3.0.0/24 } nodes: r1: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, blue] } r2: { role: router, image: deb, cpus: 1, memory: 512, segments: [red, pure] } r3: { role: router, image: deb, cpus: 1, memory: 512, segments: [pure] } hv: { role: host, image: deb, cpus: 1, memory: 512, segments: [pure, red] } l2: { role: host, image: deb, cpus: 1, memory: 512, segments: [pure] } ` func plan(t *testing.T, doc string) *topology.Plan { t.Helper() topo, err := topology.Parse([]byte(doc)) if err != nil { t.Fatalf("Parse: %v", err) } p, err := topology.Compute(topo) if err != nil { t.Fatalf("Compute: %v", err) } return p } func renderAll(t *testing.T, doc string) map[string]Node { t.Helper() nodes, err := Render(plan(t, doc), Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey}}) if err != nil { t.Fatalf("Render: %v", err) } out := map[string]Node{} for _, n := range nodes { out[n.Name] = n } return out } func nodeNamed(t *testing.T, nodes map[string]Node, name string) Node { t.Helper() n, ok := nodes[name] if !ok { t.Fatalf("no rendered node %s", name) } return n } func value(t *testing.T, args []string, flag string) []string { t.Helper() var out []string for i := 0; i+1 < len(args); i++ { if args[i] == flag { out = append(out, args[i+1]) } } if len(out) == 0 { t.Fatalf("no %s in %v", flag, args) } return out } func hasValue(args []string, flag, want string) bool { for i := 0; i+1 < len(args); i++ { if args[i] == flag && args[i+1] == want { return true } } return false } func network(t *testing.T, n Node) networkDoc { t.Helper() var doc networkDoc if err := yaml.Unmarshal(n.NetworkConfig, &doc); err != nil { t.Fatalf("network-config of %s is not yaml: %v", n.Name, err) } return doc } func iface(t *testing.T, doc networkDoc, name string) ethernet { t.Helper() e, ok := doc.Ethernets[name] if !ok { t.Fatalf("no interface %s in %v", name, doc.Ethernets) } return e } func user(t *testing.T, n Node) cloudConfig { t.Helper() if !strings.HasPrefix(string(n.UserData), "#cloud-config\n") { t.Fatalf("user-data of %s does not start with #cloud-config", n.Name) } var cfg cloudConfig if err := yaml.Unmarshal(n.UserData, &cfg); err != nil { t.Fatalf("user-data of %s is not yaml: %v", n.Name, err) } return cfg } func fileAt(t *testing.T, cfg cloudConfig, path string) writeFile { t.Helper() for _, f := range cfg.WriteFiles { if f.Path == path { return f } } t.Fatalf("no write_files entry %s", path) return writeFile{} } func TestQEMU_HypervisorArguments(t *testing.T) { hv := nodeNamed(t, renderAll(t, twoHypervisors), "hv1") for flag, want := range map[string]string{ "-name": "hv1", "-machine": "q35", "-accel": "kvm", "-cpu": "host", "-smp": "4", "-m": "16384", "-display": "none", "-serial": "file:/srv/lab/hv1/console.log", "-qmp": "unix:/srv/lab/hv1/qmp.sock,server=on,wait=off", "-pidfile": "/srv/lab/hv1/qemu.pid", } { if !hasValue(hv.QEMU, flag, want) { t.Errorf("%s %s missing from %v", flag, want, hv.QEMU) } } for _, want := range []string{ "file=/srv/lab/hv1/disk.qcow2,if=virtio,format=qcow2", "file=/srv/lab/hv1/seed.iso,media=cdrom,readonly=on", } { if !hasValue(hv.QEMU, "-drive", want) { t.Errorf("-drive %s missing", want) } } if hv.Dir != "/srv/lab/hv1" { t.Errorf("Dir = %s", hv.Dir) } nodefaults := 0 for _, a := range hv.QEMU { if a == "-nodefaults" { nodefaults++ } } if nodefaults != 1 { t.Errorf("-nodefaults appears %d times", nodefaults) } } func TestQEMU_HypervisorNetwork(t *testing.T) { hv := nodeNamed(t, renderAll(t, twoHypervisors), "hv1") wantNetdevs := []string{ "user,id=mgmt0,restrict=on,ipv6=off,hostfwd=tcp:127.0.0.1:2202-:22", "tap,id=underlay,ifname=lt2,script=no,downscript=no", } wantDevices := []string{ "virtio-net-pci,netdev=mgmt0,mac=02:4d:00:02:00:00,romfile=", "virtio-net-pci,netdev=underlay,mac=02:4c:00:02:00:00,host_mtu=9000,romfile=", } if got := value(t, hv.QEMU, "-netdev"); !reflect.DeepEqual(got, wantNetdevs) { t.Errorf("netdevs = %v\nwant %v", got, wantNetdevs) } if got := value(t, hv.QEMU, "-device"); !reflect.DeepEqual(got, wantDevices) { t.Errorf("devices = %v\nwant %v", got, wantDevices) } } func TestQEMU_RouterNetwork(t *testing.T) { sw := nodeNamed(t, renderAll(t, twoSegments), "sw") wantNetdevs := []string{ "user,id=mgmt0,restrict=off,ipv6=off,hostfwd=tcp:127.0.0.1:2200-:22", "tap,id=red,ifname=lt0,script=no,downscript=no", "tap,id=blue,ifname=lt2,script=no,downscript=no", } wantDevices := []string{ "virtio-net-pci,netdev=mgmt0,mac=02:4d:00:00:00:00,romfile=", "virtio-net-pci,netdev=red,mac=02:4c:00:00:00:00,host_mtu=9000,romfile=", "virtio-net-pci,netdev=blue,mac=02:4c:00:00:01:00,host_mtu=1500,romfile=", } if got := value(t, sw.QEMU, "-netdev"); !reflect.DeepEqual(got, wantNetdevs) { t.Errorf("netdevs = %v\nwant %v", got, wantNetdevs) } if got := value(t, sw.QEMU, "-device"); !reflect.DeepEqual(got, wantDevices) { t.Errorf("devices = %v\nwant %v", got, wantDevices) } } func TestQEMU_OnlyRoutersReachTheOutsideThroughAdministration(t *testing.T) { for name, n := range renderAll(t, withVRRP) { admin := value(t, n.QEMU, "-netdev")[0] wantRestrict := "restrict=on" if name == "r1" || name == "r2" || name == "r3" { wantRestrict = "restrict=off" } if !strings.Contains(admin, ","+wantRestrict+",") { t.Errorf("%s admin netdev %q, want %s", name, admin, wantRestrict) } if !strings.Contains(admin, "hostfwd=tcp:127.0.0.1:") { t.Errorf("%s ssh forward not bound to loopback: %q", name, admin) } } } func TestQEMU_EveryTapOfThePlanIsOpenedOnce(t *testing.T) { p := plan(t, withVRRP) opened := map[string]int{} for _, n := range renderAll(t, withVRRP) { for _, nd := range value(t, n.QEMU, "-netdev") { if strings.HasPrefix(nd, "dgram") || strings.HasPrefix(nd, "socket") { t.Errorf("%s uses a %s netdev", n.Name, nd) } for _, kv := range strings.Split(nd, ",") { if tap, ok := strings.CutPrefix(kv, "ifname="); ok { opened[tap]++ } } } } if len(opened) != len(p.Ports) { t.Errorf("%d taps opened, plan has %d ports", len(opened), len(p.Ports)) } for _, port := range p.Ports { if opened[port.Tap] != 1 { t.Errorf("tap %s opened %d times", port.Tap, opened[port.Tap]) } } } func TestNetworkConfig_Hypervisor(t *testing.T) { doc := network(t, nodeNamed(t, renderAll(t, twoHypervisors), "hv1")) if doc.Version != 2 || len(doc.Ethernets) != 2 { t.Fatalf("network-config = %+v", doc) } admin := iface(t, doc, "mgmt0") if admin.Match.MACAddress != "02:4d:00:02:00:00" || admin.SetName != "mgmt0" || !reflect.DeepEqual(admin.Addresses, []string{"10.0.2.15/24"}) || len(admin.Routes) != 0 || admin.Nameservers != nil { t.Errorf("mgmt0 = %+v", admin) } under := iface(t, doc, "underlay") if under.Match.MACAddress != "02:4c:00:02:00:00" || under.SetName != "underlay" || under.MTU != 9000 || !reflect.DeepEqual(under.Addresses, []string{"10.250.0.4/24"}) { t.Errorf("underlay = %+v", under) } if !reflect.DeepEqual(under.Routes, []route{{To: "0.0.0.0/0", Via: "10.250.0.1"}}) { t.Errorf("underlay routes = %+v", under.Routes) } if under.Nameservers == nil || !reflect.DeepEqual(under.Nameservers.Addresses, []string{"1.1.1.1", "8.8.8.8"}) { t.Errorf("underlay nameservers = %+v", under.Nameservers) } } func TestNetworkConfig_DHCPIsExplicitlyOffEverywhere(t *testing.T) { for name, n := range renderAll(t, twoHypervisors) { if c := strings.Count(string(n.NetworkConfig), "dhcp4: false"); c != len(network(t, n).Ethernets) { t.Errorf("%s: %d explicit dhcp4: false for %d interfaces", name, c, len(network(t, n).Ethernets)) } } } func TestNetworkConfig_Router(t *testing.T) { doc := network(t, nodeNamed(t, renderAll(t, twoSegments), "sw")) if len(doc.Ethernets) != 3 { t.Fatalf("network-config = %+v", doc) } admin := iface(t, doc, "mgmt0") if !reflect.DeepEqual(admin.Routes, []route{{To: "0.0.0.0/0", Via: "10.0.2.2"}}) || admin.Nameservers == nil || !reflect.DeepEqual(admin.Nameservers.Addresses, []string{"10.0.2.3"}) { t.Errorf("router mgmt0 = %+v", admin) } for name, want := range map[string]struct { mac, address string mtu int }{ "red": {"02:4c:00:00:00:00", "10.1.0.2/24", 9000}, "blue": {"02:4c:00:00:01:00", "10.2.0.2/24", 1500}, } { e := iface(t, doc, name) if e.Match.MACAddress != want.mac || e.SetName != name || e.MTU != want.mtu || !reflect.DeepEqual(e.Addresses, []string{want.address}) || len(e.Routes) != 0 || e.Nameservers != nil { t.Errorf("%s = %+v", name, e) } } } func TestNetworkConfig_DefaultRouteOnlyOnFirstSegment(t *testing.T) { doc := network(t, nodeNamed(t, renderAll(t, twoSegments), "hv")) red, blue := iface(t, doc, "red"), iface(t, doc, "blue") if !reflect.DeepEqual(red.Routes, []route{{To: "0.0.0.0/0", Via: "10.1.0.1"}}) || red.Nameservers == nil { t.Errorf("red = %+v", red) } if len(blue.Routes) != 0 || blue.Nameservers != nil || blue.MTU != 1500 { t.Errorf("blue = %+v", blue) } } func TestUserData_HostWithoutProvision(t *testing.T) { cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "hv1")) if cfg.Hostname != "hv1" || !cfg.DisableRoot || len(cfg.Packages) != 0 || cfg.Runcmd != nil || len(cfg.WriteFiles) != 0 { t.Errorf("hv1 user-data = %+v", cfg) } if !reflect.DeepEqual(cfg.SSHAuthorizedKeys, []string{labKey}) { t.Errorf("keys = %v", cfg.SSHAuthorizedKeys) } } func TestUserData_PasswordLoginIsExplicitlyOff(t *testing.T) { for name, n := range renderAll(t, twoHypervisors) { if !strings.Contains(string(n.UserData), "\nssh_pwauth: false\n") || !strings.Contains(string(n.UserData), "\ndisable_root: true\n") { t.Errorf("%s user-data:\n%s", name, n.UserData) } } } func TestNetworkConfig_DefaultRouteOnTheFirstSegmentWithAGateway(t *testing.T) { doc := network(t, nodeNamed(t, renderAll(t, withVRRP), "hv")) pure, red := iface(t, doc, "pure"), iface(t, doc, "red") if len(pure.Routes) != 0 || pure.Nameservers != nil || !reflect.DeepEqual(pure.Addresses, []string{"10.3.0.3/24"}) { t.Errorf("pure = %+v", pure) } if !reflect.DeepEqual(red.Routes, []route{{To: "0.0.0.0/0", Via: "10.1.0.1"}}) || red.Nameservers == nil || !reflect.DeepEqual(red.Addresses, []string{"10.1.0.4/24"}) { t.Errorf("red = %+v", red) } } func TestNetworkConfig_NoDefaultRouteWithoutGateway(t *testing.T) { for name, e := range network(t, nodeNamed(t, renderAll(t, withVRRP), "l2")).Ethernets { if len(e.Routes) != 0 || e.Nameservers != nil { t.Errorf("%s = %+v, want no route", name, e) } } } func TestNetworkConfig_RouterHasNoDefaultRouteOnItsSegments(t *testing.T) { doc := network(t, nodeNamed(t, renderAll(t, withVRRP), "r1")) for _, name := range []string{"red", "blue"} { if e := iface(t, doc, name); len(e.Routes) != 0 || e.Nameservers != nil { t.Errorf("%s = %+v", name, e) } } if admin := iface(t, doc, "mgmt0"); len(admin.Routes) != 1 { t.Errorf("mgmt0 = %+v", admin) } } func TestUserData_RouterBuildsGatewayAndNAT(t *testing.T) { cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "sw1")) if !reflect.DeepEqual(cfg.Packages, []string{"nftables"}) { t.Errorf("packages = %v", cfg.Packages) } script := fileAt(t, cfg, "/usr/local/sbin/lab-router") if script.Permissions != "0755" { t.Errorf("script permissions = %s", script.Permissions) } wantScript := `#!/bin/sh set -eu sysctl -qw net.ipv4.ip_forward=1 ip addr replace 10.250.0.1/24 dev underlay nft -f /etc/lab-router.nft ` if script.Content != wantScript { t.Errorf("script:\n%s\nwant:\n%s", script.Content, wantScript) } nftFile := fileAt(t, cfg, "/etc/lab-router.nft") if nftFile.Permissions != "0644" { t.Errorf("nft permissions = %s", nftFile.Permissions) } for _, want := range []string{ "add table ip lab_nat\ndelete table ip lab_nat\n", "type nat hook postrouting priority srcnat;", `ip saddr { 10.250.0.0/24 } oifname "mgmt0" masquerade`, } { if !strings.Contains(nftFile.Content, want) { t.Errorf("nft rules miss %q:\n%s", want, nftFile.Content) } } unit := fileAt(t, cfg, "/etc/systemd/system/lab-router.service").Content for _, want := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=/usr/local/sbin/lab-router", "After=network-online.target", "WantedBy=multi-user.target"} { if !strings.Contains(unit, want) { t.Errorf("unit misses %q", want) } } if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"/usr/local/sbin/lab-provision"}}) { t.Errorf("runcmd = %v", cfg.Runcmd) } if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-router.service\n" { t.Errorf("lab-provision:\n%s", got) } } func TestUserData_RouterWithTwoSegments(t *testing.T) { cfg := user(t, nodeNamed(t, renderAll(t, twoSegments), "sw")) script := fileAt(t, cfg, "/usr/local/sbin/lab-router").Content want := "sysctl -qw net.ipv4.ip_forward=1\nip addr replace 10.1.0.1/24 dev red\nip addr replace 10.2.0.1/24 dev blue\nnft -f /etc/lab-router.nft\n" if !strings.HasSuffix(script, want) { t.Errorf("script:\n%s\ndoes not end with:\n%s", script, want) } if nft := fileAt(t, cfg, "/etc/lab-router.nft").Content; !strings.Contains(nft, "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }") { t.Errorf("nft:\n%s", nft) } } func TestUserData_RouterScriptWithVRRPAndSingleSegments(t *testing.T) { script := fileAt(t, user(t, nodeNamed(t, renderAll(t, withVRRP), "r1")), "/usr/local/sbin/lab-router").Content want := `#!/bin/sh set -eu sysctl -qw net.ipv4.ip_forward=1 sysctl -qw net.ipv4.conf.red.arp_ignore=1 net.ipv4.conf.red.arp_announce=2 ip link show dev vrred >/dev/null 2>&1 || { ip link add vrred link red type macvlan mode bridge && ip link set dev vrred protodown on; } ip link set dev vrred addrgenmode random address 00:00:5e:00:01:01 ip addr replace 10.1.0.1/24 dev vrred ip link set dev vrred up ip addr replace 10.2.0.1/24 dev blue nft -f /etc/lab-router.nft ` if script != want { t.Errorf("script:\n%s\nwant:\n%s", script, want) } r2 := fileAt(t, user(t, nodeNamed(t, renderAll(t, withVRRP), "r2")), "/usr/local/sbin/lab-router").Content if !strings.Contains(r2, "ip addr replace 10.1.0.1/24 dev vrred\n") || strings.Contains(r2, "dev pure") { t.Errorf("r2 script:\n%s", r2) } } func TestUserData_RouterNATCoversTheSegmentsItRoutes(t *testing.T) { nodes := renderAll(t, withVRRP) for name, want := range map[string]string{ "r1": "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }", "r2": "ip saddr { 10.1.0.0/24 }", } { if nft := fileAt(t, user(t, nodeNamed(t, nodes, name)), "/etc/lab-router.nft").Content; !strings.Contains(nft, want) { t.Errorf("%s nft:\n%s\nmisses %s", name, nft, want) } } cfg := user(t, nodeNamed(t, nodes, "r3")) if nft := fileAt(t, cfg, "/etc/lab-router.nft").Content; nft != "add table ip lab_nat\ndelete table ip lab_nat\n" { t.Errorf("r3 nft:\n%s", nft) } if script := fileAt(t, cfg, "/usr/local/sbin/lab-router").Content; script != "#!/bin/sh\nset -eu\nsysctl -qw net.ipv4.ip_forward=1\nnft -f /etc/lab-router.nft\n" { t.Errorf("r3 script:\n%s", script) } } func TestVRRPMAC(t *testing.T) { for vrid, want := range map[int]string{1: "00:00:5e:00:01:01", 16: "00:00:5e:00:01:10", 255: "00:00:5e:00:01:ff"} { if got := vrrpMAC(vrid); got != want { t.Errorf("vrrpMAC(%d) = %s, want %s", vrid, got, want) } } } func TestVRRPPriorityFollowsTheRoutersOrder(t *testing.T) { var routers []string for i := 0; i < 22; i++ { routers = append(routers, "r"+strconv.Itoa(i)) } s := topology.SegmentPlan{Routers: routers} for name, want := range map[string]int{"r0": 200, "r1": 190, "r19": 10, "r20": 1, "r21": 1} { if got := vrrpPriority(s, name); got != want { t.Errorf("priority of %s = %d, want %d", name, got, want) } } } func TestMetaData(t *testing.T) { var doc metaDoc if err := yaml.Unmarshal(nodeNamed(t, renderAll(t, twoHypervisors), "hv2").MetaData, &doc); err != nil { t.Fatal(err) } if doc.InstanceID != "evpn-2hv-hv2" || doc.LocalHostname != "hv2" { t.Errorf("meta-data = %+v", doc) } } func TestRender_AdminMacsAreUniqueAndApartFromPortMacs(t *testing.T) { seen := map[string]string{} for name, n := range renderAll(t, twoHypervisors) { for _, d := range value(t, n.QEMU, "-device") { for _, kv := range strings.Split(d, ",") { if mac, ok := strings.CutPrefix(kv, "mac="); ok { if other, dup := seen[mac]; dup { t.Errorf("mac %s used by %s and %s", mac, other, name) } seen[mac] = name } } } } if len(seen) != 8 { t.Errorf("%d distinct macs, want 8", len(seen)) } } func TestRender_IsStable(t *testing.T) { first := renderAll(t, twoHypervisors) for i := 0; i < 20; i++ { if again := renderAll(t, twoHypervisors); !reflect.DeepEqual(first, again) { t.Fatalf("render differs on run %d", i) } } } func TestRender_RejectsBadOptions(t *testing.T) { p := plan(t, twoHypervisors) cases := []struct { name string opts Options want string }{ {"relative run dir", Options{RunDir: "lab", AuthorizedKeys: []string{labKey}}, `run dir "lab" must be an absolute path`}, {"no key", Options{RunDir: "/srv/lab"}, "at least one authorized ssh key"}, {"multi-line key", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey + "\nssh-rsa AAAA x"}}, "authorized key 1 spans several lines"}, {"not a key", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{"hello world"}}, "authorized key 1 is not an ssh public key"}, {"type only", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{"ssh-ed25519"}}, "authorized key 1 is not an ssh public key"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { _, err := Render(p, c.opts) if err == nil || !strings.Contains(err.Error(), c.want) { t.Fatalf("err = %v, want %q", err, c.want) } }) } } func TestRender_AcceptsUsualKeyTypes(t *testing.T) { p := plan(t, twoHypervisors) for _, k := range []string{ "ssh-rsa AAAAB3NzaC1yc2E user", "ecdsa-sha2-nistp256 AAAAE2VjZHNh user", "sk-ssh-ed25519@openssh.com AAAAGnNr user", } { if _, err := Render(p, Options{RunDir: "/srv/lab", AuthorizedKeys: []string{k}}); err != nil { t.Errorf("%s rejected: %v", strings.Fields(k)[0], err) } } }