diff --git a/cmd/lab/lifecycle.go b/cmd/lab/lifecycle.go new file mode 100644 index 0000000..c91205e --- /dev/null +++ b/cmd/lab/lifecycle.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "time" + + "git.g3e.fr/syonad/two/internal/lab/machine" + "git.g3e.fr/syonad/two/internal/lab/provision" +) + +const ( + topologyFile = "topology.yml" + pollInterval = 5 * time.Second + stopTimeout = 30 * time.Second +) + +var ( + execve = syscall.Exec + procDir = "/proc" +) + +func defaultDir(parts ...string) string { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(append([]string{home}, parts...)...) +} + +func flags(name string, stderr io.Writer) (*flag.FlagSet, *string) { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, usage) } + return fs, fs.String("run", defaultDir("lab-run"), "run directory of the lab") +} + +func lab(runDir string, plan string, stdout, stderr io.Writer) (machine.Lab, bool) { + dir, err := filepath.Abs(runDir) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return machine.Lab{}, false + } + p, ok := load(plan, stderr) + if !ok { + return machine.Lab{}, false + } + return machine.Lab{ + Plan: p, + RunDir: dir, + ProcDir: procDir, + Runner: provision.ExecRunner{}, + Poll: pollInterval, + Stop: stopTimeout, + Out: stdout, + }, true +} + +func current(runDir string, stdout, stderr io.Writer) (machine.Lab, bool) { + saved := filepath.Join(runDir, topologyFile) + if _, err := os.Stat(saved); err != nil { + fmt.Fprintf(stderr, "lab: no lab in %s: %v\n", runDir, err) + return machine.Lab{}, false + } + return lab(runDir, saved, stdout, stderr) +} + +func upCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("up", stderr) + cacheDir := fs.String("cache", defaultDir(".cache", "two-lab"), "image cache directory") + timeout := fs.Duration("timeout", 20*time.Minute, "how long to wait for the nodes to be ready") + if err := fs.Parse(args); err != nil || fs.NArg() != 1 { + fs.Usage() + return 2 + } + cache, err := filepath.Abs(*cacheDir) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + + if _, err := os.Stat(filepath.Join(*runDir, topologyFile)); err == nil { + previous, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + running, err := previous.Running() + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if len(running) > 0 { + fmt.Fprintf(stderr, "lab: lab %s is still running in %s: 'lab down' first\n", previous.Plan.Name, previous.RunDir) + return 1 + } + } + + l, ok := lab(*runDir, fs.Arg(0), stdout, stderr) + if !ok { + return 1 + } + source, err := os.ReadFile(fs.Arg(0)) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := os.MkdirAll(l.RunDir, 0o700); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := os.WriteFile(filepath.Join(l.RunDir, topologyFile), source, 0o600); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + fetcher := provision.Fetcher{Client: &http.Client{}, CacheDir: cache} + if err := l.Up(ctx, fetcher, *timeout); err != nil { + fmt.Fprintf(stderr, "lab: %v\nlab: started nodes keep running: 'lab status', 'lab down'\n", err) + return 1 + } + return 0 +} + +func statusCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("status", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() != 0 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + if err := l.Status(stdout); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} + +func downCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("down", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() != 0 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + if err := l.Down(context.Background()); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} + +func sshCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("ssh", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() < 1 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + argv, err := l.SSH(fs.Arg(0), isTerminal(os.Stdin), fs.Args()[1:]) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + path, err := exec.LookPath(argv[0]) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := execve(path, argv, os.Environ()); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} diff --git a/cmd/lab/main.go b/cmd/lab/main.go new file mode 100644 index 0000000..3dc6425 --- /dev/null +++ b/cmd/lab/main.go @@ -0,0 +1,164 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "git.g3e.fr/syonad/two/internal/lab/provision" + "git.g3e.fr/syonad/two/internal/lab/render" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const usage = `usage: lab [options] [dir] + + plan + validate the topology and print the deterministic plan: addresses, cables, ports + + render -key + write, for each node, //qemu.args (one argument per line) and the + cloud-init seed files meta-data, user-data and network-config + + up [-run dir] [-cache dir] [-timeout duration] + download and verify the images, create fresh disks and seeds, start every node + (switches first) and wait until cloud-init has finished on each of them; + the topology is kept in /topology.yml for the commands below + + status [-run dir] + list the nodes of the lab and whether their qemu process is running + + down [-run dir] + stop every qemu process of the lab (SIGTERM, then SIGKILL) + + ssh [-run dir] [command...] + open a shell on a node, or run a command on it, with the key generated by up + + defaults: -run $HOME/lab-run, -cache $HOME/.cache/two-lab, -timeout 20m +` + +type keyFiles []string + +func (k *keyFiles) String() string { return strings.Join(*k, ",") } +func (k *keyFiles) Set(v string) error { *k = append(*k, v); return nil } + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, usage) + return 2 + } + switch args[0] { + case "plan": + if len(args) != 2 { + fmt.Fprint(stderr, usage) + return 2 + } + return plan(args[1], stdout, stderr) + case "render": + return renderCmd(args[1:], stdout, stderr) + case "up": + return upCmd(args[1:], stdout, stderr) + case "status": + return statusCmd(args[1:], stdout, stderr) + case "down": + return downCmd(args[1:], stdout, stderr) + case "ssh": + return sshCmd(args[1:], stdout, stderr) + default: + fmt.Fprint(stderr, usage) + return 2 + } +} + +func load(path string, stderr io.Writer) (*topology.Plan, bool) { + t, err := topology.Load(path) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return nil, false + } + p, err := topology.Compute(t) + if err != nil { + fmt.Fprintf(stderr, "lab: %s:\n%v\n", path, err) + return nil, false + } + return p, true +} + +func plan(path string, stdout, stderr io.Writer) int { + p, ok := load(path, stderr) + if !ok { + return 1 + } + if err := p.Write(stdout); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} + +func renderCmd(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("render", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, usage) } + var keys keyFiles + fs.Var(&keys, "key", "public key file allowed to log in, repeatable") + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() != 2 || len(keys) == 0 { + fmt.Fprint(stderr, usage) + return 2 + } + + p, ok := load(fs.Arg(0), stderr) + if !ok { + return 1 + } + dir, err := filepath.Abs(fs.Arg(1)) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + authorized, err := readKeys(keys) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + nodes, err := render.Render(p, render.Options{RunDir: dir, AuthorizedKeys: authorized}) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + for _, n := range nodes { + if err := provision.WriteFiles(n); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "%s\n", n.Dir) + } + return 0 +} + +func readKeys(files []string) ([]string, error) { + var keys []string + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + keys = append(keys, line) + } + } + return keys, nil +} diff --git a/cmd/lab/main_test.go b/cmd/lab/main_test.go new file mode 100644 index 0000000..e8972b8 --- /dev/null +++ b/cmd/lab/main_test.go @@ -0,0 +1,301 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" +) + +func runLab(args ...string) (int, string, string) { + var stdout, stderr bytes.Buffer + code := run(args, &stdout, &stderr) + return code, stdout.String(), stderr.String() +} + +func TestRun_UsageOnMissingArguments(t *testing.T) { + for _, args := range [][]string{nil, {"plan"}, {"deploy", "x.yml"}, {"plan", "a.yml", "b.yml"}, {"render", "a.yml", "dir"}, {"render", "-key", "k.pub", "a.yml"}, {"render", "-bogus"}} { + code, _, stderr := runLab(args...) + if code != 2 || !strings.Contains(stderr, "usage: lab") { + t.Errorf("args %v: code %d, stderr %q", args, code, stderr) + } + } +} + +func TestRun_PlanOfTheShippedExampleTopology(t *testing.T) { + code, stdout, stderr := runLab("plan", filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml")) + if code != 0 { + t.Fatalf("code %d, stderr %s", code, stderr) + } + for _, want := range []string{ + "lab evpn-2hv: nodes 4, segments 1, cables 3", + "gateway 10.250.0.1", + "hv2 underlay 10.250.0.4/24 02:4c:00:03:00:00 20004 <-> sw1 p2", + } { + if !strings.Contains(stdout, want) { + t.Errorf("output does not contain %q:\n%s", want, stdout) + } + } +} + +func TestRun_InvalidTopologyExitsWithErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.yml") + doc := `name: bad +images: + deb: { url: https://example.invalid/a, sums: https://example.invalid/b } +segments: + under: { switch: sw, cidr: 10.0.0.0/31 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + rr: { role: router, image: deb, cpus: 1, memory: 512, segments: [under] } +` + if err := os.WriteFile(path, []byte(doc), 0o600); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runLab("plan", path) + if code != 1 || stdout != "" { + t.Fatalf("code %d, stdout %q", code, stdout) + } + for _, want := range []string{path, `role "router"`, "prefix length out of range"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr does not mention %q:\n%s", want, stderr) + } + } +} + +func TestRun_MissingFile(t *testing.T) { + code, _, stderr := runLab("plan", filepath.Join(t.TempDir(), "absent.yml")) + if code != 1 || !strings.Contains(stderr, "absent.yml") { + t.Errorf("code %d, stderr %q", code, stderr) + } +} + +func TestRun_RenderWritesEveryNodeFiles(t *testing.T) { + dir := t.TempDir() + key := filepath.Join(dir, "lab.pub") + if err := os.WriteFile(key, []byte("# lab key\n\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab-automation\n"), 0o600); err != nil { + t.Fatal(err) + } + out := filepath.Join(dir, "run") + code, stdout, stderr := runLab("render", "-key", key, filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml"), out) + if code != 0 { + t.Fatalf("code %d, stderr %s", code, stderr) + } + for _, node := range []string{"sw1", "rr1", "hv1", "hv2"} { + if !strings.Contains(stdout, filepath.Join(out, node)) { + t.Errorf("stdout does not list %s:\n%s", node, stdout) + } + for _, f := range []string{"qemu.args", "meta-data", "user-data", "network-config"} { + info, err := os.Stat(filepath.Join(out, node, f)) + if err != nil { + t.Errorf("%s/%s: %v", node, f, err) + continue + } + if info.Mode().Perm() != 0o600 { + t.Errorf("%s/%s mode %o, want 600", node, f, info.Mode().Perm()) + } + } + } + args, err := os.ReadFile(filepath.Join(out, "hv1", "qemu.args")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(args), "\n-netdev\nuser,id=mgmt0,restrict=on,ipv6=off,hostfwd=tcp:127.0.0.1:2202-:22\n") { + t.Errorf("qemu.args is not one argument per line:\n%s", args) + } + userData, err := os.ReadFile(filepath.Join(out, "hv1", "user-data")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(userData), "two-lab-automation") || strings.Contains(string(userData), "# lab key") { + t.Errorf("keys not read as an authorized_keys file:\n%s", userData) + } +} + +func TestRun_RenderRefusesMissingKeyFile(t *testing.T) { + code, _, stderr := runLab("render", "-key", filepath.Join(t.TempDir(), "absent.pub"), filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml"), t.TempDir()) + if code != 1 || !strings.Contains(stderr, "absent.pub") { + t.Errorf("code %d, stderr %q", code, stderr) + } +} + +func savedLab(t *testing.T) string { + t.Helper() + run := filepath.Join(t.TempDir(), "run") + if err := os.MkdirAll(run, 0o700); err != nil { + t.Fatal(err) + } + example, err := os.ReadFile(filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(run, "topology.yml"), example, 0o600); err != nil { + t.Fatal(err) + } + return run +} + +func TestRun_LifecycleUsage(t *testing.T) { + for _, args := range [][]string{{"up"}, {"up", "a.yml", "b.yml"}, {"status", "x"}, {"down", "x"}, {"ssh"}, {"up", "-bogus", "a.yml"}} { + code, _, stderr := runLab(args...) + if code != 2 || !strings.Contains(stderr, "usage: lab") { + t.Errorf("args %v: code %d, stderr %q", args, code, stderr) + } + } +} + +func TestRun_CommandsWithoutALabInTheRunDir(t *testing.T) { + run := t.TempDir() + for _, cmd := range []string{"status", "down", "ssh"} { + args := []string{cmd, "-run", run} + if cmd == "ssh" { + args = append(args, "hv1") + } + code, _, stderr := runLab(args...) + if code != 1 || !strings.Contains(stderr, "lab: no lab in "+run) { + t.Errorf("%s: code %d, stderr %q", cmd, code, stderr) + } + } +} + +func TestRun_StatusOfAStoppedLab(t *testing.T) { + code, stdout, stderr := runLab("status", "-run", savedLab(t)) + want := `node role state pid ssh +sw1 switch stopped - 127.0.0.1:2200 +rr1 rr stopped - 127.0.0.1:2201 +hv1 hypervisor stopped - 127.0.0.1:2202 +hv2 hypervisor stopped - 127.0.0.1:2203 +` + if code != 0 || stdout != want { + t.Errorf("code %d, stderr %q, stdout:\n%s", code, stderr, stdout) + } +} + +func TestRun_DownOfAStoppedLab(t *testing.T) { + code, stdout, stderr := runLab("down", "-run", savedLab(t)) + if code != 0 || stdout != "" || stderr != "" { + t.Errorf("code %d, stdout %q, stderr %q", code, stdout, stderr) + } +} + +func TestRun_SSHExecsSSHWithTheNodePort(t *testing.T) { + run := savedLab(t) + var gotPath string + var gotArgv []string + execve = func(path string, argv []string, env []string) error { + gotPath, gotArgv = path, argv + return nil + } + t.Cleanup(func() { execve = syscall.Exec }) + + code, _, stderr := runLab("ssh", "-run", run, "hv2", "ip", "-br", "a") + + if code != 0 { + t.Fatalf("code %d, stderr %q", code, stderr) + } + if filepath.Base(gotPath) != "ssh" { + t.Errorf("path = %q", gotPath) + } + want := []string{"ssh", + "-i", filepath.Join(run, "lab_ed25519"), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2203", + "debian@127.0.0.1", + "ip", "-br", "a", + } + if strings.Join(gotArgv, " ") != strings.Join(want, " ") { + t.Errorf("\n got %q\nwant %q", gotArgv, want) + } +} + +func TestRun_SSHUnknownNode(t *testing.T) { + code, _, stderr := runLab("ssh", "-run", savedLab(t), "hv9") + if code != 1 || !strings.Contains(stderr, `node "hv9" is not in lab evpn-2hv`) { + t.Errorf("code %d, stderr %q", code, stderr) + } +} + +func TestRun_UpRefusesAnInvalidTopologyAndKeepsTheSavedOne(t *testing.T) { + run := savedLab(t) + before, _ := os.ReadFile(filepath.Join(run, "topology.yml")) + bad := filepath.Join(t.TempDir(), "bad.yml") + if err := os.WriteFile(bad, []byte("name: x\n"), 0o600); err != nil { + t.Fatal(err) + } + + code, _, stderr := runLab("up", "-run", run, bad) + + if code != 1 || !strings.Contains(stderr, "at least one node is required") { + t.Errorf("code %d, stderr %q", code, stderr) + } + if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) { + t.Error("the saved topology was replaced by an invalid one") + } +} + +func TestIsTerminal_DevNullAndPipesAreNotTerminals(t *testing.T) { + null, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + defer null.Close() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + for name, f := range map[string]*os.File{"/dev/null": null, "pipe": r} { + if isTerminal(f) { + t.Errorf("%s is detected as a terminal", name) + } + } +} + +func TestRun_UpRefusesToReplaceARunningLab(t *testing.T) { + run := savedLab(t) + before, _ := os.ReadFile(filepath.Join(run, "topology.yml")) + proc := t.TempDir() + procDir = proc + t.Cleanup(func() { procDir = "/proc" }) + + cmd := exec.Command("sleep", "30") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cmd.Process.Kill(); cmd.Wait() }) + pid := strconv.Itoa(cmd.Process.Pid) + if err := os.MkdirAll(filepath.Join(proc, pid), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte("qemu-system-x86_64\x00-name\x00hv1\x00"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(run, "hv1"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(run, "hv1", "qemu.pid"), []byte(pid+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + other := filepath.Join(t.TempDir(), "other.yml") + if err := os.WriteFile(other, bytes.ReplaceAll(before, []byte("evpn-2hv"), []byte("other")), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runLab("up", "-run", run, other) + + if code != 1 || !strings.Contains(stderr, "lab evpn-2hv is still running in "+run) { + t.Errorf("code %d, stderr %q", code, stderr) + } + if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) { + t.Error("the topology of a running lab was replaced") + } +} diff --git a/cmd/lab/terminal.go b/cmd/lab/terminal.go new file mode 100644 index 0000000..fa2cb06 --- /dev/null +++ b/cmd/lab/terminal.go @@ -0,0 +1,14 @@ +//go:build linux || darwin + +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func isTerminal(f *os.File) bool { + _, err := unix.IoctlGetTermios(int(f.Fd()), ioctlReadTermios) + return err == nil +} diff --git a/cmd/lab/terminal_darwin.go b/cmd/lab/terminal_darwin.go new file mode 100644 index 0000000..4c493c7 --- /dev/null +++ b/cmd/lab/terminal_darwin.go @@ -0,0 +1,5 @@ +package main + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TIOCGETA diff --git a/cmd/lab/terminal_linux.go b/cmd/lab/terminal_linux.go new file mode 100644 index 0000000..ba4230d --- /dev/null +++ b/cmd/lab/terminal_linux.go @@ -0,0 +1,5 @@ +package main + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TCGETS diff --git a/cmd/lab/terminal_other.go b/cmd/lab/terminal_other.go new file mode 100644 index 0000000..a929525 --- /dev/null +++ b/cmd/lab/terminal_other.go @@ -0,0 +1,7 @@ +//go:build !linux && !darwin + +package main + +import "os" + +func isTerminal(*os.File) bool { return false } diff --git a/conf/lab/evpn-2hv.yml b/conf/lab/evpn-2hv.yml new file mode 100644 index 0000000..3d389f6 --- /dev/null +++ b/conf/lab/evpn-2hv.yml @@ -0,0 +1,18 @@ +name: evpn-2hv + +images: + debian12: + url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2 + sums: https://cloud.debian.org/images/cloud/bookworm/latest/SHA512SUMS + +segments: + underlay: + switch: sw1 + cidr: 10.250.0.0/24 + mtu: 9000 + +nodes: + sw1: { role: switch, image: debian12, cpus: 2, memory: 1024 } + rr1: { role: rr, image: debian12, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] } + hv2: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] } diff --git a/docs/developpement/index.rst b/docs/developpement/index.rst new file mode 100644 index 0000000..2d78704 --- /dev/null +++ b/docs/developpement/index.rst @@ -0,0 +1,10 @@ +Développement +============= + +Outillage réservé au développement de two : il ne s'installe sur aucun hyperviseur et ne sert +pas à exploiter un cluster. + +.. toctree:: + :maxdepth: 1 + + lab diff --git a/docs/developpement/lab.rst b/docs/developpement/lab.rst new file mode 100644 index 0000000..8f56789 --- /dev/null +++ b/docs/developpement/lab.rst @@ -0,0 +1,614 @@ +Lab de test multi-nœud +====================== + +Ce qui fait l'intérêt de two ne se voit qu'à partir de **deux hyperviseurs** : sur un nœud isolé, +le trafic reste sur le bridge local et l'absence de plan de contrôle passe inaperçue (voir +:doc:`/deploiement/architecture-cluster`). Le lab reproduit la topologie du cluster — +hyperviseurs, route reflector, switch L3 — sous forme de VM, sur un serveur physique loué à +l'heure. + +Le pourquoi des choix (serveur physique plutôt que VM cloud, câbles QEMU, MTU 9000, versions) est +consigné sur le ticket `#50 `_. Cette page décrit +comment s'en servir. + +.. note:: + + État actuel : étapes **E0** à **E3** livrées — le cycle de vie du serveur qui porte le lab + (``scripts/lab-host.sh``), la description de la topologie et le calcul de son plan + (``lab plan``), la génération des arguments QEMU et des fichiers cloud-init de chaque VM + (``lab render``), puis leur lancement sur le serveur (``lab up`` / ``status`` / ``down`` / + ``ssh``). Les rôles — FRR sur le switch et le route reflector, two sur les hyperviseurs — + viendront avec l'étape suivante. + +Le serveur de lab +----------------- + +Un serveur **Scaleway Elastic Metal**, créé pour une campagne de tests puis supprimé. + +.. list-table:: + :widths: 25 75 + + * - Offre + - ``EM-B212X-SSD``, zone ``fr-par-1``, **facturation horaire** : 0,321 € HT de l'heure, sans + frais de mise en service + * - Matériel + - 2 × Xeon E5-2620 v4 *or equivalent*, 256 Go, 2 × 1 To SSD + * - Système + - Debian 12, installé par Scaleway à la création + * - Pourquoi Intel + - lab3 est en Intel : two lance ses VM en ``-cpu host``, et KVM a deux implémentations + distinctes (``kvm_intel``, ``kvm_amd``) + +*Or equivalent* n'est pas une clause de style : le premier serveur livré était un +**Xeon E5-2640 v3** (Haswell, la génération de lab3), pas le E5-2620 v4 annoncé. Relever +``lscpu`` au début de chaque campagne. + +Prérequis côté Scaleway +----------------------- + +#. **Un projet dédié au lab**, séparé de toute autre ressource. ``lab-host.sh down`` supprime + tout serveur de lab du projet : il ne doit rien y avoir d'autre. +#. **Une clé d'API limitée à ce projet**, avec les droits Elastic Metal et la lecture des clés SSH + du projet. Rien d'autre. +#. **Les clés SSH publiques enregistrées dans le projet**, injectées à l'installation : sans elle, + le serveur serait facturé sans que personne puisse s'y connecter, et ``plan`` refuse de + continuer. +#. **Le quota Elastic Metal.** L'``EM-B212X-SSD`` exige un compte dont le moyen de paiement *et* + l'identité sont validés ; le quota est alors de 2. Vérifier dans la console : Organisation → + Quotas → Elastic Metal. Voir `les quotas Scaleway + `_. + +Fichiers locaux +--------------- + +Tout ce dont le script a besoin vit sous ``~/.config/two-lab/``, hors du dépôt. + +``~/.config/two-lab/scaleway.env`` + Identifiants Scaleway, une ligne ``CLÉ=valeur`` chacun : + + .. code-block:: bash + + SCW_SECRET_KEY= + SCW_DEFAULT_PROJECT_ID= + SCW_DEFAULT_ZONE=fr-par-1 + + * le fichier doit être en ``0600`` : le script refuse de s'en servir s'il est lisible par + d'autres que son propriétaire ; + * il est **lu, jamais exécuté** — pas de ``source`` ; seules ces trois clés sont reconnues ; + * une variable d'environnement du même nom l'emporte sur le fichier ; + * la clé d'accès (``SCW…``) n'est pas nécessaire : l'API REST n'authentifie que par la clé + secrète, dans l'en-tête ``X-Auth-Token``. + + Pour changer de clé, remplacer la ligne ``SCW_SECRET_KEY=`` ; rien d'autre à modifier. + +``~/.config/two-lab/ssh/lab_ed25519`` + Clé SSH dédiée au lab, **sans phrase de passe**, pour que les sessions tournent sans + intervention. Sa partie publique doit être enregistrée dans le projet. Quand elle existe, le + script l'utilise **seule** (``IdentitiesOnly``, agent désactivé) ; sinon il retombe sur + l'agent SSH. + + Elle ne doit ouvrir que les serveurs éphémères du projet de lab : **ne jamais l'installer sur + lab3 ni sur une machine durable**. En cas de doute, la retirer du projet et en générer une + autre : + + .. code-block:: bash + + mkdir -p ~/.config/two-lab/ssh && chmod 700 ~/.config/two-lab ~/.config/two-lab/ssh + ssh-keygen -t ed25519 -N '' -C two-lab-automation -f ~/.config/two-lab/ssh/lab_ed25519 + +``~/.cache/two-lab/`` + État de la session en cours : adresse et utilisateur du serveur, ``known_hosts`` dédié. Vidé + par ``down``. + +Commandes +--------- + +.. code-block:: text + + usage: lab-host.sh [arguments] + + plan résout l'offre horaire, l'OS et les clés SSH, affiche la requête de création + et le prix ; ne crée rien + up crée le serveur de lab, attend la fin de son installation et son SSH, + puis le prépare (voir prepare) + status liste les serveurs de lab du projet + ssh [commande] se connecte au serveur de lab ; avec une commande, un terminal n'est demandé + que si l'entrée standard en est un + prepare installe sur le serveur ce dont lab a besoin (qemu, genisoimage), vérifie + /dev/kvm et la virtualisation imbriquée ; lancé aussi par up + push compile cmd/lab pour linux/amd64 et dépose sur le serveur ~/lab et + ~/ ; ensuite : ssh './lab up ' + down supprime tous les serveurs de lab du projet et attend leur disparition + session [cmd] up, puis la commande distante (ou un shell), puis down quoi qu'il arrive + +``plan`` et ``status`` sont **gratuits** ; ``up`` et ``session`` **créent un serveur facturé**. + +Toujours commencer par ``plan``. Il valide la clé d'API, le quota d'offre, l'OS et les clés SSH, +et montre exactement ce qui serait commandé : + +.. code-block:: text + + $ scripts/lab-host.sh plan + == offre : EM-B212X-SSD (ddaf8ba6-b2b2-4279-8af3-51930fb602f8), facturation hourly, stock available + == prix : 0.321 EUR HT par heure, frais de mise en service 0 EUR + == os : Debian 12 (Bookworm) (83640d93-a0b8-45ad-9c9f-30cae48380a4), utilisateur root + == clés : 2 clé(s) SSH du projet + == requête : POST /baremetal/v1/zones/fr-par-1/servers + +``session`` est la forme normale d'usage : le serveur est supprimé à la fin, que la commande +réussisse, échoue, ou que la session soit interrompue (Ctrl-C, ``TERM``, fermeture du terminal). + +.. code-block:: text + + $ scripts/lab-host.sh session 'uname -a; lscpu | grep -E "Model name|^CPU\(s\)|Virtualization"; free -g | head -2; echo "nested=$(cat /sys/module/kvm_intel/parameters/nested)"; ls -l /dev/kvm' + == création de two-lab (EM-B212X-SSD, 0.321 EUR/h HT) + == serveur 2ecc1e6a-0de8-48c0-a198-93857eee5957 créé, facturé jusqu'à 'lab-host.sh down' + == serveur 2ecc1e6a-0de8-48c0-a198-93857eee5957 : ordered, installation to_install + == serveur 2ecc1e6a-0de8-48c0-a198-93857eee5957 : ready, installation installing + … + == serveur 2ecc1e6a-0de8-48c0-a198-93857eee5957 : ready, installation completed + == SSH pas encore joignable, nouvel essai dans 20s + … + == prêt : root@ + Linux two-lab 6.1.0-53-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.187-1 (2026-09-07) x86_64 GNU/Linux + CPU(s): 32 + Model name: Intel(R) Xeon(R) CPU E5-2640 v3 @ 2.60GHz + … + Virtualization: VT-x + total used free shared buff/cache available + Mem: 251 1 250 0 0 249 + nested=Y + crw-rw---- 1 root kvm 10, 232 Oct 3 17:23 /dev/kvm + == session terminée (code 0), suppression du serveur + == suppression de 2ecc1e6a-0de8-48c0-a198-93857eee5957 + == aucun serveur de lab ne reste dans le projet + +Extrait de la première campagne (lignes répétées remplacées par ``…``). Compter **environ 15 minutes** entre la création et le SSH disponible : 13 min 30 à la première +campagne, suppression comprise. SSH ne répond pas tout de suite après la fin de l'installation — +environ 100 secondes la première fois — d'où l'attente intégrée à ``up``. Le code de sortie de +``session`` est celui de la commande distante. + +``up``, ``ssh`` et ``down`` séparément servent au debug interactif — et laissent la suppression +à la charge de l'utilisateur. + +Une campagne sur le lab enchaîne ces commandes depuis le Mac ; ``lab`` s'exécute sur le serveur +(voir `Lancement des VM`_) : + +.. code-block:: text + + scripts/lab-host.sh up + scripts/lab-host.sh push conf/lab/evpn-2hv.yml + scripts/lab-host.sh ssh './lab up evpn-2hv.yml' + scripts/lab-host.sh ssh './lab ssh hv1' # shell interactif sur hv1 + scripts/lab-host.sh ssh './lab ssh hv1 ip -br a' # commande, code de retour propagé + scripts/lab-host.sh down + +``push`` transfère par la connexion SSH du script (``cat`` côté serveur, fichier renommé une fois +complet) : mêmes options, même clé, même ``known_hosts`` que ``ssh``. + +Topologie +--------- + +Un lab est décrit par un fichier YAML : des **nœuds** (les VM) et des **segments** (des réseaux L2 +portés par un switch). Exemple livré, ``conf/lab/evpn-2hv.yml`` : + +.. literalinclude:: ../../conf/lab/evpn-2hv.yml + :language: yaml + +Chaque nœud non-switch est relié au switch de chacun de ses segments par un câble virtuel QEMU ; +le switch met ces câbles dans un bridge et porte la passerelle du segment. + +Ce que le fichier déclare : + +``images`` + ``url`` de l'image qcow2 et ``sums`` du fichier de sommes à vérifier, tous deux en ``https://``. + +``segments`` + ``switch`` (un nœud de rôle ``switch``), ``cidr`` IPv4 entre ``/8`` et ``/30``, ``mtu`` + facultatif — 9000 par défaut, entre 1280 et 9000. Nom : 12 caractères au plus, minuscules et + chiffres, parce qu'il devient le nom d'interface dans les VM et, préfixé de ``br-``, celui du + bridge (15 caractères au plus sous Linux). + +``nodes`` + ``role`` (``switch``, ``rr`` ou ``hypervisor``), ``image``, ``cpus``, ``memory`` en Mio + (256 au moins), ``segments`` auxquels le nœud est relié, et ``addresses`` pour fixer + l'adresse d'un nœud sur un segment (``addresses: {underlay: 10.250.0.50}``). Un switch ne + déclare ni ``segments`` ni ``addresses`` : il porte ceux dont il est le ``switch``. + +Ce que l'outil en déduit, de façon déterministe — même fichier, même plan : + +.. list-table:: + :widths: 30 70 + + * - Passerelle d'un segment + - la première adresse du CIDR, portée par le switch sur ``br-`` + * - Adresse d'un nœud + - les suivantes, **dans l'ordre de déclaration des nœuds** ; une adresse fixée par + ``addresses`` est réservée d'abord et sautée par l'attribution automatique + * - Câbles + - un par couple (segment, nœud), segments puis nœuds dans l'ordre de déclaration ; le + câble *i* utilise les ports UDP ``20000 + 2i`` (côté nœud) et ``20001 + 2i`` (côté switch) + * - MAC + - ``02:4c::::`` — préfixe localement administré, rang du nœud + sur deux octets, rang du segment, ``00`` côté nœud et ``01`` côté switch + * - Interfaces + - côté nœud, le nom du segment ; côté switch, ``p``, du rang du câble + * - SSH d'administration + - ``127.0.0.1:<2200 + rang du nœud>`` sur l'hôte du lab + +.. warning:: + + Réordonner les nœuds ou les segments dans le fichier **change les adresses, les MAC et les + ports**. C'est assumé pour un lab ; ``lab plan`` montre le résultat avant tout lancement. + +Limites : 1000 nœuds, 256 segments, et autant de câbles que la plage UDP le permet (22 768). + +``lab plan`` valide le fichier et affiche le plan, sans rien lancer : + +.. code-block:: text + + $ go run ./cmd/lab plan conf/lab/evpn-2hv.yml + lab evpn-2hv: nodes 4, segments 1, cables 3 + + nodes + name role image cpus memory ssh + sw1 switch debian12 2 1024 MiB 127.0.0.1:2200 + rr1 rr debian12 1 1024 MiB 127.0.0.1:2201 + hv1 hypervisor debian12 4 16384 MiB 127.0.0.1:2202 + hv2 hypervisor debian12 4 16384 MiB 127.0.0.1:2203 + + segment underlay: 10.250.0.0/24, mtu 9000, switch sw1, bridge br-underlay, gateway 10.250.0.1 + node interface address mac udp switch port mac udp + rr1 underlay 10.250.0.2/24 02:4c:00:01:00:00 20000 <-> sw1 p0 02:4c:00:01:00:01 20001 + hv1 underlay 10.250.0.3/24 02:4c:00:02:00:00 20002 <-> sw1 p1 02:4c:00:02:00:01 20003 + hv2 underlay 10.250.0.4/24 02:4c:00:03:00:00 20004 <-> sw1 p2 02:4c:00:03:00:01 20005 + +Un fichier invalide est refusé avec **toutes** ses erreurs à la fois, et un code de sortie 1. Les +champs inconnus et les clés en double sont refusés aussi : + +.. code-block:: text + + $ lab plan cassee.yml + lab: cassee.yml: + segment underlay: rr1 is a rr, not a switch + segment underlay: cidr 10.250.0.0/31 prefix length out of range [/8, /30] + node sw1: switch carries no segment + +Les plages d'adresses de l'exemple sont des valeurs de travail : le plan d'adressage du lab reste à +définir (#50). + +Rendu des VM +------------ + +``lab render`` produit, pour chaque nœud, ce qu'il faut pour démarrer sa VM — sans rien lancer : + +.. code-block:: text + + $ go run ./cmd/lab render -key ~/.config/two-lab/ssh/lab_ed25519.pub conf/lab/evpn-2hv.yml + +``//`` reçoit : + +``qemu.args`` + Les arguments de ``qemu-system-x86_64``, **un par ligne** : rien à échapper, rien à + interpréter par un shell. + +``meta-data``, ``user-data``, ``network-config`` + Les trois fichiers NoCloud de cloud-init, à mettre dans une image de volume ``cidata``. + +Les chemins de la VM (``disk.qcow2``, ``seed.iso``, ``console.log``, ``qmp.sock``, ``qemu.pid``) +sont ceux du répertoire du nœud ; ``-key`` peut être répété, et accepte un fichier +``authorized_keys`` (lignes vides et commentaires ignorés). Les fichiers sont créés en ``0600``. + +Ce que contiennent les arguments QEMU d'un hyperviseur — extrait réel, côté réseau : + +.. code-block:: text + + -netdev + user,id=mgmt0,restrict=on,ipv6=off,hostfwd=tcp:127.0.0.1:2202-:22 + -device + virtio-net-pci,netdev=mgmt0,mac=02:4d:00:02:00:00,romfile= + -netdev + dgram,id=underlay,local.type=inet,local.host=127.0.0.1,local.port=20002,remote.type=inet,remote.host=127.0.0.1,remote.port=20003 + -device + virtio-net-pci,netdev=underlay,mac=02:4c:00:02:00:00,host_mtu=9000,romfile= + +Les choix qui s'y lisent : + +* **machine** ``q35``, ``-accel kvm -cpu host`` — le KVM imbriqué des hyperviseurs du lab en + dépend ; ``-nodefaults`` pour qu'aucun périphérique implicite ne s'ajoute ; +* **administration** (``mgmt0``) : le NAT de QEMU, MAC ``02:4d:::00:00``, SSH redirigé + sur la boucle locale de l'hôte. ``restrict=on`` pour tous les nœuds **sauf le switch** : un + nœud isolé ne joint ni l'hôte ni l'extérieur par là, seule la redirection SSH passe. + ``ipv6=off`` partout (voir plus bas) ; +* **câbles** : ``dgram`` sur ``127.0.0.1``, les deux extrémités d'un câble se répondent + (port local de l'une = port distant de l'autre), ``host_mtu`` annonce le MTU du segment au + guest ; +* ``romfile=`` vide sur toutes les cartes : pas de ROM de démarrage réseau, donc pas de repli + sur un démarrage PXE si le firmware ne trouve pas le disque. Pendant les essais de #50, une VM + restée bloquée sans rien écrire sur sa console, CPU au repos, avait toutes les apparences de + ce repli ; la cause n'a pas été isolée, l'option est une précaution. + +Ce que fait cloud-init : + +* **toutes les VM** : interfaces nommées d'après leur MAC (``mgmt0``, nom du segment, ``p``), + ``dhcp4: false`` partout, ``mgmt0`` en ``10.0.2.15/24`` **sans passerelle** ; connexion SSH par + clé seulement, utilisateur ``debian``, ``root`` désactivé, mot de passe refusé ; +* **un nœud** : adresse sur chaque segment, MTU du segment, route par défaut et DNS + (``1.1.1.1``, ``8.8.8.8``) sur son **premier** segment — la sortie Internet passe par le + switch ; +* **le switch** : route par défaut par ``mgmt0`` ; un service ``lab-switch`` crée un bridge + ``br-`` par segment (STP désactivé, MTU du segment), y branche ses ports, porte la + passerelle, active le routage et masque (NAT nftables) les segments vers ``mgmt0``. Le + service est rejoué à chaque démarrage. + +Vérifié sur de vraies VM +~~~~~~~~~~~~~~~~~~~~~~~~ + +Le switch et le route reflector n'ont pas besoin de KVM imbriqué : ``sw1`` et ``rr1`` de +l'exemple ont été démarrés **sur un Mac**, en émulation (TCG), avec Debian 12 ``generic`` et +les fichiers produits par ``lab render``. + +.. list-table:: + :header-rows: 1 + :widths: 60 40 + + * - Vérification + - Résultat + * - interfaces nommées et adressées, bridge ``br-underlay`` en ``10.250.0.1/24`` + - conforme + * - service ``lab-switch`` actif, y compris après redémarrage + - conforme + * - ``ping -M do -s 8972`` de ``rr1`` vers le switch (MTU 9000, sans fragmentation) + - passe + * - ``ping -M do -s 8973`` (MTU 9001) + - refusé : ``message too long, mtu=9000`` + * - Internet depuis ``rr1`` en IPv4 + - passe, par ``10.250.0.1`` + * - ``rr1`` vers un service TCP de l'hôte par ``mgmt0`` — le switch, témoin, y parvient + - bloqué + * - ``rr1`` vers Internet par ``mgmt0`` + - bloqué + +Un défaut trouvé par cet essai, et corrigé : sans ``ipv6=off``, le NAT de QEMU annonce un +préfixe IPv6 et ``mgmt0`` reçoit une **route IPv6 par défaut** — vers une impasse, puisque +``restrict=on`` bloque tout. Pas de fuite, mais chaque programme qui tente l'IPv6 d'abord (le DNS +renvoie d'abord des adresses IPv6) attend un délai avant de se rabattre sur l'IPv4. + +.. note:: + + Un ``ping`` vers ``10.0.2.2`` n'est pas un test d'isolation : c'est la passerelle virtuelle de + QEMU qui répond elle-même, ``restrict=on`` ou non. Seule une connexion vers un vrai service de + l'hôte, avec un témoin qui y parvient, le prouve. + +Reste à vérifier sur le serveur de lab : les hyperviseurs, qui exigent KVM imbriqué. + +Lancement des VM +---------------- + +``lab`` s'exécute **sur le serveur de lab**. Il garde l'état du lab dans un répertoire +(``-run``, par défaut ``~/lab-run``) : ``status``, ``down`` et ``ssh`` n'ont donc pas besoin de la +topologie. + +.. code-block:: text + + lab up [-run dir] [-cache dir] [-timeout 20m] + lab status [-run dir] + lab down [-run dir] + lab ssh [-run dir] [commande…] + +``lab up`` + 1. refuse de continuer si un lab tourne déjà dans le répertoire ; + 2. télécharge chaque image dans le cache (``-cache``, par défaut ``~/.cache/two-lab``) et la + vérifie contre ``SHA512SUMS`` ; une image déjà présente et toujours conforme n'est pas + retéléchargée, la liste des sommes est relue à chaque fois ; + 3. génère une paire de clés SSH dans le répertoire du lab, si elle n'existe pas encore ; + 4. pour chaque nœud : fichiers de ``lab render``, disque **neuf** en overlay qcow2 sur + l'image (``qemu-img create -b``, 20 Gio annoncés), image ``cidata`` (``genisoimage``) ; + 5. démarre les QEMU, **switchs d'abord**, détachés (``-daemonize``) : ils survivent à la + session SSH qui les a lancés ; + 6. attend sur chaque nœud la fin de cloud-init (``cloud-init status --wait`` par SSH), + jusqu'au délai ``-timeout``. + + La topologie est copiée dans ``/topology.yml``. Un échec laisse les nœuds démarrés en + place : ``lab status``, puis ``lab down``. + +``lab status`` + Pour chaque nœud : rôle, état du processus QEMU, PID, port SSH sur la boucle locale. + +``lab down`` + Arrête chaque QEMU par ``SIGTERM``, puis ``SIGKILL`` au bout de 30 s. Les disques sont + conservés jusqu'au prochain ``up``, qui les recrée. + +``lab ssh`` + Ouvre un shell sur un nœud, ou y exécute une commande, avec la clé générée par ``up``. ``lab`` + cède la place à ``ssh``, dont le code de retour est donc celui de la commande. Un terminal + n'est demandé (``-t``) que si l'entrée de ``lab`` en est un : depuis un script, ni + pseudo-terminal ni ``\r\n`` dans la sortie. + + Comme ``ssh``, ``lab ssh`` recolle ses arguments par des espaces et les confie à un shell + distant — et depuis le Mac, il y en a **deux** : celui du serveur, puis celui de la VM. + Une commande qui contient elle-même des guillemets se passe en une seule chaîne : + + .. code-block:: text + + $ echo | scripts/lab-host.sh ssh './lab ssh hv1 sh -c "exit 42"'; echo "rc 42=$?" + rc 42=0 + $ echo | scripts/lab-host.sh ssh "./lab ssh hv1 'sh -c \"exit 42\"'"; echo "rc 42=$?" + rc 42=42 + + Dans le premier cas, la VM reçoit ``sh -c exit 42`` : ``exit`` sans argument, ``42`` en + ``$0``. Pour plus d'une commande, passer un script sur l'entrée standard : + ``scripts/lab-host.sh ssh "./lab ssh hv1 'sudo bash -s'" < script.sh``. + +Un processus n'est tenu pour celui d'un nœud que si son PID, lu dans ``qemu.pid``, désigne un +processus vivant dont la ligne de commande (``/proc//cmdline``) contient ``-name ``. +Un PID réutilisé par un autre programme n'est donc jamais signalé. + +.. warning:: + + Le cache range une image sous son nom de fichier, et l'URL de Debian est ``latest`` : une + nouvelle publication remplace le fichier, et les overlays existants pointeraient sur une + base différente. ``up`` recrée toujours les disques, ce qui suffit avec un lab par serveur ; + **ne pas relancer un QEMU à la main** à partir d'un ``qemu.args`` après un ``up`` ultérieur. + +Une campagne réelle, de la création du serveur à la première commande sur une VM — sorties du +2026-10-04 : + +.. code-block:: text + + $ scripts/lab-host.sh up + == création de two-lab (EM-B212X-SSD, 0.321 EUR/h HT) + … + == préparation du serveur : qemu, genisoimage, KVM imbriqué + … + qemu QEMU emulator version 7.2.22 (Debian 1:7.2+dfsg-7+deb12u18+b3), nested=Y + == prêt : root@ + $ scripts/lab-host.sh push conf/lab/evpn-2hv.yml + == compilation de lab (linux/amd64) + == déposés sur le serveur : ~/lab, ~/evpn-2hv.yml — ensuite : lab-host.sh ssh './lab up evpn-2hv.yml' + $ scripts/lab-host.sh ssh './lab up evpn-2hv.yml' + sw1: started + rr1: started + hv1: started + hv2: started + sw1: ready + rr1: ready + hv1: ready + hv2: ready + $ scripts/lab-host.sh ssh './lab status' + node role state pid ssh + sw1 switch running 5158 127.0.0.1:2200 + rr1 rr running 5171 127.0.0.1:2201 + hv1 hypervisor running 5182 127.0.0.1:2202 + hv2 hypervisor running 5196 127.0.0.1:2203 + +``lab up`` a pris **49 secondes**, téléchargement et vérification de l'image (427 Mio) compris ; +l'essentiel du temps d'une campagne est la livraison du serveur (environ 25 minutes avec +``prepare``). + +Vérifié sur le serveur de lab +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Le 2026-10-04, sur un Xeon E5-2640 v3, Debian 12 et QEMU 7.2 sur le serveur, topologie +``evpn-2hv`` : + +.. list-table:: + :header-rows: 1 + :widths: 60 40 + + * - Vérification + - Résultat + * - ``ping -M do -s 8972`` de hv1 à hv2 à travers le switch + - passe + * - ``ping -M do -s 8973`` + - refusé : ``message too long, mtu=9000`` + * - sortie Internet de hv1 + - par ``10.250.0.1`` (le switch), HTTPS 200 ; aucune route IPv6 globale + * - hv1 vers un service TCP du serveur par ``mgmt0`` — sw1, témoin, y parvient (200) + - refusé + * - hv1 vers Internet par ``mgmt0``, route forcée via ``10.0.2.2`` — sw1 y parvient + - refusé + * - ``/dev/kvm`` et ``nested`` dans hv1 + - présent, ``Y`` + * - racine de hv1 (overlay de 20 Gio) + - 20 Go : ``growpart`` agrandit la partition au premier démarrage + * - code de retour à travers ``lab-host.sh ssh`` et ``lab ssh``, sans terminal + - propagé jusqu'au Mac + * - ``scripts/lab-host.sh ssh './lab ssh hv1'`` depuis un terminal + - shell interactif + * - ``lab down`` puis ``lab up`` d'une autre topologie + - conforme + +Facturation +----------- + +.. warning:: + + Un serveur Elastic Metal est facturé **de sa création à sa suppression, éteint compris**. + Éteindre ne suffit pas : il faut supprimer. La granularité n'est pas documentée par Scaleway — + compter chaque heure entamée comme une heure pleine. + +Ce que le script garantit : + +* il ne commande **jamais** d'offre mensuelle : il exige une seule offre au nom demandé, en + facturation horaire, en stock et sans frais de mise en service, sinon il refuse avant toute + création. La CLI ``scw`` n'est pas utilisée pour cette raison : son ``server create type=…`` + choisit l'offre par son seul nom et peut tomber sur la mensuelle, qui engage un mois ; +* il refuse de créer un second serveur si un serveur de lab existe déjà ; +* ``down`` agit sur **tous** les serveurs portant le tag ``two-lab`` dans le projet, et + ``session`` y ajoute l'identifiant reçu à la création : un serveur créé juste avant une + interruption est rattrapé ; +* une suppression refusée pendant la livraison ou l'installation est réessayée tant qu'elle dure, + dans la limite du délai d'installation augmenté du délai de suppression ; +* le serveur n'est déclaré supprimé qu'au 404 de l'API, jamais sur une erreur passagère ; +* un échec de suppression se termine par ``SERVEUR(S) DE LAB TOUJOURS FACTURÉ(S)`` et un code + d'erreur. + +Ce qu'il ne peut pas garantir : un ``SIGKILL``, une coupure de courant ou une mise en veille du +poste qui lance la session. En cas de doute, toujours : + +.. code-block:: bash + + scripts/lab-host.sh status + scripts/lab-host.sh down + +Diagnostic +---------- + +``aucune clé SSH active dans le projet`` + Aucune clé SSH n'est enregistrée dans le projet de lab. En ajouter une dans la console (projet + → Clés SSH). + +``offre horaire EM-B212X-SSD : 0 correspondance(s)`` + L'offre n'existe pas dans la zone en facturation horaire. Vérifier ``SCW_DEFAULT_ZONE``. + +``création incertaine`` + La création a échoué ou n'a pas rendu d'identifiant. Le serveur a pu être créé malgré tout — + cas typique : le quota (identité non validée), ou une réponse perdue. Le script indique s'il + voit un serveur de lab ; dans tous les cas, lancer ``status`` puis ``down``. + +``SSH injoignable`` + L'installation est terminée mais SSH ne répond pas après 10 minutes. Avec une clé matérielle + (Yubikey), chaque connexion demande un PIN ou un toucher : utiliser la clé dédiée du lab. Le + serveur est toujours facturé : ``down``. + +``SERVEUR(S) DE LAB TOUJOURS FACTURÉ(S)`` + La suppression n'a pas abouti dans les délais. Relancer ``down`` ; si l'erreur persiste, + supprimer depuis la console Scaleway. + +``HTTP 403 insufficient permissions`` + La clé d'API est authentifiée mais n'a pas le droit demandé — en général la lecture des clés + SSH du projet. Compléter la politique de la clé, limitée au projet. + +Sécurité +-------- + +* La clé secrète n'apparaît ni dans les arguments des processus (elle est passée à ``curl`` par + un descripteur de fichier), ni dans les journaux, ni dans l'environnement de ``ssh``. +* **Ne jamais lancer le script sous** ``bash -x`` : la trace afficherait la clé. +* Le serveur n'expose que SSH, par clé. Le lab n'a aucune donnée personnelle ni secret de + production. +* Une clé secrète qui a circulé ailleurs que dans ``scaleway.env`` (conversation, terminal + partagé, capture d'écran) se régénère. +* **Clé SSH des VM** : générée par ``lab up`` sur le serveur, c'est la seule clé autorisée dans + les VM. Elle ne quitte jamais le serveur, n'ouvre que les VM du lab — qui n'écoutent qu'en + boucle locale — et disparaît avec lui. La clé publique du Mac n'est jamais envoyée aux VM. +* **Clés d'hôte des VM non vérifiées** par ``lab ssh`` (``known_hosts`` jetable) : elles changent + à chaque ``up``. Acceptable uniquement parce que la connexion reste sur la boucle locale d'un + serveur auquel on s'est authentifié. +* **Image** : ``SHA512SUMS`` vient de la même origine que l'image, en HTTPS. La vérification + protège contre la corruption, pas contre une origine compromise ; la signature GPG de Debian + (``SHA512SUMS.sign``) n'est pas encore vérifiée. + +Tests +----- + +.. code-block:: bash + + bash scripts/lab-host_test.sh + go test ./internal/lab/... ./cmd/lab/ + +Environ une minute et demie, sans réseau : la suite remplace ``curl`` par une fausse API Scaleway +qui se place dans le pire cas (offre mensuelle listée avant l'horaire, serveurs d'autres projets, +suppressions refusées, erreurs 503, serveur qui tarde à disparaître) et ``ssh`` par un faux client. +Elle tourne sous bash 5 comme sous le bash 3.2 de macOS. diff --git a/docs/index.rst b/docs/index.rst index b72be04..8eba581 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,9 @@ Par où commencer :doc:`/exploitation/index` Configuration, services, API de l'agent, métriques et diagnostic sur un nœud en service. +:doc:`/developpement/index` + Outillage de développement de two, dont le lab de test multi-nœud sur serveur loué à l'heure. + :doc:`/concepts/index` Comment les éléments fonctionnent entre eux : modèle de données, modes réseau, cycle de vie, metadata. À lire avant de diagnostiquer un comportement inattendu. @@ -57,6 +60,12 @@ Par où commencer exploitation/index +.. toctree:: + :hidden: + :caption: Développement + + developpement/index + .. toctree:: :hidden: :caption: Interne diff --git a/go.mod b/go.mod index 0e59eb9..9bcc86b 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/sys v0.45.0 ) @@ -46,7 +47,6 @@ require ( go.opentelemetry.io/otel/metric v1.37.0 // indirect go.opentelemetry.io/otel/trace v1.37.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/protobuf v1.36.8 // indirect diff --git a/internal/lab/machine/machine.go b/internal/lab/machine/machine.go new file mode 100644 index 0000000..1630ed7 --- /dev/null +++ b/internal/lab/machine/machine.go @@ -0,0 +1,269 @@ +package machine + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "text/tabwriter" + "time" + + "git.g3e.fr/syonad/two/internal/lab/provision" + "git.g3e.fr/syonad/two/internal/lab/render" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const ( + User = "debian" + + sshUnreachable = 255 + cloudInitRecoverable = 2 +) + +type Lab struct { + Plan *topology.Plan + RunDir string + ProcDir string + Runner provision.Runner + Poll time.Duration + Stop time.Duration + Out io.Writer + + signal func(pid int, sig syscall.Signal) error +} + +func (l Lab) node(name string) (topology.NodePlan, error) { + for _, n := range l.Plan.Nodes { + if n.Name == name { + return n, nil + } + } + return topology.NodePlan{}, fmt.Errorf("node %q is not in lab %s", name, l.Plan.Name) +} + +func (l Lab) pidFile(name string) string { + return filepath.Join(l.RunDir, name, render.PIDFile) +} + +func (l Lab) PID(name string) (int, error) { + data, err := os.ReadFile(l.pidFile(name)) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0, fmt.Errorf("%s: not a pid: %q", l.pidFile(name), data) + } + if !l.runs(pid, name) { + return 0, nil + } + return pid, nil +} + +func (l Lab) runs(pid int, name string) bool { + if err := syscall.Kill(pid, 0); err != nil && !errors.Is(err, syscall.EPERM) { + return false + } + cmdline, err := os.ReadFile(filepath.Join(l.ProcDir, strconv.Itoa(pid), "cmdline")) + if err != nil { + return false + } + return bytes.Contains(cmdline, []byte("\x00-name\x00"+name+"\x00")) +} + +func (l Lab) Running() ([]string, error) { + var running []string + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + return nil, err + } + if pid != 0 { + running = append(running, n.Name) + } + } + return running, nil +} + +func (l Lab) Up(ctx context.Context, fetcher provision.Fetcher, timeout time.Duration) error { + running, err := l.Running() + if err != nil { + return err + } + if len(running) > 0 { + return fmt.Errorf("lab %s is already running (%s): 'lab down' first", l.Plan.Name, strings.Join(running, ", ")) + } + + nodes, err := provision.Prepare(ctx, l.Plan, provision.Options{RunDir: l.RunDir, Fetcher: fetcher, Runner: l.Runner}) + if err != nil { + return err + } + for _, n := range switchesFirst(l.Plan, nodes) { + if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := l.Runner.Run(ctx, render.QEMUBinary, append(n.QEMU, "-daemonize")...); err != nil { + return fmt.Errorf("node %s: %w", n.Name, err) + } + fmt.Fprintf(l.Out, "%s: started\n", n.Name) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var errs []error + for _, n := range l.Plan.Nodes { + if err := l.waitReady(ctx, n); err != nil { + errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err)) + continue + } + fmt.Fprintf(l.Out, "%s: ready\n", n.Name) + } + return errors.Join(errs...) +} + +func switchesFirst(p *topology.Plan, nodes []render.Node) []render.Node { + var first, rest []render.Node + for i, n := range nodes { + if p.Nodes[i].Role == topology.RoleSwitch { + first = append(first, n) + } else { + rest = append(rest, n) + } + } + return append(first, rest...) +} + +func (l Lab) waitReady(ctx context.Context, n topology.NodePlan) error { + args := l.sshArgs(n, true, []string{"cloud-init", "status", "--wait"}) + for { + err := l.Runner.Run(ctx, "ssh", args...) + switch code := exitCode(err); { + case err == nil: + return nil + case code == cloudInitRecoverable: + fmt.Fprintf(l.Out, "%s: cloud-init finished with recoverable errors: %v\n", n.Name, err) + return nil + case code != sshUnreachable: + return fmt.Errorf("cloud-init failed: %w", err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("not reachable over ssh: %w", ctx.Err()) + case <-time.After(l.Poll): + } + } +} + +func exitCode(err error) int { + var coded interface{ ExitCode() int } + if errors.As(err, &coded) { + return coded.ExitCode() + } + return -1 +} + +func (l Lab) sshArgs(n topology.NodePlan, batch bool, command []string) []string { + args := []string{ + "-i", filepath.Join(l.RunDir, provision.KeyFile), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", strconv.Itoa(n.SSHPort), + } + if batch { + args = append(args, "-o", "BatchMode=yes", "-o", "ConnectTimeout=5") + } + args = append(args, User+"@127.0.0.1") + return append(args, command...) +} + +func (l Lab) SSH(name string, terminal bool, command []string) ([]string, error) { + n, err := l.node(name) + if err != nil { + return nil, err + } + args := []string{"ssh"} + if terminal && len(command) > 0 { + args = append(args, "-t") + } + return append(args, l.sshArgs(n, false, command)...), nil +} + +func (l Lab) Status(w io.Writer) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintf(tw, "node\trole\tstate\tpid\tssh\n") + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + return err + } + state, shown := "stopped", "-" + if pid != 0 { + state, shown = "running", strconv.Itoa(pid) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t127.0.0.1:%d\n", n.Name, n.Role, state, shown, n.SSHPort) + } + return tw.Flush() +} + +func (l Lab) Down(ctx context.Context) error { + var errs []error + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + errs = append(errs, err) + continue + } + if pid == 0 { + continue + } + if err := l.stop(ctx, n.Name, pid); err != nil { + errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err)) + continue + } + if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, err) + continue + } + fmt.Fprintf(l.Out, "%s: stopped\n", n.Name) + } + return errors.Join(errs...) +} + +func (l Lab) stop(ctx context.Context, name string, pid int) error { + if pid <= 0 { + return fmt.Errorf("refusing to signal pid %d", pid) + } + send := l.signal + if send == nil { + send = syscall.Kill + } + for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} { + if err := send(pid, sig); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + deadline := time.Now().Add(l.Stop) + for l.runs(pid, name) && time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(l.Poll): + } + } + if !l.runs(pid, name) { + return nil + } + } + return fmt.Errorf("pid %d still running after SIGKILL", pid) +} diff --git a/internal/lab/machine/machine_test.go b/internal/lab/machine/machine_test.go new file mode 100644 index 0000000..22635d6 --- /dev/null +++ b/internal/lab/machine/machine_test.go @@ -0,0 +1,543 @@ +package machine + +import ( + "bytes" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "git.g3e.fr/syonad/two/internal/lab/provision" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const labKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab" + +type exitErr int + +func (e exitErr) Error() string { return "exit status " + strconv.Itoa(int(e)) } +func (e exitErr) ExitCode() int { return int(e) } + +type fakeRunner struct { + mu sync.Mutex + calls [][]string + fail string + ssh map[string][]error + always map[string]error +} + +func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, append([]string{name}, args...)) + if name == f.fail { + return errors.New(name + " failed") + } + switch name { + case "ssh-keygen": + private := args[len(args)-1] + if err := os.WriteFile(private, []byte("private"), 0o600); err != nil { + return err + } + return os.WriteFile(private+".pub", []byte(labKey+"\n"), 0o644) + case "ssh": + port := args[indexOf(args, "-p")+1] + answers := f.ssh[port] + if len(answers) == 0 { + return f.always[port] + } + f.ssh[port] = answers[1:] + return answers[0] + } + return nil +} + +func (f *fakeRunner) commands(name string) [][]string { + f.mu.Lock() + defer f.mu.Unlock() + var out [][]string + for _, c := range f.calls { + if c[0] == name { + out = append(out, c) + } + } + return out +} + +func indexOf(list []string, value string) int { + for i, v := range list { + if v == value { + return i + } + } + return -1 +} + +func mirror(t *testing.T) (*httptest.Server, string) { + t.Helper() + content := []byte("qcow2 image") + h := sha512.Sum512(content) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "SHA512SUMS") { + fmt.Fprintf(w, "%s deb.qcow2\n", hex.EncodeToString(h[:])) + return + } + w.Write(content) + })) + t.Cleanup(server.Close) + return server, server.URL +} + +func labPlan(t *testing.T, base, nodes string) *topology.Plan { + t.Helper() + doc := `name: evpn-2hv +images: + deb: + url: ` + base + `/deb.qcow2 + sums: ` + base + `/SHA512SUMS +segments: + underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 } +nodes: +` + nodes + topo, err := topology.Parse([]byte(doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + p, err := topology.Compute(topo) + if err != nil { + t.Fatalf("Compute: %v", err) + } + return p +} + +const switchLast = ` rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } + sw1: { role: switch, image: deb, cpus: 2, memory: 1024 } +` + +type fixture struct { + lab Lab + runner *fakeRunner + out *bytes.Buffer + fetcher provision.Fetcher +} + +func newFixture(t *testing.T, nodes string) *fixture { + t.Helper() + server, base := mirror(t) + root := t.TempDir() + runner := &fakeRunner{ssh: map[string][]error{}, always: map[string]error{}} + out := &bytes.Buffer{} + return &fixture{ + lab: Lab{ + Plan: labPlan(t, base, nodes), + RunDir: filepath.Join(root, "run"), + ProcDir: filepath.Join(root, "proc"), + Runner: runner, + Poll: time.Millisecond, + Stop: 500 * time.Millisecond, + Out: out, + }, + runner: runner, + out: out, + fetcher: provision.Fetcher{Client: server.Client(), CacheDir: filepath.Join(root, "cache")}, + } +} + +func (f *fixture) process(t *testing.T, node, script string, cmdline ...string) int { + t.Helper() + cmd := exec.Command("sh", "-c", script) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { cmd.Wait(); close(done) }() + t.Cleanup(func() { + cmd.Process.Kill() + <-done + }) + pid := cmd.Process.Pid + proc := filepath.Join(f.lab.ProcDir, strconv.Itoa(pid)) + if err := os.MkdirAll(proc, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, "cmdline"), []byte(strings.Join(cmdline, "\x00")+"\x00"), 0o644); err != nil { + t.Fatal(err) + } + f.pidFile(t, node, strconv.Itoa(pid)+"\n") + return pid +} + +func (f *fixture) pidFile(t *testing.T, node, content string) { + t.Helper() + dir := filepath.Join(f.lab.RunDir, node) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "qemu.pid"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func qemu(node string) []string { + return []string{"qemu-system-x86_64", "-name", node, "-machine", "q35"} +} + +func alive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +func waitDead(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for alive(pid) && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if alive(pid) { + t.Errorf("pid %d is still alive", pid) + } +} + +func TestPID_NoPidFileMeansStopped(t *testing.T) { + f := newFixture(t, switchLast) + if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil { + t.Errorf("PID = %d, %v", pid, err) + } +} + +func TestPID_RefusesACorruptedPidFile(t *testing.T) { + for _, content := range []string{"", "abc", "0", "-12"} { + f := newFixture(t, switchLast) + f.pidFile(t, "hv1", content) + if _, err := f.lab.PID("hv1"); err == nil || !strings.Contains(err.Error(), "not a pid") { + t.Errorf("pid file %q: error = %v", content, err) + } + } +} + +func TestPID_RecognisesTheNodeProcess(t *testing.T) { + f := newFixture(t, switchLast) + want := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + if pid, err := f.lab.PID("hv1"); pid != want || err != nil { + t.Errorf("PID = %d, %v, want %d", pid, err, want) + } +} + +func TestPID_IgnoresAPidReusedByAnotherProcess(t *testing.T) { + cases := map[string][]string{ + "another program": {"/usr/sbin/sshd", "-D"}, + "another node": qemu("hv10"), + "name as a value": {"qemu-system-x86_64", "-serial", "-name", "-name", "hv2"}, + } + for name, cmdline := range cases { + t.Run(name, func(t *testing.T) { + f := newFixture(t, switchLast) + f.process(t, "hv1", "sleep 30", cmdline...) + if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil { + t.Errorf("PID = %d, %v, want 0", pid, err) + } + }) + } +} + +func TestPID_IgnoresADeadProcess(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "exit 0", qemu("hv1")...) + waitDead(t, pid) + if got, err := f.lab.PID("hv1"); got != 0 || err != nil { + t.Errorf("PID = %d, %v, want 0", got, err) + } +} + +func TestStatus(t *testing.T) { + f := newFixture(t, switchLast) + pid := strconv.Itoa(f.process(t, "hv1", "sleep 30", qemu("hv1")...)) + var out bytes.Buffer + if err := f.lab.Status(&out); err != nil { + t.Fatal(err) + } + w := len(pid) + if w < 3 { + w = 3 + } + want := fmt.Sprintf("node role state %-*s ssh\n", w, "pid") + + fmt.Sprintf("rr1 rr stopped %-*s 127.0.0.1:2200\n", w, "-") + + fmt.Sprintf("hv1 hypervisor running %-*s 127.0.0.1:2201\n", w, pid) + + fmt.Sprintf("sw1 switch stopped %-*s 127.0.0.1:2202\n", w, "-") + if out.String() != want { + t.Errorf("status:\n%s\nwant:\n%s", out.String(), want) + } +} + +func TestDown_StopsWithSIGTERM(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + waitDead(t, pid) + if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) { + t.Errorf("pid file still there: %v", err) + } + if f.out.String() != "hv1: stopped\n" { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestDown_FallsBackToSIGKILL(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.Stop = 100 * time.Millisecond + pid := f.process(t, "hv1", `trap "" TERM; while :; do sleep 0.05; done`, qemu("hv1")...) + time.Sleep(50 * time.Millisecond) + + start := time.Now() + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + waitDead(t, pid) + if elapsed := time.Since(start); elapsed < 100*time.Millisecond { + t.Errorf("SIGKILL after %v, want at least the 100ms grace period", elapsed) + } +} + +func TestDown_NeverSignalsAProcessThatIsNotTheNode(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "sleep 30", "/usr/sbin/sshd", "-D") + + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + time.Sleep(50 * time.Millisecond) + if !alive(pid) { + t.Error("a process that is not the node was killed") + } + if f.out.Len() != 0 { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestDown_ReportsACorruptedPidFileAndStopsTheOthers(t *testing.T) { + f := newFixture(t, switchLast) + f.pidFile(t, "rr1", "garbage") + pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + err := f.lab.Down(context.Background()) + + if err == nil || !strings.Contains(err.Error(), "not a pid") { + t.Errorf("error = %v", err) + } + waitDead(t, pid) +} + +func TestSSH_Arguments(t *testing.T) { + f := newFixture(t, switchLast) + key := filepath.Join(f.lab.RunDir, "lab_ed25519") + base := []string{ + "-i", key, + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2201", + "debian@127.0.0.1", + } + cases := map[string]struct { + terminal bool + command []string + want []string + }{ + "interactive shell": {true, nil, append([]string{"ssh"}, base...)}, + "command from a terminal": {true, []string{"top"}, append(append([]string{"ssh", "-t"}, base...), "top")}, + "command from a script": {false, []string{"ip", "-br", "a"}, append(append([]string{"ssh"}, base...), "ip", "-br", "a")}, + "shell from a script": {false, nil, append([]string{"ssh"}, base...)}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + got, err := f.lab.SSH("hv1", c.terminal, c.command) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("\n got %q\nwant %q", got, c.want) + } + }) + } +} + +func TestSSH_UnknownNode(t *testing.T) { + f := newFixture(t, switchLast) + if _, err := f.lab.SSH("hv9", true, nil); err == nil || !strings.Contains(err.Error(), `node "hv9" is not in lab evpn-2hv`) { + t.Errorf("error = %v", err) + } +} + +func TestUp_StartsSwitchesFirstAndWaitsForEveryNode(t *testing.T) { + f := newFixture(t, switchLast) + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + + var started []string + for _, c := range f.runner.commands("qemu-system-x86_64") { + if c[len(c)-1] != "-daemonize" { + t.Errorf("qemu not daemonized: %q", c) + } + started = append(started, c[indexOf(c, "-name")+1]) + } + if !reflect.DeepEqual(started, []string{"sw1", "rr1", "hv1"}) { + t.Errorf("start order = %v, want the switch first", started) + } + + ssh := f.runner.commands("ssh") + if len(ssh) != 3 { + t.Fatalf("%d ssh calls, want 3", len(ssh)) + } + want := []string{"ssh", + "-i", filepath.Join(f.lab.RunDir, "lab_ed25519"), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2200", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=5", + "debian@127.0.0.1", + "cloud-init", "status", "--wait", + } + if !reflect.DeepEqual(ssh[0], want) { + t.Errorf("readiness check:\n got %q\nwant %q", ssh[0], want) + } + if f.out.String() != "sw1: started\nrr1: started\nhv1: started\nrr1: ready\nhv1: ready\nsw1: ready\n" { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestUp_RetriesWhileSSHIsUnreachable(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.ssh["2201"] = []error{exitErr(255), exitErr(255), nil} + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if n := len(f.runner.commands("ssh")); n != 5 { + t.Errorf("%d ssh calls, want 5", n) + } +} + +func TestUp_ReadinessFailures(t *testing.T) { + cases := map[string]struct { + answers []error + want string + }{ + "cloud-init error": {[]error{exitErr(1)}, "node hv1: cloud-init failed: exit status 1"}, + "not an exit status": {[]error{errors.New("fork failed")}, "node hv1: cloud-init failed: fork failed"}, + "ssh never reachable": {nil, "node hv1: not reachable over ssh: context deadline exceeded"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.Poll = 10 * time.Millisecond + f.runner.ssh["2201"] = c.answers + f.runner.always["2201"] = exitErr(255) + + err := f.lab.Up(context.Background(), f.fetcher, 300*time.Millisecond) + + if err == nil || err.Error() != c.want { + t.Errorf("error = %v, want %q", err, c.want) + } + if !strings.Contains(f.out.String(), "rr1: ready\n") || !strings.Contains(f.out.String(), "sw1: ready\n") { + t.Errorf("the other nodes were not waited for: %q", f.out.String()) + } + }) + } +} + +func TestUp_RecoverableCloudInitErrorsAreReportedNotFatal(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.ssh["2201"] = []error{exitErr(2)} + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if !strings.Contains(f.out.String(), "hv1: cloud-init finished with recoverable errors: exit status 2\nhv1: ready\n") { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestUp_RefusesARunningLab(t *testing.T) { + f := newFixture(t, switchLast) + f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + err := f.lab.Up(context.Background(), f.fetcher, time.Second) + + if err == nil || err.Error() != "lab evpn-2hv is already running (hv1): 'lab down' first" { + t.Errorf("error = %v", err) + } + if len(f.runner.calls) != 0 { + t.Errorf("commands ran: %q", f.runner.calls) + } +} + +func TestUp_RemovesAStalePidFileBeforeStarting(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "exit 0", qemu("hv1")...) + waitDead(t, pid) + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) { + t.Errorf("stale pid file kept: %v", err) + } +} + +func TestUp_StopsAtTheFirstQEMUFailure(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.fail = "qemu-system-x86_64" + + err := f.lab.Up(context.Background(), f.fetcher, time.Second) + + if err == nil || err.Error() != "node sw1: qemu-system-x86_64 failed" { + t.Errorf("error = %v", err) + } + if n := len(f.runner.commands("qemu-system-x86_64")); n != 1 { + t.Errorf("%d qemu starts, want 1", n) + } + if n := len(f.runner.commands("ssh")); n != 0 { + t.Errorf("%d ssh calls after a failed start", n) + } +} + +func TestStop_RefusesPidsThatTargetAGroup(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.signal = func(pid int, sig syscall.Signal) error { + t.Fatalf("signal %v sent to pid %d", sig, pid) + return nil + } + for _, pid := range []int{0, -1} { + if err := f.lab.stop(context.Background(), "hv1", pid); err == nil || err.Error() != fmt.Sprintf("refusing to signal pid %d", pid) { + t.Errorf("pid %d: error = %v", pid, err) + } + } +} diff --git a/internal/lab/provision/image.go b/internal/lab/provision/image.go new file mode 100644 index 0000000..c4e8347 --- /dev/null +++ b/internal/lab/provision/image.go @@ -0,0 +1,185 @@ +package provision + +import ( + "bufio" + "bytes" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const maxSumsSize = 1 << 20 + +var sha512Pattern = regexp.MustCompile(`^[0-9a-f]{128}$`) + +type Fetcher struct { + Client *http.Client + CacheDir string +} + +func (f Fetcher) Image(ctx context.Context, img topology.Image) (string, error) { + if !filepath.IsAbs(f.CacheDir) { + return "", fmt.Errorf("image cache dir %q must be an absolute path", f.CacheDir) + } + name, err := fileName(img.URL) + if err != nil { + return "", fmt.Errorf("image %s: %w", img.Name, err) + } + + sums, err := f.get(ctx, img.Sums, maxSumsSize) + if err != nil { + return "", fmt.Errorf("image %s: sums: %w", img.Name, err) + } + want, err := expectedSum(sums, name) + if err != nil { + return "", fmt.Errorf("image %s: %s: %w", img.Name, img.Sums, err) + } + + dir := filepath.Join(f.CacheDir, img.Name) + target := filepath.Join(dir, name) + if got, err := fileSum(target); err == nil && got == want { + return target, nil + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("image %s: %w", img.Name, err) + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + if err := f.download(ctx, img.URL, dir, target, want); err != nil { + return "", fmt.Errorf("image %s: %w", img.Name, err) + } + return target, nil +} + +func (f Fetcher) client() *http.Client { + if f.Client != nil { + return f.Client + } + return http.DefaultClient +} + +func (f Fetcher) open(ctx context.Context, rawURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := f.client().Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: %s", rawURL, resp.Status) + } + return resp, nil +} + +func (f Fetcher) get(ctx context.Context, rawURL string, limit int64) ([]byte, error) { + resp, err := f.open(ctx, rawURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("GET %s: larger than %d bytes", rawURL, limit) + } + return data, nil +} + +func (f Fetcher) download(ctx context.Context, rawURL, dir, target, want string) error { + resp, err := f.open(ctx, rawURL) + if err != nil { + return err + } + defer resp.Body.Close() + + tmp, err := os.CreateTemp(dir, filepath.Base(target)+".part-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + + h := sha512.New() + if _, err := io.Copy(io.MultiWriter(tmp, h), resp.Body); err != nil { + tmp.Close() + return fmt.Errorf("GET %s: %w", rawURL, err) + } + if err := tmp.Close(); err != nil { + return err + } + if got := hex.EncodeToString(h.Sum(nil)); got != want { + return fmt.Errorf("GET %s: sha512 %s, want %s", rawURL, got, want) + } + if err := os.Chmod(tmp.Name(), 0o644); err != nil { + return err + } + return os.Rename(tmp.Name(), target) +} + +func fileName(rawURL string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + name := path.Base(u.Path) + if name == "" || name == "." || name == "/" { + return "", fmt.Errorf("url %q does not name a file", rawURL) + } + return name, nil +} + +func expectedSum(sums []byte, name string) (string, error) { + var found []string + sc := bufio.NewScanner(bytes.NewReader(sums)) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) != 2 || strings.TrimPrefix(fields[1], "*") != name { + continue + } + sum := strings.ToLower(fields[0]) + if !sha512Pattern.MatchString(sum) { + return "", fmt.Errorf("entry for %s is not a sha512 sum", name) + } + found = append(found, sum) + } + if err := sc.Err(); err != nil { + return "", err + } + switch { + case len(found) == 0: + return "", fmt.Errorf("no sum for %s", name) + case len(found) > 1: + return "", fmt.Errorf("%d sums for %s", len(found), name) + } + return found[0], nil +} + +func fileSum(p string) (string, error) { + file, err := os.Open(p) + if err != nil { + return "", err + } + defer file.Close() + h := sha512.New() + if _, err := io.Copy(h, file); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/lab/provision/image_test.go b/internal/lab/provision/image_test.go new file mode 100644 index 0000000..cd1ab20 --- /dev/null +++ b/internal/lab/provision/image_test.go @@ -0,0 +1,334 @@ +package provision + +import ( + "context" + "crypto/sha512" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const imageName = "debian-12-generic-amd64.qcow2" + +type mirror struct { + mu sync.Mutex + files map[string][]byte + status map[string]int + hits map[string]int + server *httptest.Server +} + +func newMirror(t *testing.T) *mirror { + t.Helper() + m := &mirror{files: map[string][]byte{}, status: map[string]int{}, hits: map[string]int{}} + m.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + m.hits[r.URL.Path]++ + if code := m.status[r.URL.Path]; code != 0 { + w.WriteHeader(code) + return + } + data, ok := m.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Write(data) + })) + t.Cleanup(m.server.Close) + return m +} + +func (m *mirror) put(path string, data []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.files[path] = data +} + +func (m *mirror) count(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.hits[path] +} + +func (m *mirror) image() topology.Image { + return topology.Image{ + Name: "debian12", + URL: m.server.URL + "/bookworm/latest/" + imageName, + Sums: m.server.URL + "/bookworm/latest/SHA512SUMS", + } +} + +func sum(data []byte) string { + h := sha512.Sum512(data) + return hex.EncodeToString(h[:]) +} + +func publish(m *mirror, content []byte) { + m.put("/bookworm/latest/"+imageName, content) + m.put("/bookworm/latest/SHA512SUMS", []byte( + sum([]byte("other"))+" debian-12-genericcloud-amd64.qcow2\n"+ + sum(content)+" "+imageName+"\n"+ + sum([]byte("raw"))+" debian-12-generic-amd64.raw\n")) +} + +func fetch(t *testing.T, m *mirror, cache string) string { + t.Helper() + got, err := Fetcher{Client: m.server.Client(), CacheDir: cache}.Image(context.Background(), m.image()) + if err != nil { + t.Fatalf("Image: %v", err) + } + return got +} + +func fetchError(t *testing.T, m *mirror, cache string) string { + t.Helper() + _, err := Fetcher{Client: m.server.Client(), CacheDir: cache}.Image(context.Background(), m.image()) + if err == nil { + t.Fatal("Image: no error") + } + return err.Error() +} + +func entries(t *testing.T, dir string) []string { + t.Helper() + list, err := os.ReadDir(dir) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var names []string + for _, e := range list { + names = append(names, e.Name()) + } + return names +} + +func TestImage_DownloadsAndVerifies(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + cache := t.TempDir() + + got := fetch(t, m, cache) + + if want := filepath.Join(cache, "debian12", "debian-12-generic-amd64.qcow2"); got != want { + t.Errorf("path = %s, want %s", got, want) + } + data, err := os.ReadFile(got) + if err != nil || string(data) != "qcow2 image" { + t.Errorf("content = %q, %v", data, err) + } + info, err := os.Stat(got) + if err != nil || info.Mode().Perm() != 0o644 { + t.Errorf("mode = %v, %v", info.Mode().Perm(), err) + } + if names := entries(t, filepath.Join(cache, "debian12")); len(names) != 1 { + t.Errorf("cache dir holds %v, want only the image", names) + } +} + +func TestImage_ReusesAVerifiedCacheButAlwaysRereadsTheSums(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + cache := t.TempDir() + + fetch(t, m, cache) + fetch(t, m, cache) + + if n := m.count("/bookworm/latest/" + imageName); n != 1 { + t.Errorf("image downloaded %d times, want 1", n) + } + if n := m.count("/bookworm/latest/SHA512SUMS"); n != 2 { + t.Errorf("sums read %d times, want 2", n) + } +} + +func TestImage_ReplacesACorruptedCache(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + cache := t.TempDir() + path := fetch(t, m, cache) + if err := os.WriteFile(path, []byte("bit rot"), 0o644); err != nil { + t.Fatal(err) + } + + fetch(t, m, cache) + + if data, _ := os.ReadFile(path); string(data) != "qcow2 image" { + t.Errorf("content = %q, want the published image", data) + } + if n := m.count("/bookworm/latest/" + imageName); n != 2 { + t.Errorf("image downloaded %d times, want 2", n) + } +} + +func TestImage_FollowsANewReleaseOfTheSameFile(t *testing.T) { + m := newMirror(t) + publish(m, []byte("release 1")) + cache := t.TempDir() + path := fetch(t, m, cache) + + publish(m, []byte("release 2")) + fetch(t, m, cache) + + if data, _ := os.ReadFile(path); string(data) != "release 2" { + t.Errorf("content = %q, want release 2", data) + } +} + +func TestImage_RejectsAMismatchAndLeavesNothing(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + m.put("/bookworm/latest/"+imageName, []byte("tampered")) + cache := t.TempDir() + + msg := fetchError(t, m, cache) + + if !strings.Contains(msg, "sha512") { + t.Errorf("error = %q, want a sha512 mismatch", msg) + } + if names := entries(t, filepath.Join(cache, "debian12")); len(names) != 0 { + t.Errorf("cache dir holds %v after a mismatch, want nothing", names) + } +} + +func TestImage_KeepsTheVerifiedCacheWhenANewDownloadFails(t *testing.T) { + m := newMirror(t) + publish(m, []byte("release 1")) + cache := t.TempDir() + path := fetch(t, m, cache) + + publish(m, []byte("release 2")) + m.put("/bookworm/latest/"+imageName, []byte("truncated")) + fetchError(t, m, cache) + + if data, _ := os.ReadFile(path); string(data) != "release 1" { + t.Errorf("content = %q, want release 1 kept", data) + } + if names := entries(t, filepath.Join(cache, "debian12")); len(names) != 1 { + t.Errorf("cache dir holds %v, want only the image", names) + } +} + +func TestImage_AcceptsBinaryModeSumLines(t *testing.T) { + m := newMirror(t) + content := []byte("qcow2 image") + m.put("/bookworm/latest/"+imageName, content) + m.put("/bookworm/latest/SHA512SUMS", []byte(strings.ToUpper(sum(content))+" *"+imageName+"\n")) + + fetch(t, m, t.TempDir()) +} + +func TestImage_SumsRejections(t *testing.T) { + content := []byte("qcow2 image") + cases := map[string]struct { + sums string + want string + }{ + "no entry": {sum(content) + " debian-12-generic-arm64.qcow2\n", "no sum for " + imageName}, + "two entries": {sum(content) + " " + imageName + "\n" + sum(content) + " " + imageName + "\n", "2 sums for " + imageName}, + "not sha512": {"d41d8cd98f00b204e9800998ecf8427e " + imageName + "\n", "is not a sha512 sum"}, + "prefix of name": {sum(content) + " " + imageName + ".sig\n", "no sum for " + imageName}, + "name in directory": {sum(content) + " nested/" + imageName + "\n", "no sum for " + imageName}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + m := newMirror(t) + m.put("/bookworm/latest/"+imageName, content) + m.put("/bookworm/latest/SHA512SUMS", []byte(c.sums)) + if msg := fetchError(t, m, t.TempDir()); !strings.Contains(msg, c.want) { + t.Errorf("error = %q, want %q", msg, c.want) + } + if n := m.count("/bookworm/latest/" + imageName); n != 0 { + t.Errorf("image downloaded %d times before the sums were trusted", n) + } + }) + } +} + +func TestImage_RefusesOversizedSums(t *testing.T) { + m := newMirror(t) + m.put("/bookworm/latest/SHA512SUMS", make([]byte, 1<<20+1)) + if msg := fetchError(t, m, t.TempDir()); !strings.Contains(msg, "larger than 1048576 bytes") { + t.Errorf("error = %q", msg) + } +} + +func TestImage_AcceptsSumsOfExactlyTheLimit(t *testing.T) { + m := newMirror(t) + content := []byte("qcow2 image") + line := sum(content) + " " + imageName + "\n" + m.put("/bookworm/latest/"+imageName, content) + m.put("/bookworm/latest/SHA512SUMS", []byte(strings.Repeat("\n", 1<<20-len(line))+line)) + + fetch(t, m, t.TempDir()) +} + +func TestImage_HTTPErrors(t *testing.T) { + for _, path := range []string{"/bookworm/latest/SHA512SUMS", "/bookworm/latest/" + imageName} { + t.Run(path, func(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + m.status[path] = http.StatusServiceUnavailable + cache := t.TempDir() + if msg := fetchError(t, m, cache); !strings.Contains(msg, "503") { + t.Errorf("error = %q, want the http status", msg) + } + if names := entries(t, filepath.Join(cache, "debian12")); len(names) != 0 { + t.Errorf("cache dir holds %v, want nothing", names) + } + }) + } +} + +func TestImage_RefusesARelativeCacheDir(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + if msg := fetchError(t, m, "cache"); !strings.Contains(msg, "must be an absolute path") { + t.Errorf("error = %q", msg) + } + if n := m.count("/bookworm/latest/SHA512SUMS"); n != 0 { + t.Errorf("sums read %d times", n) + } +} + +func TestImage_RefusesAURLWithoutFileName(t *testing.T) { + m := newMirror(t) + img := m.image() + img.URL = m.server.URL + "/" + _, err := Fetcher{Client: m.server.Client(), CacheDir: t.TempDir()}.Image(context.Background(), img) + if err == nil || !strings.Contains(err.Error(), "does not name a file") { + t.Errorf("error = %v", err) + } +} + +func TestImage_RejectsAnInterruptedDownloadAndLeavesNothing(t *testing.T) { + content := []byte("qcow2 image") + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "SHA512SUMS") { + w.Write([]byte(sum(content) + " " + imageName + "\n")) + return + } + w.Header().Set("Content-Length", "1000") + w.Write(content[:4]) + })) + defer server.Close() + cache := t.TempDir() + img := topology.Image{Name: "debian12", URL: server.URL + "/" + imageName, Sums: server.URL + "/SHA512SUMS"} + + _, err := Fetcher{Client: server.Client(), CacheDir: cache}.Image(context.Background(), img) + + if err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Errorf("error = %v, want an interrupted transfer", err) + } + if names := entries(t, filepath.Join(cache, "debian12")); len(names) != 0 { + t.Errorf("cache dir holds %v, want nothing", names) + } +} diff --git a/internal/lab/provision/provision.go b/internal/lab/provision/provision.go new file mode 100644 index 0000000..98c1b2f --- /dev/null +++ b/internal/lab/provision/provision.go @@ -0,0 +1,151 @@ +package provision + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.g3e.fr/syonad/two/internal/lab/render" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const ( + KeyFile = "lab_ed25519" + KeyLabel = "two-lab" + DiskSize = "20G" + SeedLabel = "cidata" + + ArgsFile = "qemu.args" +) + +var SeedFiles = []string{"user-data", "meta-data", "network-config"} + +type Runner interface { + Run(ctx context.Context, name string, args ...string) error +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(string(out))) + } + return nil +} + +type Options struct { + RunDir string + Fetcher Fetcher + Runner Runner +} + +func Prepare(ctx context.Context, p *topology.Plan, o Options) ([]render.Node, error) { + if !filepath.IsAbs(o.RunDir) { + return nil, fmt.Errorf("run dir %q must be an absolute path", o.RunDir) + } + if err := os.MkdirAll(o.RunDir, 0o700); err != nil { + return nil, err + } + + images := map[string]string{} + for _, n := range p.Nodes { + if _, ok := images[n.Image]; ok { + continue + } + img, ok := imageNamed(p, n.Image) + if !ok { + return nil, fmt.Errorf("node %s: image %q is not declared", n.Name, n.Image) + } + base, err := o.Fetcher.Image(ctx, img) + if err != nil { + return nil, err + } + images[n.Image] = base + } + + key, err := EnsureKey(ctx, o.Runner, o.RunDir) + if err != nil { + return nil, err + } + nodes, err := render.Render(p, render.Options{RunDir: o.RunDir, AuthorizedKeys: []string{key}}) + if err != nil { + return nil, err + } + for i, n := range nodes { + if err := Stage(ctx, o.Runner, n, images[p.Nodes[i].Image]); err != nil { + return nil, fmt.Errorf("node %s: %w", n.Name, err) + } + } + return nodes, nil +} + +func EnsureKey(ctx context.Context, r Runner, dir string) (string, error) { + private := filepath.Join(dir, KeyFile) + if _, err := os.Stat(private); errors.Is(err, os.ErrNotExist) { + if err := r.Run(ctx, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", KeyLabel, "-f", private); err != nil { + return "", err + } + } else if err != nil { + return "", err + } + data, err := os.ReadFile(private + ".pub") + if err != nil { + return "", err + } + key := strings.TrimSpace(string(data)) + if key == "" || strings.ContainsAny(key, "\r\n") { + return "", fmt.Errorf("%s.pub: not a single public key", private) + } + return key, nil +} + +func WriteFiles(n render.Node) error { + if err := os.MkdirAll(n.Dir, 0o700); err != nil { + return err + } + files := map[string][]byte{ + ArgsFile: []byte(strings.Join(n.QEMU, "\n") + "\n"), + "meta-data": n.MetaData, + "user-data": n.UserData, + "network-config": n.NetworkConfig, + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(n.Dir, name), content, 0o600); err != nil { + return err + } + } + return nil +} + +func Stage(ctx context.Context, r Runner, n render.Node, base string) error { + if !filepath.IsAbs(base) { + return fmt.Errorf("base image %q must be an absolute path", base) + } + if err := WriteFiles(n); err != nil { + return err + } + disk := filepath.Join(n.Dir, render.DiskFile) + seed := filepath.Join(n.Dir, render.SeedFile) + if err := r.Run(ctx, "qemu-img", "create", "-q", "-f", "qcow2", "-F", "qcow2", "-b", base, disk, DiskSize); err != nil { + return err + } + args := []string{"-quiet", "-output", seed, "-volid", SeedLabel, "-joliet", "-rock"} + for _, f := range SeedFiles { + args = append(args, filepath.Join(n.Dir, f)) + } + return r.Run(ctx, "genisoimage", args...) +} + +func imageNamed(p *topology.Plan, name string) (topology.Image, bool) { + for _, i := range p.Images { + if i.Name == name { + return i, true + } + } + return topology.Image{}, false +} diff --git a/internal/lab/provision/provision_test.go b/internal/lab/provision/provision_test.go new file mode 100644 index 0000000..b05d693 --- /dev/null +++ b/internal/lab/provision/provision_test.go @@ -0,0 +1,317 @@ +package provision + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "go.yaml.in/yaml/v3" + + "git.g3e.fr/syonad/two/internal/lab/render" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const generatedKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab" + +type fakeRunner struct { + calls [][]string + fail string +} + +func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error { + f.calls = append(f.calls, append([]string{name}, args...)) + if name == f.fail { + return errors.New(name + " failed") + } + if name == "ssh-keygen" { + private := args[len(args)-1] + if err := os.WriteFile(private, []byte("private"), 0o600); err != nil { + return err + } + return os.WriteFile(private+".pub", []byte(generatedKey+"\n"), 0o644) + } + return nil +} + +func (f *fakeRunner) commands(name string) [][]string { + var out [][]string + for _, c := range f.calls { + if c[0] == name { + out = append(out, c) + } + } + return out +} + +func labPlan(t *testing.T, m *mirror) *topology.Plan { + t.Helper() + img := m.image() + doc := `name: evpn-2hv +images: + debian12: + url: ` + img.URL + ` + sums: ` + img.Sums + ` +segments: + underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 } +nodes: + sw1: { role: switch, image: debian12, cpus: 2, memory: 1024 } + rr1: { role: rr, image: debian12, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] } + hv2: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] } +` + topo, err := topology.Parse([]byte(doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + p, err := topology.Compute(topo) + if err != nil { + t.Fatalf("Compute: %v", err) + } + return p +} + +func prepare(t *testing.T, runner *fakeRunner) (string, string, *mirror, []render.Node, error) { + t.Helper() + m := newMirror(t) + publish(m, []byte("qcow2 image")) + root := t.TempDir() + run := filepath.Join(root, "run") + cache := filepath.Join(root, "cache") + nodes, err := Prepare(context.Background(), labPlan(t, m), Options{ + RunDir: run, + Fetcher: Fetcher{Client: m.server.Client(), CacheDir: cache}, + Runner: runner, + }) + return run, cache, m, nodes, err +} + +func TestPrepare_StagesEveryNode(t *testing.T) { + runner := &fakeRunner{} + run, cache, _, nodes, err := prepare(t, runner) + if err != nil { + t.Fatalf("Prepare: %v", err) + } + + base := filepath.Join(cache, "debian12", "debian-12-generic-amd64.qcow2") + want := [][]string{ + {"ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "two-lab", "-f", filepath.Join(run, "lab_ed25519")}, + } + for _, n := range []string{"sw1", "rr1", "hv1", "hv2"} { + dir := filepath.Join(run, n) + want = append(want, + []string{"qemu-img", "create", "-q", "-f", "qcow2", "-F", "qcow2", "-b", base, filepath.Join(dir, "disk.qcow2"), "20G"}, + []string{"genisoimage", "-quiet", "-output", filepath.Join(dir, "seed.iso"), "-volid", "cidata", "-joliet", "-rock", + filepath.Join(dir, "user-data"), filepath.Join(dir, "meta-data"), filepath.Join(dir, "network-config")}, + ) + } + if !reflect.DeepEqual(runner.calls, want) { + t.Errorf("commands:\n got %q\nwant %q", runner.calls, want) + } + + var names []string + for _, n := range nodes { + names = append(names, n.Name) + } + if !reflect.DeepEqual(names, []string{"sw1", "rr1", "hv1", "hv2"}) { + t.Errorf("nodes = %v", names) + } + if info, err := os.Stat(run); err != nil || info.Mode().Perm() != 0o700 { + t.Errorf("run dir mode = %v, %v", info.Mode().Perm(), err) + } +} + +func TestPrepare_DownloadsASharedImageOnce(t *testing.T) { + _, _, m, _, err := prepare(t, &fakeRunner{}) + if err != nil { + t.Fatalf("Prepare: %v", err) + } + if n := m.count("/bookworm/latest/" + imageName); n != 1 { + t.Errorf("image downloaded %d times for four nodes, want 1", n) + } + if n := m.count("/bookworm/latest/SHA512SUMS"); n != 1 { + t.Errorf("sums read %d times for four nodes, want 1", n) + } +} + +func TestPrepare_TheGeneratedKeyIsTheOnlyAuthorizedKey(t *testing.T) { + _, _, _, nodes, err := prepare(t, &fakeRunner{}) + if err != nil { + t.Fatalf("Prepare: %v", err) + } + for _, n := range nodes { + data, err := os.ReadFile(filepath.Join(n.Dir, "user-data")) + if err != nil { + t.Fatal(err) + } + var cfg struct { + Keys []string `yaml:"ssh_authorized_keys"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + t.Fatalf("%s user-data: %v", n.Name, err) + } + if !reflect.DeepEqual(cfg.Keys, []string{generatedKey}) { + t.Errorf("%s authorized keys = %q, want only the generated key", n.Name, cfg.Keys) + } + } +} + +func TestPrepare_WritesTheRenderedFilesPrivately(t *testing.T) { + run, _, _, nodes, err := prepare(t, &fakeRunner{}) + if err != nil { + t.Fatalf("Prepare: %v", err) + } + hv1 := filepath.Join(run, "hv1") + if nodes[2].Dir != hv1 { + t.Fatalf("hv1 dir = %s, want %s", nodes[2].Dir, hv1) + } + for _, f := range []string{"qemu.args", "user-data", "meta-data", "network-config"} { + info, err := os.Stat(filepath.Join(hv1, f)) + if err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("%s mode = %v, %v", f, info.Mode().Perm(), err) + } + } + args, err := os.ReadFile(filepath.Join(hv1, "qemu.args")) + if err != nil || !strings.HasPrefix(string(args), "-name\nhv1\n") || !strings.HasSuffix(string(args), "\n") { + t.Errorf("qemu.args = %q, %v", args, err) + } +} + +func TestPrepare_StopsAtTheFirstFailedCommand(t *testing.T) { + runner := &fakeRunner{fail: "qemu-img"} + _, _, _, _, err := prepare(t, runner) + if err == nil || !strings.Contains(err.Error(), "node sw1: qemu-img failed") { + t.Errorf("error = %v", err) + } + if n := len(runner.commands("genisoimage")); n != 0 { + t.Errorf("genisoimage ran %d times after qemu-img failed", n) + } +} + +func TestPrepare_StopsBeforeAnyCommandWhenTheImageIsWrong(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + m.put("/bookworm/latest/"+imageName, []byte("tampered")) + runner := &fakeRunner{} + root := t.TempDir() + _, err := Prepare(context.Background(), labPlan(t, m), Options{ + RunDir: filepath.Join(root, "run"), + Fetcher: Fetcher{Client: m.server.Client(), CacheDir: filepath.Join(root, "cache")}, + Runner: runner, + }) + if err == nil || !strings.Contains(err.Error(), "sha512") { + t.Errorf("error = %v", err) + } + if len(runner.calls) != 0 { + t.Errorf("commands ran after a bad image: %q", runner.calls) + } +} + +func TestPrepare_RefusesARelativeRunDir(t *testing.T) { + m := newMirror(t) + publish(m, []byte("qcow2 image")) + runner := &fakeRunner{} + _, err := Prepare(context.Background(), labPlan(t, m), Options{ + RunDir: "run", + Fetcher: Fetcher{Client: m.server.Client(), CacheDir: t.TempDir()}, + Runner: runner, + }) + if err == nil || !strings.Contains(err.Error(), `run dir "run" must be an absolute path`) { + t.Errorf("error = %v", err) + } + if n := m.count("/bookworm/latest/SHA512SUMS"); n != 0 { + t.Errorf("sums read %d times before the run dir was checked", n) + } +} + +func TestPrepare_RefusesAnUndeclaredImage(t *testing.T) { + m := newMirror(t) + p := labPlan(t, m) + p.Images = nil + _, err := Prepare(context.Background(), p, Options{RunDir: t.TempDir(), Runner: &fakeRunner{}}) + if err == nil || !strings.Contains(err.Error(), `node sw1: image "debian12" is not declared`) { + t.Errorf("error = %v", err) + } +} + +func TestEnsureKey_GeneratesOnlyOnce(t *testing.T) { + dir := t.TempDir() + runner := &fakeRunner{} + + for i := 0; i < 2; i++ { + key, err := EnsureKey(context.Background(), runner, dir) + if err != nil || key != generatedKey { + t.Fatalf("EnsureKey #%d = %q, %v", i+1, key, err) + } + } + if n := len(runner.commands("ssh-keygen")); n != 1 { + t.Errorf("ssh-keygen ran %d times, want 1", n) + } +} + +func TestEnsureKey_Rejections(t *testing.T) { + cases := map[string]string{ + "empty": "\n", + "two keys": generatedKey + "\n" + generatedKey + "\n", + "stray break": "ssh-ed25519\rAAAA", + } + for name, pub := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "lab_ed25519"), []byte("private"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "lab_ed25519.pub"), []byte(pub), 0o644); err != nil { + t.Fatal(err) + } + if _, err := EnsureKey(context.Background(), &fakeRunner{}, dir); err == nil || !strings.Contains(err.Error(), "not a single public key") { + t.Errorf("error = %v", err) + } + }) + } +} + +func TestEnsureKey_ReportsAFailedGeneration(t *testing.T) { + if _, err := EnsureKey(context.Background(), &fakeRunner{fail: "ssh-keygen"}, t.TempDir()); err == nil || !strings.Contains(err.Error(), "ssh-keygen failed") { + t.Errorf("error = %v", err) + } +} + +func TestStage_RefusesARelativeBaseImage(t *testing.T) { + runner := &fakeRunner{} + err := Stage(context.Background(), runner, render.Node{Name: "hv1", Dir: t.TempDir()}, "debian.qcow2") + if err == nil || !strings.Contains(err.Error(), "must be an absolute path") { + t.Errorf("error = %v", err) + } + if len(runner.calls) != 0 { + t.Errorf("commands ran: %q", runner.calls) + } +} + +func TestExecRunner_ReportsTheCommandOutput(t *testing.T) { + err := ExecRunner{}.Run(context.Background(), "sh", "-c", "echo boom >&2; exit 3") + if err == nil || !strings.Contains(err.Error(), "sh: exit status 3: boom") { + t.Errorf("error = %v", err) + } + if err := (ExecRunner{}).Run(context.Background(), "true"); err != nil { + t.Errorf("true: %v", err) + } +} + +func TestEnsureKey_WithTheRealSSHKeygen(t *testing.T) { + dir := t.TempDir() + key, err := EnsureKey(context.Background(), ExecRunner{}, dir) + if err != nil { + t.Fatalf("EnsureKey: %v", err) + } + if !strings.HasPrefix(key, "ssh-ed25519 ") || !strings.HasSuffix(key, " two-lab") { + t.Errorf("key = %q", key) + } + info, err := os.Stat(filepath.Join(dir, "lab_ed25519")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("private key mode = %v, %v", info.Mode().Perm(), err) + } +} diff --git a/internal/lab/render/qemu.go b/internal/lab/render/qemu.go new file mode 100644 index 0000000..9a870d2 --- /dev/null +++ b/internal/lab/render/qemu.go @@ -0,0 +1,54 @@ +package render + +import ( + "fmt" + "path/filepath" + "strconv" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const loopback = "127.0.0.1" + +func qemuArgs(p *topology.Plan, n topology.NodePlan, index int, dir string) []string { + args := []string{ + "-name", n.Name, + "-machine", "q35", + "-accel", "kvm", + "-cpu", "host", + "-smp", strconv.Itoa(n.CPUs), + "-m", strconv.Itoa(n.Memory), + "-nodefaults", + "-display", "none", + "-serial", "file:" + filepath.Join(dir, ConsoleFile), + "-qmp", "unix:" + filepath.Join(dir, QMPFile) + ",server=on,wait=off", + "-pidfile", filepath.Join(dir, PIDFile), + "-drive", "file=" + filepath.Join(dir, DiskFile) + ",if=virtio,format=qcow2", + "-drive", "file=" + filepath.Join(dir, SeedFile) + ",media=cdrom,readonly=on", + } + + restrict := "on" + if n.Role == topology.RoleSwitch { + restrict = "off" + } + args = append(args, + "-netdev", fmt.Sprintf("user,id=%s,restrict=%s,ipv6=off,hostfwd=tcp:%s:%d-:22", AdminInterface, restrict, loopback, n.SSHPort), + "-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,romfile=", AdminInterface, adminMAC(index)), + ) + + for _, c := range nodeCables(p, n.Name) { + args = append(args, cable(c.NodeInterface, c.NodePort, c.SwitchPort, c.NodeMAC.String(), c.MTU)...) + } + for _, c := range switchCables(p, n.Name) { + args = append(args, cable(c.SwitchInterface, c.SwitchPort, c.NodePort, c.SwitchMAC.String(), c.MTU)...) + } + return args +} + +func cable(id string, local, remote int, mac string, mtu int) []string { + return []string{ + "-netdev", fmt.Sprintf("dgram,id=%s,local.type=inet,local.host=%s,local.port=%d,remote.type=inet,remote.host=%s,remote.port=%d", + id, loopback, local, loopback, remote), + "-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,host_mtu=%d,romfile=", id, mac, mtu), + } +} diff --git a/internal/lab/render/render.go b/internal/lab/render/render.go new file mode 100644 index 0000000..49b3089 --- /dev/null +++ b/internal/lab/render/render.go @@ -0,0 +1,132 @@ +package render + +import ( + "errors" + "fmt" + "net" + "path/filepath" + "strings" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const ( + QEMUBinary = "qemu-system-x86_64" + + AdminInterface = topology.ReservedInterface + AdminAddress = "10.0.2.15/24" + AdminGateway = "10.0.2.2" + AdminDNS = "10.0.2.3" + + DiskFile = "disk.qcow2" + SeedFile = "seed.iso" + ConsoleFile = "console.log" + QMPFile = "qmp.sock" + PIDFile = "qemu.pid" +) + +var Nameservers = []string{"1.1.1.1", "8.8.8.8"} + +type Options struct { + RunDir string + AuthorizedKeys []string +} + +type Node struct { + Name string + Dir string + QEMU []string + MetaData []byte + UserData []byte + NetworkConfig []byte +} + +func Render(p *topology.Plan, o Options) ([]Node, error) { + if err := o.validate(); err != nil { + return nil, err + } + var nodes []Node + for index, n := range p.Nodes { + dir := filepath.Join(o.RunDir, n.Name) + meta, err := metaData(p, n) + if err != nil { + return nil, err + } + user, err := userData(p, n, o.AuthorizedKeys) + if err != nil { + return nil, err + } + network, err := networkConfig(p, n, index) + if err != nil { + return nil, err + } + nodes = append(nodes, Node{ + Name: n.Name, + Dir: dir, + QEMU: qemuArgs(p, n, index, dir), + MetaData: meta, + UserData: user, + NetworkConfig: network, + }) + } + return nodes, nil +} + +func (o Options) validate() error { + var errs []error + if !filepath.IsAbs(o.RunDir) { + errs = append(errs, fmt.Errorf("run dir %q must be an absolute path", o.RunDir)) + } + if len(o.AuthorizedKeys) == 0 { + errs = append(errs, errors.New("at least one authorized ssh key is required")) + } + for i, k := range o.AuthorizedKeys { + if strings.ContainsAny(k, "\r\n") { + errs = append(errs, fmt.Errorf("authorized key %d spans several lines", i+1)) + continue + } + fields := strings.Fields(k) + if len(fields) < 2 || !validKeyType(fields[0]) { + errs = append(errs, fmt.Errorf("authorized key %d is not an ssh public key", i+1)) + } + } + return errors.Join(errs...) +} + +func validKeyType(t string) bool { + return strings.HasPrefix(t, "ssh-") || strings.HasPrefix(t, "ecdsa-sha2-") || strings.HasPrefix(t, "sk-") +} + +func adminMAC(index int) net.HardwareAddr { + return net.HardwareAddr{0x02, 0x4d, byte(index >> 8), byte(index), 0x00, 0x00} +} + +func nodeCables(p *topology.Plan, name string) []topology.Cable { + var out []topology.Cable + for _, c := range p.Cables { + if c.Node == name { + out = append(out, c) + } + } + return out +} + +func switchCables(p *topology.Plan, name string) []topology.Cable { + var out []topology.Cable + for _, c := range p.Cables { + if c.Switch == name { + out = append(out, c) + } + } + return out +} + +func switchSegments(p *topology.Plan, name string) []topology.SegmentPlan { + var out []topology.SegmentPlan + for _, s := range p.Segments { + if s.Switch == name { + out = append(out, s) + } + } + return out +} diff --git a/internal/lab/render/render_test.go b/internal/lab/render/render_test.go new file mode 100644 index 0000000..f270d13 --- /dev/null +++ b/internal/lab/render/render_test.go @@ -0,0 +1,477 @@ +package render + +import ( + "reflect" + "strings" + "testing" + + "go.yaml.in/yaml/v3" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const labKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab-automation" + +const twoHypervisors = `name: evpn-2hv +images: + deb: + url: https://example.invalid/deb.qcow2 + sums: https://example.invalid/SHA512SUMS +segments: + underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 } +nodes: + sw1: { role: switch, image: deb, cpus: 2, memory: 1024 } + rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } + hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } +` + +const twoSegments = `name: two-seg +images: + deb: + url: https://example.invalid/deb.qcow2 + sums: https://example.invalid/SHA512SUMS +segments: + red: { switch: sw, cidr: 10.1.0.0/24 } + blue: { switch: sw, cidr: 10.2.0.0/24, mtu: 1500 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red] } +` + +func plan(t *testing.T, doc string) *topology.Plan { + t.Helper() + topo, err := topology.Parse([]byte(doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + p, err := topology.Compute(topo) + if err != nil { + t.Fatalf("Compute: %v", err) + } + return p +} + +func renderAll(t *testing.T, doc string) map[string]Node { + t.Helper() + nodes, err := Render(plan(t, doc), Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey}}) + if err != nil { + t.Fatalf("Render: %v", err) + } + out := map[string]Node{} + for _, n := range nodes { + out[n.Name] = n + } + return out +} + +func nodeNamed(t *testing.T, nodes map[string]Node, name string) Node { + t.Helper() + n, ok := nodes[name] + if !ok { + t.Fatalf("no rendered node %s", name) + } + return n +} + +func value(t *testing.T, args []string, flag string) []string { + t.Helper() + var out []string + for i := 0; i+1 < len(args); i++ { + if args[i] == flag { + out = append(out, args[i+1]) + } + } + if len(out) == 0 { + t.Fatalf("no %s in %v", flag, args) + } + return out +} + +func hasValue(args []string, flag, want string) bool { + for i := 0; i+1 < len(args); i++ { + if args[i] == flag && args[i+1] == want { + return true + } + } + return false +} + +func network(t *testing.T, n Node) networkDoc { + t.Helper() + var doc networkDoc + if err := yaml.Unmarshal(n.NetworkConfig, &doc); err != nil { + t.Fatalf("network-config of %s is not yaml: %v", n.Name, err) + } + return doc +} + +func iface(t *testing.T, doc networkDoc, name string) ethernet { + t.Helper() + e, ok := doc.Ethernets[name] + if !ok { + t.Fatalf("no interface %s in %v", name, doc.Ethernets) + } + return e +} + +func user(t *testing.T, n Node) cloudConfig { + t.Helper() + if !strings.HasPrefix(string(n.UserData), "#cloud-config\n") { + t.Fatalf("user-data of %s does not start with #cloud-config", n.Name) + } + var cfg cloudConfig + if err := yaml.Unmarshal(n.UserData, &cfg); err != nil { + t.Fatalf("user-data of %s is not yaml: %v", n.Name, err) + } + return cfg +} + +func fileAt(t *testing.T, cfg cloudConfig, path string) writeFile { + t.Helper() + for _, f := range cfg.WriteFiles { + if f.Path == path { + return f + } + } + t.Fatalf("no write_files entry %s", path) + return writeFile{} +} + +func TestQEMU_HypervisorArguments(t *testing.T) { + hv := nodeNamed(t, renderAll(t, twoHypervisors), "hv1") + for flag, want := range map[string]string{ + "-name": "hv1", + "-machine": "q35", + "-accel": "kvm", + "-cpu": "host", + "-smp": "4", + "-m": "16384", + "-display": "none", + "-serial": "file:/srv/lab/hv1/console.log", + "-qmp": "unix:/srv/lab/hv1/qmp.sock,server=on,wait=off", + "-pidfile": "/srv/lab/hv1/qemu.pid", + } { + if !hasValue(hv.QEMU, flag, want) { + t.Errorf("%s %s missing from %v", flag, want, hv.QEMU) + } + } + for _, want := range []string{ + "file=/srv/lab/hv1/disk.qcow2,if=virtio,format=qcow2", + "file=/srv/lab/hv1/seed.iso,media=cdrom,readonly=on", + } { + if !hasValue(hv.QEMU, "-drive", want) { + t.Errorf("-drive %s missing", want) + } + } + if hv.Dir != "/srv/lab/hv1" { + t.Errorf("Dir = %s", hv.Dir) + } + nodefaults := 0 + for _, a := range hv.QEMU { + if a == "-nodefaults" { + nodefaults++ + } + } + if nodefaults != 1 { + t.Errorf("-nodefaults appears %d times", nodefaults) + } +} + +func TestQEMU_HypervisorNetwork(t *testing.T) { + hv := nodeNamed(t, renderAll(t, twoHypervisors), "hv1") + wantNetdevs := []string{ + "user,id=mgmt0,restrict=on,ipv6=off,hostfwd=tcp:127.0.0.1:2202-:22", + "dgram,id=underlay,local.type=inet,local.host=127.0.0.1,local.port=20002,remote.type=inet,remote.host=127.0.0.1,remote.port=20003", + } + wantDevices := []string{ + "virtio-net-pci,netdev=mgmt0,mac=02:4d:00:02:00:00,romfile=", + "virtio-net-pci,netdev=underlay,mac=02:4c:00:02:00:00,host_mtu=9000,romfile=", + } + if got := value(t, hv.QEMU, "-netdev"); !reflect.DeepEqual(got, wantNetdevs) { + t.Errorf("netdevs = %v\nwant %v", got, wantNetdevs) + } + if got := value(t, hv.QEMU, "-device"); !reflect.DeepEqual(got, wantDevices) { + t.Errorf("devices = %v\nwant %v", got, wantDevices) + } +} + +func TestQEMU_SwitchNetwork(t *testing.T) { + sw := nodeNamed(t, renderAll(t, twoHypervisors), "sw1") + wantNetdevs := []string{ + "user,id=mgmt0,restrict=off,ipv6=off,hostfwd=tcp:127.0.0.1:2200-:22", + "dgram,id=p0,local.type=inet,local.host=127.0.0.1,local.port=20001,remote.type=inet,remote.host=127.0.0.1,remote.port=20000", + "dgram,id=p1,local.type=inet,local.host=127.0.0.1,local.port=20003,remote.type=inet,remote.host=127.0.0.1,remote.port=20002", + "dgram,id=p2,local.type=inet,local.host=127.0.0.1,local.port=20005,remote.type=inet,remote.host=127.0.0.1,remote.port=20004", + } + if got := value(t, sw.QEMU, "-netdev"); !reflect.DeepEqual(got, wantNetdevs) { + t.Errorf("netdevs = %v\nwant %v", got, wantNetdevs) + } + if !hasValue(sw.QEMU, "-device", "virtio-net-pci,netdev=p2,mac=02:4c:00:03:00:01,host_mtu=9000,romfile=") { + t.Errorf("p2 device missing: %v", sw.QEMU) + } +} + +func TestQEMU_OnlyTheSwitchReachesTheOutsideThroughAdministration(t *testing.T) { + for name, n := range renderAll(t, twoHypervisors) { + admin := value(t, n.QEMU, "-netdev")[0] + wantRestrict := "restrict=on" + if name == "sw1" { + wantRestrict = "restrict=off" + } + if !strings.Contains(admin, ","+wantRestrict+",") { + t.Errorf("%s admin netdev %q, want %s", name, admin, wantRestrict) + } + if !strings.Contains(admin, "hostfwd=tcp:127.0.0.1:") { + t.Errorf("%s ssh forward not bound to loopback: %q", name, admin) + } + } +} + +func TestQEMU_EveryCableEndsMatch(t *testing.T) { + ends := map[string]int{} + for _, n := range renderAll(t, twoHypervisors) { + for _, nd := range value(t, n.QEMU, "-netdev") { + if !strings.HasPrefix(nd, "dgram,") { + continue + } + var local, remote string + for _, kv := range strings.Split(nd, ",") { + if v, ok := strings.CutPrefix(kv, "local.port="); ok { + local = v + } + if v, ok := strings.CutPrefix(kv, "remote.port="); ok { + remote = v + } + } + ends[local+">"+remote]++ + } + } + for pair, count := range ends { + local, remote, _ := strings.Cut(pair, ">") + if count != 1 || ends[remote+">"+local] != 1 { + t.Errorf("cable end %s has no single matching end", pair) + } + } + if len(ends) != 6 { + t.Errorf("%d cable ends, want 6", len(ends)) + } +} + +func TestNetworkConfig_Hypervisor(t *testing.T) { + doc := network(t, nodeNamed(t, renderAll(t, twoHypervisors), "hv1")) + if doc.Version != 2 || len(doc.Ethernets) != 2 { + t.Fatalf("network-config = %+v", doc) + } + admin := iface(t, doc, "mgmt0") + if admin.Match.MACAddress != "02:4d:00:02:00:00" || admin.SetName != "mgmt0" || + !reflect.DeepEqual(admin.Addresses, []string{"10.0.2.15/24"}) || len(admin.Routes) != 0 || admin.Nameservers != nil { + t.Errorf("mgmt0 = %+v", admin) + } + under := iface(t, doc, "underlay") + if under.Match.MACAddress != "02:4c:00:02:00:00" || under.SetName != "underlay" || under.MTU != 9000 || + !reflect.DeepEqual(under.Addresses, []string{"10.250.0.3/24"}) { + t.Errorf("underlay = %+v", under) + } + if !reflect.DeepEqual(under.Routes, []route{{To: "0.0.0.0/0", Via: "10.250.0.1"}}) { + t.Errorf("underlay routes = %+v", under.Routes) + } + if under.Nameservers == nil || !reflect.DeepEqual(under.Nameservers.Addresses, []string{"1.1.1.1", "8.8.8.8"}) { + t.Errorf("underlay nameservers = %+v", under.Nameservers) + } +} + +func TestNetworkConfig_DHCPIsExplicitlyOffEverywhere(t *testing.T) { + for name, n := range renderAll(t, twoHypervisors) { + if c := strings.Count(string(n.NetworkConfig), "dhcp4: false"); c != len(network(t, n).Ethernets) { + t.Errorf("%s: %d explicit dhcp4: false for %d interfaces", name, c, len(network(t, n).Ethernets)) + } + } +} + +func TestNetworkConfig_Switch(t *testing.T) { + doc := network(t, nodeNamed(t, renderAll(t, twoHypervisors), "sw1")) + admin := iface(t, doc, "mgmt0") + if !reflect.DeepEqual(admin.Routes, []route{{To: "0.0.0.0/0", Via: "10.0.2.2"}}) || + admin.Nameservers == nil || !reflect.DeepEqual(admin.Nameservers.Addresses, []string{"10.0.2.3"}) { + t.Errorf("switch mgmt0 = %+v", admin) + } + for port, mac := range map[string]string{"p0": "02:4c:00:01:00:01", "p1": "02:4c:00:02:00:01", "p2": "02:4c:00:03:00:01"} { + e := iface(t, doc, port) + if e.Match.MACAddress != mac || e.SetName != port || e.MTU != 9000 || len(e.Addresses) != 0 || len(e.Routes) != 0 { + t.Errorf("%s = %+v", port, e) + } + } +} + +func TestNetworkConfig_DefaultRouteOnlyOnFirstSegment(t *testing.T) { + doc := network(t, nodeNamed(t, renderAll(t, twoSegments), "hv")) + red, blue := iface(t, doc, "red"), iface(t, doc, "blue") + if !reflect.DeepEqual(red.Routes, []route{{To: "0.0.0.0/0", Via: "10.1.0.1"}}) || red.Nameservers == nil { + t.Errorf("red = %+v", red) + } + if len(blue.Routes) != 0 || blue.Nameservers != nil || blue.MTU != 1500 { + t.Errorf("blue = %+v", blue) + } +} + +func TestUserData_Hypervisor(t *testing.T) { + cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "hv1")) + if cfg.Hostname != "hv1" || !cfg.DisableRoot || len(cfg.WriteFiles) != 0 || len(cfg.Runcmd) != 0 || len(cfg.Packages) != 0 { + t.Errorf("hv1 user-data = %+v", cfg) + } + if !reflect.DeepEqual(cfg.SSHAuthorizedKeys, []string{labKey}) { + t.Errorf("keys = %v", cfg.SSHAuthorizedKeys) + } +} + +func TestUserData_PasswordLoginIsExplicitlyOff(t *testing.T) { + for name, n := range renderAll(t, twoHypervisors) { + if !strings.Contains(string(n.UserData), "\nssh_pwauth: false\n") || !strings.Contains(string(n.UserData), "\ndisable_root: true\n") { + t.Errorf("%s user-data:\n%s", name, n.UserData) + } + } +} + +func TestUserData_SwitchBuildsBridgeGatewayAndNAT(t *testing.T) { + cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "sw1")) + if !reflect.DeepEqual(cfg.Packages, []string{"nftables"}) { + t.Errorf("packages = %v", cfg.Packages) + } + script := fileAt(t, cfg, "/usr/local/sbin/lab-switch") + if script.Permissions != "0755" { + t.Errorf("script permissions = %s", script.Permissions) + } + wantScript := `#!/bin/sh +set -eu +sysctl -qw net.ipv4.ip_forward=1 +ip link add br-underlay type bridge stp_state 0 2>/dev/null || true +ip link set dev p0 master br-underlay +ip link set dev p0 up +ip link set dev p1 master br-underlay +ip link set dev p1 up +ip link set dev p2 master br-underlay +ip link set dev p2 up +ip link set dev br-underlay mtu 9000 +ip addr replace 10.250.0.1/24 dev br-underlay +ip link set dev br-underlay up +nft -f /etc/lab-switch.nft +` + if script.Content != wantScript { + t.Errorf("script:\n%s\nwant:\n%s", script.Content, wantScript) + } + nft := fileAt(t, cfg, "/etc/lab-switch.nft").Content + for _, want := range []string{ + "add table ip lab_nat\ndelete table ip lab_nat\n", + "type nat hook postrouting priority srcnat;", + `ip saddr { 10.250.0.0/24 } oifname "mgmt0" masquerade`, + } { + if !strings.Contains(nft, want) { + t.Errorf("nft rules miss %q:\n%s", want, nft) + } + } + unit := fileAt(t, cfg, "/etc/systemd/system/lab-switch.service").Content + for _, want := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=/usr/local/sbin/lab-switch", "After=network-online.target", "WantedBy=multi-user.target"} { + if !strings.Contains(unit, want) { + t.Errorf("unit misses %q", want) + } + } + if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"systemctl", "daemon-reload"}, {"systemctl", "enable", "--now", "lab-switch.service"}}) { + t.Errorf("runcmd = %v", cfg.Runcmd) + } +} + +func TestUserData_SwitchWithTwoSegments(t *testing.T) { + cfg := user(t, nodeNamed(t, renderAll(t, twoSegments), "sw")) + script := fileAt(t, cfg, "/usr/local/sbin/lab-switch").Content + for _, want := range []string{ + "ip link set dev p0 master br-red\n", + "ip link set dev p1 master br-blue\n", + "ip link set dev br-blue mtu 1500\n", + "ip addr replace 10.2.0.1/24 dev br-blue\n", + } { + if !strings.Contains(script, want) { + t.Errorf("script misses %q:\n%s", want, script) + } + } + if strings.Contains(script, "p1 master br-red") || strings.Contains(script, "p0 master br-blue") { + t.Errorf("port bridged on the wrong segment:\n%s", script) + } + if nft := fileAt(t, cfg, "/etc/lab-switch.nft").Content; !strings.Contains(nft, "ip saddr { 10.1.0.0/24, 10.2.0.0/24 }") { + t.Errorf("nft:\n%s", nft) + } +} + +func TestMetaData(t *testing.T) { + var doc metaDoc + if err := yaml.Unmarshal(nodeNamed(t, renderAll(t, twoHypervisors), "hv2").MetaData, &doc); err != nil { + t.Fatal(err) + } + if doc.InstanceID != "evpn-2hv-hv2" || doc.LocalHostname != "hv2" { + t.Errorf("meta-data = %+v", doc) + } +} + +func TestRender_AdminMacsAreUniqueAndApartFromCableMacs(t *testing.T) { + seen := map[string]string{} + for name, n := range renderAll(t, twoHypervisors) { + for _, d := range value(t, n.QEMU, "-device") { + for _, kv := range strings.Split(d, ",") { + if mac, ok := strings.CutPrefix(kv, "mac="); ok { + if other, dup := seen[mac]; dup { + t.Errorf("mac %s used by %s and %s", mac, other, name) + } + seen[mac] = name + } + } + } + } + if len(seen) != 10 { + t.Errorf("%d distinct macs, want 10", len(seen)) + } +} + +func TestRender_IsStable(t *testing.T) { + first := renderAll(t, twoHypervisors) + for i := 0; i < 20; i++ { + if again := renderAll(t, twoHypervisors); !reflect.DeepEqual(first, again) { + t.Fatalf("render differs on run %d", i) + } + } +} + +func TestRender_RejectsBadOptions(t *testing.T) { + p := plan(t, twoHypervisors) + cases := []struct { + name string + opts Options + want string + }{ + {"relative run dir", Options{RunDir: "lab", AuthorizedKeys: []string{labKey}}, `run dir "lab" must be an absolute path`}, + {"no key", Options{RunDir: "/srv/lab"}, "at least one authorized ssh key"}, + {"multi-line key", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{labKey + "\nssh-rsa AAAA x"}}, "authorized key 1 spans several lines"}, + {"not a key", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{"hello world"}}, "authorized key 1 is not an ssh public key"}, + {"type only", Options{RunDir: "/srv/lab", AuthorizedKeys: []string{"ssh-ed25519"}}, "authorized key 1 is not an ssh public key"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := Render(p, c.opts) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Fatalf("err = %v, want %q", err, c.want) + } + }) + } +} + +func TestRender_AcceptsUsualKeyTypes(t *testing.T) { + p := plan(t, twoHypervisors) + for _, k := range []string{ + "ssh-rsa AAAAB3NzaC1yc2E user", + "ecdsa-sha2-nistp256 AAAAE2VjZHNh user", + "sk-ssh-ed25519@openssh.com AAAAGnNr user", + } { + if _, err := Render(p, Options{RunDir: "/srv/lab", AuthorizedKeys: []string{k}}); err != nil { + t.Errorf("%s rejected: %v", strings.Fields(k)[0], err) + } + } +} diff --git a/internal/lab/render/seed.go b/internal/lab/render/seed.go new file mode 100644 index 0000000..c959d3c --- /dev/null +++ b/internal/lab/render/seed.go @@ -0,0 +1,192 @@ +package render + +import ( + "fmt" + "strings" + + "go.yaml.in/yaml/v3" + + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const ( + SwitchScript = "/usr/local/sbin/lab-switch" + SwitchNFT = "/etc/lab-switch.nft" + SwitchUnit = "/etc/systemd/system/lab-switch.service" +) + +type metaDoc struct { + InstanceID string `yaml:"instance-id"` + LocalHostname string `yaml:"local-hostname"` +} + +type writeFile struct { + Path string `yaml:"path"` + Permissions string `yaml:"permissions"` + Content string `yaml:"content"` +} + +type cloudConfig struct { + Hostname string `yaml:"hostname"` + SSHPwauth bool `yaml:"ssh_pwauth"` + DisableRoot bool `yaml:"disable_root"` + SSHAuthorizedKeys []string `yaml:"ssh_authorized_keys"` + Packages []string `yaml:"packages,omitempty"` + WriteFiles []writeFile `yaml:"write_files,omitempty"` + Runcmd [][]string `yaml:"runcmd,omitempty"` +} + +type match struct { + MACAddress string `yaml:"macaddress"` +} + +type route struct { + To string `yaml:"to"` + Via string `yaml:"via"` +} + +type nameservers struct { + Addresses []string `yaml:"addresses"` +} + +type ethernet struct { + Match match `yaml:"match"` + SetName string `yaml:"set-name"` + DHCP4 bool `yaml:"dhcp4"` + MTU int `yaml:"mtu,omitempty"` + Addresses []string `yaml:"addresses,omitempty"` + Routes []route `yaml:"routes,omitempty"` + Nameservers *nameservers `yaml:"nameservers,omitempty"` +} + +type networkDoc struct { + Version int `yaml:"version"` + Ethernets map[string]ethernet `yaml:"ethernets"` +} + +func metaData(p *topology.Plan, n topology.NodePlan) ([]byte, error) { + return yaml.Marshal(metaDoc{InstanceID: p.Name + "-" + n.Name, LocalHostname: n.Name}) +} + +func userData(p *topology.Plan, n topology.NodePlan, keys []string) ([]byte, error) { + cfg := cloudConfig{ + Hostname: n.Name, + SSHPwauth: false, + DisableRoot: true, + SSHAuthorizedKeys: keys, + } + if n.Role == topology.RoleSwitch { + cfg.Packages = []string{"nftables"} + cfg.WriteFiles = []writeFile{ + {Path: SwitchScript, Permissions: "0755", Content: switchScript(p, n.Name)}, + {Path: SwitchNFT, Permissions: "0644", Content: switchNFT(p, n.Name)}, + {Path: SwitchUnit, Permissions: "0644", Content: switchUnit()}, + } + cfg.Runcmd = [][]string{ + {"systemctl", "daemon-reload"}, + {"systemctl", "enable", "--now", "lab-switch.service"}, + } + } + body, err := yaml.Marshal(cfg) + if err != nil { + return nil, err + } + return append([]byte("#cloud-config\n"), body...), nil +} + +func networkConfig(p *topology.Plan, n topology.NodePlan, index int) ([]byte, error) { + doc := networkDoc{Version: 2, Ethernets: map[string]ethernet{}} + admin := ethernet{ + Match: match{MACAddress: adminMAC(index).String()}, + SetName: AdminInterface, + Addresses: []string{AdminAddress}, + } + if n.Role == topology.RoleSwitch { + admin.Routes = []route{{To: "0.0.0.0/0", Via: AdminGateway}} + admin.Nameservers = &nameservers{Addresses: []string{AdminDNS}} + for _, c := range switchCables(p, n.Name) { + doc.Ethernets[c.SwitchInterface] = ethernet{ + Match: match{MACAddress: c.SwitchMAC.String()}, + SetName: c.SwitchInterface, + MTU: c.MTU, + } + } + } + doc.Ethernets[AdminInterface] = admin + + for i, c := range nodeCables(p, n.Name) { + e := ethernet{ + Match: match{MACAddress: c.NodeMAC.String()}, + SetName: c.NodeInterface, + MTU: c.MTU, + Addresses: []string{c.NodeAddress.String()}, + } + if i == 0 { + e.Routes = []route{{To: "0.0.0.0/0", Via: gatewayOf(p, c.Segment)}} + e.Nameservers = &nameservers{Addresses: Nameservers} + } + doc.Ethernets[c.NodeInterface] = e + } + return yaml.Marshal(doc) +} + +func gatewayOf(p *topology.Plan, segment string) string { + for _, s := range p.Segments { + if s.Name == segment { + return s.Gateway.String() + } + } + return "" +} + +func switchScript(p *topology.Plan, name string) string { + var b strings.Builder + b.WriteString("#!/bin/sh\nset -eu\nsysctl -qw net.ipv4.ip_forward=1\n") + for _, s := range switchSegments(p, name) { + fmt.Fprintf(&b, "ip link add %s type bridge stp_state 0 2>/dev/null || true\n", s.Bridge) + for _, c := range switchCables(p, name) { + if c.Segment != s.Name { + continue + } + fmt.Fprintf(&b, "ip link set dev %s master %s\n", c.SwitchInterface, s.Bridge) + fmt.Fprintf(&b, "ip link set dev %s up\n", c.SwitchInterface) + } + fmt.Fprintf(&b, "ip link set dev %s mtu %d\n", s.Bridge, s.MTU) + fmt.Fprintf(&b, "ip addr replace %s/%d dev %s\n", s.Gateway, s.Network.Bits(), s.Bridge) + fmt.Fprintf(&b, "ip link set dev %s up\n", s.Bridge) + } + fmt.Fprintf(&b, "nft -f %s\n", SwitchNFT) + return b.String() +} + +func switchNFT(p *topology.Plan, name string) string { + var networks []string + for _, s := range switchSegments(p, name) { + networks = append(networks, s.Network.String()) + } + return fmt.Sprintf(`add table ip lab_nat +delete table ip lab_nat +table ip lab_nat { + chain postrouting { + type nat hook postrouting priority srcnat; policy accept; + ip saddr { %s } oifname "%s" masquerade + } +} +`, strings.Join(networks, ", "), AdminInterface) +} + +func switchUnit() string { + return fmt.Sprintf(`[Unit] +Description=Lab switch: bridges, gateways and NAT +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=%s + +[Install] +WantedBy=multi-user.target +`, SwitchScript) +} diff --git a/internal/lab/topology/format.go b/internal/lab/topology/format.go new file mode 100644 index 0000000..e501bff --- /dev/null +++ b/internal/lab/topology/format.go @@ -0,0 +1,34 @@ +package topology + +import ( + "fmt" + "io" + "text/tabwriter" +) + +func (p *Plan) Write(w io.Writer) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + + fmt.Fprintf(tw, "lab %s: nodes %d, segments %d, cables %d\n", p.Name, len(p.Nodes), len(p.Segments), len(p.Cables)) + + fmt.Fprintf(tw, "\nnodes\n") + fmt.Fprintf(tw, " name\trole\timage\tcpus\tmemory\tssh\n") + for _, n := range p.Nodes { + fmt.Fprintf(tw, " %s\t%s\t%s\t%d\t%d MiB\t127.0.0.1:%d\n", n.Name, n.Role, n.Image, n.CPUs, n.Memory, n.SSHPort) + } + + for _, s := range p.Segments { + fmt.Fprintf(tw, "\nsegment %s: %s, mtu %d, switch %s, bridge %s, gateway %s\n", + s.Name, s.Network, s.MTU, s.Switch, s.Bridge, s.Gateway) + fmt.Fprintf(tw, " node\tinterface\taddress\tmac\tudp\t\tswitch port\tmac\tudp\n") + for _, c := range p.Cables { + if c.Segment != s.Name { + continue + } + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%d\t<->\t%s %s\t%s\t%d\n", + c.Node, c.NodeInterface, c.NodeAddress, c.NodeMAC, c.NodePort, + c.Switch, c.SwitchInterface, c.SwitchMAC, c.SwitchPort) + } + } + return tw.Flush() +} diff --git a/internal/lab/topology/plan.go b/internal/lab/topology/plan.go new file mode 100644 index 0000000..0fd0ee6 --- /dev/null +++ b/internal/lab/topology/plan.go @@ -0,0 +1,196 @@ +package topology + +import ( + "errors" + "fmt" + "net" + "net/netip" + "strconv" +) + +const ( + SSHBasePort = 2200 + CableBasePort = 20000 + + maxNodes = 1000 + maxSegments = 256 +) + +type Plan struct { + Name string + Images []Image + Segments []SegmentPlan + Nodes []NodePlan + Cables []Cable +} + +type SegmentPlan struct { + Name string + Switch string + Bridge string + Network netip.Prefix + Gateway netip.Addr + MTU int +} + +type NodePlan struct { + Name string + Role string + Image string + CPUs int + Memory int + SSHPort int +} + +type Cable struct { + Segment string + MTU int + + Node string + NodeInterface string + NodeMAC net.HardwareAddr + NodeAddress netip.Prefix + NodePort int + + Switch string + SwitchInterface string + SwitchMAC net.HardwareAddr + SwitchPort int +} + +func Compute(t *Topology) (*Plan, error) { + if err := t.Validate(); err != nil { + return nil, err + } + + p := &Plan{Name: t.Name, Images: append([]Image(nil), t.Images...)} + for i, n := range t.Nodes { + p.Nodes = append(p.Nodes, NodePlan{ + Name: n.Name, + Role: n.Role, + Image: n.Image, + CPUs: n.CPUs, + Memory: n.Memory, + SSHPort: SSHBasePort + i, + }) + } + + var errs []error + for segIndex, s := range t.Segments { + network := netip.MustParsePrefix(s.CIDR) + gateway := network.Addr().Next() + p.Segments = append(p.Segments, SegmentPlan{ + Name: s.Name, + Switch: s.Switch, + Bridge: "br-" + s.Name, + Network: network, + Gateway: gateway, + MTU: s.MTU, + }) + + addresses, err := allocate(t, s, network, gateway) + if err != nil { + errs = append(errs, err) + continue + } + + for nodeIndex, n := range t.Nodes { + address, ok := addresses[n.Name] + if !ok { + continue + } + cable := len(p.Cables) + p.Cables = append(p.Cables, Cable{ + Segment: s.Name, + MTU: s.MTU, + Node: n.Name, + NodeInterface: s.Name, + NodeMAC: mac(nodeIndex, segIndex, 0), + NodeAddress: netip.PrefixFrom(address, network.Bits()), + NodePort: CableBasePort + 2*cable, + Switch: s.Switch, + SwitchInterface: "p" + strconv.Itoa(cable), + SwitchMAC: mac(nodeIndex, segIndex, 1), + SwitchPort: CableBasePort + 2*cable + 1, + }) + } + } + + if last := CableBasePort + 2*len(p.Cables) - 1; last > 65535 { + errs = append(errs, fmt.Errorf("cables: %d cables need udp ports up to %d, beyond 65535", len(p.Cables), last)) + } + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + return p, nil +} + +func allocate(t *Topology, s Segment, network netip.Prefix, gateway netip.Addr) (map[string]netip.Addr, error) { + broadcast := lastAddr(network) + used := map[netip.Addr]string{gateway: s.Switch + " (gateway)"} + result := map[string]netip.Addr{} + var errs []error + + var auto []string + for _, n := range t.Nodes { + if !contains(n.Segments, s.Name) { + continue + } + raw, ok := n.Addresses[s.Name] + if !ok { + auto = append(auto, n.Name) + continue + } + addr := netip.MustParseAddr(raw) + switch { + case !network.Contains(addr): + errs = append(errs, fmt.Errorf("node %s: address %s is outside segment %s (%s)", n.Name, addr, s.Name, network)) + case addr == network.Addr() || addr == broadcast: + errs = append(errs, fmt.Errorf("node %s: address %s is the network or broadcast address of segment %s", n.Name, addr, s.Name)) + case used[addr] != "": + errs = append(errs, fmt.Errorf("node %s: address %s on segment %s is already taken by %s", n.Name, addr, s.Name, used[addr])) + default: + used[addr] = n.Name + result[n.Name] = addr + } + } + + next := gateway.Next() + for _, name := range auto { + for next != broadcast && used[next] != "" { + next = next.Next() + } + if next == broadcast { + errs = append(errs, fmt.Errorf("segment %s: no address left in %s for node %s", s.Name, network, name)) + break + } + used[next] = name + result[name] = next + next = next.Next() + } + + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + return result, nil +} + +func lastAddr(network netip.Prefix) netip.Addr { + a := network.Addr().As4() + host := uint32(1)<<(32-network.Bits()) - 1 + v := (uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])) | host + return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}) +} + +func mac(node, segment, side int) net.HardwareAddr { + return net.HardwareAddr{0x02, 0x4c, byte(node >> 8), byte(node), byte(segment), byte(side)} +} + +func contains(list []string, value string) bool { + for _, v := range list { + if v == value { + return true + } + } + return false +} diff --git a/internal/lab/topology/plan_test.go b/internal/lab/topology/plan_test.go new file mode 100644 index 0000000..1621621 --- /dev/null +++ b/internal/lab/topology/plan_test.go @@ -0,0 +1,359 @@ +package topology + +import ( + "bytes" + "fmt" + "net/netip" + "reflect" + "strings" + "testing" +) + +func compute(t *testing.T, doc string) *Plan { + t.Helper() + p, err := Compute(parse(t, doc)) + if err != nil { + t.Fatalf("Compute: %v", err) + } + return p +} + +func computeError(t *testing.T, doc string) string { + t.Helper() + _, err := Compute(parse(t, doc)) + if err == nil { + t.Fatalf("Compute accepted:\n%s", doc) + } + return err.Error() +} + +func cableOf(t *testing.T, p *Plan, node, segment string) Cable { + t.Helper() + for _, c := range p.Cables { + if c.Node == node && c.Segment == segment { + return c + } + } + t.Fatalf("no cable for %s on %s", node, segment) + return Cable{} +} + +func nodeOf(t *testing.T, p *Plan, name string) NodePlan { + t.Helper() + for _, n := range p.Nodes { + if n.Name == name { + return n + } + } + t.Fatalf("no node %s", name) + return NodePlan{} +} + +const twoHypervisors = header + ` +segments: + underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 } +nodes: + sw1: { role: switch, image: deb, cpus: 2, memory: 1024 } + rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } + hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } +` + +func TestCompute_TwoHypervisorsPlan(t *testing.T) { + p := compute(t, twoHypervisors) + + if len(p.Segments) != 1 { + t.Fatalf("%d segments, want 1", len(p.Segments)) + } + s := p.Segments[0] + if s.Bridge != "br-underlay" || s.Gateway != netip.MustParseAddr("10.250.0.1") || s.MTU != 9000 || s.Switch != "sw1" { + t.Errorf("segment = %+v", s) + } + + want := []struct { + node, address, nodeMAC, switchMAC, switchIface string + nodePort, switchPort int + }{ + {"rr1", "10.250.0.2/24", "02:4c:00:01:00:00", "02:4c:00:01:00:01", "p0", 20000, 20001}, + {"hv1", "10.250.0.3/24", "02:4c:00:02:00:00", "02:4c:00:02:00:01", "p1", 20002, 20003}, + {"hv2", "10.250.0.4/24", "02:4c:00:03:00:00", "02:4c:00:03:00:01", "p2", 20004, 20005}, + } + if len(p.Cables) != len(want) { + t.Fatalf("%d cables, want %d", len(p.Cables), len(want)) + } + for _, w := range want { + c := cableOf(t, p, w.node, "underlay") + if c.NodeAddress.String() != w.address { + t.Errorf("%s address = %s, want %s", w.node, c.NodeAddress, w.address) + } + if c.NodeMAC.String() != w.nodeMAC || c.SwitchMAC.String() != w.switchMAC { + t.Errorf("%s macs = %s / %s, want %s / %s", w.node, c.NodeMAC, c.SwitchMAC, w.nodeMAC, w.switchMAC) + } + if c.NodePort != w.nodePort || c.SwitchPort != w.switchPort { + t.Errorf("%s ports = %d / %d, want %d / %d", w.node, c.NodePort, c.SwitchPort, w.nodePort, w.switchPort) + } + if c.NodeInterface != "underlay" || c.SwitchInterface != w.switchIface || c.Switch != "sw1" || c.MTU != 9000 { + t.Errorf("%s cable = %+v", w.node, c) + } + } + + for name, port := range map[string]int{"sw1": 2200, "rr1": 2201, "hv1": 2202, "hv2": 2203} { + if got := nodeOf(t, p, name).SSHPort; got != port { + t.Errorf("%s ssh port = %d, want %d", name, got, port) + } + } + if hv := nodeOf(t, p, "hv1"); hv.Role != "hypervisor" || hv.CPUs != 4 || hv.Memory != 16384 || hv.Image != "deb" { + t.Errorf("hv1 = %+v", hv) + } +} + +func TestCompute_CarriesTheDeclaredImages(t *testing.T) { + p := compute(t, twoHypervisors) + want := []Image{{Name: "deb", URL: "https://example.invalid/deb.qcow2", Sums: "https://example.invalid/SHA512SUMS"}} + if !reflect.DeepEqual(p.Images, want) { + t.Errorf("images = %+v, want %+v", p.Images, want) + } +} + +func TestCompute_IsStableAcrossRuns(t *testing.T) { + first := compute(t, twoHypervisors) + for i := 0; i < 20; i++ { + if again := compute(t, twoHypervisors); !reflect.DeepEqual(first, again) { + t.Fatalf("plan differs on run %d", i) + } + } +} + +func TestCompute_AddressesFollowDeclarationOrderNotNames(t *testing.T) { + p := compute(t, header+` +segments: + under: { switch: sw, cidr: 10.0.0.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + zeta: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under] } + alfa: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under] } +`) + if got := cableOf(t, p, "zeta", "under").NodeAddress.String(); got != "10.0.0.2/24" { + t.Errorf("zeta = %s, want 10.0.0.2/24", got) + } + if got := cableOf(t, p, "alfa", "under").NodeAddress.String(); got != "10.0.0.3/24" { + t.Errorf("alfa = %s, want 10.0.0.3/24", got) + } +} + +func TestCompute_ExplicitAddressIsKeptAndSkippedByAllocation(t *testing.T) { + p := compute(t, header+` +segments: + under: { switch: sw, cidr: 10.0.0.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + a: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under] } + b: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under], addresses: {under: 10.0.0.3} } + c: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under] } +`) + for node, want := range map[string]string{"a": "10.0.0.2/24", "b": "10.0.0.3/24", "c": "10.0.0.4/24"} { + if got := cableOf(t, p, node, "under").NodeAddress.String(); got != want { + t.Errorf("%s = %s, want %s", node, got, want) + } + } +} + +func TestCompute_TwoSegmentsCableOrderMacsAndPorts(t *testing.T) { + p := compute(t, header+` +segments: + red: { switch: sw, cidr: 10.1.0.0/24 } + blue: { switch: sw, cidr: 10.2.0.0/24, mtu: 1500 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red] } + rr: { role: rr, image: deb, cpus: 1, memory: 512, segments: [red] } +`) + got := make([]string, 0, len(p.Cables)) + for _, c := range p.Cables { + got = append(got, c.Segment+"/"+c.Node+"/"+c.SwitchInterface) + } + if strings.Join(got, ",") != "red/hv/p0,red/rr/p1,blue/hv/p2" { + t.Errorf("cable order = %v", got) + } + blue := cableOf(t, p, "hv", "blue") + if blue.NodeMAC.String() != "02:4c:00:01:01:00" || blue.NodePort != 20004 || blue.MTU != 1500 || blue.NodeAddress.String() != "10.2.0.2/24" { + t.Errorf("hv on blue = %+v", blue) + } + if red := cableOf(t, p, "hv", "red"); red.NodeMAC.String() != "02:4c:00:01:00:00" { + t.Errorf("hv on red mac = %s", red.NodeMAC) + } +} + +func TestCompute_EveryMacAndPortIsUnique(t *testing.T) { + p := compute(t, header+` +segments: + a: { switch: sw, cidr: 10.1.0.0/24 } + b: { switch: sw, cidr: 10.2.0.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + n1: { role: rr, image: deb, cpus: 1, memory: 512, segments: [a, b] } + n2: { role: rr, image: deb, cpus: 1, memory: 512, segments: [a, b] } + n3: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [b, a] } +`) + macs := map[string]bool{} + ports := map[int]bool{} + for _, n := range p.Nodes { + ports[n.SSHPort] = true + } + for _, c := range p.Cables { + for _, m := range []string{c.NodeMAC.String(), c.SwitchMAC.String()} { + if macs[m] { + t.Errorf("mac %s used twice", m) + } + macs[m] = true + } + for _, port := range []int{c.NodePort, c.SwitchPort} { + if ports[port] { + t.Errorf("port %d used twice", port) + } + ports[port] = true + } + } + if len(macs) != 12 { + t.Errorf("%d distinct macs, want 12", len(macs)) + } +} + +func TestCompute_MacsAreLocallyAdministeredUnicast(t *testing.T) { + for _, c := range compute(t, twoHypervisors).Cables { + for _, m := range []string{c.NodeMAC.String(), c.SwitchMAC.String()} { + if !strings.HasPrefix(m, "02:") { + t.Errorf("mac %s is not locally administered unicast", m) + } + } + } +} + +func TestCompute_AddressRejections(t *testing.T) { + base := func(addresses string) string { + return header + ` +segments: + under: { switch: sw, cidr: 10.0.0.0/29 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + a: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under]` + addresses + ` } + b: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under], addresses: {under: 10.0.0.5} } +` + } + cases := []struct{ name, addresses, want string }{ + {"outside", ", addresses: {under: 10.0.1.2}", "node a: address 10.0.1.2 is outside segment under"}, + {"network", ", addresses: {under: 10.0.0.0}", "node a: address 10.0.0.0 is the network or broadcast"}, + {"broadcast", ", addresses: {under: 10.0.0.7}", "node a: address 10.0.0.7 is the network or broadcast"}, + {"gateway", ", addresses: {under: 10.0.0.1}", "node a: address 10.0.0.1 on segment under is already taken by sw (gateway)"}, + {"duplicate", ", addresses: {under: 10.0.0.5}", "node b: address 10.0.0.5 on segment under is already taken by a"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + requireContains(t, computeError(t, base(c.addresses)), c.want) + }) + } +} + +func TestCompute_ReportsExhaustedSegment(t *testing.T) { + got := computeError(t, header+` +segments: + tiny: { switch: sw, cidr: 10.0.0.0/30 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + a: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } + b: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } +`) + requireContains(t, got, "segment tiny: no address left in 10.0.0.0/30 for node b") +} + +func TestCompute_FillsSegmentExactly(t *testing.T) { + p := compute(t, header+` +segments: + tiny: { switch: sw, cidr: 10.0.0.0/29 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + a: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } + b: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } + c: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } + d: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } + e: { role: rr, image: deb, cpus: 1, memory: 512, segments: [tiny] } +`) + if got := cableOf(t, p, "e", "tiny").NodeAddress.String(); got != "10.0.0.6/29" { + t.Errorf("last node = %s, want 10.0.0.6/29", got) + } +} + +func TestCompute_RefusesInvalidTopology(t *testing.T) { + requireContains(t, computeError(t, strings.Replace(valid, "role: rr", "role: router", 1)), `role "router"`) +} + +func TestLastAddr(t *testing.T) { + for cidr, want := range map[string]string{ + "10.0.0.0/24": "10.0.0.255", + "10.0.0.0/30": "10.0.0.3", + "10.250.0.0/16": "10.250.255.255", + "192.168.4.8/29": "192.168.4.15", + } { + if got := lastAddr(netip.MustParsePrefix(cidr)).String(); got != want { + t.Errorf("lastAddr(%s) = %s, want %s", cidr, got, want) + } + } +} + +func TestWrite_TwoHypervisorsPlan(t *testing.T) { + var buf bytes.Buffer + if err := compute(t, twoHypervisors).Write(&buf); err != nil { + t.Fatalf("Write: %v", err) + } + want := `lab lab-test: nodes 4, segments 1, cables 3 + +nodes + name role image cpus memory ssh + sw1 switch deb 2 1024 MiB 127.0.0.1:2200 + rr1 rr deb 1 1024 MiB 127.0.0.1:2201 + hv1 hypervisor deb 4 16384 MiB 127.0.0.1:2202 + hv2 hypervisor deb 4 16384 MiB 127.0.0.1:2203 + +segment underlay: 10.250.0.0/24, mtu 9000, switch sw1, bridge br-underlay, gateway 10.250.0.1 + node interface address mac udp switch port mac udp + rr1 underlay 10.250.0.2/24 02:4c:00:01:00:00 20000 <-> sw1 p0 02:4c:00:01:00:01 20001 + hv1 underlay 10.250.0.3/24 02:4c:00:02:00:00 20002 <-> sw1 p1 02:4c:00:02:00:01 20003 + hv2 underlay 10.250.0.4/24 02:4c:00:03:00:00 20004 <-> sw1 p2 02:4c:00:03:00:01 20005 +` + if buf.String() != want { + t.Errorf("output:\n%s\nwant:\n%s", buf.String(), want) + } +} + +func generated(segments, nodes int) string { + var b strings.Builder + b.WriteString(header + "segments:\n") + names := make([]string, segments) + for i := range names { + names[i] = fmt.Sprintf("s%d", i) + fmt.Fprintf(&b, " %s: { switch: sw, cidr: 10.%d.0.0/16 }\n", names[i], i%250) + } + b.WriteString("nodes:\n sw: { role: switch, image: deb, cpus: 1, memory: 512 }\n") + for i := 1; i < nodes; i++ { + fmt.Fprintf(&b, " n%d: { role: rr, image: deb, cpus: 1, memory: 512, segments: [%s] }\n", i, strings.Join(names, ", ")) + } + return b.String() +} + +func TestValidate_RejectsTooManyNodes(t *testing.T) { + requireContains(t, validationError(t, generated(1, 1001)), "nodes: 1001 declared, at most 1000") +} + +func TestValidate_AcceptsExactlyMaxNodes(t *testing.T) { + if err := parse(t, generated(1, 1000)).Validate(); err != nil { + t.Fatalf("1000 nodes rejected: %v", err) + } +} + +func TestValidate_RejectsTooManySegments(t *testing.T) { + requireContains(t, validationError(t, generated(257, 2)), "segments: 257 declared, at most 256") +} + +func TestCompute_RejectsCablesBeyondUDPPortRange(t *testing.T) { + requireContains(t, computeError(t, generated(23, 1000)), "cables: 22977 cables need udp ports up to 65953, beyond 65535") +} diff --git a/internal/lab/topology/topology.go b/internal/lab/topology/topology.go new file mode 100644 index 0000000..bd184d7 --- /dev/null +++ b/internal/lab/topology/topology.go @@ -0,0 +1,151 @@ +package topology + +import ( + "bytes" + "errors" + "fmt" + "os" + + "go.yaml.in/yaml/v3" +) + +const ( + RoleSwitch = "switch" + RoleRR = "rr" + RoleHypervisor = "hypervisor" + + DefaultMTU = 9000 +) + +type Topology struct { + Name string + Images []Image + Segments []Segment + Nodes []Node +} + +type Image struct { + Name string + URL string + Sums string +} + +type Segment struct { + Name string + Switch string + CIDR string + MTU int +} + +type Node struct { + Name string + Role string + Image string + CPUs int + Memory int + Segments []string + Addresses map[string]string +} + +type fileImage struct { + URL string `yaml:"url"` + Sums string `yaml:"sums"` +} + +type fileSegment struct { + Switch string `yaml:"switch"` + CIDR string `yaml:"cidr"` + MTU int `yaml:"mtu"` +} + +type fileNode struct { + Role string `yaml:"role"` + Image string `yaml:"image"` + CPUs int `yaml:"cpus"` + Memory int `yaml:"memory"` + Segments []string `yaml:"segments"` + Addresses map[string]string `yaml:"addresses"` +} + +type file struct { + Name string `yaml:"name"` + Images map[string]fileImage `yaml:"images"` + Segments map[string]fileSegment `yaml:"segments"` + Nodes map[string]fileNode `yaml:"nodes"` +} + +func Load(path string) (*Topology, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + t, err := Parse(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return t, nil +} + +func Parse(data []byte) (*Topology, error) { + var f file + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&f); err != nil { + return nil, fmt.Errorf("yaml: %w", err) + } + + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + return nil, fmt.Errorf("yaml: %w", err) + } + order, err := keyOrder(&root) + if err != nil { + return nil, err + } + + t := &Topology{Name: f.Name} + for _, name := range order["images"] { + i := f.Images[name] + t.Images = append(t.Images, Image{Name: name, URL: i.URL, Sums: i.Sums}) + } + for _, name := range order["segments"] { + s := f.Segments[name] + mtu := s.MTU + if mtu == 0 { + mtu = DefaultMTU + } + t.Segments = append(t.Segments, Segment{Name: name, Switch: s.Switch, CIDR: s.CIDR, MTU: mtu}) + } + for _, name := range order["nodes"] { + n := f.Nodes[name] + t.Nodes = append(t.Nodes, Node{ + Name: name, + Role: n.Role, + Image: n.Image, + CPUs: n.CPUs, + Memory: n.Memory, + Segments: n.Segments, + Addresses: n.Addresses, + }) + } + return t, nil +} + +func keyOrder(root *yaml.Node) (map[string][]string, error) { + order := map[string][]string{} + if root.Kind != yaml.DocumentNode || len(root.Content) != 1 || root.Content[0].Kind != yaml.MappingNode { + return nil, errors.New("yaml: the document must be a mapping") + } + top := root.Content[0] + for i := 0; i+1 < len(top.Content); i += 2 { + section := top.Content[i].Value + value := top.Content[i+1] + if value.Kind != yaml.MappingNode { + continue + } + for j := 0; j+1 < len(value.Content); j += 2 { + order[section] = append(order[section], value.Content[j].Value) + } + } + return order, nil +} diff --git a/internal/lab/topology/topology_test.go b/internal/lab/topology/topology_test.go new file mode 100644 index 0000000..d050a0f --- /dev/null +++ b/internal/lab/topology/topology_test.go @@ -0,0 +1,173 @@ +package topology + +import ( + "strings" + "testing" +) + +const header = `name: lab-test +images: + deb: + url: https://example.invalid/deb.qcow2 + sums: https://example.invalid/SHA512SUMS +` + +func parse(t *testing.T, doc string) *Topology { + t.Helper() + topo, err := Parse([]byte(doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + return topo +} + +func validationError(t *testing.T, doc string) string { + t.Helper() + err := parse(t, doc).Validate() + if err == nil { + t.Fatalf("Validate accepted an invalid topology:\n%s", doc) + } + return err.Error() +} + +func requireContains(t *testing.T, got, want string) { + t.Helper() + if !strings.Contains(got, want) { + t.Fatalf("error does not mention %q:\n%s", want, got) + } +} + +func TestParse_KeepsDeclarationOrderOfNodesAndSegments(t *testing.T) { + topo := parse(t, header+` +segments: + zulu: { switch: sw, cidr: 10.0.1.0/24 } + alpha: { switch: sw, cidr: 10.0.2.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + zeta: { role: rr, image: deb, cpus: 1, memory: 512, segments: [zulu, alpha] } + beta: { role: rr, image: deb, cpus: 1, memory: 512, segments: [zulu] } +`) + var nodes, segments []string + for _, n := range topo.Nodes { + nodes = append(nodes, n.Name) + } + for _, s := range topo.Segments { + segments = append(segments, s.Name) + } + if got := strings.Join(nodes, ","); got != "sw,zeta,beta" { + t.Errorf("node order = %s, want sw,zeta,beta", got) + } + if got := strings.Join(segments, ","); got != "zulu,alpha" { + t.Errorf("segment order = %s, want zulu,alpha", got) + } +} + +func TestParse_DefaultsMTUTo9000(t *testing.T) { + topo := parse(t, header+` +segments: + under: { switch: sw, cidr: 10.0.0.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } +`) + if topo.Segments[0].MTU != 9000 { + t.Errorf("MTU = %d, want 9000", topo.Segments[0].MTU) + } +} + +func TestParse_RejectsUnknownField(t *testing.T) { + _, err := Parse([]byte(header + ` +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512, ram: 4 } +`)) + if err == nil || !strings.Contains(err.Error(), "ram") { + t.Fatalf("unknown field not rejected: %v", err) + } +} + +func TestParse_RejectsDuplicateNode(t *testing.T) { + _, err := Parse([]byte(header + ` +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + sw: { role: rr, image: deb, cpus: 1, memory: 512 } +`)) + if err == nil || !strings.Contains(err.Error(), "already defined") { + t.Fatalf("duplicate node not rejected: %v", err) + } +} + +func TestParse_RejectsNonMappingDocument(t *testing.T) { + if _, err := Parse([]byte("- a\n- b\n")); err == nil { + t.Fatal("a list document was accepted") + } +} + +func TestLoad_ReportsPathOnError(t *testing.T) { + if _, err := Load("/nonexistent/lab.yml"); err == nil { + t.Fatal("missing file accepted") + } +} + +const valid = header + ` +segments: + under: { switch: sw, cidr: 10.0.0.0/24 } +nodes: + sw: { role: switch, image: deb, cpus: 1, memory: 512 } + rr: { role: rr, image: deb, cpus: 1, memory: 512, segments: [under] } +` + +func TestValidate_AcceptsMinimalTopology(t *testing.T) { + if err := parse(t, valid).Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } +} + +func TestValidate_Rejections(t *testing.T) { + cases := []struct { + name, doc, want string + }{ + {"bad lab name", strings.Replace(valid, "name: lab-test", "name: Lab_Test", 1), `name "Lab_Test"`}, + {"image url not https", strings.Replace(valid, "url: https://", "url: http://", 1), "image deb: url"}, + {"image sums not https", strings.Replace(valid, "sums: https://", "sums: ftp://", 1), "image deb: sums"}, + {"no node", header + "\nsegments: {}\nnodes: {}\n", "at least one node"}, + {"segment switch missing", strings.Replace(valid, "switch: sw, cidr", "cidr", 1), "segment under: switch is required"}, + {"segment switch unknown", strings.Replace(valid, "switch: sw, cidr", "switch: ghost, cidr", 1), "switch ghost is not a declared node"}, + {"segment switch not a switch", strings.Replace(valid, "switch: sw, cidr", "switch: rr, cidr", 1), "rr is a rr, not a switch"}, + {"segment name too long", strings.ReplaceAll(valid, "under", "underlayunder"), "segment underlayunder: name must match"}, + {"segment named like the admin interface", strings.ReplaceAll(valid, "under", "mgmt0"), "segment mgmt0: name is reserved"}, + {"segment name with dash", strings.ReplaceAll(valid, "under", "un-der"), "segment un-der: name must match"}, + {"mtu too high", strings.Replace(valid, "cidr: 10.0.0.0/24", "cidr: 10.0.0.0/24, mtu: 9001", 1), "mtu 9001 out of range"}, + {"mtu too low", strings.Replace(valid, "cidr: 10.0.0.0/24", "cidr: 10.0.0.0/24, mtu: 1279", 1), "mtu 1279 out of range"}, + {"cidr unparsable", strings.Replace(valid, "10.0.0.0/24", "10.0.0/24", 1), `cidr "10.0.0/24"`}, + {"cidr ipv6", strings.Replace(valid, "10.0.0.0/24", "fd00::/64", 1), "is not IPv4"}, + {"cidr host bits", strings.Replace(valid, "10.0.0.0/24", "10.0.0.5/24", 1), "network is 10.0.0.0/24"}, + {"cidr too small", strings.Replace(valid, "10.0.0.0/24", "10.0.0.0/31", 1), "prefix length out of range"}, + {"cidr too large", strings.Replace(valid, "10.0.0.0/24", "10.0.0.0/7", 1), "prefix length out of range"}, + {"bad node name", strings.Replace(valid, " rr: {", " RR: {", 1), "node RR: name must match"}, + {"bad role", strings.Replace(valid, "role: rr", "role: router", 1), `role "router"`}, + {"unknown image", strings.Replace(valid, "role: rr, image: deb", "role: rr, image: ubuntu", 1), `image "ubuntu" is not declared`}, + {"no cpu", strings.Replace(valid, "role: rr, image: deb, cpus: 1", "role: rr, image: deb, cpus: 0", 1), "node rr: cpus must be at least 1"}, + {"memory too low", strings.Replace(valid, "cpus: 1, memory: 512, segments", "cpus: 1, memory: 255, segments", 1), "node rr: memory must be at least 256"}, + {"switch with segments", strings.Replace(valid, "role: switch, image: deb, cpus: 1, memory: 512 }", "role: switch, image: deb, cpus: 1, memory: 512, segments: [under] }", 1), "node sw: a switch carries its segments"}, + {"switch with addresses", strings.Replace(valid, "role: switch, image: deb, cpus: 1, memory: 512 }", "role: switch, image: deb, cpus: 1, memory: 512, addresses: {under: 10.0.0.9} }", 1), "node sw: a switch carries its segments"}, + {"switch without segment", valid + " sw2: { role: switch, image: deb, cpus: 1, memory: 512 }\n", "node sw2: switch carries no segment"}, + {"node without segment", valid + " rr2: { role: rr, image: deb, cpus: 1, memory: 512 }\n", "node rr2: must be attached to at least one segment"}, + {"node on unknown segment", strings.Replace(valid, "segments: [under]", "segments: [over]", 1), "node rr: segment over is not declared"}, + {"node on segment twice", strings.Replace(valid, "segments: [under]", "segments: [under, under]", 1), "node rr: segment under listed twice"}, + {"segment without node", strings.Replace(valid, " under: { switch: sw, cidr: 10.0.0.0/24 }", " under: { switch: sw, cidr: 10.0.0.0/24 }\n empty: { switch: sw, cidr: 10.0.9.0/24 }", 1), "segment empty: no node is attached"}, + {"address on foreign segment", strings.Replace(valid, "segments: [under]", "segments: [under], addresses: {over: 10.0.0.9}", 1), "address given for segment over"}, + {"address unparsable", strings.Replace(valid, "segments: [under]", "segments: [under], addresses: {under: 10.0.0}", 1), `address "10.0.0" on under`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + requireContains(t, validationError(t, c.doc), c.want) + }) + } +} + +func TestValidate_ReportsEveryErrorAtOnce(t *testing.T) { + doc := strings.Replace(valid, "role: rr", "role: router", 1) + doc = strings.Replace(doc, "10.0.0.0/24", "10.0.0.0/31", 1) + got := validationError(t, doc) + requireContains(t, got, `role "router"`) + requireContains(t, got, "prefix length out of range") +} diff --git a/internal/lab/topology/validate.go b/internal/lab/topology/validate.go new file mode 100644 index 0000000..6a7d53e --- /dev/null +++ b/internal/lab/topology/validate.go @@ -0,0 +1,175 @@ +package topology + +import ( + "errors" + "fmt" + "net/netip" + "regexp" + "sort" + "strings" +) + +const ( + MinMTU = 1280 + MaxMTU = 9000 + MinMemory = 256 + MaxPrefix = 30 + MinPrefix = 8 + + ReservedInterface = "mgmt0" +) + +var ( + namePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,14}$`) + segmentPattern = regexp.MustCompile(`^[a-z][a-z0-9]{0,11}$`) +) + +func (t *Topology) Validate() error { + var errs []error + add := func(format string, args ...any) { + errs = append(errs, fmt.Errorf(format, args...)) + } + + if !namePattern.MatchString(t.Name) { + add("name %q: must match %s", t.Name, namePattern) + } + + images := map[string]bool{} + for _, i := range t.Images { + images[i.Name] = true + if !strings.HasPrefix(i.URL, "https://") { + add("image %s: url must be an https:// URL", i.Name) + } + if !strings.HasPrefix(i.Sums, "https://") { + add("image %s: sums must be an https:// URL", i.Name) + } + } + + nodes := map[string]Node{} + for _, n := range t.Nodes { + nodes[n.Name] = n + } + if len(t.Nodes) == 0 { + add("nodes: at least one node is required") + } + if len(t.Segments) > maxSegments { + add("segments: %d declared, at most %d", len(t.Segments), maxSegments) + } + if len(t.Nodes) > maxNodes { + add("nodes: %d declared, at most %d", len(t.Nodes), maxNodes) + } + + segments := map[string]Segment{} + for _, s := range t.Segments { + segments[s.Name] = s + if !segmentPattern.MatchString(s.Name) { + add("segment %s: name must match %s", s.Name, segmentPattern) + } + if s.Name == ReservedInterface { + add("segment %s: name is reserved for the administration interface", s.Name) + } + sw, ok := nodes[s.Switch] + switch { + case s.Switch == "": + add("segment %s: switch is required", s.Name) + case !ok: + add("segment %s: switch %s is not a declared node", s.Name, s.Switch) + case sw.Role != RoleSwitch: + add("segment %s: %s is a %s, not a switch", s.Name, s.Switch, sw.Role) + } + if s.MTU < MinMTU || s.MTU > MaxMTU { + add("segment %s: mtu %d out of range [%d, %d]", s.Name, s.MTU, MinMTU, MaxMTU) + } + prefix, err := netip.ParsePrefix(s.CIDR) + switch { + case err != nil: + add("segment %s: cidr %q: %v", s.Name, s.CIDR, err) + case !prefix.Addr().Is4(): + add("segment %s: cidr %s is not IPv4", s.Name, s.CIDR) + case prefix.Masked() != prefix: + add("segment %s: cidr %s has host bits set, network is %s", s.Name, s.CIDR, prefix.Masked()) + case prefix.Bits() < MinPrefix || prefix.Bits() > MaxPrefix: + add("segment %s: cidr %s prefix length out of range [/%d, /%d]", s.Name, s.CIDR, MinPrefix, MaxPrefix) + } + } + + attached := map[string]int{} + hosting := map[string]int{} + for _, s := range t.Segments { + hosting[s.Switch]++ + } + for _, n := range t.Nodes { + if !namePattern.MatchString(n.Name) { + add("node %s: name must match %s", n.Name, namePattern) + } + switch n.Role { + case RoleSwitch, RoleRR, RoleHypervisor: + default: + add("node %s: role %q must be one of %s, %s, %s", n.Name, n.Role, RoleSwitch, RoleRR, RoleHypervisor) + } + if !images[n.Image] { + add("node %s: image %q is not declared", n.Name, n.Image) + } + if n.CPUs < 1 { + add("node %s: cpus must be at least 1", n.Name) + } + if n.Memory < MinMemory { + add("node %s: memory must be at least %d MiB", n.Name, MinMemory) + } + if n.Role == RoleSwitch { + if len(n.Segments) > 0 || len(n.Addresses) > 0 { + add("node %s: a switch carries its segments through segments..switch, not through segments or addresses", n.Name) + } + if hosting[n.Name] == 0 { + add("node %s: switch carries no segment", n.Name) + } + continue + } + if len(n.Segments) == 0 { + add("node %s: must be attached to at least one segment", n.Name) + } + seen := map[string]bool{} + for _, name := range n.Segments { + if seen[name] { + add("node %s: segment %s listed twice", n.Name, name) + continue + } + seen[name] = true + if _, ok := segments[name]; !ok { + add("node %s: segment %s is not declared", n.Name, name) + continue + } + attached[name]++ + } + for _, name := range sortedKeys(n.Addresses) { + address := n.Addresses[name] + if !seen[name] { + add("node %s: address given for segment %s it is not attached to", n.Name, name) + continue + } + if _, err := netip.ParseAddr(address); err != nil { + add("node %s: address %q on %s: %v", n.Name, address, name, err) + } + } + } + + for _, s := range t.Segments { + if attached[s.Name] == 0 { + add("segment %s: no node is attached", s.Name) + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d6b1828..b654197 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -387,5 +387,6 @@ main () { switch_binaries "${FLAGS_dryrun}" "${FLAGS_profile}" "${ASSETS}" } -[[ "${BASH_SOURCE[0]}" == "${0}" ]] && (main "$@" || exit 1) -[[ "${BASH_SOURCE[0]}" == "" ]] && (main "$@" || exit 1) +if [[ "${BASH_SOURCE[0]}" == "${0}" || -z "${BASH_SOURCE[0]}" ]]; then + (main "$@" || exit 1) +fi diff --git a/scripts/lab-host.sh b/scripts/lab-host.sh new file mode 100755 index 0000000..412fd6a --- /dev/null +++ b/scripts/lab-host.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash + +API_URL="${SCW_API_URL:-https://api.scaleway.com}" +CREDENTIALS_FILE="${LAB_CREDENTIALS_FILE:-${HOME}/.config/two-lab/scaleway.env}" +OFFER_NAME="${LAB_OFFER:-EM-B212X-SSD}" +OS_NAME="${LAB_OS_NAME:-Debian}" +OS_VERSION="${LAB_OS_VERSION:-12}" +LAB_NAME="${LAB_NAME:-two-lab}" +LAB_TAG="${LAB_TAG:-two-lab}" +INSTALL_TIMEOUT="${LAB_INSTALL_TIMEOUT:-3600}" +DELETE_TIMEOUT="${LAB_DELETE_TIMEOUT:-900}" +SSH_TIMEOUT="${LAB_SSH_TIMEOUT:-600}" +POLL_INTERVAL="${LAB_POLL_INTERVAL:-20}" +HTTP_CONNECT_TIMEOUT="${LAB_HTTP_CONNECT_TIMEOUT:-10}" +HTTP_TIMEOUT="${LAB_HTTP_TIMEOUT:-60}" +STATE_DIR="${LAB_STATE_DIR:-${HOME}/.cache/two-lab}" +SSH_KEY="${LAB_SSH_KEY:-${HOME}/.config/two-lab/ssh/lab_ed25519}" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CREATED_ID="" + +info () { echo "== ${1}" >&2; } +warn () { echo "!! ${1}" >&2; } +die () { echo "!! ${1}" >&2; exit 1; } + +usage () { + cat >&2 < [arguments] + + plan résout l'offre horaire, l'OS et les clés SSH, affiche la requête de création + et le prix ; ne crée rien + up crée le serveur de lab, attend la fin de son installation et son SSH + status liste les serveurs de lab du projet + ssh [commande] se connecte au serveur de lab ; avec une commande, un terminal n'est demandé + que si l'entrée standard en est un + prepare installe sur le serveur ce dont lab a besoin (qemu, genisoimage), vérifie + /dev/kvm et la virtualisation imbriquée ; lancé aussi par up + push compile cmd/lab pour linux/amd64 et dépose sur le serveur ~/lab et + ~/ ; ensuite : ssh './lab up ' + down supprime tous les serveurs de lab du projet et attend leur disparition + session [cmd] up, puis la commande distante (ou un shell), puis down quoi qu'il arrive + +environnement : SCW_SECRET_KEY et SCW_DEFAULT_PROJECT_ID (requis, à défaut lus dans ${CREDENTIALS_FILE}, + fichier en 0600, lignes CLÉ=valeur), SCW_DEFAULT_ZONE (${ZONE}), + LAB_OFFER (${OFFER_NAME}), LAB_OS_NAME (${OS_NAME}), LAB_OS_VERSION (${OS_VERSION}), + LAB_SSH_KEY (${SSH_KEY} ; à défaut, l'agent SSH) + +ne jamais lancer sous 'bash -x' : la trace afficherait SCW_SECRET_KEY. +EOF + exit 2 +} + +load_credentials () { + local KEY VALUE + if [[ -z "${SCW_SECRET_KEY:-}" || -z "${SCW_DEFAULT_PROJECT_ID:-}" ]] && [[ -f "${CREDENTIALS_FILE}" ]]; then + [[ $(ls -l "${CREDENTIALS_FILE}" | cut -c5-10) == "------" ]] \ + || die "${CREDENTIALS_FILE} est lisible par d'autres que son propriétaire : chmod 600" + while IFS='=' read -r KEY VALUE || [[ -n "${KEY}" ]]; do + case "${KEY}" in + SCW_SECRET_KEY) [[ -n "${SCW_SECRET_KEY:-}" ]] || SCW_SECRET_KEY="${VALUE}" ;; + SCW_DEFAULT_PROJECT_ID) [[ -n "${SCW_DEFAULT_PROJECT_ID:-}" ]] || SCW_DEFAULT_PROJECT_ID="${VALUE}" ;; + SCW_DEFAULT_ZONE) [[ -n "${SCW_DEFAULT_ZONE:-}" ]] || SCW_DEFAULT_ZONE="${VALUE}" ;; + esac + done < "${CREDENTIALS_FILE}" + fi + ZONE="${SCW_DEFAULT_ZONE:-fr-par-1}" + PROJECT_ID="${SCW_DEFAULT_PROJECT_ID:-}" +} + +require_env () { + [[ -n "${SCW_SECRET_KEY:-}" ]] || die "SCW_SECRET_KEY absent de l'environnement" + [[ -n "${PROJECT_ID}" ]] || die "SCW_DEFAULT_PROJECT_ID absent de l'environnement" + command -v jq >/dev/null 2>&1 || die "jq est requis" + command -v curl >/dev/null 2>&1 || die "curl est requis" +} + +http_code () { + cat "${STATE_DIR}/last_http_code" 2>/dev/null +} + +api () { + local METHOD="${1}" + local URL_PATH="${2}" + local BODY="${3:-}" + local OUT CODE + local -a ARGS=(-sS -X "${METHOD}" -H "Content-Type: application/json" -w '\n%{http_code}' + --connect-timeout "${HTTP_CONNECT_TIMEOUT}" --max-time "${HTTP_TIMEOUT}") + + : > "${STATE_DIR}/last_http_code" + [[ -n "${BODY}" ]] && ARGS+=(--data "${BODY}") + OUT=$(curl "${ARGS[@]}" -H @<(printf 'X-Auth-Token: %s\n' "${SCW_SECRET_KEY}") "${API_URL}${URL_PATH}") \ + || { warn "${METHOD} ${URL_PATH} : échec réseau"; return 1; } + CODE="${OUT##*$'\n'}" + OUT="${OUT%$'\n'*}" + printf '%s' "${CODE}" > "${STATE_DIR}/last_http_code" + if [[ "${CODE}" -lt 200 || "${CODE}" -ge 300 ]]; then + warn "${METHOD} ${URL_PATH} : HTTP ${CODE} ${OUT}" + return 1 + fi + printf '%s\n' "${OUT}" +} + +http_failure_is_final () { + local CODE + CODE=$(http_code) + [[ "${CODE}" =~ ^4[0-9][0-9]$ && "${CODE}" != "429" ]] +} + +money () { + jq -r "${1} | if . == null then \"?\" else \"\\(.units + .nanos / 1000000000) \\(.currency_code)\" end" +} + +resolve_offer () { + local JSON OFFER COUNT + JSON=$(api GET "/baremetal/v1/zones/${ZONE}/offers?subscription_period=hourly&name=${OFFER_NAME}&page_size=100") \ + || die "impossible de lister les offres" + OFFER=$(jq -c --arg n "${OFFER_NAME}" \ + '[.offers[] | select(.name == $n and .subscription_period == "hourly")]' <<< "${JSON}") + COUNT=$(jq 'length' <<< "${OFFER}") + [[ "${COUNT}" -eq 1 ]] || die "offre horaire ${OFFER_NAME} : ${COUNT} correspondance(s) en ${ZONE}, une seule attendue" + OFFER=$(jq -c '.[0]' <<< "${OFFER}") + [[ $(jq -r '.enable' <<< "${OFFER}") == "true" ]] || die "offre ${OFFER_NAME} désactivée" + [[ $(jq -r '.stock' <<< "${OFFER}") != "empty" ]] || die "offre ${OFFER_NAME} en rupture de stock en ${ZONE}" + [[ $(jq -r '((.fee.units // 0) == 0) and ((.fee.nanos // 0) == 0)' <<< "${OFFER}") == "true" ]] \ + || die "offre ${OFFER_NAME} : frais de mise en service non nuls ($(money '.fee' <<< "${OFFER}")), refus" + printf '%s\n' "${OFFER}" +} + +resolve_os () { + local OFFER_ID="${1}" + local JSON OS COUNT + JSON=$(api GET "/baremetal/v1/zones/${ZONE}/os?offer_id=${OFFER_ID}&page_size=100") \ + || die "impossible de lister les OS" + OS=$(jq -c --arg n "${OS_NAME}" --arg v "${OS_VERSION}" \ + '[.os[] | select(.enabled and .allowed and (.name | ascii_downcase | contains($n | ascii_downcase)) + and (.version | split(" ")[0] | split(".")[0]) == $v)]' <<< "${JSON}") + COUNT=$(jq 'length' <<< "${OS}") + if [[ "${COUNT}" -ne 1 ]]; then + warn "OS disponibles pour ${OFFER_NAME} :" + jq -r '.os[] | " \(.name) \(.version) (\(.id))"' <<< "${JSON}" >&2 + die "OS ${OS_NAME} ${OS_VERSION} : ${COUNT} correspondance(s), une seule attendue" + fi + jq -c '.[0]' <<< "${OS}" +} + +resolve_ssh_keys () { + local JSON + JSON=$(api GET "/iam/v1alpha1/ssh-keys?project_id=${PROJECT_ID}&disabled=false&page_size=100") \ + || die "impossible de lister les clés SSH du projet" + JSON=$(jq -c '[.ssh_keys[] | select(.disabled | not) | .id]' <<< "${JSON}") + [[ $(jq 'length' <<< "${JSON}") -gt 0 ]] || die "aucune clé SSH active dans le projet ${PROJECT_ID}" + printf '%s\n' "${JSON}" +} + +build_request () { + local OFFER="${1}" + local OS="${2}" + local KEYS="${3}" + jq -nc --argjson offer "${OFFER}" --argjson os "${OS}" --argjson keys "${KEYS}" \ + --arg project "${PROJECT_ID}" --arg name "${LAB_NAME}" --arg tag "${LAB_TAG}" \ + '{offer_id: $offer.id, project_id: $project, name: $name, description: "lab two (#50)", + tags: [$tag], install: {os_id: $os.id, hostname: $name, ssh_key_ids: $keys}}' +} + +lab_servers () { + local JSON + JSON=$(api GET "/baremetal/v1/zones/${ZONE}/servers?project_id=${PROJECT_ID}&tags=${LAB_TAG}&page_size=100") \ + || return 1 + jq -c --arg p "${PROJECT_ID}" --arg t "${LAB_TAG}" \ + '[.servers[] | select(.project_id == $p and (.tags | index($t)))]' <<< "${JSON}" +} + +server_ip () { + jq -r '[.ips[] | select(.version == "IPv4") | .address][0] // empty' +} + +server_is_installing () { + jq -e '.status == "delivering" or .status == "ordered" + or (.install.status // "") == "to_install" or (.install.status // "") == "installing"' >/dev/null +} + +cmd_plan () { + local OFFER OS KEYS + OFFER=$(resolve_offer) || exit 1 + OS=$(resolve_os "$(jq -r '.id' <<< "${OFFER}")") || exit 1 + KEYS=$(resolve_ssh_keys) || exit 1 + info "offre : ${OFFER_NAME} ($(jq -r '.id' <<< "${OFFER}")), facturation $(jq -r '.subscription_period' <<< "${OFFER}"), stock $(jq -r '.stock' <<< "${OFFER}")" + info "prix : $(money '.price_per_hour' <<< "${OFFER}") HT par heure, frais de mise en service $(money '.fee' <<< "${OFFER}")" + info "os : $(jq -r '"\(.name) \(.version) (\(.id))"' <<< "${OS}"), utilisateur $(jq -r '.user.default_value // "?"' <<< "${OS}")" + info "clés : $(jq -r 'length' <<< "${KEYS}") clé(s) SSH du projet" + info "requête : POST /baremetal/v1/zones/${ZONE}/servers" + build_request "${OFFER}" "${OS}" "${KEYS}" | jq . +} + +wait_installed () { + local ID="${1}" + local DEADLINE=$(( SECONDS + INSTALL_TIMEOUT )) + local JSON STATUS INSTALL + while (( SECONDS < DEADLINE )); do + if ! JSON=$(api GET "/baremetal/v1/zones/${ZONE}/servers/${ID}"); then + if http_failure_is_final; then + warn "erreur définitive HTTP $(http_code) en attendant l'installation" + return 1 + fi + sleep "${POLL_INTERVAL}" + continue + fi + STATUS=$(jq -r '.status' <<< "${JSON}") + INSTALL=$(jq -r '.install.status // "none"' <<< "${JSON}") + info "serveur ${ID} : ${STATUS}, installation ${INSTALL}" + case "${STATUS}/${INSTALL}" in + ready/completed) printf '%s\n' "${JSON}"; return 0 ;; + error/*|*/error|out_of_stock/*|locked/*|deleting/*|stopped/*|stopping/*) + warn "état terminal en échec : ${STATUS}/${INSTALL}"; return 1 ;; + esac + sleep "${POLL_INTERVAL}" + done + warn "installation non terminée après ${INSTALL_TIMEOUT}s" + return 1 +} + +ssh_run () { + local -a OPTS=() + while [[ $# -gt 0 && "${1}" != "--" ]]; do + OPTS+=("${1}") + shift + done + [[ "${1:-}" == "--" ]] && shift + [[ -f "${SSH_KEY}" ]] && OPTS+=(-i "${SSH_KEY}" -o IdentitiesOnly=yes -o IdentityAgent=none) + env -u SCW_SECRET_KEY ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${STATE_DIR}/known_hosts" \ + "${OPTS[@]}" "$(cat "${STATE_DIR}/user")@$(cat "${STATE_DIR}/ip")" "$@" +} + +wait_ssh () { + local DEADLINE=$(( SECONDS + SSH_TIMEOUT )) + while (( SECONDS < DEADLINE )); do + ssh_run -o BatchMode=yes -o ConnectTimeout=5 -- true >/dev/null 2>&1 && return 0 + info "SSH pas encore joignable, nouvel essai dans ${POLL_INTERVAL}s" + sleep "${POLL_INTERVAL}" + done + return 1 +} + +require_no_lab_server () { + local EXISTING + EXISTING=$(lab_servers) || die "impossible de lister les serveurs de lab" + [[ $(jq 'length' <<< "${EXISTING}") -eq 0 ]] \ + || die "un serveur de lab existe déjà dans le projet ; '${0##*/} status' puis '${0##*/} down'" +} + +report_uncertain_creation () { + local FOUND + if FOUND=$(lab_servers) && [[ $(jq 'length' <<< "${FOUND}") -gt 0 ]]; then + warn "un serveur de lab existe bien : $(jq -r '[.[].id] | join(", ")' <<< "${FOUND}")" + fi + die "création incertaine : le serveur a pu être créé et être facturé ; '${0##*/} status' puis '${0##*/} down'" +} + +cmd_up () { + local OFFER OS KEYS BODY JSON ID IP SSH_USER + require_no_lab_server + OFFER=$(resolve_offer) || exit 1 + OS=$(resolve_os "$(jq -r '.id' <<< "${OFFER}")") || exit 1 + KEYS=$(resolve_ssh_keys) || exit 1 + BODY=$(build_request "${OFFER}" "${OS}" "${KEYS}") + info "création de ${LAB_NAME} (${OFFER_NAME}, $(money '.price_per_hour' <<< "${OFFER}")/h HT)" + JSON=$(api POST "/baremetal/v1/zones/${ZONE}/servers" "${BODY}") || report_uncertain_creation + ID=$(jq -r '.id // empty' <<< "${JSON}" 2>/dev/null) + [[ -n "${ID}" ]] || report_uncertain_creation + CREATED_ID="${ID}" + info "serveur ${ID} créé, facturé jusqu'à '${0##*/} down'" + JSON=$(wait_installed "${ID}") || die "serveur ${ID} inutilisable — il est toujours facturé : '${0##*/} down'" + IP=$(server_ip <<< "${JSON}") + [[ -n "${IP}" ]] || die "serveur ${ID} sans IPv4 publique — il est toujours facturé : '${0##*/} down'" + SSH_USER=$(jq -r '.user.default_value // empty' <<< "${OS}") + printf '%s\n' "${IP}" > "${STATE_DIR}/ip" + printf '%s\n' "${SSH_USER:-root}" > "${STATE_DIR}/user" + : > "${STATE_DIR}/known_hosts" + wait_ssh || die "SSH injoignable après ${SSH_TIMEOUT}s — le serveur est toujours facturé : '${0##*/} down'" + cmd_prepare || die "préparation du serveur échouée — il est toujours facturé : '${0##*/} down'" + info "prêt : ${SSH_USER:-root}@${IP}" +} + +cmd_status () { + local SERVERS + SERVERS=$(lab_servers) || die "impossible de lister les serveurs de lab" + if [[ $(jq 'length' <<< "${SERVERS}") -eq 0 ]]; then + info "aucun serveur de lab dans le projet ${PROJECT_ID} (${ZONE})" + return 0 + fi + jq -r '.[] | "\(.id) \(.name) \(.offer_name) \(.status) installation \(.install.status // "none") \(.created_at) \([.ips[] | select(.version == "IPv4") | .address] | join(","))"' <<< "${SERVERS}" +} + +require_known_server () { + [[ -s "${STATE_DIR}/ip" ]] || die "aucun serveur de lab connu ; '${0##*/} up' d'abord" +} + +cmd_ssh () { + require_known_server + if [[ $# -gt 0 && -t 0 ]]; then + ssh_run -t -- "$@" + else + ssh_run -- "$@" + fi +} + +prepare_script () { + cat <<'EOF' +set -eu +SUDO= +[ "$(id -u)" -eq 0 ] || SUDO=sudo +$SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq +$SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends qemu-system-x86 qemu-utils genisoimage +[ "$(id -u)" -eq 0 ] || $SUDO usermod -aG kvm "$(id -un)" +[ -c /dev/kvm ] || { echo "/dev/kvm absent" >&2; exit 1; } +NESTED=$(cat /sys/module/kvm_intel/parameters/nested 2>/dev/null || cat /sys/module/kvm_amd/parameters/nested 2>/dev/null || true) +case "${NESTED}" in + Y|1) ;; + *) echo "virtualisation imbriquée désactivée (nested=${NESTED:-absent})" >&2; exit 1 ;; +esac +echo "qemu $(qemu-system-x86_64 --version | head -n 1), nested=${NESTED}" +EOF +} + +cmd_prepare () { + require_known_server + info "préparation du serveur : qemu, genisoimage, KVM imbriqué" + prepare_script | ssh_run -- bash -s +} + +push_file () { + local SOURCE="${1}" TARGET="${2}" MODE="${3}" + ssh_run -- "cat > '${TARGET}.part' && chmod ${MODE} '${TARGET}.part' && mv '${TARGET}.part' '${TARGET}'" < "${SOURCE}" +} + +cmd_push () { + local TOPOLOGY="${1:-}" + local NAME="${TOPOLOGY##*/}" + local BINARY="${STATE_DIR}/lab" + [[ $# -eq 1 && -n "${TOPOLOGY}" ]] || usage + [[ -f "${TOPOLOGY}" ]] || die "topologie introuvable : ${TOPOLOGY}" + [[ "${NAME}" =~ ^[A-Za-z0-9._-]+$ ]] || die "nom de topologie refusé : ${NAME} (lettres, chiffres, '.', '_', '-')" + require_known_server + info "compilation de lab (linux/amd64)" + (cd "${REPO_DIR}" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "${BINARY}" ./cmd/lab) \ + || die "compilation de lab échouée" + push_file "${BINARY}" lab 755 || die "envoi de lab échoué" + push_file "${TOPOLOGY}" "${NAME}" 644 || die "envoi de ${NAME} échoué" + info "déposés sur le serveur : ~/lab, ~/${NAME} — ensuite : ${0##*/} ssh './lab up ${NAME}'" +} + +delete_server () { + local ID="${1}" + local DEADLINE=$(( SECONDS + DELETE_TIMEOUT )) + local HARD_DEADLINE=$(( SECONDS + DELETE_TIMEOUT + INSTALL_TIMEOUT )) + local JSON + while (( SECONDS < DEADLINE )); do + api DELETE "/baremetal/v1/zones/${ZONE}/servers/${ID}" >/dev/null && return 0 + [[ $(http_code) == "404" ]] && return 0 + if JSON=$(api GET "/baremetal/v1/zones/${ZONE}/servers/${ID}" 2>/dev/null) && server_is_installing <<< "${JSON}"; then + DEADLINE=$(( SECONDS + DELETE_TIMEOUT )) + (( DEADLINE > HARD_DEADLINE )) && DEADLINE="${HARD_DEADLINE}" + warn "suppression de ${ID} refusée pendant la livraison ou l'installation, nouvel essai dans ${POLL_INTERVAL}s" + else + warn "suppression de ${ID} refusée, nouvel essai dans ${POLL_INTERVAL}s" + fi + sleep "${POLL_INTERVAL}" + done + return 1 +} + +wait_gone () { + local ID="${1}" + local DEADLINE=$(( SECONDS + DELETE_TIMEOUT )) + while (( SECONDS < DEADLINE )); do + if ! api GET "/baremetal/v1/zones/${ZONE}/servers/${ID}" >/dev/null 2>&1; then + [[ $(http_code) == "404" ]] && return 0 + fi + sleep "${POLL_INTERVAL}" + done + return 1 +} + +lab_server_ids () { + local DEADLINE=$(( SECONDS + DELETE_TIMEOUT )) + local SERVERS + while (( SECONDS < DEADLINE )); do + if SERVERS=$(lab_servers); then + jq -r '.[].id' <<< "${SERVERS}" + return 0 + fi + warn "liste des serveurs de lab indisponible, nouvel essai dans ${POLL_INTERVAL}s" + sleep "${POLL_INTERVAL}" + done + return 1 +} + +cmd_down () { + local IDS ID FAILED=0 + if ! IDS=$(lab_server_ids); then + warn "impossible de lister les serveurs de lab après ${DELETE_TIMEOUT}s" + FAILED=1 + IDS="" + fi + IDS=$(printf '%s\n%s\n' "${IDS}" "${CREATED_ID}" | grep -v '^$' | sort -u) + for ID in ${IDS}; do + info "suppression de ${ID}" + if ! delete_server "${ID}"; then + warn "suppression de ${ID} toujours refusée" + FAILED=1 + continue + fi + if ! wait_gone "${ID}"; then + warn "${ID} toujours présent après ${DELETE_TIMEOUT}s" + FAILED=1 + fi + done + rm -f "${STATE_DIR}/ip" "${STATE_DIR}/user" "${STATE_DIR}/known_hosts" + [[ "${FAILED}" -eq 0 ]] || die "SERVEUR(S) DE LAB TOUJOURS FACTURÉ(S) — '${0##*/} status', puis la console Scaleway" + info "aucun serveur de lab ne reste dans le projet" +} + +cmd_session () { + local RC + require_no_lab_server + trap 'trap - EXIT; trap "" INT TERM HUP; cmd_down' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + trap 'exit 129' HUP + cmd_up + cmd_ssh "$@" + RC=$? + info "session terminée (code ${RC}), suppression du serveur" + return "${RC}" +} + +main () { + local COMMAND="${1:-}" + [[ $# -gt 0 ]] && shift + load_credentials + [[ -n "${COMMAND}" ]] || usage + mkdir -p "${STATE_DIR}" || die "impossible de créer ${STATE_DIR}" + chmod 700 "${STATE_DIR}" + case "${COMMAND}" in + plan) require_env; cmd_plan ;; + up) require_env; cmd_up ;; + status) require_env; cmd_status ;; + ssh) cmd_ssh "$@" ;; + prepare) cmd_prepare || die "préparation du serveur échouée" ;; + push) cmd_push "$@" ;; + down) require_env; cmd_down ;; + session) require_env; cmd_session "$@" ;; + *) usage ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" || -z "${BASH_SOURCE[0]}" ]]; then + main "$@" +fi diff --git a/scripts/lab-host_test.sh b/scripts/lab-host_test.sh new file mode 100755 index 0000000..9db9776 --- /dev/null +++ b/scripts/lab-host_test.sh @@ -0,0 +1,816 @@ +#!/usr/bin/env bash + +SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lab-host.sh" +PASS=0 +FAIL=0 +SECRET="secret-de-test-a1b2c3" +PROJECT="11111111-1111-1111-1111-111111111111" +OTHER_PROJECT="22222222-2222-2222-2222-222222222222" +HOURLY_ID="offer-hourly-0001" +MONTHLY_ID="offer-monthly-0001" + +fail () { echo "FAIL: ${CURRENT}: ${1}"; FAIL=$(( FAIL + 1 )); CASE_OK=0; } + +write_fakes () { + local DIR="${1}" + mkdir -p "${DIR}/bin" + cat > "${DIR}/bin/curl" <<'PY' +#!/usr/bin/env python3 +import json, os, sys, time, urllib.parse + +d = os.environ["FAKE_DIR"] +args = sys.argv[1:] +with open(os.path.join(d, "argv.log"), "a") as f: + f.write(json.dumps(args) + "\n") + +method, body, headers, url = "GET", None, [], None +i = 0 +while i < len(args): + a = args[i] + if a == "-X": + method = args[i + 1]; i += 2; continue + if a == "-H": + h = args[i + 1] + if h.startswith("@"): + with open(h[1:]) as f: + headers += [l.strip() for l in f if l.strip()] + else: + headers.append(h) + i += 2; continue + if a == "--data": + body = args[i + 1]; i += 2; continue + if a in ("-w", "--max-time", "--connect-timeout"): + i += 2; continue + if a.startswith("-"): + i += 1; continue + url = a; i += 1 + +state_path = os.path.join(d, "state.json") +with open(state_path) as f: + st = json.load(f) + +def reply(code, obj): + with open(state_path, "w") as f: + json.dump(st, f) + sys.stdout.write(json.dumps(obj) + "\n" + str(code)) + sys.exit(0) + +u = urllib.parse.urlparse(url) +path = u.path +with open(os.path.join(d, "calls.log"), "a") as f: + f.write(method + " " + path + ("?" + u.query if u.query else "") + "\n") + +if "X-Auth-Token: " + os.environ["FAKE_SECRET"] not in headers: + reply(401, {"message": "unauthorized"}) + +zone = "/baremetal/v1/zones/fr-par-1" +if method == "GET" and path == zone + "/offers": + reply(200, {"offers": st["offers"], "total_count": len(st["offers"])}) +if method == "GET" and path == zone + "/os": + reply(200, {"os": st["os"], "total_count": len(st["os"])}) +if method == "GET" and path == "/iam/v1alpha1/ssh-keys": + reply(200, {"ssh_keys": st["keys"], "total_count": len(st["keys"])}) +if method == "GET" and path == zone + "/servers": + if st["list_errors"] > 0: + st["list_errors"] -= 1 + reply(503, {"message": "service unavailable"}) + listed = [v for v in st["servers"].values() if not v.get("hidden")] + reply(200, {"servers": listed, "total_count": len(listed)}) +if method == "POST" and path == zone + "/servers": + req = json.loads(body) + with open(os.path.join(d, "create.json"), "w") as f: + json.dump(req, f) + st["seq"] += 1 + sid = "srv-%d" % st["seq"] + st["servers"][sid] = {"id": sid, "name": req["name"], "project_id": req["project_id"], + "tags": req["tags"], "offer_name": "EM-B212X-SSD", "status": "delivering", + "install": {"status": "to_install"}, "ips": [], "created_at": "now", + "hidden": st["hide_from_list"]} + if st["post_fail_after_create"]: + reply(502, {"message": "bad gateway"}) + if st["post_no_id"]: + reply(200, {}) + reply(200, st["servers"][sid]) +if path.startswith(zone + "/servers/"): + sid = path.rsplit("/", 1)[1] + if sid not in st["servers"]: + reply(404, {"message": "not found"}) + srv = st["servers"][sid] + if method == "GET" and st["get_errors"] > 0: + st["get_errors"] -= 1 + reply(503, {"message": "service unavailable"}) + if method == "GET" and srv["status"] == "deleting": + if st["linger"] > 0: + st["linger"] -= 1 + reply(200, srv) + del st["servers"][sid] + reply(404, {"message": "not found"}) + if method == "GET" and st["vanish_after"] == 0: + del st["servers"][sid] + reply(404, {"message": "not found"}) + if method == "GET" and st["vanish_after"] > 0: + st["vanish_after"] -= 1 + if method == "GET": + if st["progress"]: + status, install = st["progress"].pop(0) + srv["status"] = status + srv["install"]["status"] = install + if status == "ready": + srv["ips"] = [{"address": "2001:db8::1", "version": "IPv6"}, + {"address": "203.0.113.7", "version": "IPv4"}] + reply(200, srv) + if method == "DELETE": + if time.time() < st["refuse_until"]: + reply(409, {"message": "server is installing"}) + if st["delete_refusals"] > 0: + st["delete_refusals"] -= 1 + reply(409, {"message": "server is installing"}) + if st["linger"] > 0: + srv["status"] = "deleting" + else: + del st["servers"][sid] + reply(200, srv) +reply(500, {"message": "route inconnue du faux serveur : " + method + " " + path}) +PY + cat > "${DIR}/bin/ssh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "${FAKE_DIR}/ssh.log" +printf 'key=%s\n' "${SCW_SECRET_KEY:-}" >> "${FAKE_DIR}/ssh-env.log" +case "$*" in + *BatchMode=yes*) + N=$(cat "${FAKE_DIR}/probe_fails" 2>/dev/null || echo 0) + if [ "${N}" -gt 0 ]; then + echo $(( N - 1 )) > "${FAKE_DIR}/probe_fails" + exit 255 + fi + exit 0 ;; + *"bash -s"*) + cat > "${FAKE_DIR}/prepare.sh" + exit "${FAKE_PREPARE_RC:-0}" ;; + *"cat > "*) + N=$(ls "${FAKE_DIR}" | grep -c '^pushed\.') + cat > "${FAKE_DIR}/pushed.$(( N + 1 ))" + exit "${FAKE_PUSH_RC:-0}" ;; +esac +[ -n "${FAKE_SSH_SLEEP:-}" ] && sleep "${FAKE_SSH_SLEEP}" +exit "${FAKE_SSH_RC:-0}" +SH + cat > "${DIR}/bin/go" <<'SH' +#!/usr/bin/env bash +printf '%s|%s|CGO_ENABLED=%s GOOS=%s GOARCH=%s\n' "${PWD}" "$*" "${CGO_ENABLED:-}" "${GOOS:-}" "${GOARCH:-}" >> "${FAKE_DIR}/go.log" +[ -n "${FAKE_GO_FAIL:-}" ] && exit 1 +while [ $# -gt 0 ]; do + [ "${1}" = "-o" ] && printf 'binaire-lab\n' > "${2}" + shift +done +exit 0 +SH + chmod +x "${DIR}/bin/curl" "${DIR}/bin/ssh" "${DIR}/bin/go" +} + +base_state () { + jq -n --arg h "${HOURLY_ID}" --arg m "${MONTHLY_ID}" --arg p "${PROJECT}" --arg o "${OTHER_PROJECT}" '{ + seq: 0, progress: [], delete_refusals: 0, linger: 0, get_errors: 0, list_errors: 0, + hide_from_list: false, refuse_until: 0, post_fail_after_create: false, post_no_id: false, vanish_after: -1, + offers: [ + {id: $m, name: "EM-B212X-SSD", subscription_period: "monthly", enable: true, stock: "available", + price_per_hour: null, price_per_month: {currency_code: "EUR", units: 115, nanos: 990000000}, + fee: {currency_code: "EUR", units: 115, nanos: 990000000}}, + {id: "offer-other", name: "EM-A610R-NVMe", subscription_period: "hourly", enable: true, stock: "available", + price_per_hour: {currency_code: "EUR", units: 0, nanos: 110000000}, fee: {currency_code: "EUR", units: 0, nanos: 0}}, + {id: $h, name: "EM-B212X-SSD", subscription_period: "hourly", enable: true, stock: "available", + price_per_hour: {currency_code: "EUR", units: 0, nanos: 321000000}, fee: {currency_code: "EUR", units: 0, nanos: 0}} + ], + os: [ + {id: "os-deb11", name: "Debian", version: "11 (Bullseye)", enabled: true, allowed: true, user: {default_value: "debian"}}, + {id: "os-deb12", name: "Debian", version: "12 (Bookworm)", enabled: true, allowed: true, user: {default_value: "debian"}}, + {id: "os-deb13", name: "Debian", version: "13 (Trixie)", enabled: true, allowed: true, user: {default_value: "debian"}}, + {id: "os-ubu", name: "Ubuntu", version: "24.04 LTS (Noble Numbat)", enabled: true, allowed: true, user: {default_value: "ubuntu"}} + ], + keys: [{id: "key-1", disabled: false}, {id: "key-off", disabled: true}], + servers: { + "foreign-1": {id: "foreign-1", name: "prod-db", project_id: $o, tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}, + "foreign-2": {id: "foreign-2", name: "autre", project_id: $p, tags: ["autre"], status: "ready", install: {status: "completed"}, ips: []} + } + }' +} + +setup () { + CURRENT="${1}" + CASE_OK=1 + WORK=$(mktemp -d "${TMPDIR:-/tmp}/labhost.XXXXXX") + write_fakes "${WORK}" + base_state > "${WORK}/state.json" + : > "${WORK}/calls.log" + : > "${WORK}/argv.log" + : > "${WORK}/ssh.log" + : > "${WORK}/ssh-env.log" +} + +mutate () { + local TMP + TMP=$(jq "${1}" "${WORK}/state.json") && printf '%s\n' "${TMP}" > "${WORK}/state.json" +} + +lab_env () { + local EXEC="" + [[ "${1}" == "exec" ]] && { EXEC="exec"; shift; } + ${EXEC} env -i PATH="${WORK}/bin:${PATH}" HOME="${WORK}" TMPDIR="${TMPDIR:-/tmp}" \ + FAKE_DIR="${WORK}" FAKE_SECRET="${SECRET}" FAKE_SSH_RC="${FAKE_SSH_RC:-0}" \ + FAKE_SSH_SLEEP="${FAKE_SSH_SLEEP:-}" FAKE_PREPARE_RC="${FAKE_PREPARE_RC:-0}" \ + FAKE_PUSH_RC="${FAKE_PUSH_RC:-0}" FAKE_GO_FAIL="${FAKE_GO_FAIL:-}" \ + SCW_API_URL="https://api.example.invalid" SCW_SECRET_KEY="${SECRET}" \ + SCW_DEFAULT_PROJECT_ID="${PROJECT}" LAB_POLL_INTERVAL="${LAB_POLL_INTERVAL:-0}" \ + LAB_INSTALL_TIMEOUT="${LAB_INSTALL_TIMEOUT:-60}" LAB_DELETE_TIMEOUT="${LAB_DELETE_TIMEOUT:-60}" \ + LAB_SSH_TIMEOUT="${LAB_SSH_TIMEOUT:-60}" \ + "$@" +} + +run_lab () { + lab_env bash "${SCRIPT}" "$@" > "${WORK}/out.log" 2>&1 +} + +remaining () { + jq -r '.servers | keys | sort | join(",")' "${WORK}/state.json" +} + +teardown () { + [[ "${CASE_OK}" -eq 1 ]] && PASS=$(( PASS + 1 )) && echo "ok: ${CURRENT}" + [[ "${CASE_OK}" -eq 1 ]] || { echo " sortie :"; sed 's/^/ | /' "${WORK}/out.log"; } + rm -rf "${WORK}" +} + +test_plan_picks_hourly_offer_even_when_monthly_is_listed_first () { + setup "plan : offre horaire choisie même si la mensuelle est listée d'abord" + run_lab plan || fail "code de sortie $?" + grep -q "\"offer_id\": \"${HOURLY_ID}\"" "${WORK}/out.log" || fail "offre horaire absente de la requête" + grep -q "${MONTHLY_ID}" "${WORK}/out.log" && fail "offre mensuelle présente dans la sortie" + grep -q "^POST" "${WORK}/calls.log" && fail "plan a créé quelque chose" + teardown +} + +test_plan_selects_debian_12_only () { + setup "plan : Debian 12 et rien d'autre" + run_lab plan || fail "code de sortie $?" + grep -q '"os_id": "os-deb12"' "${WORK}/out.log" || fail "os-deb12 non retenu" + teardown +} + +test_plan_passes_only_enabled_ssh_keys () { + setup "plan : seules les clés SSH actives sont transmises" + run_lab plan || fail "code de sortie $?" + grep -q '"key-1"' "${WORK}/out.log" || fail "key-1 absente" + grep -q '"key-off"' "${WORK}/out.log" && fail "clé désactivée transmise" + teardown +} + +test_refuses_offer_with_setup_fee () { + setup "refus d'une offre horaire avec frais de mise en service" + mutate '(.offers[] | select(.id == "'"${HOURLY_ID}"'") | .fee.units) = 30' + run_lab up && fail "up a réussi malgré des frais" + grep -q "^POST" "${WORK}/calls.log" && fail "serveur créé malgré des frais" + teardown +} + +test_refuses_when_hourly_offer_missing () { + setup "refus quand seule l'offre mensuelle existe" + mutate 'del(.offers[] | select(.id == "'"${HOURLY_ID}"'"))' + run_lab up && fail "up a réussi sans offre horaire" + grep -q "^POST" "${WORK}/calls.log" && fail "serveur créé sur une offre mensuelle" + teardown +} + +test_refuses_ambiguous_os () { + setup "refus d'un OS ambigu" + mutate '.os += [{id: "os-deb12b", name: "Debian", version: "12 (Bookworm)", enabled: true, allowed: true}]' + run_lab plan && fail "plan a réussi avec deux Debian 12" + teardown +} + +test_refuses_out_of_stock () { + setup "refus d'une offre en rupture" + mutate '(.offers[] | select(.id == "'"${HOURLY_ID}"'") | .stock) = "empty"' + run_lab up && fail "up a réussi en rupture de stock" + grep -q "^POST" "${WORK}/calls.log" && fail "serveur créé en rupture de stock" + teardown +} + +test_up_creates_and_waits_for_installation () { + setup "up : création puis attente de la fin d'installation" + mutate '.progress = [["delivering","to_install"],["ready","installing"],["ready","completed"]]' + run_lab up || fail "code de sortie $?" + [[ $(jq -r '.offer_id' "${WORK}/create.json") == "${HOURLY_ID}" ]] || fail "offre créée incorrecte" + [[ $(jq -r '.tags | join(",")' "${WORK}/create.json") == "two-lab" ]] || fail "tag absent" + [[ $(jq -r '.project_id' "${WORK}/create.json") == "${PROJECT}" ]] || fail "projet incorrect" + [[ $(cat "${WORK}/.cache/two-lab/ip" 2>/dev/null) == "203.0.113.7" ]] || fail "IPv4 non retenue" + [[ $(cat "${WORK}/.cache/two-lab/user" 2>/dev/null) == "debian" ]] || fail "utilisateur non retenu" + teardown +} + +test_up_refuses_when_lab_server_exists () { + setup "up : refus si un serveur de lab existe déjà" + mutate '.servers["srv-old"] = {id: "srv-old", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + run_lab up && fail "up a réussi" + grep -q "^POST" "${WORK}/calls.log" && fail "second serveur créé" + teardown +} + +test_secret_never_in_argv () { + setup "la clé d'API n'apparaît jamais dans les arguments de curl" + mutate '.progress = [["ready","completed"]]' + run_lab up || fail "code de sortie $?" + grep -q "${SECRET}" "${WORK}/argv.log" && fail "secret visible dans argv" + grep -q "${SECRET}" "${WORK}/out.log" && fail "secret affiché" + teardown +} + +test_down_deletes_only_lab_servers_of_project () { + setup "down : ne supprime que les serveurs de lab du projet" + mutate '.servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + run_lab down || fail "code de sortie $?" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_down_retries_refused_deletion () { + setup "down : réessaie une suppression refusée" + mutate '.delete_refusals = 2 | .servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "installing"}, ips: []}' + run_lab down || fail "code de sortie $?" + [[ $(grep -c "^DELETE" "${WORK}/calls.log") -eq 3 ]] || fail "$(grep -c "^DELETE" "${WORK}/calls.log") DELETE au lieu de 3" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_down_fails_loudly_when_deletion_keeps_failing () { + setup "down : échec bruyant si la suppression n'aboutit jamais" + mutate '.delete_refusals = 1000 | .servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "installing"}, ips: []}' + LAB_DELETE_TIMEOUT=1 run_lab down && fail "down a réussi" + grep -q "TOUJOURS FACTURÉ" "${WORK}/out.log" || fail "pas d'alerte de facturation" + teardown +} + +test_down_waits_until_server_is_gone () { + setup "down : attend que le serveur ait disparu" + mutate '.linger = 3 | .servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + run_lab down || fail "code de sortie $?" + [[ $(grep -c "^GET /baremetal/v1/zones/fr-par-1/servers/srv-lab$" "${WORK}/calls.log") -ge 4 ]] || fail "disparition non attendue" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_down_fails_loudly_when_server_never_disappears () { + setup "down : échec bruyant si le serveur ne disparaît jamais" + mutate '.linger = 100000 | .servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + LAB_DELETE_TIMEOUT=1 run_lab down && fail "down a réussi" + grep -q "TOUJOURS FACTURÉ" "${WORK}/out.log" || fail "pas d'alerte de facturation" + teardown +} + +test_down_does_not_mistake_api_error_for_disappearance () { + setup "down : une erreur de l'API n'est pas prise pour une disparition" + mutate '.linger = 100000 | .get_errors = 1 | .servers["srv-lab"] = {id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + LAB_DELETE_TIMEOUT=1 run_lab down && fail "down a réussi alors que le serveur existe encore" + grep -q "TOUJOURS FACTURÉ" "${WORK}/out.log" || fail "pas d'alerte de facturation" + teardown +} + +test_session_deletes_server_after_remote_command () { + setup "session : serveur supprimé après la commande distante" + mutate '.progress = [["ready","completed"]]' + run_lab session uname -a || fail "code de sortie $?" + grep -q "uname -a" "${WORK}/ssh.log" || fail "commande distante non lancée" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_session_deletes_server_and_propagates_failure_of_remote_command () { + setup "session : serveur supprimé et code d'échec propagé" + mutate '.progress = [["ready","completed"]]' + FAKE_SSH_RC=7 run_lab session false + [[ $? -eq 7 ]] || fail "code de sortie différent de 7" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_session_deletes_server_when_installation_fails () { + setup "session : serveur supprimé quand l'installation échoue" + mutate '.progress = [["ready","installing"],["ready","error"]]' + run_lab session true && fail "session a réussi" + grep -q "true" "${WORK}/ssh.log" && fail "commande lancée sur un serveur en échec" + grep -q "état terminal en échec : ready/error" "${WORK}/out.log" || fail "échec d'installation non reconnu comme terminal" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_session_deletes_server_when_installation_times_out () { + setup "session : serveur supprimé quand l'installation n'aboutit pas" + mutate '.progress = [range(0; 100000) | ["delivering","to_install"]]' + LAB_INSTALL_TIMEOUT=1 run_lab session true && fail "session a réussi" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_session_does_not_touch_existing_lab_server () { + setup "session : ne supprime pas un serveur de lab lancé à côté" + mutate '.servers["srv-old"] = {id: "srv-old", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' + run_lab session true && fail "session a réussi" + grep -q "^DELETE" "${WORK}/calls.log" && fail "suppression lancée" + [[ $(remaining) == "foreign-1,foreign-2,srv-old" ]] || fail "restants : $(remaining)" + teardown +} + +test_missing_credentials_are_refused_before_any_call () { + setup "refus sans clé d'API, avant tout appel" + env -i PATH="${WORK}/bin:${PATH}" HOME="${WORK}" FAKE_DIR="${WORK}" SCW_DEFAULT_PROJECT_ID="${PROJECT}" bash "${SCRIPT}" plan > "${WORK}/out.log" 2>&1 \ + && fail "plan a réussi sans clé" + [[ -s "${WORK}/calls.log" ]] && fail "appel réseau sans clé" + teardown +} + +LAB_SERVER='{id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "completed"}, ips: []}' +INSTALLING_LAB_SERVER='{id: "srv-lab", name: "two-lab", project_id: "'"${PROJECT}"'", tags: ["two-lab"], status: "ready", install: {status: "installing"}, ips: []}' + +test_down_keeps_retrying_while_server_is_installing () { + setup "down : réessaie au-delà du délai tant que le serveur s'installe" + mutate ".servers[\"srv-lab\"] = ${INSTALLING_LAB_SERVER} | .refuse_until = (now + 5)" + LAB_DELETE_TIMEOUT=2 LAB_POLL_INTERVAL=0.2 run_lab down || fail "code de sortie $?" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + grep -q "pendant la livraison ou l'installation" "${WORK}/out.log" || fail "installation non reconnue" + teardown +} + +test_down_gives_up_on_installing_server_after_hard_deadline () { + local RC + setup "down : abandonne quand même après délai d'installation + délai de suppression" + mutate ".servers[\"srv-lab\"] = ${INSTALLING_LAB_SERVER} | .refuse_until = (now + 100000)" + LAB_DELETE_TIMEOUT=2 LAB_INSTALL_TIMEOUT=2 LAB_POLL_INTERVAL=0.2 lab_env timeout 30 bash "${SCRIPT}" down > "${WORK}/out.log" 2>&1 + RC=$? + [[ "${RC}" -ne 124 ]] || fail "aucun plafond : down tourne encore après 30 s" + [[ "${RC}" -ne 0 ]] || fail "down a réussi" + grep -q "TOUJOURS FACTURÉ" "${WORK}/out.log" || fail "pas d'alerte de facturation" + teardown +} + +test_down_retries_listing_servers () { + setup "down : réessaie la liste des serveurs" + mutate ".servers[\"srv-lab\"] = ${LAB_SERVER} | .list_errors = 2" + run_lab down || fail "code de sortie $?" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_down_fails_loudly_when_listing_never_works () { + setup "down : échec bruyant si la liste reste indisponible" + mutate ".servers[\"srv-lab\"] = ${LAB_SERVER} | .list_errors = 100000" + LAB_DELETE_TIMEOUT=1 run_lab down && fail "down a réussi" + grep -q "TOUJOURS FACTURÉ" "${WORK}/out.log" || fail "pas d'alerte de facturation" + teardown +} + +test_session_deletes_created_server_even_if_missing_from_list () { + setup "session : supprime le serveur créé même absent de la liste" + mutate '.hide_from_list = true | .progress = [["ready","completed"]]' + run_lab session true || fail "code de sortie $?" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_every_api_call_has_timeouts () { + setup "chaque appel à l'API a un délai de connexion et un délai total" + mutate '.progress = [["ready","completed"]]' + run_lab up || fail "code de sortie $?" + [[ -s "${WORK}/argv.log" ]] || fail "aucun appel" + [[ $(jq -s '[.[] | select((index("--max-time") | not) or (index("--connect-timeout") | not))] | length' "${WORK}/argv.log") -eq 0 ]] \ + || fail "appel sans délai" + teardown +} + +test_up_reports_uncertain_creation_when_post_fails () { + setup "up : création incertaine signalée quand le POST échoue" + mutate '.post_fail_after_create = true' + run_lab up && fail "up a réussi" + grep -q "création incertaine" "${WORK}/out.log" || fail "pas de message de création incertaine" + grep -q "un serveur de lab existe bien : srv-1" "${WORK}/out.log" || fail "serveur créé non signalé" + teardown +} + +test_session_deletes_server_when_post_fails_after_creation () { + setup "session : serveur supprimé quand le POST échoue après création" + mutate '.post_fail_after_create = true' + run_lab session true && fail "session a réussi" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_up_refuses_response_without_id () { + setup "up : réponse de création sans identifiant refusée" + mutate '.post_no_id = true' + run_lab up && fail "up a réussi" + grep -q "création incertaine" "${WORK}/out.log" || fail "pas de message de création incertaine" + grep -q "servers/null" "${WORK}/calls.log" && fail "appel sur un identifiant null" + teardown +} + +test_up_stops_at_once_when_server_vanishes () { + setup "up : arrêt immédiat si le serveur disparaît pendant l'installation" + mutate '.vanish_after = 1 | .progress = [["delivering","to_install"]]' + LAB_INSTALL_TIMEOUT=5 run_lab up && fail "up a réussi" + grep -q "erreur définitive HTTP 404" "${WORK}/out.log" || fail "404 non traité comme définitif" + teardown +} + +test_up_stops_at_once_when_server_is_stopped_during_installation () { + setup "up : arrêt immédiat si le serveur passe à l'arrêt pendant l'installation" + mutate '.progress = [["delivering","to_install"],["stopped","installing"]]' + LAB_INSTALL_TIMEOUT=5 run_lab up && fail "up a réussi" + grep -q "état terminal en échec : stopped/installing" "${WORK}/out.log" || fail "arrêt non reconnu comme terminal" + teardown +} + +test_session_cleanup_survives_a_second_signal () { + local PID RC I + setup "session : un second signal n'interrompt pas le nettoyage" + mutate '.progress = [["ready","completed"]] | .linger = 15' + set -m + FAKE_SSH_SLEEP=5 LAB_POLL_INTERVAL=0.2 lab_env exec bash "${SCRIPT}" session sleep-long > "${WORK}/out.log" 2>&1 & + PID=$! + set +m + for I in $(seq 1 100); do + grep -q "sleep-long" "${WORK}/ssh.log" && break + sleep 0.1 + done + kill -TERM -- "-${PID}" + for I in $(seq 1 100); do + grep -q "suppression de srv-1" "${WORK}/out.log" && break + sleep 0.1 + done + grep -q "suppression de srv-1" "${WORK}/out.log" || fail "nettoyage jamais commencé" + kill -INT -- "-${PID}" 2>/dev/null + kill -TERM -- "-${PID}" 2>/dev/null + wait "${PID}" + RC=$? + [[ "${RC}" -eq 143 ]] || fail "code ${RC} au lieu de 143" + grep -q "aucun serveur de lab ne reste" "${WORK}/out.log" || fail "nettoyage interrompu" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_ssh_never_receives_api_key () { + setup "ssh ne reçoit jamais la clé d'API dans son environnement" + mutate '.progress = [["ready","completed"]]' + run_lab session uname || fail "code de sortie $?" + [[ -s "${WORK}/ssh-env.log" ]] || fail "ssh jamais lancé" + grep -qv '^key=$' "${WORK}/ssh-env.log" && fail "clé transmise à ssh" + teardown +} + +test_session_waits_for_ssh_before_running_command () { + setup "session : attend que SSH réponde avant la commande" + mutate '.progress = [["ready","completed"]]' + echo 3 > "${WORK}/probe_fails" + run_lab session uname || fail "code de sortie $?" + [[ $(grep -c "BatchMode=yes" "${WORK}/ssh.log") -eq 4 ]] || fail "$(grep -c "BatchMode=yes" "${WORK}/ssh.log") sondes au lieu de 4" + [[ $(tail -n 1 "${WORK}/ssh.log") == *" uname" ]] || fail "commande non lancée en dernier" + teardown +} + +test_session_deletes_server_when_ssh_never_answers () { + setup "session : serveur supprimé si SSH ne répond jamais" + mutate '.progress = [["ready","completed"]]' + echo 100000 > "${WORK}/probe_fails" + LAB_SSH_TIMEOUT=1 LAB_POLL_INTERVAL=0.2 run_lab session uname && fail "session a réussi" + grep -qv "BatchMode=yes" "${WORK}/ssh.log" && fail "commande lancée sans SSH joignable" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_ssh_uses_dedicated_lab_key_only_when_present () { + setup "ssh : clé dédiée du lab utilisée seule quand elle existe" + mutate '.progress = [["ready","completed"]]' + mkdir -p "${WORK}/.config/two-lab/ssh" + : > "${WORK}/.config/two-lab/ssh/lab_ed25519" + run_lab session uname || fail "code de sortie $?" + grep -q " uname" "${WORK}/ssh.log" || fail "commande non lancée" + [[ $(grep -c -- "-i ${WORK}/.config/two-lab/ssh/lab_ed25519 -o IdentitiesOnly=yes -o IdentityAgent=none" "${WORK}/ssh.log") -eq $(wc -l < "${WORK}/ssh.log") ]] \ + || fail "une connexion n'a pas utilisé la clé dédiée seule" + teardown +} + +test_ssh_falls_back_to_agent_without_lab_key () { + setup "ssh : agent SSH utilisé quand la clé dédiée est absente" + mutate '.progress = [["ready","completed"]]' + run_lab session uname || fail "code de sortie $?" + grep -q -- "-i " "${WORK}/ssh.log" && fail "clé imposée alors qu'elle n'existe pas" + grep -q "IdentityAgent=none" "${WORK}/ssh.log" && fail "agent désactivé sans clé dédiée" + teardown +} + +known_server () { + mkdir -p "${WORK}/.cache/two-lab" + echo 203.0.113.7 > "${WORK}/.cache/two-lab/ip" + echo debian > "${WORK}/.cache/two-lab/user" + : > "${WORK}/.cache/two-lab/known_hosts" +} + +test_up_prepares_the_server_after_ssh () { + setup "up : prépare le serveur (qemu, genisoimage, KVM imbriqué) une fois SSH joignable" + mutate '.progress = [["ready","completed"]]' + run_lab up || fail "code de sortie $?" + [[ $(tail -n 1 "${WORK}/ssh.log") == *"debian@203.0.113.7 bash -s" ]] || fail "préparation non lancée en dernier : $(tail -n 1 "${WORK}/ssh.log")" + grep -q "apt-get install -y -qq --no-install-recommends qemu-system-x86 qemu-utils genisoimage" "${WORK}/prepare.sh" || fail "paquets absents du script" + grep -q 'usermod -aG kvm' "${WORK}/prepare.sh" || fail "groupe kvm absent du script" + grep -q -- '-c /dev/kvm' "${WORK}/prepare.sh" || fail "contrôle de /dev/kvm absent" + grep -q 'kvm_intel/parameters/nested' "${WORK}/prepare.sh" || fail "contrôle de nested absent" + teardown +} + +test_up_reports_a_billed_server_when_preparation_fails () { + setup "up : préparation échouée signalée, serveur toujours facturé" + mutate '.progress = [["ready","completed"]]' + FAKE_PREPARE_RC=1 run_lab up && fail "up a réussi" + grep -q "préparation du serveur échouée — il est toujours facturé" "${WORK}/out.log" || fail "message absent" + grep -q "prêt :" "${WORK}/out.log" && fail "serveur annoncé prêt" + teardown +} + +test_session_deletes_server_when_preparation_fails () { + setup "session : serveur supprimé si la préparation échoue, commande jamais lancée" + mutate '.progress = [["ready","completed"]]' + FAKE_PREPARE_RC=1 run_lab session uname && fail "session a réussi" + grep -q " uname" "${WORK}/ssh.log" && fail "commande lancée malgré la préparation échouée" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_prepare_script_is_valid_shell () { + setup "prepare : le script distant est du shell valide" + known_server + run_lab prepare || fail "code de sortie $?" + bash -n "${WORK}/prepare.sh" || fail "bash -n refuse le script" + sh -n "${WORK}/prepare.sh" || fail "sh -n refuse le script" + teardown +} + +test_prepare_without_known_server_is_refused () { + setup "prepare : refusé sans serveur connu" + run_lab prepare && fail "prepare a réussi" + [[ -s "${WORK}/ssh.log" ]] && fail "ssh lancé" + teardown +} + +test_push_builds_for_linux_and_sends_binary_and_topology () { + setup "push : compile lab pour linux/amd64, envoie le binaire et la topologie" + known_server + printf 'name: evpn-2hv\n' > "${WORK}/evpn-2hv.yml" + run_lab push "${WORK}/evpn-2hv.yml" || fail "code de sortie $?" + local REPO + REPO="$(cd "$(dirname "${SCRIPT}")/.." && pwd)" + [[ $(cat "${WORK}/go.log") == "${REPO}|build -o ${WORK}/.cache/two-lab/lab ./cmd/lab|CGO_ENABLED=0 GOOS=linux GOARCH=amd64" ]] \ + || fail "compilation : $(cat "${WORK}/go.log")" + grep -q "debian@203.0.113.7 cat > 'lab.part' && chmod 755 'lab.part' && mv 'lab.part' 'lab'" "${WORK}/ssh.log" || fail "envoi de lab absent" + grep -q "debian@203.0.113.7 cat > 'evpn-2hv.yml.part' && chmod 644 'evpn-2hv.yml.part' && mv 'evpn-2hv.yml.part' 'evpn-2hv.yml'" "${WORK}/ssh.log" || fail "envoi de la topologie absent" + [[ $(cat "${WORK}/pushed.1") == "binaire-lab" ]] || fail "contenu de lab : $(cat "${WORK}/pushed.1")" + [[ $(cat "${WORK}/pushed.2") == "name: evpn-2hv" ]] || fail "contenu de la topologie : $(cat "${WORK}/pushed.2")" + teardown +} + +test_push_stops_when_the_build_fails () { + setup "push : rien n'est envoyé si la compilation échoue" + known_server + printf 'name: x\n' > "${WORK}/x.yml" + FAKE_GO_FAIL=1 run_lab push "${WORK}/x.yml" && fail "push a réussi" + [[ -s "${WORK}/ssh.log" ]] && fail "ssh lancé" + grep -q "compilation de lab échouée" "${WORK}/out.log" || fail "message absent" + teardown +} + +test_push_fails_when_a_transfer_fails () { + setup "push : un envoi échoué fait échouer push" + known_server + printf 'name: x\n' > "${WORK}/x.yml" + FAKE_PUSH_RC=1 run_lab push "${WORK}/x.yml" && fail "push a réussi" + grep -q "envoi de lab échoué" "${WORK}/out.log" || fail "message absent" + teardown +} + +test_push_refusals () { + local CASE + for CASE in absent quote nothing; do + setup "push : refus (${CASE})" + known_server + case "${CASE}" in + absent) run_lab push "${WORK}/absent.yml" && fail "push a réussi" ;; + quote) printf 'x\n' > "${WORK}/a'b.yml"; run_lab push "${WORK}/a'b.yml" && fail "push a réussi" ;; + nothing) run_lab push && fail "push a réussi" ;; + esac + [[ -s "${WORK}/go.log" ]] && fail "compilation lancée" + [[ -s "${WORK}/ssh.log" ]] && fail "ssh lancé" + teardown + done +} + +test_ssh_without_terminal_does_not_ask_for_one () { + setup "ssh : pas de -t quand l'entrée n'est pas un terminal" + known_server + run_lab ssh './lab status' < /dev/null || fail "code de sortie $?" + [[ $(cat "${WORK}/ssh.log") == *"debian@203.0.113.7 ./lab status" ]] || fail "commande : $(cat "${WORK}/ssh.log")" + grep -q -- " -t " "${WORK}/ssh.log" && fail "-t demandé sans terminal" + teardown +} + +run_lab_without_env_credentials () { + env -i PATH="${WORK}/bin:${PATH}" HOME="${WORK}" TMPDIR="${TMPDIR:-/tmp}" \ + FAKE_DIR="${WORK}" FAKE_SECRET="${SECRET}" SCW_API_URL="https://api.example.invalid" \ + LAB_POLL_INTERVAL=0 "$@" bash "${SCRIPT}" plan > "${WORK}/out.log" 2>&1 +} + +write_credentials () { + mkdir -p "${WORK}/.config/two-lab" + printf '%s\n' "$@" > "${WORK}/.config/two-lab/scaleway.env" + chmod 600 "${WORK}/.config/two-lab/scaleway.env" +} + +test_credentials_file_is_used_when_environment_is_empty () { + setup "identifiants : fichier lu quand l'environnement est vide" + write_credentials "SCW_SECRET_KEY=${SECRET}" "SCW_DEFAULT_PROJECT_ID=${PROJECT}" "SCW_DEFAULT_ZONE=fr-par-1" + run_lab_without_env_credentials || fail "code de sortie $?" + grep -q "\"project_id\": \"${PROJECT}\"" "${WORK}/out.log" || fail "projet du fichier non utilisé" + grep -q "${SECRET}" "${WORK}/out.log" && fail "secret affiché" + teardown +} + +test_credentials_file_without_final_newline_is_read () { + setup "identifiants : dernière ligne sans retour à la ligne lue quand même" + mkdir -p "${WORK}/.config/two-lab" + printf 'SCW_DEFAULT_PROJECT_ID=%s\nSCW_SECRET_KEY=%s' "${PROJECT}" "${SECRET}" > "${WORK}/.config/two-lab/scaleway.env" + chmod 600 "${WORK}/.config/two-lab/scaleway.env" + run_lab_without_env_credentials || fail "code de sortie $?" + teardown +} + +test_environment_wins_over_credentials_file () { + setup "identifiants : l'environnement l'emporte sur le fichier" + write_credentials "SCW_SECRET_KEY=mauvais-secret" "SCW_DEFAULT_PROJECT_ID=${OTHER_PROJECT}" + run_lab_without_env_credentials SCW_SECRET_KEY="${SECRET}" SCW_DEFAULT_PROJECT_ID="${PROJECT}" || fail "code de sortie $?" + grep -q "\"project_id\": \"${PROJECT}\"" "${WORK}/out.log" || fail "projet de l'environnement non retenu" + teardown +} + +test_environment_value_kept_when_file_completes_the_rest () { + setup "identifiants : une valeur de l'environnement n'est pas écrasée quand le fichier complète" + write_credentials "SCW_SECRET_KEY=${SECRET}" "SCW_DEFAULT_PROJECT_ID=${OTHER_PROJECT}" + run_lab_without_env_credentials SCW_DEFAULT_PROJECT_ID="${PROJECT}" || fail "code de sortie $?" + grep -q "\"project_id\": \"${PROJECT}\"" "${WORK}/out.log" || fail "projet de l'environnement écrasé par le fichier" + teardown +} + +test_credentials_file_readable_by_others_is_refused () { + setup "identifiants : fichier lisible par d'autres refusé, sans aucun appel" + write_credentials "SCW_SECRET_KEY=${SECRET}" "SCW_DEFAULT_PROJECT_ID=${PROJECT}" + chmod 644 "${WORK}/.config/two-lab/scaleway.env" + run_lab_without_env_credentials && fail "plan a réussi" + grep -q "chmod 600" "${WORK}/out.log" || fail "pas de consigne de droits" + [[ -s "${WORK}/calls.log" ]] && fail "appel réseau avec un fichier exposé" + teardown +} + +test_credentials_file_is_never_executed () { + setup "identifiants : le fichier est lu, jamais exécuté" + write_credentials "SCW_SECRET_KEY=\$(touch ${WORK}/pwned)" "touch ${WORK}/pwned2" "SCW_DEFAULT_PROJECT_ID=${PROJECT}" + run_lab_without_env_credentials && fail "plan a réussi avec un secret invalide" + [[ -e "${WORK}/pwned" || -e "${WORK}/pwned2" ]] && fail "contenu du fichier exécuté" + teardown +} + +signal_during_session () { + local SIGNAL="${1}" + local EXPECTED="${2}" + local PID RC I + setup "session : ${SIGNAL} pendant la commande distante, serveur supprimé, code ${EXPECTED}" + mutate '.progress = [["ready","completed"]]' + set -m + FAKE_SSH_SLEEP=5 lab_env exec bash "${SCRIPT}" session sleep-long > "${WORK}/out.log" 2>&1 & + PID=$! + set +m + for I in $(seq 1 100); do + grep -q "sleep-long" "${WORK}/ssh.log" && break + sleep 0.1 + done + grep -q "sleep-long" "${WORK}/ssh.log" || fail "commande distante jamais lancée" + kill "-${SIGNAL}" -- "-${PID}" + wait "${PID}" + RC=$? + [[ "${RC}" -eq "${EXPECTED}" ]] || fail "code ${RC} au lieu de ${EXPECTED}" + [[ $(remaining) == "foreign-1,foreign-2" ]] || fail "restants : $(remaining)" + teardown +} + +test_session_handles_term () { signal_during_session TERM 143; } +test_session_handles_hup () { signal_during_session HUP 129; } +test_session_handles_int () { signal_during_session INT 130; } + +for T in $(declare -F | awk '{print $3}' | grep '^test_'); do + "${T}" +done +echo "${PASS} réussi(s), ${FAIL} échec(s)" +[[ "${FAIL}" -eq 0 ]]