From 1146d13d45d7a19cf97d0c59ba51ca9e448342a7 Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 4 Oct 2026 14:49:34 +0200 Subject: [PATCH] =?UTF-8?q?f-50:=20lab:=20lancement,=20=C3=A9tat,=20arr?= =?UTF-8?q?=C3=AAt=20et=20acc=C3=A8s=20SSH=20des=20VM=20#50?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: GnomeZworc --- cmd/lab/lifecycle.go | 193 ++++++++++ cmd/lab/main.go | 24 ++ cmd/lab/main_test.go | 180 +++++++++ cmd/lab/terminal.go | 14 + cmd/lab/terminal_darwin.go | 5 + cmd/lab/terminal_linux.go | 5 + cmd/lab/terminal_other.go | 7 + docs/developpement/lab.rst | 73 +++- internal/lab/machine/machine.go | 269 +++++++++++++ internal/lab/machine/machine_test.go | 543 +++++++++++++++++++++++++++ 10 files changed, 1309 insertions(+), 4 deletions(-) create mode 100644 cmd/lab/lifecycle.go create mode 100644 cmd/lab/terminal.go create mode 100644 cmd/lab/terminal_darwin.go create mode 100644 cmd/lab/terminal_linux.go create mode 100644 cmd/lab/terminal_other.go create mode 100644 internal/lab/machine/machine.go create mode 100644 internal/lab/machine/machine_test.go diff --git a/cmd/lab/lifecycle.go b/cmd/lab/lifecycle.go new file mode 100644 index 0000000..c91205e --- /dev/null +++ b/cmd/lab/lifecycle.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "time" + + "git.g3e.fr/syonad/two/internal/lab/machine" + "git.g3e.fr/syonad/two/internal/lab/provision" +) + +const ( + topologyFile = "topology.yml" + pollInterval = 5 * time.Second + stopTimeout = 30 * time.Second +) + +var ( + execve = syscall.Exec + procDir = "/proc" +) + +func defaultDir(parts ...string) string { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(append([]string{home}, parts...)...) +} + +func flags(name string, stderr io.Writer) (*flag.FlagSet, *string) { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, usage) } + return fs, fs.String("run", defaultDir("lab-run"), "run directory of the lab") +} + +func lab(runDir string, plan string, stdout, stderr io.Writer) (machine.Lab, bool) { + dir, err := filepath.Abs(runDir) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return machine.Lab{}, false + } + p, ok := load(plan, stderr) + if !ok { + return machine.Lab{}, false + } + return machine.Lab{ + Plan: p, + RunDir: dir, + ProcDir: procDir, + Runner: provision.ExecRunner{}, + Poll: pollInterval, + Stop: stopTimeout, + Out: stdout, + }, true +} + +func current(runDir string, stdout, stderr io.Writer) (machine.Lab, bool) { + saved := filepath.Join(runDir, topologyFile) + if _, err := os.Stat(saved); err != nil { + fmt.Fprintf(stderr, "lab: no lab in %s: %v\n", runDir, err) + return machine.Lab{}, false + } + return lab(runDir, saved, stdout, stderr) +} + +func upCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("up", stderr) + cacheDir := fs.String("cache", defaultDir(".cache", "two-lab"), "image cache directory") + timeout := fs.Duration("timeout", 20*time.Minute, "how long to wait for the nodes to be ready") + if err := fs.Parse(args); err != nil || fs.NArg() != 1 { + fs.Usage() + return 2 + } + cache, err := filepath.Abs(*cacheDir) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + + if _, err := os.Stat(filepath.Join(*runDir, topologyFile)); err == nil { + previous, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + running, err := previous.Running() + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if len(running) > 0 { + fmt.Fprintf(stderr, "lab: lab %s is still running in %s: 'lab down' first\n", previous.Plan.Name, previous.RunDir) + return 1 + } + } + + l, ok := lab(*runDir, fs.Arg(0), stdout, stderr) + if !ok { + return 1 + } + source, err := os.ReadFile(fs.Arg(0)) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := os.MkdirAll(l.RunDir, 0o700); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := os.WriteFile(filepath.Join(l.RunDir, topologyFile), source, 0o600); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + fetcher := provision.Fetcher{Client: &http.Client{}, CacheDir: cache} + if err := l.Up(ctx, fetcher, *timeout); err != nil { + fmt.Fprintf(stderr, "lab: %v\nlab: started nodes keep running: 'lab status', 'lab down'\n", err) + return 1 + } + return 0 +} + +func statusCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("status", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() != 0 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + if err := l.Status(stdout); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} + +func downCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("down", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() != 0 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + if err := l.Down(context.Background()); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} + +func sshCmd(args []string, stdout, stderr io.Writer) int { + fs, runDir := flags("ssh", stderr) + if err := fs.Parse(args); err != nil || fs.NArg() < 1 { + fs.Usage() + return 2 + } + l, ok := current(*runDir, stdout, stderr) + if !ok { + return 1 + } + argv, err := l.SSH(fs.Arg(0), isTerminal(os.Stdin), fs.Args()[1:]) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + path, err := exec.LookPath(argv[0]) + if err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + if err := execve(path, argv, os.Environ()); err != nil { + fmt.Fprintf(stderr, "lab: %v\n", err) + return 1 + } + return 0 +} diff --git a/cmd/lab/main.go b/cmd/lab/main.go index 13db735..3dc6425 100644 --- a/cmd/lab/main.go +++ b/cmd/lab/main.go @@ -21,6 +21,22 @@ const usage = `usage: lab [options] [dir] render -key write, for each node, //qemu.args (one argument per line) and the cloud-init seed files meta-data, user-data and network-config + + up [-run dir] [-cache dir] [-timeout duration] + download and verify the images, create fresh disks and seeds, start every node + (switches first) and wait until cloud-init has finished on each of them; + the topology is kept in /topology.yml for the commands below + + status [-run dir] + list the nodes of the lab and whether their qemu process is running + + down [-run dir] + stop every qemu process of the lab (SIGTERM, then SIGKILL) + + ssh [-run dir] [command...] + open a shell on a node, or run a command on it, with the key generated by up + + defaults: -run $HOME/lab-run, -cache $HOME/.cache/two-lab, -timeout 20m ` type keyFiles []string @@ -46,6 +62,14 @@ func run(args []string, stdout, stderr io.Writer) int { return plan(args[1], stdout, stderr) case "render": return renderCmd(args[1:], stdout, stderr) + case "up": + return upCmd(args[1:], stdout, stderr) + case "status": + return statusCmd(args[1:], stdout, stderr) + case "down": + return downCmd(args[1:], stdout, stderr) + case "ssh": + return sshCmd(args[1:], stdout, stderr) default: fmt.Fprint(stderr, usage) return 2 diff --git a/cmd/lab/main_test.go b/cmd/lab/main_test.go index a1cc588..e8972b8 100644 --- a/cmd/lab/main_test.go +++ b/cmd/lab/main_test.go @@ -3,8 +3,11 @@ package main import ( "bytes" "os" + "os/exec" "path/filepath" + "strconv" "strings" + "syscall" "testing" ) @@ -119,3 +122,180 @@ func TestRun_RenderRefusesMissingKeyFile(t *testing.T) { t.Errorf("code %d, stderr %q", code, stderr) } } + +func savedLab(t *testing.T) string { + t.Helper() + run := filepath.Join(t.TempDir(), "run") + if err := os.MkdirAll(run, 0o700); err != nil { + t.Fatal(err) + } + example, err := os.ReadFile(filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(run, "topology.yml"), example, 0o600); err != nil { + t.Fatal(err) + } + return run +} + +func TestRun_LifecycleUsage(t *testing.T) { + for _, args := range [][]string{{"up"}, {"up", "a.yml", "b.yml"}, {"status", "x"}, {"down", "x"}, {"ssh"}, {"up", "-bogus", "a.yml"}} { + code, _, stderr := runLab(args...) + if code != 2 || !strings.Contains(stderr, "usage: lab") { + t.Errorf("args %v: code %d, stderr %q", args, code, stderr) + } + } +} + +func TestRun_CommandsWithoutALabInTheRunDir(t *testing.T) { + run := t.TempDir() + for _, cmd := range []string{"status", "down", "ssh"} { + args := []string{cmd, "-run", run} + if cmd == "ssh" { + args = append(args, "hv1") + } + code, _, stderr := runLab(args...) + if code != 1 || !strings.Contains(stderr, "lab: no lab in "+run) { + t.Errorf("%s: code %d, stderr %q", cmd, code, stderr) + } + } +} + +func TestRun_StatusOfAStoppedLab(t *testing.T) { + code, stdout, stderr := runLab("status", "-run", savedLab(t)) + want := `node role state pid ssh +sw1 switch stopped - 127.0.0.1:2200 +rr1 rr stopped - 127.0.0.1:2201 +hv1 hypervisor stopped - 127.0.0.1:2202 +hv2 hypervisor stopped - 127.0.0.1:2203 +` + if code != 0 || stdout != want { + t.Errorf("code %d, stderr %q, stdout:\n%s", code, stderr, stdout) + } +} + +func TestRun_DownOfAStoppedLab(t *testing.T) { + code, stdout, stderr := runLab("down", "-run", savedLab(t)) + if code != 0 || stdout != "" || stderr != "" { + t.Errorf("code %d, stdout %q, stderr %q", code, stdout, stderr) + } +} + +func TestRun_SSHExecsSSHWithTheNodePort(t *testing.T) { + run := savedLab(t) + var gotPath string + var gotArgv []string + execve = func(path string, argv []string, env []string) error { + gotPath, gotArgv = path, argv + return nil + } + t.Cleanup(func() { execve = syscall.Exec }) + + code, _, stderr := runLab("ssh", "-run", run, "hv2", "ip", "-br", "a") + + if code != 0 { + t.Fatalf("code %d, stderr %q", code, stderr) + } + if filepath.Base(gotPath) != "ssh" { + t.Errorf("path = %q", gotPath) + } + want := []string{"ssh", + "-i", filepath.Join(run, "lab_ed25519"), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2203", + "debian@127.0.0.1", + "ip", "-br", "a", + } + if strings.Join(gotArgv, " ") != strings.Join(want, " ") { + t.Errorf("\n got %q\nwant %q", gotArgv, want) + } +} + +func TestRun_SSHUnknownNode(t *testing.T) { + code, _, stderr := runLab("ssh", "-run", savedLab(t), "hv9") + if code != 1 || !strings.Contains(stderr, `node "hv9" is not in lab evpn-2hv`) { + t.Errorf("code %d, stderr %q", code, stderr) + } +} + +func TestRun_UpRefusesAnInvalidTopologyAndKeepsTheSavedOne(t *testing.T) { + run := savedLab(t) + before, _ := os.ReadFile(filepath.Join(run, "topology.yml")) + bad := filepath.Join(t.TempDir(), "bad.yml") + if err := os.WriteFile(bad, []byte("name: x\n"), 0o600); err != nil { + t.Fatal(err) + } + + code, _, stderr := runLab("up", "-run", run, bad) + + if code != 1 || !strings.Contains(stderr, "at least one node is required") { + t.Errorf("code %d, stderr %q", code, stderr) + } + if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) { + t.Error("the saved topology was replaced by an invalid one") + } +} + +func TestIsTerminal_DevNullAndPipesAreNotTerminals(t *testing.T) { + null, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + defer null.Close() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + for name, f := range map[string]*os.File{"/dev/null": null, "pipe": r} { + if isTerminal(f) { + t.Errorf("%s is detected as a terminal", name) + } + } +} + +func TestRun_UpRefusesToReplaceARunningLab(t *testing.T) { + run := savedLab(t) + before, _ := os.ReadFile(filepath.Join(run, "topology.yml")) + proc := t.TempDir() + procDir = proc + t.Cleanup(func() { procDir = "/proc" }) + + cmd := exec.Command("sleep", "30") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cmd.Process.Kill(); cmd.Wait() }) + pid := strconv.Itoa(cmd.Process.Pid) + if err := os.MkdirAll(filepath.Join(proc, pid), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte("qemu-system-x86_64\x00-name\x00hv1\x00"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(run, "hv1"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(run, "hv1", "qemu.pid"), []byte(pid+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + other := filepath.Join(t.TempDir(), "other.yml") + if err := os.WriteFile(other, bytes.ReplaceAll(before, []byte("evpn-2hv"), []byte("other")), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runLab("up", "-run", run, other) + + if code != 1 || !strings.Contains(stderr, "lab evpn-2hv is still running in "+run) { + t.Errorf("code %d, stderr %q", code, stderr) + } + if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) { + t.Error("the topology of a running lab was replaced") + } +} diff --git a/cmd/lab/terminal.go b/cmd/lab/terminal.go new file mode 100644 index 0000000..fa2cb06 --- /dev/null +++ b/cmd/lab/terminal.go @@ -0,0 +1,14 @@ +//go:build linux || darwin + +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func isTerminal(f *os.File) bool { + _, err := unix.IoctlGetTermios(int(f.Fd()), ioctlReadTermios) + return err == nil +} diff --git a/cmd/lab/terminal_darwin.go b/cmd/lab/terminal_darwin.go new file mode 100644 index 0000000..4c493c7 --- /dev/null +++ b/cmd/lab/terminal_darwin.go @@ -0,0 +1,5 @@ +package main + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TIOCGETA diff --git a/cmd/lab/terminal_linux.go b/cmd/lab/terminal_linux.go new file mode 100644 index 0000000..ba4230d --- /dev/null +++ b/cmd/lab/terminal_linux.go @@ -0,0 +1,5 @@ +package main + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TCGETS diff --git a/cmd/lab/terminal_other.go b/cmd/lab/terminal_other.go new file mode 100644 index 0000000..a929525 --- /dev/null +++ b/cmd/lab/terminal_other.go @@ -0,0 +1,7 @@ +//go:build !linux && !darwin + +package main + +import "os" + +func isTerminal(*os.File) bool { return false } diff --git a/docs/developpement/lab.rst b/docs/developpement/lab.rst index 7c0371d..c81cdd5 100644 --- a/docs/developpement/lab.rst +++ b/docs/developpement/lab.rst @@ -13,10 +13,12 @@ comment s'en servir. .. note:: - État actuel : étapes **E0** à **E2** livrées — le cycle de vie du serveur qui portera le lab - (``scripts/lab-host.sh``), la description de la topologie et le calcul de son plan - (``lab plan``), puis la génération des arguments QEMU et des fichiers cloud-init de chaque VM - (``lab render``). Le lancement des VM viendra avec l'étape suivante, et cette page avec elle. + État actuel : étapes **E0** à **E2** livrées, **E3** en cours — le cycle de vie du serveur qui + portera le lab (``scripts/lab-host.sh``), la description de la topologie et le calcul de son + plan (``lab plan``), la génération des arguments QEMU et des fichiers cloud-init de chaque VM + (``lab render``), puis leur lancement (``lab up`` / ``status`` / ``down`` / ``ssh``). Reste à + apprendre à ``lab-host.sh`` à préparer le serveur et à y déposer ``lab``, puis à valider le + tout sur le serveur loué. Le serveur de lab ----------------- @@ -359,6 +361,60 @@ renvoie d'abord des adresses IPv6) attend un délai avant de se rabattre sur l'I Reste à vérifier sur le serveur de lab : les hyperviseurs, qui exigent KVM imbriqué. +Lancement des VM +---------------- + +``lab`` s'exécute **sur le serveur de lab**. Il garde l'état du lab dans un répertoire +(``-run``, par défaut ``~/lab-run``) : ``status``, ``down`` et ``ssh`` n'ont donc pas besoin de la +topologie. + +.. code-block:: text + + lab up [-run dir] [-cache dir] [-timeout 20m] + lab status [-run dir] + lab down [-run dir] + lab ssh [-run dir] [commande…] + +``lab up`` + 1. refuse de continuer si un lab tourne déjà dans le répertoire ; + 2. télécharge chaque image dans le cache (``-cache``, par défaut ``~/.cache/two-lab``) et la + vérifie contre ``SHA512SUMS`` ; une image déjà présente et toujours conforme n'est pas + retéléchargée, la liste des sommes est relue à chaque fois ; + 3. génère une paire de clés SSH dans le répertoire du lab, si elle n'existe pas encore ; + 4. pour chaque nœud : fichiers de ``lab render``, disque **neuf** en overlay qcow2 sur + l'image (``qemu-img create -b``, 20 Gio annoncés), image ``cidata`` (``genisoimage``) ; + 5. démarre les QEMU, **switchs d'abord**, détachés (``-daemonize``) : ils survivent à la + session SSH qui les a lancés ; + 6. attend sur chaque nœud la fin de cloud-init (``cloud-init status --wait`` par SSH), + jusqu'au délai ``-timeout``. + + La topologie est copiée dans ``/topology.yml``. Un échec laisse les nœuds démarrés en + place : ``lab status``, puis ``lab down``. + +``lab status`` + Pour chaque nœud : rôle, état du processus QEMU, PID, port SSH sur la boucle locale. + +``lab down`` + Arrête chaque QEMU par ``SIGTERM``, puis ``SIGKILL`` au bout de 30 s. Les disques sont + conservés jusqu'au prochain ``up``, qui les recrée. + +``lab ssh`` + Ouvre un shell sur un nœud, ou y exécute une commande, avec la clé générée par ``up``. ``lab`` + cède la place à ``ssh``, dont le code de retour est donc celui de la commande. Un terminal + n'est demandé (``-t``) que si l'entrée de ``lab`` en est un : depuis un script, ni + pseudo-terminal ni ``\r\n`` dans la sortie. + +Un processus n'est tenu pour celui d'un nœud que si son PID, lu dans ``qemu.pid``, désigne un +processus vivant dont la ligne de commande (``/proc//cmdline``) contient ``-name ``. +Un PID réutilisé par un autre programme n'est donc jamais signalé. + +.. warning:: + + Le cache range une image sous son nom de fichier, et l'URL de Debian est ``latest`` : une + nouvelle publication remplace le fichier, et les overlays existants pointeraient sur une + base différente. ``up`` recrée toujours les disques, ce qui suffit avec un lab par serveur ; + **ne pas relancer un QEMU à la main** à partir d'un ``qemu.args`` après un ``up`` ultérieur. + Facturation ----------- @@ -430,6 +486,15 @@ Sécurité production. * Une clé secrète qui a circulé ailleurs que dans ``scaleway.env`` (conversation, terminal partagé, capture d'écran) se régénère. +* **Clé SSH des VM** : générée par ``lab up`` sur le serveur, c'est la seule clé autorisée dans + les VM. Elle ne quitte jamais le serveur, n'ouvre que les VM du lab — qui n'écoutent qu'en + boucle locale — et disparaît avec lui. La clé publique du Mac n'est jamais envoyée aux VM. +* **Clés d'hôte des VM non vérifiées** par ``lab ssh`` (``known_hosts`` jetable) : elles changent + à chaque ``up``. Acceptable uniquement parce que la connexion reste sur la boucle locale d'un + serveur auquel on s'est authentifié. +* **Image** : ``SHA512SUMS`` vient de la même origine que l'image, en HTTPS. La vérification + protège contre la corruption, pas contre une origine compromise ; la signature GPG de Debian + (``SHA512SUMS.sign``) n'est pas encore vérifiée. Tests ----- diff --git a/internal/lab/machine/machine.go b/internal/lab/machine/machine.go new file mode 100644 index 0000000..1630ed7 --- /dev/null +++ b/internal/lab/machine/machine.go @@ -0,0 +1,269 @@ +package machine + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "text/tabwriter" + "time" + + "git.g3e.fr/syonad/two/internal/lab/provision" + "git.g3e.fr/syonad/two/internal/lab/render" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const ( + User = "debian" + + sshUnreachable = 255 + cloudInitRecoverable = 2 +) + +type Lab struct { + Plan *topology.Plan + RunDir string + ProcDir string + Runner provision.Runner + Poll time.Duration + Stop time.Duration + Out io.Writer + + signal func(pid int, sig syscall.Signal) error +} + +func (l Lab) node(name string) (topology.NodePlan, error) { + for _, n := range l.Plan.Nodes { + if n.Name == name { + return n, nil + } + } + return topology.NodePlan{}, fmt.Errorf("node %q is not in lab %s", name, l.Plan.Name) +} + +func (l Lab) pidFile(name string) string { + return filepath.Join(l.RunDir, name, render.PIDFile) +} + +func (l Lab) PID(name string) (int, error) { + data, err := os.ReadFile(l.pidFile(name)) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + return 0, fmt.Errorf("%s: not a pid: %q", l.pidFile(name), data) + } + if !l.runs(pid, name) { + return 0, nil + } + return pid, nil +} + +func (l Lab) runs(pid int, name string) bool { + if err := syscall.Kill(pid, 0); err != nil && !errors.Is(err, syscall.EPERM) { + return false + } + cmdline, err := os.ReadFile(filepath.Join(l.ProcDir, strconv.Itoa(pid), "cmdline")) + if err != nil { + return false + } + return bytes.Contains(cmdline, []byte("\x00-name\x00"+name+"\x00")) +} + +func (l Lab) Running() ([]string, error) { + var running []string + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + return nil, err + } + if pid != 0 { + running = append(running, n.Name) + } + } + return running, nil +} + +func (l Lab) Up(ctx context.Context, fetcher provision.Fetcher, timeout time.Duration) error { + running, err := l.Running() + if err != nil { + return err + } + if len(running) > 0 { + return fmt.Errorf("lab %s is already running (%s): 'lab down' first", l.Plan.Name, strings.Join(running, ", ")) + } + + nodes, err := provision.Prepare(ctx, l.Plan, provision.Options{RunDir: l.RunDir, Fetcher: fetcher, Runner: l.Runner}) + if err != nil { + return err + } + for _, n := range switchesFirst(l.Plan, nodes) { + if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := l.Runner.Run(ctx, render.QEMUBinary, append(n.QEMU, "-daemonize")...); err != nil { + return fmt.Errorf("node %s: %w", n.Name, err) + } + fmt.Fprintf(l.Out, "%s: started\n", n.Name) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var errs []error + for _, n := range l.Plan.Nodes { + if err := l.waitReady(ctx, n); err != nil { + errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err)) + continue + } + fmt.Fprintf(l.Out, "%s: ready\n", n.Name) + } + return errors.Join(errs...) +} + +func switchesFirst(p *topology.Plan, nodes []render.Node) []render.Node { + var first, rest []render.Node + for i, n := range nodes { + if p.Nodes[i].Role == topology.RoleSwitch { + first = append(first, n) + } else { + rest = append(rest, n) + } + } + return append(first, rest...) +} + +func (l Lab) waitReady(ctx context.Context, n topology.NodePlan) error { + args := l.sshArgs(n, true, []string{"cloud-init", "status", "--wait"}) + for { + err := l.Runner.Run(ctx, "ssh", args...) + switch code := exitCode(err); { + case err == nil: + return nil + case code == cloudInitRecoverable: + fmt.Fprintf(l.Out, "%s: cloud-init finished with recoverable errors: %v\n", n.Name, err) + return nil + case code != sshUnreachable: + return fmt.Errorf("cloud-init failed: %w", err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("not reachable over ssh: %w", ctx.Err()) + case <-time.After(l.Poll): + } + } +} + +func exitCode(err error) int { + var coded interface{ ExitCode() int } + if errors.As(err, &coded) { + return coded.ExitCode() + } + return -1 +} + +func (l Lab) sshArgs(n topology.NodePlan, batch bool, command []string) []string { + args := []string{ + "-i", filepath.Join(l.RunDir, provision.KeyFile), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", strconv.Itoa(n.SSHPort), + } + if batch { + args = append(args, "-o", "BatchMode=yes", "-o", "ConnectTimeout=5") + } + args = append(args, User+"@127.0.0.1") + return append(args, command...) +} + +func (l Lab) SSH(name string, terminal bool, command []string) ([]string, error) { + n, err := l.node(name) + if err != nil { + return nil, err + } + args := []string{"ssh"} + if terminal && len(command) > 0 { + args = append(args, "-t") + } + return append(args, l.sshArgs(n, false, command)...), nil +} + +func (l Lab) Status(w io.Writer) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintf(tw, "node\trole\tstate\tpid\tssh\n") + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + return err + } + state, shown := "stopped", "-" + if pid != 0 { + state, shown = "running", strconv.Itoa(pid) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t127.0.0.1:%d\n", n.Name, n.Role, state, shown, n.SSHPort) + } + return tw.Flush() +} + +func (l Lab) Down(ctx context.Context) error { + var errs []error + for _, n := range l.Plan.Nodes { + pid, err := l.PID(n.Name) + if err != nil { + errs = append(errs, err) + continue + } + if pid == 0 { + continue + } + if err := l.stop(ctx, n.Name, pid); err != nil { + errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err)) + continue + } + if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, err) + continue + } + fmt.Fprintf(l.Out, "%s: stopped\n", n.Name) + } + return errors.Join(errs...) +} + +func (l Lab) stop(ctx context.Context, name string, pid int) error { + if pid <= 0 { + return fmt.Errorf("refusing to signal pid %d", pid) + } + send := l.signal + if send == nil { + send = syscall.Kill + } + for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} { + if err := send(pid, sig); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + deadline := time.Now().Add(l.Stop) + for l.runs(pid, name) && time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(l.Poll): + } + } + if !l.runs(pid, name) { + return nil + } + } + return fmt.Errorf("pid %d still running after SIGKILL", pid) +} diff --git a/internal/lab/machine/machine_test.go b/internal/lab/machine/machine_test.go new file mode 100644 index 0000000..22635d6 --- /dev/null +++ b/internal/lab/machine/machine_test.go @@ -0,0 +1,543 @@ +package machine + +import ( + "bytes" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "git.g3e.fr/syonad/two/internal/lab/provision" + "git.g3e.fr/syonad/two/internal/lab/topology" +) + +const labKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab" + +type exitErr int + +func (e exitErr) Error() string { return "exit status " + strconv.Itoa(int(e)) } +func (e exitErr) ExitCode() int { return int(e) } + +type fakeRunner struct { + mu sync.Mutex + calls [][]string + fail string + ssh map[string][]error + always map[string]error +} + +func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, append([]string{name}, args...)) + if name == f.fail { + return errors.New(name + " failed") + } + switch name { + case "ssh-keygen": + private := args[len(args)-1] + if err := os.WriteFile(private, []byte("private"), 0o600); err != nil { + return err + } + return os.WriteFile(private+".pub", []byte(labKey+"\n"), 0o644) + case "ssh": + port := args[indexOf(args, "-p")+1] + answers := f.ssh[port] + if len(answers) == 0 { + return f.always[port] + } + f.ssh[port] = answers[1:] + return answers[0] + } + return nil +} + +func (f *fakeRunner) commands(name string) [][]string { + f.mu.Lock() + defer f.mu.Unlock() + var out [][]string + for _, c := range f.calls { + if c[0] == name { + out = append(out, c) + } + } + return out +} + +func indexOf(list []string, value string) int { + for i, v := range list { + if v == value { + return i + } + } + return -1 +} + +func mirror(t *testing.T) (*httptest.Server, string) { + t.Helper() + content := []byte("qcow2 image") + h := sha512.Sum512(content) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "SHA512SUMS") { + fmt.Fprintf(w, "%s deb.qcow2\n", hex.EncodeToString(h[:])) + return + } + w.Write(content) + })) + t.Cleanup(server.Close) + return server, server.URL +} + +func labPlan(t *testing.T, base, nodes string) *topology.Plan { + t.Helper() + doc := `name: evpn-2hv +images: + deb: + url: ` + base + `/deb.qcow2 + sums: ` + base + `/SHA512SUMS +segments: + underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 } +nodes: +` + nodes + topo, err := topology.Parse([]byte(doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + p, err := topology.Compute(topo) + if err != nil { + t.Fatalf("Compute: %v", err) + } + return p +} + +const switchLast = ` rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] } + hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] } + sw1: { role: switch, image: deb, cpus: 2, memory: 1024 } +` + +type fixture struct { + lab Lab + runner *fakeRunner + out *bytes.Buffer + fetcher provision.Fetcher +} + +func newFixture(t *testing.T, nodes string) *fixture { + t.Helper() + server, base := mirror(t) + root := t.TempDir() + runner := &fakeRunner{ssh: map[string][]error{}, always: map[string]error{}} + out := &bytes.Buffer{} + return &fixture{ + lab: Lab{ + Plan: labPlan(t, base, nodes), + RunDir: filepath.Join(root, "run"), + ProcDir: filepath.Join(root, "proc"), + Runner: runner, + Poll: time.Millisecond, + Stop: 500 * time.Millisecond, + Out: out, + }, + runner: runner, + out: out, + fetcher: provision.Fetcher{Client: server.Client(), CacheDir: filepath.Join(root, "cache")}, + } +} + +func (f *fixture) process(t *testing.T, node, script string, cmdline ...string) int { + t.Helper() + cmd := exec.Command("sh", "-c", script) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { cmd.Wait(); close(done) }() + t.Cleanup(func() { + cmd.Process.Kill() + <-done + }) + pid := cmd.Process.Pid + proc := filepath.Join(f.lab.ProcDir, strconv.Itoa(pid)) + if err := os.MkdirAll(proc, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, "cmdline"), []byte(strings.Join(cmdline, "\x00")+"\x00"), 0o644); err != nil { + t.Fatal(err) + } + f.pidFile(t, node, strconv.Itoa(pid)+"\n") + return pid +} + +func (f *fixture) pidFile(t *testing.T, node, content string) { + t.Helper() + dir := filepath.Join(f.lab.RunDir, node) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "qemu.pid"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func qemu(node string) []string { + return []string{"qemu-system-x86_64", "-name", node, "-machine", "q35"} +} + +func alive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +func waitDead(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for alive(pid) && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if alive(pid) { + t.Errorf("pid %d is still alive", pid) + } +} + +func TestPID_NoPidFileMeansStopped(t *testing.T) { + f := newFixture(t, switchLast) + if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil { + t.Errorf("PID = %d, %v", pid, err) + } +} + +func TestPID_RefusesACorruptedPidFile(t *testing.T) { + for _, content := range []string{"", "abc", "0", "-12"} { + f := newFixture(t, switchLast) + f.pidFile(t, "hv1", content) + if _, err := f.lab.PID("hv1"); err == nil || !strings.Contains(err.Error(), "not a pid") { + t.Errorf("pid file %q: error = %v", content, err) + } + } +} + +func TestPID_RecognisesTheNodeProcess(t *testing.T) { + f := newFixture(t, switchLast) + want := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + if pid, err := f.lab.PID("hv1"); pid != want || err != nil { + t.Errorf("PID = %d, %v, want %d", pid, err, want) + } +} + +func TestPID_IgnoresAPidReusedByAnotherProcess(t *testing.T) { + cases := map[string][]string{ + "another program": {"/usr/sbin/sshd", "-D"}, + "another node": qemu("hv10"), + "name as a value": {"qemu-system-x86_64", "-serial", "-name", "-name", "hv2"}, + } + for name, cmdline := range cases { + t.Run(name, func(t *testing.T) { + f := newFixture(t, switchLast) + f.process(t, "hv1", "sleep 30", cmdline...) + if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil { + t.Errorf("PID = %d, %v, want 0", pid, err) + } + }) + } +} + +func TestPID_IgnoresADeadProcess(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "exit 0", qemu("hv1")...) + waitDead(t, pid) + if got, err := f.lab.PID("hv1"); got != 0 || err != nil { + t.Errorf("PID = %d, %v, want 0", got, err) + } +} + +func TestStatus(t *testing.T) { + f := newFixture(t, switchLast) + pid := strconv.Itoa(f.process(t, "hv1", "sleep 30", qemu("hv1")...)) + var out bytes.Buffer + if err := f.lab.Status(&out); err != nil { + t.Fatal(err) + } + w := len(pid) + if w < 3 { + w = 3 + } + want := fmt.Sprintf("node role state %-*s ssh\n", w, "pid") + + fmt.Sprintf("rr1 rr stopped %-*s 127.0.0.1:2200\n", w, "-") + + fmt.Sprintf("hv1 hypervisor running %-*s 127.0.0.1:2201\n", w, pid) + + fmt.Sprintf("sw1 switch stopped %-*s 127.0.0.1:2202\n", w, "-") + if out.String() != want { + t.Errorf("status:\n%s\nwant:\n%s", out.String(), want) + } +} + +func TestDown_StopsWithSIGTERM(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + waitDead(t, pid) + if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) { + t.Errorf("pid file still there: %v", err) + } + if f.out.String() != "hv1: stopped\n" { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestDown_FallsBackToSIGKILL(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.Stop = 100 * time.Millisecond + pid := f.process(t, "hv1", `trap "" TERM; while :; do sleep 0.05; done`, qemu("hv1")...) + time.Sleep(50 * time.Millisecond) + + start := time.Now() + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + waitDead(t, pid) + if elapsed := time.Since(start); elapsed < 100*time.Millisecond { + t.Errorf("SIGKILL after %v, want at least the 100ms grace period", elapsed) + } +} + +func TestDown_NeverSignalsAProcessThatIsNotTheNode(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "sleep 30", "/usr/sbin/sshd", "-D") + + if err := f.lab.Down(context.Background()); err != nil { + t.Fatalf("Down: %v", err) + } + + time.Sleep(50 * time.Millisecond) + if !alive(pid) { + t.Error("a process that is not the node was killed") + } + if f.out.Len() != 0 { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestDown_ReportsACorruptedPidFileAndStopsTheOthers(t *testing.T) { + f := newFixture(t, switchLast) + f.pidFile(t, "rr1", "garbage") + pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + err := f.lab.Down(context.Background()) + + if err == nil || !strings.Contains(err.Error(), "not a pid") { + t.Errorf("error = %v", err) + } + waitDead(t, pid) +} + +func TestSSH_Arguments(t *testing.T) { + f := newFixture(t, switchLast) + key := filepath.Join(f.lab.RunDir, "lab_ed25519") + base := []string{ + "-i", key, + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2201", + "debian@127.0.0.1", + } + cases := map[string]struct { + terminal bool + command []string + want []string + }{ + "interactive shell": {true, nil, append([]string{"ssh"}, base...)}, + "command from a terminal": {true, []string{"top"}, append(append([]string{"ssh", "-t"}, base...), "top")}, + "command from a script": {false, []string{"ip", "-br", "a"}, append(append([]string{"ssh"}, base...), "ip", "-br", "a")}, + "shell from a script": {false, nil, append([]string{"ssh"}, base...)}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + got, err := f.lab.SSH("hv1", c.terminal, c.command) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Errorf("\n got %q\nwant %q", got, c.want) + } + }) + } +} + +func TestSSH_UnknownNode(t *testing.T) { + f := newFixture(t, switchLast) + if _, err := f.lab.SSH("hv9", true, nil); err == nil || !strings.Contains(err.Error(), `node "hv9" is not in lab evpn-2hv`) { + t.Errorf("error = %v", err) + } +} + +func TestUp_StartsSwitchesFirstAndWaitsForEveryNode(t *testing.T) { + f := newFixture(t, switchLast) + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + + var started []string + for _, c := range f.runner.commands("qemu-system-x86_64") { + if c[len(c)-1] != "-daemonize" { + t.Errorf("qemu not daemonized: %q", c) + } + started = append(started, c[indexOf(c, "-name")+1]) + } + if !reflect.DeepEqual(started, []string{"sw1", "rr1", "hv1"}) { + t.Errorf("start order = %v, want the switch first", started) + } + + ssh := f.runner.commands("ssh") + if len(ssh) != 3 { + t.Fatalf("%d ssh calls, want 3", len(ssh)) + } + want := []string{"ssh", + "-i", filepath.Join(f.lab.RunDir, "lab_ed25519"), + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-p", "2200", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=5", + "debian@127.0.0.1", + "cloud-init", "status", "--wait", + } + if !reflect.DeepEqual(ssh[0], want) { + t.Errorf("readiness check:\n got %q\nwant %q", ssh[0], want) + } + if f.out.String() != "sw1: started\nrr1: started\nhv1: started\nrr1: ready\nhv1: ready\nsw1: ready\n" { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestUp_RetriesWhileSSHIsUnreachable(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.ssh["2201"] = []error{exitErr(255), exitErr(255), nil} + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if n := len(f.runner.commands("ssh")); n != 5 { + t.Errorf("%d ssh calls, want 5", n) + } +} + +func TestUp_ReadinessFailures(t *testing.T) { + cases := map[string]struct { + answers []error + want string + }{ + "cloud-init error": {[]error{exitErr(1)}, "node hv1: cloud-init failed: exit status 1"}, + "not an exit status": {[]error{errors.New("fork failed")}, "node hv1: cloud-init failed: fork failed"}, + "ssh never reachable": {nil, "node hv1: not reachable over ssh: context deadline exceeded"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.Poll = 10 * time.Millisecond + f.runner.ssh["2201"] = c.answers + f.runner.always["2201"] = exitErr(255) + + err := f.lab.Up(context.Background(), f.fetcher, 300*time.Millisecond) + + if err == nil || err.Error() != c.want { + t.Errorf("error = %v, want %q", err, c.want) + } + if !strings.Contains(f.out.String(), "rr1: ready\n") || !strings.Contains(f.out.String(), "sw1: ready\n") { + t.Errorf("the other nodes were not waited for: %q", f.out.String()) + } + }) + } +} + +func TestUp_RecoverableCloudInitErrorsAreReportedNotFatal(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.ssh["2201"] = []error{exitErr(2)} + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if !strings.Contains(f.out.String(), "hv1: cloud-init finished with recoverable errors: exit status 2\nhv1: ready\n") { + t.Errorf("output = %q", f.out.String()) + } +} + +func TestUp_RefusesARunningLab(t *testing.T) { + f := newFixture(t, switchLast) + f.process(t, "hv1", "sleep 30", qemu("hv1")...) + + err := f.lab.Up(context.Background(), f.fetcher, time.Second) + + if err == nil || err.Error() != "lab evpn-2hv is already running (hv1): 'lab down' first" { + t.Errorf("error = %v", err) + } + if len(f.runner.calls) != 0 { + t.Errorf("commands ran: %q", f.runner.calls) + } +} + +func TestUp_RemovesAStalePidFileBeforeStarting(t *testing.T) { + f := newFixture(t, switchLast) + pid := f.process(t, "hv1", "exit 0", qemu("hv1")...) + waitDead(t, pid) + + if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil { + t.Fatalf("Up: %v", err) + } + if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) { + t.Errorf("stale pid file kept: %v", err) + } +} + +func TestUp_StopsAtTheFirstQEMUFailure(t *testing.T) { + f := newFixture(t, switchLast) + f.runner.fail = "qemu-system-x86_64" + + err := f.lab.Up(context.Background(), f.fetcher, time.Second) + + if err == nil || err.Error() != "node sw1: qemu-system-x86_64 failed" { + t.Errorf("error = %v", err) + } + if n := len(f.runner.commands("qemu-system-x86_64")); n != 1 { + t.Errorf("%d qemu starts, want 1", n) + } + if n := len(f.runner.commands("ssh")); n != 0 { + t.Errorf("%d ssh calls after a failed start", n) + } +} + +func TestStop_RefusesPidsThatTargetAGroup(t *testing.T) { + f := newFixture(t, switchLast) + f.lab.signal = func(pid int, sig syscall.Signal) error { + t.Fatalf("signal %v sent to pid %d", sig, pid) + return nil + } + for _, pid := range []int{0, -1} { + if err := f.lab.stop(context.Background(), "hv1", pid); err == nil || err.Error() != fmt.Sprintf("refusing to signal pid %d", pid) { + t.Errorf("pid %d: error = %v", pid, err) + } + } +}