f-50: lab: lancement, état, arrêt et accès SSH des VM #50
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
d1cd943d86
commit
1146d13d45
GPG key ID:
4406BBBF8845D632
10 changed files with 1309 additions and 4 deletions
193
cmd/lab/lifecycle.go
Normal file
193
cmd/lab/lifecycle.go
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/lab/machine"
|
||||
"git.g3e.fr/syonad/two/internal/lab/provision"
|
||||
)
|
||||
|
||||
const (
|
||||
topologyFile = "topology.yml"
|
||||
pollInterval = 5 * time.Second
|
||||
stopTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
execve = syscall.Exec
|
||||
procDir = "/proc"
|
||||
)
|
||||
|
||||
func defaultDir(parts ...string) string {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(append([]string{home}, parts...)...)
|
||||
}
|
||||
|
||||
func flags(name string, stderr io.Writer) (*flag.FlagSet, *string) {
|
||||
fs := flag.NewFlagSet(name, flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
fs.Usage = func() { fmt.Fprint(stderr, usage) }
|
||||
return fs, fs.String("run", defaultDir("lab-run"), "run directory of the lab")
|
||||
}
|
||||
|
||||
func lab(runDir string, plan string, stdout, stderr io.Writer) (machine.Lab, bool) {
|
||||
dir, err := filepath.Abs(runDir)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return machine.Lab{}, false
|
||||
}
|
||||
p, ok := load(plan, stderr)
|
||||
if !ok {
|
||||
return machine.Lab{}, false
|
||||
}
|
||||
return machine.Lab{
|
||||
Plan: p,
|
||||
RunDir: dir,
|
||||
ProcDir: procDir,
|
||||
Runner: provision.ExecRunner{},
|
||||
Poll: pollInterval,
|
||||
Stop: stopTimeout,
|
||||
Out: stdout,
|
||||
}, true
|
||||
}
|
||||
|
||||
func current(runDir string, stdout, stderr io.Writer) (machine.Lab, bool) {
|
||||
saved := filepath.Join(runDir, topologyFile)
|
||||
if _, err := os.Stat(saved); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: no lab in %s: %v\n", runDir, err)
|
||||
return machine.Lab{}, false
|
||||
}
|
||||
return lab(runDir, saved, stdout, stderr)
|
||||
}
|
||||
|
||||
func upCmd(args []string, stdout, stderr io.Writer) int {
|
||||
fs, runDir := flags("up", stderr)
|
||||
cacheDir := fs.String("cache", defaultDir(".cache", "two-lab"), "image cache directory")
|
||||
timeout := fs.Duration("timeout", 20*time.Minute, "how long to wait for the nodes to be ready")
|
||||
if err := fs.Parse(args); err != nil || fs.NArg() != 1 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
cache, err := filepath.Abs(*cacheDir)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(*runDir, topologyFile)); err == nil {
|
||||
previous, ok := current(*runDir, stdout, stderr)
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
running, err := previous.Running()
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintf(stderr, "lab: lab %s is still running in %s: 'lab down' first\n", previous.Plan.Name, previous.RunDir)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
l, ok := lab(*runDir, fs.Arg(0), stdout, stderr)
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
source, err := os.ReadFile(fs.Arg(0))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := os.MkdirAll(l.RunDir, 0o700); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(l.RunDir, topologyFile), source, 0o600); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
fetcher := provision.Fetcher{Client: &http.Client{}, CacheDir: cache}
|
||||
if err := l.Up(ctx, fetcher, *timeout); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\nlab: started nodes keep running: 'lab status', 'lab down'\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func statusCmd(args []string, stdout, stderr io.Writer) int {
|
||||
fs, runDir := flags("status", stderr)
|
||||
if err := fs.Parse(args); err != nil || fs.NArg() != 0 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
l, ok := current(*runDir, stdout, stderr)
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
if err := l.Status(stdout); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func downCmd(args []string, stdout, stderr io.Writer) int {
|
||||
fs, runDir := flags("down", stderr)
|
||||
if err := fs.Parse(args); err != nil || fs.NArg() != 0 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
l, ok := current(*runDir, stdout, stderr)
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
if err := l.Down(context.Background()); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func sshCmd(args []string, stdout, stderr io.Writer) int {
|
||||
fs, runDir := flags("ssh", stderr)
|
||||
if err := fs.Parse(args); err != nil || fs.NArg() < 1 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
l, ok := current(*runDir, stdout, stderr)
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
argv, err := l.SSH(fs.Arg(0), isTerminal(os.Stdin), fs.Args()[1:])
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
path, err := exec.LookPath(argv[0])
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := execve(path, argv, os.Environ()); err != nil {
|
||||
fmt.Fprintf(stderr, "lab: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -21,6 +21,22 @@ const usage = `usage: lab <command> [options] <topology.yml> [dir]
|
|||
render -key <public key file> <topology.yml> <dir>
|
||||
write, for each node, <dir>/<node>/qemu.args (one argument per line) and the
|
||||
cloud-init seed files meta-data, user-data and network-config
|
||||
|
||||
up [-run dir] [-cache dir] [-timeout duration] <topology.yml>
|
||||
download and verify the images, create fresh disks and seeds, start every node
|
||||
(switches first) and wait until cloud-init has finished on each of them;
|
||||
the topology is kept in <run>/topology.yml for the commands below
|
||||
|
||||
status [-run dir]
|
||||
list the nodes of the lab and whether their qemu process is running
|
||||
|
||||
down [-run dir]
|
||||
stop every qemu process of the lab (SIGTERM, then SIGKILL)
|
||||
|
||||
ssh [-run dir] <node> [command...]
|
||||
open a shell on a node, or run a command on it, with the key generated by up
|
||||
|
||||
defaults: -run $HOME/lab-run, -cache $HOME/.cache/two-lab, -timeout 20m
|
||||
`
|
||||
|
||||
type keyFiles []string
|
||||
|
|
@ -46,6 +62,14 @@ func run(args []string, stdout, stderr io.Writer) int {
|
|||
return plan(args[1], stdout, stderr)
|
||||
case "render":
|
||||
return renderCmd(args[1:], stdout, stderr)
|
||||
case "up":
|
||||
return upCmd(args[1:], stdout, stderr)
|
||||
case "status":
|
||||
return statusCmd(args[1:], stdout, stderr)
|
||||
case "down":
|
||||
return downCmd(args[1:], stdout, stderr)
|
||||
case "ssh":
|
||||
return sshCmd(args[1:], stdout, stderr)
|
||||
default:
|
||||
fmt.Fprint(stderr, usage)
|
||||
return 2
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@ package main
|
|||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -119,3 +122,180 @@ func TestRun_RenderRefusesMissingKeyFile(t *testing.T) {
|
|||
t.Errorf("code %d, stderr %q", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func savedLab(t *testing.T) string {
|
||||
t.Helper()
|
||||
run := filepath.Join(t.TempDir(), "run")
|
||||
if err := os.MkdirAll(run, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
example, err := os.ReadFile(filepath.Join("..", "..", "conf", "lab", "evpn-2hv.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(run, "topology.yml"), example, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return run
|
||||
}
|
||||
|
||||
func TestRun_LifecycleUsage(t *testing.T) {
|
||||
for _, args := range [][]string{{"up"}, {"up", "a.yml", "b.yml"}, {"status", "x"}, {"down", "x"}, {"ssh"}, {"up", "-bogus", "a.yml"}} {
|
||||
code, _, stderr := runLab(args...)
|
||||
if code != 2 || !strings.Contains(stderr, "usage: lab") {
|
||||
t.Errorf("args %v: code %d, stderr %q", args, code, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_CommandsWithoutALabInTheRunDir(t *testing.T) {
|
||||
run := t.TempDir()
|
||||
for _, cmd := range []string{"status", "down", "ssh"} {
|
||||
args := []string{cmd, "-run", run}
|
||||
if cmd == "ssh" {
|
||||
args = append(args, "hv1")
|
||||
}
|
||||
code, _, stderr := runLab(args...)
|
||||
if code != 1 || !strings.Contains(stderr, "lab: no lab in "+run) {
|
||||
t.Errorf("%s: code %d, stderr %q", cmd, code, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_StatusOfAStoppedLab(t *testing.T) {
|
||||
code, stdout, stderr := runLab("status", "-run", savedLab(t))
|
||||
want := `node role state pid ssh
|
||||
sw1 switch stopped - 127.0.0.1:2200
|
||||
rr1 rr stopped - 127.0.0.1:2201
|
||||
hv1 hypervisor stopped - 127.0.0.1:2202
|
||||
hv2 hypervisor stopped - 127.0.0.1:2203
|
||||
`
|
||||
if code != 0 || stdout != want {
|
||||
t.Errorf("code %d, stderr %q, stdout:\n%s", code, stderr, stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_DownOfAStoppedLab(t *testing.T) {
|
||||
code, stdout, stderr := runLab("down", "-run", savedLab(t))
|
||||
if code != 0 || stdout != "" || stderr != "" {
|
||||
t.Errorf("code %d, stdout %q, stderr %q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_SSHExecsSSHWithTheNodePort(t *testing.T) {
|
||||
run := savedLab(t)
|
||||
var gotPath string
|
||||
var gotArgv []string
|
||||
execve = func(path string, argv []string, env []string) error {
|
||||
gotPath, gotArgv = path, argv
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { execve = syscall.Exec })
|
||||
|
||||
code, _, stderr := runLab("ssh", "-run", run, "hv2", "ip", "-br", "a")
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("code %d, stderr %q", code, stderr)
|
||||
}
|
||||
if filepath.Base(gotPath) != "ssh" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
want := []string{"ssh",
|
||||
"-i", filepath.Join(run, "lab_ed25519"),
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", "2203",
|
||||
"debian@127.0.0.1",
|
||||
"ip", "-br", "a",
|
||||
}
|
||||
if strings.Join(gotArgv, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("\n got %q\nwant %q", gotArgv, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_SSHUnknownNode(t *testing.T) {
|
||||
code, _, stderr := runLab("ssh", "-run", savedLab(t), "hv9")
|
||||
if code != 1 || !strings.Contains(stderr, `node "hv9" is not in lab evpn-2hv`) {
|
||||
t.Errorf("code %d, stderr %q", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_UpRefusesAnInvalidTopologyAndKeepsTheSavedOne(t *testing.T) {
|
||||
run := savedLab(t)
|
||||
before, _ := os.ReadFile(filepath.Join(run, "topology.yml"))
|
||||
bad := filepath.Join(t.TempDir(), "bad.yml")
|
||||
if err := os.WriteFile(bad, []byte("name: x\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, _, stderr := runLab("up", "-run", run, bad)
|
||||
|
||||
if code != 1 || !strings.Contains(stderr, "at least one node is required") {
|
||||
t.Errorf("code %d, stderr %q", code, stderr)
|
||||
}
|
||||
if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) {
|
||||
t.Error("the saved topology was replaced by an invalid one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTerminal_DevNullAndPipesAreNotTerminals(t *testing.T) {
|
||||
null, err := os.Open(os.DevNull)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer null.Close()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
defer w.Close()
|
||||
for name, f := range map[string]*os.File{"/dev/null": null, "pipe": r} {
|
||||
if isTerminal(f) {
|
||||
t.Errorf("%s is detected as a terminal", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_UpRefusesToReplaceARunningLab(t *testing.T) {
|
||||
run := savedLab(t)
|
||||
before, _ := os.ReadFile(filepath.Join(run, "topology.yml"))
|
||||
proc := t.TempDir()
|
||||
procDir = proc
|
||||
t.Cleanup(func() { procDir = "/proc" })
|
||||
|
||||
cmd := exec.Command("sleep", "30")
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cmd.Process.Kill(); cmd.Wait() })
|
||||
pid := strconv.Itoa(cmd.Process.Pid)
|
||||
if err := os.MkdirAll(filepath.Join(proc, pid), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(proc, pid, "cmdline"), []byte("qemu-system-x86_64\x00-name\x00hv1\x00"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(run, "hv1"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(run, "hv1", "qemu.pid"), []byte(pid+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
other := filepath.Join(t.TempDir(), "other.yml")
|
||||
if err := os.WriteFile(other, bytes.ReplaceAll(before, []byte("evpn-2hv"), []byte("other")), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runLab("up", "-run", run, other)
|
||||
|
||||
if code != 1 || !strings.Contains(stderr, "lab evpn-2hv is still running in "+run) {
|
||||
t.Errorf("code %d, stderr %q", code, stderr)
|
||||
}
|
||||
if after, _ := os.ReadFile(filepath.Join(run, "topology.yml")); string(after) != string(before) {
|
||||
t.Error("the topology of a running lab was replaced")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
14
cmd/lab/terminal.go
Normal file
14
cmd/lab/terminal.go
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//go:build linux || darwin
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func isTerminal(f *os.File) bool {
|
||||
_, err := unix.IoctlGetTermios(int(f.Fd()), ioctlReadTermios)
|
||||
return err == nil
|
||||
}
|
||||
5
cmd/lab/terminal_darwin.go
Normal file
5
cmd/lab/terminal_darwin.go
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
package main
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
const ioctlReadTermios = unix.TIOCGETA
|
||||
5
cmd/lab/terminal_linux.go
Normal file
5
cmd/lab/terminal_linux.go
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
package main
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
const ioctlReadTermios = unix.TCGETS
|
||||
7
cmd/lab/terminal_other.go
Normal file
7
cmd/lab/terminal_other.go
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
//go:build !linux && !darwin
|
||||
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func isTerminal(*os.File) bool { return false }
|
||||
|
|
@ -13,10 +13,12 @@ comment s'en servir.
|
|||
|
||||
.. note::
|
||||
|
||||
État actuel : étapes **E0** à **E2** livrées — le cycle de vie du serveur qui portera le lab
|
||||
(``scripts/lab-host.sh``), la description de la topologie et le calcul de son plan
|
||||
(``lab plan``), puis la génération des arguments QEMU et des fichiers cloud-init de chaque VM
|
||||
(``lab render``). Le lancement des VM viendra avec l'étape suivante, et cette page avec elle.
|
||||
État actuel : étapes **E0** à **E2** livrées, **E3** en cours — le cycle de vie du serveur qui
|
||||
portera le lab (``scripts/lab-host.sh``), la description de la topologie et le calcul de son
|
||||
plan (``lab plan``), la génération des arguments QEMU et des fichiers cloud-init de chaque VM
|
||||
(``lab render``), puis leur lancement (``lab up`` / ``status`` / ``down`` / ``ssh``). Reste à
|
||||
apprendre à ``lab-host.sh`` à préparer le serveur et à y déposer ``lab``, puis à valider le
|
||||
tout sur le serveur loué.
|
||||
|
||||
Le serveur de lab
|
||||
-----------------
|
||||
|
|
@ -359,6 +361,60 @@ renvoie d'abord des adresses IPv6) attend un délai avant de se rabattre sur l'I
|
|||
|
||||
Reste à vérifier sur le serveur de lab : les hyperviseurs, qui exigent KVM imbriqué.
|
||||
|
||||
Lancement des VM
|
||||
----------------
|
||||
|
||||
``lab`` s'exécute **sur le serveur de lab**. Il garde l'état du lab dans un répertoire
|
||||
(``-run``, par défaut ``~/lab-run``) : ``status``, ``down`` et ``ssh`` n'ont donc pas besoin de la
|
||||
topologie.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
lab up [-run dir] [-cache dir] [-timeout 20m] <topologie.yml>
|
||||
lab status [-run dir]
|
||||
lab down [-run dir]
|
||||
lab ssh [-run dir] <nœud> [commande…]
|
||||
|
||||
``lab up``
|
||||
1. refuse de continuer si un lab tourne déjà dans le répertoire ;
|
||||
2. télécharge chaque image dans le cache (``-cache``, par défaut ``~/.cache/two-lab``) et la
|
||||
vérifie contre ``SHA512SUMS`` ; une image déjà présente et toujours conforme n'est pas
|
||||
retéléchargée, la liste des sommes est relue à chaque fois ;
|
||||
3. génère une paire de clés SSH dans le répertoire du lab, si elle n'existe pas encore ;
|
||||
4. pour chaque nœud : fichiers de ``lab render``, disque **neuf** en overlay qcow2 sur
|
||||
l'image (``qemu-img create -b``, 20 Gio annoncés), image ``cidata`` (``genisoimage``) ;
|
||||
5. démarre les QEMU, **switchs d'abord**, détachés (``-daemonize``) : ils survivent à la
|
||||
session SSH qui les a lancés ;
|
||||
6. attend sur chaque nœud la fin de cloud-init (``cloud-init status --wait`` par SSH),
|
||||
jusqu'au délai ``-timeout``.
|
||||
|
||||
La topologie est copiée dans ``<run>/topology.yml``. Un échec laisse les nœuds démarrés en
|
||||
place : ``lab status``, puis ``lab down``.
|
||||
|
||||
``lab status``
|
||||
Pour chaque nœud : rôle, état du processus QEMU, PID, port SSH sur la boucle locale.
|
||||
|
||||
``lab down``
|
||||
Arrête chaque QEMU par ``SIGTERM``, puis ``SIGKILL`` au bout de 30 s. Les disques sont
|
||||
conservés jusqu'au prochain ``up``, qui les recrée.
|
||||
|
||||
``lab ssh``
|
||||
Ouvre un shell sur un nœud, ou y exécute une commande, avec la clé générée par ``up``. ``lab``
|
||||
cède la place à ``ssh``, dont le code de retour est donc celui de la commande. Un terminal
|
||||
n'est demandé (``-t``) que si l'entrée de ``lab`` en est un : depuis un script, ni
|
||||
pseudo-terminal ni ``\r\n`` dans la sortie.
|
||||
|
||||
Un processus n'est tenu pour celui d'un nœud que si son PID, lu dans ``qemu.pid``, désigne un
|
||||
processus vivant dont la ligne de commande (``/proc/<pid>/cmdline``) contient ``-name <nœud>``.
|
||||
Un PID réutilisé par un autre programme n'est donc jamais signalé.
|
||||
|
||||
.. warning::
|
||||
|
||||
Le cache range une image sous son nom de fichier, et l'URL de Debian est ``latest`` : une
|
||||
nouvelle publication remplace le fichier, et les overlays existants pointeraient sur une
|
||||
base différente. ``up`` recrée toujours les disques, ce qui suffit avec un lab par serveur ;
|
||||
**ne pas relancer un QEMU à la main** à partir d'un ``qemu.args`` après un ``up`` ultérieur.
|
||||
|
||||
Facturation
|
||||
-----------
|
||||
|
||||
|
|
@ -430,6 +486,15 @@ Sécurité
|
|||
production.
|
||||
* Une clé secrète qui a circulé ailleurs que dans ``scaleway.env`` (conversation, terminal
|
||||
partagé, capture d'écran) se régénère.
|
||||
* **Clé SSH des VM** : générée par ``lab up`` sur le serveur, c'est la seule clé autorisée dans
|
||||
les VM. Elle ne quitte jamais le serveur, n'ouvre que les VM du lab — qui n'écoutent qu'en
|
||||
boucle locale — et disparaît avec lui. La clé publique du Mac n'est jamais envoyée aux VM.
|
||||
* **Clés d'hôte des VM non vérifiées** par ``lab ssh`` (``known_hosts`` jetable) : elles changent
|
||||
à chaque ``up``. Acceptable uniquement parce que la connexion reste sur la boucle locale d'un
|
||||
serveur auquel on s'est authentifié.
|
||||
* **Image** : ``SHA512SUMS`` vient de la même origine que l'image, en HTTPS. La vérification
|
||||
protège contre la corruption, pas contre une origine compromise ; la signature GPG de Debian
|
||||
(``SHA512SUMS.sign``) n'est pas encore vérifiée.
|
||||
|
||||
Tests
|
||||
-----
|
||||
|
|
|
|||
269
internal/lab/machine/machine.go
Normal file
269
internal/lab/machine/machine.go
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
package machine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/lab/provision"
|
||||
"git.g3e.fr/syonad/two/internal/lab/render"
|
||||
"git.g3e.fr/syonad/two/internal/lab/topology"
|
||||
)
|
||||
|
||||
const (
|
||||
User = "debian"
|
||||
|
||||
sshUnreachable = 255
|
||||
cloudInitRecoverable = 2
|
||||
)
|
||||
|
||||
type Lab struct {
|
||||
Plan *topology.Plan
|
||||
RunDir string
|
||||
ProcDir string
|
||||
Runner provision.Runner
|
||||
Poll time.Duration
|
||||
Stop time.Duration
|
||||
Out io.Writer
|
||||
|
||||
signal func(pid int, sig syscall.Signal) error
|
||||
}
|
||||
|
||||
func (l Lab) node(name string) (topology.NodePlan, error) {
|
||||
for _, n := range l.Plan.Nodes {
|
||||
if n.Name == name {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
return topology.NodePlan{}, fmt.Errorf("node %q is not in lab %s", name, l.Plan.Name)
|
||||
}
|
||||
|
||||
func (l Lab) pidFile(name string) string {
|
||||
return filepath.Join(l.RunDir, name, render.PIDFile)
|
||||
}
|
||||
|
||||
func (l Lab) PID(name string) (int, error) {
|
||||
data, err := os.ReadFile(l.pidFile(name))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
pid, err := strconv.Atoi(strings.TrimSpace(string(data)))
|
||||
if err != nil || pid <= 0 {
|
||||
return 0, fmt.Errorf("%s: not a pid: %q", l.pidFile(name), data)
|
||||
}
|
||||
if !l.runs(pid, name) {
|
||||
return 0, nil
|
||||
}
|
||||
return pid, nil
|
||||
}
|
||||
|
||||
func (l Lab) runs(pid int, name string) bool {
|
||||
if err := syscall.Kill(pid, 0); err != nil && !errors.Is(err, syscall.EPERM) {
|
||||
return false
|
||||
}
|
||||
cmdline, err := os.ReadFile(filepath.Join(l.ProcDir, strconv.Itoa(pid), "cmdline"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Contains(cmdline, []byte("\x00-name\x00"+name+"\x00"))
|
||||
}
|
||||
|
||||
func (l Lab) Running() ([]string, error) {
|
||||
var running []string
|
||||
for _, n := range l.Plan.Nodes {
|
||||
pid, err := l.PID(n.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pid != 0 {
|
||||
running = append(running, n.Name)
|
||||
}
|
||||
}
|
||||
return running, nil
|
||||
}
|
||||
|
||||
func (l Lab) Up(ctx context.Context, fetcher provision.Fetcher, timeout time.Duration) error {
|
||||
running, err := l.Running()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(running) > 0 {
|
||||
return fmt.Errorf("lab %s is already running (%s): 'lab down' first", l.Plan.Name, strings.Join(running, ", "))
|
||||
}
|
||||
|
||||
nodes, err := provision.Prepare(ctx, l.Plan, provision.Options{RunDir: l.RunDir, Fetcher: fetcher, Runner: l.Runner})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range switchesFirst(l.Plan, nodes) {
|
||||
if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := l.Runner.Run(ctx, render.QEMUBinary, append(n.QEMU, "-daemonize")...); err != nil {
|
||||
return fmt.Errorf("node %s: %w", n.Name, err)
|
||||
}
|
||||
fmt.Fprintf(l.Out, "%s: started\n", n.Name)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
var errs []error
|
||||
for _, n := range l.Plan.Nodes {
|
||||
if err := l.waitReady(ctx, n); err != nil {
|
||||
errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(l.Out, "%s: ready\n", n.Name)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func switchesFirst(p *topology.Plan, nodes []render.Node) []render.Node {
|
||||
var first, rest []render.Node
|
||||
for i, n := range nodes {
|
||||
if p.Nodes[i].Role == topology.RoleSwitch {
|
||||
first = append(first, n)
|
||||
} else {
|
||||
rest = append(rest, n)
|
||||
}
|
||||
}
|
||||
return append(first, rest...)
|
||||
}
|
||||
|
||||
func (l Lab) waitReady(ctx context.Context, n topology.NodePlan) error {
|
||||
args := l.sshArgs(n, true, []string{"cloud-init", "status", "--wait"})
|
||||
for {
|
||||
err := l.Runner.Run(ctx, "ssh", args...)
|
||||
switch code := exitCode(err); {
|
||||
case err == nil:
|
||||
return nil
|
||||
case code == cloudInitRecoverable:
|
||||
fmt.Fprintf(l.Out, "%s: cloud-init finished with recoverable errors: %v\n", n.Name, err)
|
||||
return nil
|
||||
case code != sshUnreachable:
|
||||
return fmt.Errorf("cloud-init failed: %w", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("not reachable over ssh: %w", ctx.Err())
|
||||
case <-time.After(l.Poll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func exitCode(err error) int {
|
||||
var coded interface{ ExitCode() int }
|
||||
if errors.As(err, &coded) {
|
||||
return coded.ExitCode()
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (l Lab) sshArgs(n topology.NodePlan, batch bool, command []string) []string {
|
||||
args := []string{
|
||||
"-i", filepath.Join(l.RunDir, provision.KeyFile),
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", strconv.Itoa(n.SSHPort),
|
||||
}
|
||||
if batch {
|
||||
args = append(args, "-o", "BatchMode=yes", "-o", "ConnectTimeout=5")
|
||||
}
|
||||
args = append(args, User+"@127.0.0.1")
|
||||
return append(args, command...)
|
||||
}
|
||||
|
||||
func (l Lab) SSH(name string, terminal bool, command []string) ([]string, error) {
|
||||
n, err := l.node(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args := []string{"ssh"}
|
||||
if terminal && len(command) > 0 {
|
||||
args = append(args, "-t")
|
||||
}
|
||||
return append(args, l.sshArgs(n, false, command)...), nil
|
||||
}
|
||||
|
||||
func (l Lab) Status(w io.Writer) error {
|
||||
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintf(tw, "node\trole\tstate\tpid\tssh\n")
|
||||
for _, n := range l.Plan.Nodes {
|
||||
pid, err := l.PID(n.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state, shown := "stopped", "-"
|
||||
if pid != 0 {
|
||||
state, shown = "running", strconv.Itoa(pid)
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t127.0.0.1:%d\n", n.Name, n.Role, state, shown, n.SSHPort)
|
||||
}
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func (l Lab) Down(ctx context.Context) error {
|
||||
var errs []error
|
||||
for _, n := range l.Plan.Nodes {
|
||||
pid, err := l.PID(n.Name)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if pid == 0 {
|
||||
continue
|
||||
}
|
||||
if err := l.stop(ctx, n.Name, pid); err != nil {
|
||||
errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(l.pidFile(n.Name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(l.Out, "%s: stopped\n", n.Name)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (l Lab) stop(ctx context.Context, name string, pid int) error {
|
||||
if pid <= 0 {
|
||||
return fmt.Errorf("refusing to signal pid %d", pid)
|
||||
}
|
||||
send := l.signal
|
||||
if send == nil {
|
||||
send = syscall.Kill
|
||||
}
|
||||
for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} {
|
||||
if err := send(pid, sig); err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(l.Stop)
|
||||
for l.runs(pid, name) && time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(l.Poll):
|
||||
}
|
||||
}
|
||||
if !l.runs(pid, name) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("pid %d still running after SIGKILL", pid)
|
||||
}
|
||||
543
internal/lab/machine/machine_test.go
Normal file
543
internal/lab/machine/machine_test.go
Normal file
|
|
@ -0,0 +1,543 @@
|
|||
package machine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/lab/provision"
|
||||
"git.g3e.fr/syonad/two/internal/lab/topology"
|
||||
)
|
||||
|
||||
const labKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFG/JMmjfko96WkJV8DiL6rip/H/q/R++y8s27Z+Cj6O two-lab"
|
||||
|
||||
type exitErr int
|
||||
|
||||
func (e exitErr) Error() string { return "exit status " + strconv.Itoa(int(e)) }
|
||||
func (e exitErr) ExitCode() int { return int(e) }
|
||||
|
||||
type fakeRunner struct {
|
||||
mu sync.Mutex
|
||||
calls [][]string
|
||||
fail string
|
||||
ssh map[string][]error
|
||||
always map[string]error
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, append([]string{name}, args...))
|
||||
if name == f.fail {
|
||||
return errors.New(name + " failed")
|
||||
}
|
||||
switch name {
|
||||
case "ssh-keygen":
|
||||
private := args[len(args)-1]
|
||||
if err := os.WriteFile(private, []byte("private"), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(private+".pub", []byte(labKey+"\n"), 0o644)
|
||||
case "ssh":
|
||||
port := args[indexOf(args, "-p")+1]
|
||||
answers := f.ssh[port]
|
||||
if len(answers) == 0 {
|
||||
return f.always[port]
|
||||
}
|
||||
f.ssh[port] = answers[1:]
|
||||
return answers[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRunner) commands(name string) [][]string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out [][]string
|
||||
for _, c := range f.calls {
|
||||
if c[0] == name {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexOf(list []string, value string) int {
|
||||
for i, v := range list {
|
||||
if v == value {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func mirror(t *testing.T) (*httptest.Server, string) {
|
||||
t.Helper()
|
||||
content := []byte("qcow2 image")
|
||||
h := sha512.Sum512(content)
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "SHA512SUMS") {
|
||||
fmt.Fprintf(w, "%s deb.qcow2\n", hex.EncodeToString(h[:]))
|
||||
return
|
||||
}
|
||||
w.Write(content)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server, server.URL
|
||||
}
|
||||
|
||||
func labPlan(t *testing.T, base, nodes string) *topology.Plan {
|
||||
t.Helper()
|
||||
doc := `name: evpn-2hv
|
||||
images:
|
||||
deb:
|
||||
url: ` + base + `/deb.qcow2
|
||||
sums: ` + base + `/SHA512SUMS
|
||||
segments:
|
||||
underlay: { switch: sw1, cidr: 10.250.0.0/24, mtu: 9000 }
|
||||
nodes:
|
||||
` + nodes
|
||||
topo, err := topology.Parse([]byte(doc))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
p, err := topology.Compute(topo)
|
||||
if err != nil {
|
||||
t.Fatalf("Compute: %v", err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
const switchLast = ` rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay] }
|
||||
hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
|
||||
sw1: { role: switch, image: deb, cpus: 2, memory: 1024 }
|
||||
`
|
||||
|
||||
type fixture struct {
|
||||
lab Lab
|
||||
runner *fakeRunner
|
||||
out *bytes.Buffer
|
||||
fetcher provision.Fetcher
|
||||
}
|
||||
|
||||
func newFixture(t *testing.T, nodes string) *fixture {
|
||||
t.Helper()
|
||||
server, base := mirror(t)
|
||||
root := t.TempDir()
|
||||
runner := &fakeRunner{ssh: map[string][]error{}, always: map[string]error{}}
|
||||
out := &bytes.Buffer{}
|
||||
return &fixture{
|
||||
lab: Lab{
|
||||
Plan: labPlan(t, base, nodes),
|
||||
RunDir: filepath.Join(root, "run"),
|
||||
ProcDir: filepath.Join(root, "proc"),
|
||||
Runner: runner,
|
||||
Poll: time.Millisecond,
|
||||
Stop: 500 * time.Millisecond,
|
||||
Out: out,
|
||||
},
|
||||
runner: runner,
|
||||
out: out,
|
||||
fetcher: provision.Fetcher{Client: server.Client(), CacheDir: filepath.Join(root, "cache")},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fixture) process(t *testing.T, node, script string, cmdline ...string) int {
|
||||
t.Helper()
|
||||
cmd := exec.Command("sh", "-c", script)
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { cmd.Wait(); close(done) }()
|
||||
t.Cleanup(func() {
|
||||
cmd.Process.Kill()
|
||||
<-done
|
||||
})
|
||||
pid := cmd.Process.Pid
|
||||
proc := filepath.Join(f.lab.ProcDir, strconv.Itoa(pid))
|
||||
if err := os.MkdirAll(proc, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(proc, "cmdline"), []byte(strings.Join(cmdline, "\x00")+"\x00"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.pidFile(t, node, strconv.Itoa(pid)+"\n")
|
||||
return pid
|
||||
}
|
||||
|
||||
func (f *fixture) pidFile(t *testing.T, node, content string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(f.lab.RunDir, node)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "qemu.pid"), []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func qemu(node string) []string {
|
||||
return []string{"qemu-system-x86_64", "-name", node, "-machine", "q35"}
|
||||
}
|
||||
|
||||
func alive(pid int) bool {
|
||||
err := syscall.Kill(pid, 0)
|
||||
return err == nil || errors.Is(err, syscall.EPERM)
|
||||
}
|
||||
|
||||
func waitDead(t *testing.T, pid int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for alive(pid) && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if alive(pid) {
|
||||
t.Errorf("pid %d is still alive", pid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPID_NoPidFileMeansStopped(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil {
|
||||
t.Errorf("PID = %d, %v", pid, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPID_RefusesACorruptedPidFile(t *testing.T) {
|
||||
for _, content := range []string{"", "abc", "0", "-12"} {
|
||||
f := newFixture(t, switchLast)
|
||||
f.pidFile(t, "hv1", content)
|
||||
if _, err := f.lab.PID("hv1"); err == nil || !strings.Contains(err.Error(), "not a pid") {
|
||||
t.Errorf("pid file %q: error = %v", content, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPID_RecognisesTheNodeProcess(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
want := f.process(t, "hv1", "sleep 30", qemu("hv1")...)
|
||||
if pid, err := f.lab.PID("hv1"); pid != want || err != nil {
|
||||
t.Errorf("PID = %d, %v, want %d", pid, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPID_IgnoresAPidReusedByAnotherProcess(t *testing.T) {
|
||||
cases := map[string][]string{
|
||||
"another program": {"/usr/sbin/sshd", "-D"},
|
||||
"another node": qemu("hv10"),
|
||||
"name as a value": {"qemu-system-x86_64", "-serial", "-name", "-name", "hv2"},
|
||||
}
|
||||
for name, cmdline := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.process(t, "hv1", "sleep 30", cmdline...)
|
||||
if pid, err := f.lab.PID("hv1"); pid != 0 || err != nil {
|
||||
t.Errorf("PID = %d, %v, want 0", pid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPID_IgnoresADeadProcess(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
pid := f.process(t, "hv1", "exit 0", qemu("hv1")...)
|
||||
waitDead(t, pid)
|
||||
if got, err := f.lab.PID("hv1"); got != 0 || err != nil {
|
||||
t.Errorf("PID = %d, %v, want 0", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatus(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
pid := strconv.Itoa(f.process(t, "hv1", "sleep 30", qemu("hv1")...))
|
||||
var out bytes.Buffer
|
||||
if err := f.lab.Status(&out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := len(pid)
|
||||
if w < 3 {
|
||||
w = 3
|
||||
}
|
||||
want := fmt.Sprintf("node role state %-*s ssh\n", w, "pid") +
|
||||
fmt.Sprintf("rr1 rr stopped %-*s 127.0.0.1:2200\n", w, "-") +
|
||||
fmt.Sprintf("hv1 hypervisor running %-*s 127.0.0.1:2201\n", w, pid) +
|
||||
fmt.Sprintf("sw1 switch stopped %-*s 127.0.0.1:2202\n", w, "-")
|
||||
if out.String() != want {
|
||||
t.Errorf("status:\n%s\nwant:\n%s", out.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDown_StopsWithSIGTERM(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...)
|
||||
|
||||
if err := f.lab.Down(context.Background()); err != nil {
|
||||
t.Fatalf("Down: %v", err)
|
||||
}
|
||||
|
||||
waitDead(t, pid)
|
||||
if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) {
|
||||
t.Errorf("pid file still there: %v", err)
|
||||
}
|
||||
if f.out.String() != "hv1: stopped\n" {
|
||||
t.Errorf("output = %q", f.out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDown_FallsBackToSIGKILL(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.lab.Stop = 100 * time.Millisecond
|
||||
pid := f.process(t, "hv1", `trap "" TERM; while :; do sleep 0.05; done`, qemu("hv1")...)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
start := time.Now()
|
||||
if err := f.lab.Down(context.Background()); err != nil {
|
||||
t.Fatalf("Down: %v", err)
|
||||
}
|
||||
|
||||
waitDead(t, pid)
|
||||
if elapsed := time.Since(start); elapsed < 100*time.Millisecond {
|
||||
t.Errorf("SIGKILL after %v, want at least the 100ms grace period", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDown_NeverSignalsAProcessThatIsNotTheNode(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
pid := f.process(t, "hv1", "sleep 30", "/usr/sbin/sshd", "-D")
|
||||
|
||||
if err := f.lab.Down(context.Background()); err != nil {
|
||||
t.Fatalf("Down: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if !alive(pid) {
|
||||
t.Error("a process that is not the node was killed")
|
||||
}
|
||||
if f.out.Len() != 0 {
|
||||
t.Errorf("output = %q", f.out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDown_ReportsACorruptedPidFileAndStopsTheOthers(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.pidFile(t, "rr1", "garbage")
|
||||
pid := f.process(t, "hv1", "sleep 30", qemu("hv1")...)
|
||||
|
||||
err := f.lab.Down(context.Background())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "not a pid") {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
waitDead(t, pid)
|
||||
}
|
||||
|
||||
func TestSSH_Arguments(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
key := filepath.Join(f.lab.RunDir, "lab_ed25519")
|
||||
base := []string{
|
||||
"-i", key,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", "2201",
|
||||
"debian@127.0.0.1",
|
||||
}
|
||||
cases := map[string]struct {
|
||||
terminal bool
|
||||
command []string
|
||||
want []string
|
||||
}{
|
||||
"interactive shell": {true, nil, append([]string{"ssh"}, base...)},
|
||||
"command from a terminal": {true, []string{"top"}, append(append([]string{"ssh", "-t"}, base...), "top")},
|
||||
"command from a script": {false, []string{"ip", "-br", "a"}, append(append([]string{"ssh"}, base...), "ip", "-br", "a")},
|
||||
"shell from a script": {false, nil, append([]string{"ssh"}, base...)},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := f.lab.SSH("hv1", c.terminal, c.command)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("\n got %q\nwant %q", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSH_UnknownNode(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
if _, err := f.lab.SSH("hv9", true, nil); err == nil || !strings.Contains(err.Error(), `node "hv9" is not in lab evpn-2hv`) {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_StartsSwitchesFirstAndWaitsForEveryNode(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
|
||||
if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
|
||||
t.Fatalf("Up: %v", err)
|
||||
}
|
||||
|
||||
var started []string
|
||||
for _, c := range f.runner.commands("qemu-system-x86_64") {
|
||||
if c[len(c)-1] != "-daemonize" {
|
||||
t.Errorf("qemu not daemonized: %q", c)
|
||||
}
|
||||
started = append(started, c[indexOf(c, "-name")+1])
|
||||
}
|
||||
if !reflect.DeepEqual(started, []string{"sw1", "rr1", "hv1"}) {
|
||||
t.Errorf("start order = %v, want the switch first", started)
|
||||
}
|
||||
|
||||
ssh := f.runner.commands("ssh")
|
||||
if len(ssh) != 3 {
|
||||
t.Fatalf("%d ssh calls, want 3", len(ssh))
|
||||
}
|
||||
want := []string{"ssh",
|
||||
"-i", filepath.Join(f.lab.RunDir, "lab_ed25519"),
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", "2200",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"debian@127.0.0.1",
|
||||
"cloud-init", "status", "--wait",
|
||||
}
|
||||
if !reflect.DeepEqual(ssh[0], want) {
|
||||
t.Errorf("readiness check:\n got %q\nwant %q", ssh[0], want)
|
||||
}
|
||||
if f.out.String() != "sw1: started\nrr1: started\nhv1: started\nrr1: ready\nhv1: ready\nsw1: ready\n" {
|
||||
t.Errorf("output = %q", f.out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_RetriesWhileSSHIsUnreachable(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.runner.ssh["2201"] = []error{exitErr(255), exitErr(255), nil}
|
||||
|
||||
if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
|
||||
t.Fatalf("Up: %v", err)
|
||||
}
|
||||
if n := len(f.runner.commands("ssh")); n != 5 {
|
||||
t.Errorf("%d ssh calls, want 5", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_ReadinessFailures(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
answers []error
|
||||
want string
|
||||
}{
|
||||
"cloud-init error": {[]error{exitErr(1)}, "node hv1: cloud-init failed: exit status 1"},
|
||||
"not an exit status": {[]error{errors.New("fork failed")}, "node hv1: cloud-init failed: fork failed"},
|
||||
"ssh never reachable": {nil, "node hv1: not reachable over ssh: context deadline exceeded"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.lab.Poll = 10 * time.Millisecond
|
||||
f.runner.ssh["2201"] = c.answers
|
||||
f.runner.always["2201"] = exitErr(255)
|
||||
|
||||
err := f.lab.Up(context.Background(), f.fetcher, 300*time.Millisecond)
|
||||
|
||||
if err == nil || err.Error() != c.want {
|
||||
t.Errorf("error = %v, want %q", err, c.want)
|
||||
}
|
||||
if !strings.Contains(f.out.String(), "rr1: ready\n") || !strings.Contains(f.out.String(), "sw1: ready\n") {
|
||||
t.Errorf("the other nodes were not waited for: %q", f.out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_RecoverableCloudInitErrorsAreReportedNotFatal(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.runner.ssh["2201"] = []error{exitErr(2)}
|
||||
|
||||
if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
|
||||
t.Fatalf("Up: %v", err)
|
||||
}
|
||||
if !strings.Contains(f.out.String(), "hv1: cloud-init finished with recoverable errors: exit status 2\nhv1: ready\n") {
|
||||
t.Errorf("output = %q", f.out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_RefusesARunningLab(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.process(t, "hv1", "sleep 30", qemu("hv1")...)
|
||||
|
||||
err := f.lab.Up(context.Background(), f.fetcher, time.Second)
|
||||
|
||||
if err == nil || err.Error() != "lab evpn-2hv is already running (hv1): 'lab down' first" {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
if len(f.runner.calls) != 0 {
|
||||
t.Errorf("commands ran: %q", f.runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_RemovesAStalePidFileBeforeStarting(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
pid := f.process(t, "hv1", "exit 0", qemu("hv1")...)
|
||||
waitDead(t, pid)
|
||||
|
||||
if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
|
||||
t.Fatalf("Up: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.lab.RunDir, "hv1", "qemu.pid")); !os.IsNotExist(err) {
|
||||
t.Errorf("stale pid file kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUp_StopsAtTheFirstQEMUFailure(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.runner.fail = "qemu-system-x86_64"
|
||||
|
||||
err := f.lab.Up(context.Background(), f.fetcher, time.Second)
|
||||
|
||||
if err == nil || err.Error() != "node sw1: qemu-system-x86_64 failed" {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
if n := len(f.runner.commands("qemu-system-x86_64")); n != 1 {
|
||||
t.Errorf("%d qemu starts, want 1", n)
|
||||
}
|
||||
if n := len(f.runner.commands("ssh")); n != 0 {
|
||||
t.Errorf("%d ssh calls after a failed start", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStop_RefusesPidsThatTargetAGroup(t *testing.T) {
|
||||
f := newFixture(t, switchLast)
|
||||
f.lab.signal = func(pid int, sig syscall.Signal) error {
|
||||
t.Fatalf("signal %v sent to pid %d", sig, pid)
|
||||
return nil
|
||||
}
|
||||
for _, pid := range []int{0, -1} {
|
||||
if err := f.lab.stop(context.Background(), "hv1", pid); err == nil || err.Error() != fmt.Sprintf("refusing to signal pid %d", pid) {
|
||||
t.Errorf("pid %d: error = %v", pid, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue