f-50: lab: déploiement de two sur les hyperviseurs et provisionnement en un script #50

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>

cmd/lab/main_test.go

@@ -342,7 +342,7 @@ segments:
   underlay: { switch: sw1, cidr: 10.1.0.0/24 }
 nodes:
   sw1: { role: switch, image: deb, cpus: 1, memory: 512, frr: frr/absent.conf }
-  hv1: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [underlay] }
+  hv1: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [underlay], release: 0.2.0rc002 }
 `
 	if err := os.WriteFile(topo, []byte(doc), 0o600); err != nil {
 		t.Fatal(err)

conf/lab/evpn-2hv.yml

@@ -21,7 +21,7 @@ nodes:
       loopback: 10.255.255.1/32, frr: frr/rr1.conf }
   hv1:
     { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay],
-      addresses: { underlay: 192.168.14.11 }, frr: frr/hv1.conf }
+      addresses: { underlay: 192.168.14.11 }, frr: frr/hv1.conf, release: 0.2.0rc002 }
   hv2:
     { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay],
-      addresses: { underlay: 192.168.14.12 }, frr: frr/hv2.conf }
+      addresses: { underlay: 192.168.14.12 }, frr: frr/hv2.conf, release: 0.2.0rc002 }

docs/developpement/lab.rst

@@ -229,6 +229,9 @@ Ce que le fichier déclare :
      (``loopback: 10.255.255.1/32``) ;
    * ``frr`` — le chemin d'un ``frr.conf``, relatif au fichier de topologie : FRR est installé
      au démarrage et la configuration déposée **telle quelle** (voir `Rôles`_).
+   * ``release`` — **obligatoire pour un hyperviseur**, refusé ailleurs : le tag de la release de
+     two que ``deploy.sh`` installe (``release: 0.2.0rc002``). Sans lui, ``deploy.sh`` prendrait la
+     dernière release, et le lab ne serait plus reproductible.
 
    ``mgmt0`` et ``lo1`` sont réservés : aucun segment ne peut porter ces noms.
 
@@ -275,11 +278,11 @@ Limites : 1000 nœuds, 256 segments, et autant de câbles que la plage UDP le pe
      hv2   hypervisor  debian12  4     16384 MiB  127.0.0.1:2203
 
    roles
-     name  loopback             secondary                frr
+     name  loopback             secondary                frr       release
-     sw1   -                    underlay 169.254.0.1/28  sw1.conf
+     sw1   -                    underlay 169.254.0.1/28  sw1.conf  -
-     rr1   lo1 10.255.255.1/32  underlay 169.254.0.3/28  rr1.conf
+     rr1   lo1 10.255.255.1/32  underlay 169.254.0.3/28  rr1.conf  -
-     hv1   -                    -                        hv1.conf
+     hv1   -                    -                        hv1.conf  0.2.0rc002
-     hv2   -                    -                        hv2.conf
+     hv2   -                    -                        hv2.conf  0.2.0rc002
 
    segment underlay: 192.168.14.0/24, mtu 9000, switch sw1, bridge br-underlay, gateway 192.168.14.1
      node  interface  address           mac                udp         switch port  mac                udp
@@ -336,6 +339,39 @@ même valeur sur ``keys.openpgp.org`` et ``keyserver.ubuntu.com`` :
 Une clé renouvelée par FRR fera échouer l'installation (signature inconnue) : remplacer le fichier
 après avoir vérifié les nouvelles empreintes.
 
+**Hyperviseurs.** Ils se déploient comme en production, par ``deploy.sh`` — celui **du dépôt**,
+embarqué dans ``lab`` avec ``bootstrap_kvm.sh`` (paquet ``scripts``) et déposé dans
+``/opt/two/scripts/``, où ``deploy.sh`` cherche d'abord ``bootstrap_kvm.sh`` :
+
+.. code-block:: text
+
+   deploy.sh --noup_script -i -u <segment> -t <release>
+
+``--noup_script`` empêche l'auto-mise à jour de remplacer le script par celui de ``main`` : le lab
+teste les scripts de sa branche. L'uplink ``-u`` est l'interface qui porte la route par défaut —
+celle du premier segment de l'hyperviseur dans l'ordre de déclaration des segments — parce que
+``deploy.sh`` y lit l'adresse et la passerelle qu'il déplace sur ``br-000000``. ``deploy.sh``
+télécharge la release sur ``git.g3e.fr`` sans réessayer : le lancement attend d'abord que le serveur
+réponde, à travers le switch. FRR est installé **après** : il démarre sur le réseau final.
+
+**Un seul script de provisionnement par nœud.** cloud-init exécute ``runcmd`` comme un script
+``sh`` sans ``set -e`` : seule la dernière commande compte, et un ``deploy.sh`` en échec suivi d'un
+FRR installé avec succès passerait pour un démarrage réussi. Chaque nœud reçoit donc
+``/usr/local/sbin/lab-provision``, en ``set -eu``, qui enchaîne ses étapes ; ``runcmd`` n'appelle
+que lui, et la première étape en échec met cloud-init en erreur.
+
+**Ce que vérifie** ``lab up``, une fois cloud-init terminé sans erreur : ``agent.service`` actif
+sur chaque hyperviseur, ``frr`` actif sur chaque nœud qui en a un. Ce contrôle couvre ce que
+cloud-init ne voit pas — si la migration réseau échoue, ``deploy.sh`` arme un redémarrage de
+secours, et la VM redémarrée ne rejoue pas ``runcmd``.
+
+.. warning::
+
+   **Un hyperviseur du lab ne survit pas à un redémarrage.** En production, la racine est en
+   tmpfs et ``deploy.sh --bootstrap`` est rejoué à chaque démarrage ; dans le lab, ``-i`` n'est
+   exécuté qu'au premier, et la migration réseau, qui n'est pas persistée, est perdue. Recréer le
+   lab : ``lab down`` puis ``lab up``.
+
 .. note::
 
    La configuration du switch (``conf/lab/frr/sw1.conf``) **n'est pas celle des routeurs** : écrite
@@ -661,6 +697,10 @@ Sécurité
 * **Clés d'hôte des VM non vérifiées** par ``lab ssh`` (``known_hosts`` jetable) : elles changent
   à chaque ``up``. Acceptable uniquement parce que la connexion reste sur la boucle locale d'un
   serveur auquel on s'est authentifié.
+* **Code exécuté sans épinglage par** ``deploy.sh`` : la bibliothèque ``shflags`` est récupérée
+  par ``curl`` sur la branche ``main`` d'un autre dépôt (``H6N/tools``) et exécutée par ``eval``,
+  sans vérification d'intégrité — dans le lab comme en production. Les artefacts de la release
+  sont, eux, vérifiés contre ``SHA256SUMS``.
 * **Image** : ``SHA512SUMS`` vient de la même origine que l'image, en HTTPS. La vérification
   protège contre la corruption, pas contre une origine compromise ; la signature GPG de Debian
   (``SHA512SUMS.sign``) n'est pas encore vérifiée.

internal/lab/machine/machine.go

@@ -125,6 +125,10 @@ func (l Lab) Up(ctx context.Context, fetcher provision.Fetcher, timeout time.Dur
 			errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err))
 			continue
 		}
+		if err := l.checkServices(ctx, n); err != nil {
+			errs = append(errs, fmt.Errorf("node %s: %w", n.Name, err))
+			continue
+		}
 		fmt.Fprintf(l.Out, "%s: ready\n", n.Name)
 	}
 	return errors.Join(errs...)
@@ -163,6 +167,26 @@ func (l Lab) waitReady(ctx context.Context, n topology.NodePlan) error {
 	}
 }
 
+func services(n topology.NodePlan) []string {
+	var units []string
+	if n.Role == topology.RoleHypervisor {
+		units = append(units, "agent.service")
+	}
+	if n.FRR != "" {
+		units = append(units, "frr.service")
+	}
+	return units
+}
+
+func (l Lab) checkServices(ctx context.Context, n topology.NodePlan) error {
+	for _, unit := range services(n) {
+		if err := l.Runner.Run(ctx, "ssh", l.sshArgs(n, true, []string{"systemctl", "is-active", "--quiet", unit})...); err != nil {
+			return fmt.Errorf("%s is not active: %w", unit, err)
+		}
+	}
+	return nil
+}
+
 func exitCode(err error) int {
 	var coded interface{ ExitCode() int }
 	if errors.As(err, &coded) {

internal/lab/machine/machine_test.go

@@ -37,6 +37,7 @@ type fakeRunner struct {
 	fail     string
 	ssh      map[string][]error
 	always   map[string]error
+	inactive string
 }
 
 func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error {
@@ -55,6 +56,9 @@ func (f *fakeRunner) Run(_ context.Context, name string, args ...string) error {
 		return os.WriteFile(private+".pub", []byte(labKey+"\n"), 0o644)
 	case "ssh":
 		port := args[indexOf(args, "-p")+1]
+		if args[len(args)-1] == f.inactive {
+			return exitErr(3)
+		}
 		answers := f.ssh[port]
 		if len(answers) == 0 {
 			return f.always[port]
@@ -124,7 +128,7 @@ nodes:
 }
 
 const switchLast = `  rr1: { role: rr,         image: deb, cpus: 1, memory: 1024, segments: [underlay] }
-  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
   sw1: { role: switch,     image: deb, cpus: 2, memory: 1024 }
 `
 
@@ -409,8 +413,8 @@ func TestUp_StartsSwitchesFirstAndWaitsForEveryNode(t *testing.T) {
 	}
 
 	ssh := f.runner.commands("ssh")
-	if len(ssh) != 3 {
+	if len(ssh) != 4 {
-		t.Fatalf("%d ssh calls, want 3", len(ssh))
+		t.Fatalf("%d ssh calls, want 4", len(ssh))
 	}
 	want := []string{"ssh",
 		"-i", filepath.Join(f.lab.RunDir, "lab_ed25519"),
@@ -440,8 +444,8 @@ func TestUp_RetriesWhileSSHIsUnreachable(t *testing.T) {
 	if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
 		t.Fatalf("Up: %v", err)
 	}
-	if n := len(f.runner.commands("ssh")); n != 5 {
+	if n := len(f.runner.commands("ssh")); n != 6 {
-		t.Errorf("%d ssh calls, want 5", n)
+		t.Errorf("%d ssh calls, want 6", n)
 	}
 }
 
@@ -541,3 +545,44 @@ func TestStop_RefusesPidsThatTargetAGroup(t *testing.T) {
 		}
 	}
 }
+
+func TestUp_ChecksTheServicesOfEachRole(t *testing.T) {
+	f := newFixture(t, switchLast)
+	conf := filepath.Join(t.TempDir(), "sw1.conf")
+	if err := os.WriteFile(conf, []byte("hostname sw1\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	f.lab.Plan.Nodes[2].FRR = conf
+
+	if err := f.lab.Up(context.Background(), f.fetcher, time.Second); err != nil {
+		t.Fatalf("Up: %v", err)
+	}
+
+	var checks []string
+	for _, c := range f.runner.commands("ssh") {
+		if c[len(c)-4] == "systemctl" {
+			checks = append(checks, c[indexOf(c, "-p")+1]+" "+strings.Join(c[len(c)-4:], " "))
+		}
+	}
+	want := []string{
+		"2201 systemctl is-active --quiet agent.service",
+		"2202 systemctl is-active --quiet frr.service",
+	}
+	if !reflect.DeepEqual(checks, want) {
+		t.Errorf("checks = %q, want %q", checks, want)
+	}
+}
+
+func TestUp_ReportsAnInactiveServiceAndWaitsForTheOthers(t *testing.T) {
+	f := newFixture(t, switchLast)
+	f.runner.inactive = "agent.service"
+
+	err := f.lab.Up(context.Background(), f.fetcher, time.Second)
+
+	if err == nil || err.Error() != "node hv1: agent.service is not active: exit status 3" {
+		t.Errorf("error = %v", err)
+	}
+	if !strings.Contains(f.out.String(), "rr1: ready\n") || !strings.Contains(f.out.String(), "sw1: ready\n") || strings.Contains(f.out.String(), "hv1: ready") {
+		t.Errorf("output = %q", f.out.String())
+	}
+}

internal/lab/provision/provision_test.go

@@ -60,8 +60,8 @@ segments:
 nodes:
   sw1: { role: switch,     image: debian12, cpus: 2, memory: 1024 }
   rr1: { role: rr,         image: debian12, cpus: 1, memory: 1024, segments: [underlay] }
-  hv1: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] }
+  hv1: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
-  hv2: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay] }
+  hv2: { role: hypervisor, image: debian12, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
 `
 	topo, err := topology.Parse([]byte(doc))
 	if err != nil {

internal/lab/render/render_test.go

@@ -22,8 +22,8 @@ segments:
 nodes:
   sw1: { role: switch,     image: deb, cpus: 2, memory: 1024 }
   rr1: { role: rr,         image: deb, cpus: 1, memory: 1024, segments: [underlay] }
-  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
-  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
 `
 
 const twoSegments = `name: two-seg
@@ -36,7 +36,7 @@ segments:
   blue: { switch: sw, cidr: 10.2.0.0/24, mtu: 1500 }
 nodes:
   sw: { role: switch, image: deb, cpus: 1, memory: 512 }
-  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red] }
+  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red], release: 0.2.0rc002 }
 `
 
 func plan(t *testing.T, doc string) *topology.Plan {
@@ -317,7 +317,7 @@ func TestNetworkConfig_DefaultRouteOnlyOnFirstSegment(t *testing.T) {
 
 func TestUserData_Hypervisor(t *testing.T) {
 	cfg := user(t, nodeNamed(t, renderAll(t, twoHypervisors), "hv1"))
-	if cfg.Hostname != "hv1" || !cfg.DisableRoot || len(cfg.WriteFiles) != 0 || len(cfg.Runcmd) != 0 || len(cfg.Packages) != 0 {
+	if cfg.Hostname != "hv1" || !cfg.DisableRoot || len(cfg.Packages) != 0 || !reflect.DeepEqual(cfg.Runcmd, [][]string{{"/usr/local/sbin/lab-provision"}}) {
 		t.Errorf("hv1 user-data = %+v", cfg)
 	}
 	if !reflect.DeepEqual(cfg.SSHAuthorizedKeys, []string{labKey}) {
@@ -376,9 +376,12 @@ nft -f /etc/lab-switch.nft
 			t.Errorf("unit misses %q", want)
 		}
 	}
-	if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"systemctl", "daemon-reload"}, {"systemctl", "enable", "--now", "lab-switch.service"}}) {
+	if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"/usr/local/sbin/lab-provision"}}) {
 		t.Errorf("runcmd = %v", cfg.Runcmd)
 	}
+	if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-switch.service\n" {
+		t.Errorf("lab-provision:\n%s", got)
+	}
 }
 
 func TestUserData_SwitchWithTwoSegments(t *testing.T) {

internal/lab/render/roles_test.go

@@ -1,9 +1,11 @@
 package render
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -19,8 +21,8 @@ segments:
 nodes:
   sw1: { role: switch, image: deb, cpus: 2, memory: 1024, secondary: { underlay: [169.254.0.1/28] }, frr: sw1.conf }
   rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay], secondary: { underlay: [169.254.0.3/28] }, loopback: 10.255.255.1/32, frr: rr1.conf }
-  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], frr: hv1.conf }
+  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], frr: hv1.conf, release: 0.2.0rc002 }
-  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
 `
 
 var frrConfigs = map[string]string{
@@ -95,9 +97,9 @@ func TestRoles_LoopbackOnADummyInterfaceReplayedAtBoot(t *testing.T) {
 	if !strings.Contains(unit, "ExecStart=/usr/local/sbin/lab-node\n") || !strings.Contains(unit, "WantedBy=multi-user.target\n") {
 		t.Errorf("lab-node.service:\n%s", unit)
 	}
-	wantCmds := []string{"systemctl daemon-reload", "systemctl enable --now lab-node.service", "/usr/local/sbin/lab-frr"}
+	want = "#!/bin/sh\nset -eu\nsystemctl daemon-reload\nsystemctl enable --now lab-node.service\n/usr/local/sbin/lab-frr\n"
-	if got := runcmd(t, rr1); !reflect.DeepEqual(got, wantCmds) {
+	if got := fileAt(t, cfg, "/usr/local/sbin/lab-provision").Content; got != want {
-		t.Errorf("runcmd = %q, want %q", got, wantCmds)
+		t.Errorf("lab-provision:\n%s\nwant:\n%s", got, want)
 	}
 }
 
@@ -141,22 +143,96 @@ func TestRoles_FRRKeyIsThePinnedRepositoryKey(t *testing.T) {
 	}
 }
 
-func TestRoles_FRRInstallRunsLast(t *testing.T) {
+func TestRoles_ProvisioningIsOneScriptThatStopsAtTheFirstFailure(t *testing.T) {
 	nodes := renderRoles(t)
-	for name, want := range map[string][]string{
+	for name, steps := range map[string]string{
-		"sw1": {"systemctl daemon-reload", "systemctl enable --now lab-switch.service", "/usr/local/sbin/lab-frr"},
+		"sw1": "systemctl daemon-reload\nsystemctl enable --now lab-switch.service\n/usr/local/sbin/lab-frr\n",
-		"hv1": {"/usr/local/sbin/lab-frr"},
+		"hv1": "/usr/local/sbin/lab-deploy\n/usr/local/sbin/lab-frr\n",
+		"hv2": "/usr/local/sbin/lab-deploy\n",
 	} {
-		if got := runcmd(t, nodeNamed(t, nodes, name)); !reflect.DeepEqual(got, want) {
+		cfg := user(t, nodeNamed(t, nodes, name))
-			t.Errorf("%s runcmd = %q, want %q", name, got, want)
+		if !reflect.DeepEqual(cfg.Runcmd, [][]string{{"/usr/local/sbin/lab-provision"}}) {
+			t.Errorf("%s runcmd = %q", name, cfg.Runcmd)
+		}
+		f := fileAt(t, cfg, "/usr/local/sbin/lab-provision")
+		if f.Content != "#!/bin/sh\nset -eu\n"+steps || f.Permissions != "0755" {
+			t.Errorf("%s lab-provision (%s):\n%s", name, f.Permissions, f.Content)
 		}
 	}
 }
 
-func TestRoles_NodeWithoutRoleFieldsGetsNothingExtra(t *testing.T) {
+func TestRoles_HypervisorDeploysTheReleaseThroughTheRepositoryScripts(t *testing.T) {
-	cfg := user(t, nodeNamed(t, renderRoles(t), "hv2"))
+	cfg := user(t, nodeNamed(t, renderRoles(t), "hv1"))
-	if len(cfg.WriteFiles) != 0 || len(cfg.Runcmd) != 0 {
+
-		t.Errorf("hv2 write_files %d, runcmd %q", len(cfg.WriteFiles), cfg.Runcmd)
+	f := fileAt(t, cfg, "/usr/local/sbin/lab-deploy")
+	want := `#!/bin/sh
+set -eu
+n=0
+until curl -fsS -o /dev/null https://git.g3e.fr/; do
+    n=$((n + 1))
+    [ "$n" -lt 30 ] || exit 1
+    sleep 10
+done
+cd /opt/two/scripts
+bash ./deploy.sh --noup_script -i -u underlay -t 0.2.0rc002
+`
+	if f.Content != want || f.Permissions != "0755" {
+		t.Errorf("lab-deploy (%s):\n%s\nwant:\n%s", f.Permissions, f.Content, want)
+	}
+
+	for path, source := range map[string]string{
+		"/opt/two/scripts/deploy.sh":        "../../../scripts/deploy.sh",
+		"/opt/two/scripts/bootstrap_kvm.sh": "../../../scripts/bootstrap_kvm.sh",
+	} {
+		f := fileAt(t, cfg, path)
+		got, err := base64.StdEncoding.DecodeString(f.Content)
+		if err != nil {
+			t.Fatal(err)
+		}
+		repo, err := os.ReadFile(source)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if f.Encoding != "b64" || f.Permissions != "0755" || !bytes.Equal(got, repo) {
+			t.Errorf("%s: encoding %q, permissions %q, identical to %s: %v", path, f.Encoding, f.Permissions, source, bytes.Equal(got, repo))
+		}
+	}
+}
+
+func TestRoles_UplinkIsTheInterfaceCarryingTheDefaultRoute(t *testing.T) {
+	hv := nodeNamed(t, renderAll(t, twoSegments), "hv")
+	var withDefault []string
+	for name, e := range network(t, hv).Ethernets {
+		for _, r := range e.Routes {
+			if r.To == "0.0.0.0/0" {
+				withDefault = append(withDefault, name)
+			}
+		}
+	}
+	if !reflect.DeepEqual(withDefault, []string{"red"}) {
+		t.Fatalf("default route on %v, want [red]", withDefault)
+	}
+	if got := fileAt(t, user(t, hv), "/usr/local/sbin/lab-deploy").Content; !strings.HasSuffix(got, "bash ./deploy.sh --noup_script -i -u red -t 0.2.0rc002\n") {
+		t.Errorf("lab-deploy:\n%s", got)
+	}
+}
+
+func TestRoles_OnlyHypervisorsDeployTwo(t *testing.T) {
+	nodes := renderRoles(t)
+	for _, name := range []string{"sw1", "rr1"} {
+		for _, f := range user(t, nodeNamed(t, nodes, name)).WriteFiles {
+			if strings.HasPrefix(f.Path, "/opt/two/") || f.Path == "/usr/local/sbin/lab-deploy" {
+				t.Errorf("%s receives %s", name, f.Path)
+			}
+		}
+	}
+}
+
+func TestRoles_NodeWithoutRoleFieldsGetsNoFRR(t *testing.T) {
+	for _, f := range user(t, nodeNamed(t, renderRoles(t), "hv2")).WriteFiles {
+		if strings.Contains(f.Path, "frr") {
+			t.Errorf("hv2 receives %s", f.Path)
+		}
 	}
 }
 

internal/lab/render/seed.go

@@ -9,6 +9,7 @@ import (
 	"go.yaml.in/yaml/v3"
 
 	"git.g3e.fr/syonad/two/internal/lab/topology"
+	"git.g3e.fr/syonad/two/scripts"
 )
 
 const (
@@ -25,6 +26,11 @@ const (
 	FRRSuite    = "frr-stable"
 	FRRRepo     = "https://deb.frrouting.org/frr"
 	FRRPackages = "frr frr-pythontools"
+
+	ProvisionScript = "/usr/local/sbin/lab-provision"
+	DeployScript    = "/usr/local/sbin/lab-deploy"
+	TwoScriptsDir   = "/opt/two/scripts"
+	TwoGitServer    = "https://git.g3e.fr/"
 )
 
 //go:embed frrouting.gpg
@@ -91,6 +97,7 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
 		DisableRoot:       true,
 		SSHAuthorizedKeys: o.AuthorizedKeys,
 	}
+	var steps []string
 	switch {
 	case n.Role == topology.RoleSwitch:
 		cfg.Packages = []string{"nftables"}
@@ -99,19 +106,21 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
 			{Path: SwitchNFT, Permissions: "0644", Content: switchNFT(p, n.Name)},
 			{Path: SwitchUnit, Permissions: "0644", Content: unit("Lab switch: bridges, gateways and NAT", SwitchScript)},
 		}
-		cfg.Runcmd = [][]string{
+		steps = append(steps, "systemctl daemon-reload", "systemctl enable --now lab-switch.service")
-			{"systemctl", "daemon-reload"},
-			{"systemctl", "enable", "--now", "lab-switch.service"},
-		}
 	case n.Loopback.IsValid():
 		cfg.WriteFiles = []writeFile{
 			{Path: NodeScript, Permissions: "0755", Content: "#!/bin/sh\nset -eu\n" + loopbackLines(n)},
 			{Path: NodeUnit, Permissions: "0644", Content: unit("Lab node: loopback", NodeScript)},
 		}
-		cfg.Runcmd = [][]string{
+		steps = append(steps, "systemctl daemon-reload", "systemctl enable --now lab-node.service")
-			{"systemctl", "daemon-reload"},
-			{"systemctl", "enable", "--now", "lab-node.service"},
 	}
+	if n.Role == topology.RoleHypervisor {
+		cfg.WriteFiles = append(cfg.WriteFiles,
+			writeFile{Path: TwoScriptsDir + "/deploy.sh", Permissions: "0755", Encoding: "b64", Content: base64.StdEncoding.EncodeToString(scripts.Deploy)},
+			writeFile{Path: TwoScriptsDir + "/bootstrap_kvm.sh", Permissions: "0755", Encoding: "b64", Content: base64.StdEncoding.EncodeToString(scripts.BootstrapKVM)},
+			writeFile{Path: DeployScript, Permissions: "0755", Content: deployScript(p, n)},
+		)
+		steps = append(steps, DeployScript)
 	}
 	if n.FRR != "" {
 		cfg.WriteFiles = append(cfg.WriteFiles,
@@ -119,7 +128,13 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
 			writeFile{Path: FRRConfig, Permissions: "0640", Content: o.FRR[n.Name]},
 			writeFile{Path: FRRScript, Permissions: "0755", Content: frrScript(n)},
 		)
-		cfg.Runcmd = append(cfg.Runcmd, []string{FRRScript})
+		steps = append(steps, FRRScript)
+	}
+	if len(steps) > 0 {
+		cfg.WriteFiles = append(cfg.WriteFiles,
+			writeFile{Path: ProvisionScript, Permissions: "0755", Content: "#!/bin/sh\nset -eu\n" + strings.Join(steps, "\n") + "\n"},
+		)
+		cfg.Runcmd = [][]string{{ProvisionScript}}
 	}
 	body, err := yaml.Marshal(cfg)
 	if err != nil {
@@ -128,6 +143,18 @@ func userData(p *topology.Plan, n topology.NodePlan, o Options) ([]byte, error)
 	return append([]byte("#cloud-config\n"), body...), nil
 }
 
+func retry(command string) string {
+	return "n=0\nuntil " + command + "; do\n    n=$((n + 1))\n    [ \"$n\" -lt 30 ] || exit 1\n    sleep 10\ndone\n"
+}
+
+func deployScript(p *topology.Plan, n topology.NodePlan) string {
+	uplink := nodeCables(p, n.Name)[0].NodeInterface
+	return "#!/bin/sh\nset -eu\n" +
+		retry("curl -fsS -o /dev/null "+TwoGitServer) +
+		"cd " + TwoScriptsDir + "\n" +
+		fmt.Sprintf("bash ./deploy.sh --noup_script -i -u %s -t %s\n", uplink, n.Release)
+}
+
 func frrDaemons(n topology.NodePlan) []string {
 	if n.Role == topology.RoleHypervisor {
 		return []string{"bgpd"}
@@ -139,7 +166,7 @@ func frrScript(n topology.NodePlan) string {
 	var b strings.Builder
 	b.WriteString("#!/bin/sh\nset -eu\nexport DEBIAN_FRONTEND=noninteractive\n. /etc/os-release\n")
 	fmt.Fprintf(&b, "echo \"deb [signed-by=%s] %s ${VERSION_CODENAME} %s\" > /etc/apt/sources.list.d/frr.list\n", FRRKey, FRRRepo, FRRSuite)
-	b.WriteString("n=0\nuntil apt-get update -qq --error-on=any; do\n    n=$((n + 1))\n    [ \"$n\" -lt 30 ] || exit 1\n    sleep 10\ndone\n")
+	b.WriteString(retry("apt-get update -qq --error-on=any"))
 	fmt.Fprintf(&b, "apt-get install -y -qq --no-install-recommends %s\n", FRRPackages)
 	for _, d := range frrDaemons(n) {
 		fmt.Fprintf(&b, "sed -i 's/^%s=no/%s=yes/' /etc/frr/daemons\n", d, d)

internal/lab/topology/format.go

@@ -22,15 +22,15 @@ func (p *Plan) Write(w io.Writer) error {
 
 	var extras []NodePlan
 	for _, n := range p.Nodes {
-		if len(n.Secondary) > 0 || n.Loopback.IsValid() || n.FRR != "" {
+		if len(n.Secondary) > 0 || n.Loopback.IsValid() || n.FRR != "" || n.Release != "" {
 			extras = append(extras, n)
 		}
 	}
 	if len(extras) > 0 {
 		fmt.Fprintf(tw, "\nroles\n")
-		fmt.Fprintf(tw, "  name\tloopback\tsecondary\tfrr\n")
+		fmt.Fprintf(tw, "  name\tloopback\tsecondary\tfrr\trelease\n")
 		for _, n := range extras {
-			fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\n", n.Name, orDash(loopback(n)), orDash(secondary(n)), orDash(filepath.Base(n.FRR)))
+			fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\t%s\n", n.Name, orDash(loopback(n)), orDash(secondary(n)), orDash(filepath.Base(n.FRR)), orDash(n.Release))
 		}
 	}
 

internal/lab/topology/plan.go

@@ -43,6 +43,7 @@ type NodePlan struct {
 	Secondary map[string][]netip.Prefix
 	Loopback  netip.Prefix
 	FRR       string
+	Release   string
 }
 
 type Cable struct {
@@ -76,6 +77,7 @@ func Compute(t *Topology) (*Plan, error) {
 			Memory:  n.Memory,
 			SSHPort: SSHBasePort + i,
 			FRR:     n.FRR,
+			Release: n.Release,
 		}
 		for segment, raws := range n.Secondary {
 			for _, raw := range raws {

internal/lab/topology/plan_test.go

@@ -55,8 +55,8 @@ segments:
 nodes:
   sw1: { role: switch,     image: deb, cpus: 2, memory: 1024 }
   rr1: { role: rr,         image: deb, cpus: 1, memory: 1024, segments: [underlay] }
-  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
-  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay] }
+  hv2: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], release: 0.2.0rc002 }
 `
 
 func TestCompute_TwoHypervisorsPlan(t *testing.T) {
@@ -165,7 +165,7 @@ segments:
   blue: { switch: sw, cidr: 10.2.0.0/24, mtu: 1500 }
 nodes:
   sw: { role: switch, image: deb, cpus: 1, memory: 512 }
-  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red] }
+  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [blue, red], release: 0.2.0rc002 }
   rr: { role: rr, image: deb, cpus: 1, memory: 512, segments: [red] }
 `)
 	got := make([]string, 0, len(p.Cables))
@@ -193,7 +193,7 @@ nodes:
   sw: { role: switch, image: deb, cpus: 1, memory: 512 }
   n1: { role: rr, image: deb, cpus: 1, memory: 512, segments: [a, b] }
   n2: { role: rr, image: deb, cpus: 1, memory: 512, segments: [a, b] }
-  n3: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [b, a] }
+  n3: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [b, a], release: 0.2.0rc002 }
 `)
 	macs := map[string]bool{}
 	ports := map[int]bool{}
@@ -314,6 +314,11 @@ nodes
   hv1   hypervisor  deb    4     16384 MiB  127.0.0.1:2202
   hv2   hypervisor  deb    4     16384 MiB  127.0.0.1:2203
 
+roles
+  name  loopback  secondary  frr  release
+  hv1   -         -          -    0.2.0rc002
+  hv2   -         -          -    0.2.0rc002
+
 segment underlay: 10.250.0.0/24, mtu 9000, switch sw1, bridge br-underlay, gateway 10.250.0.1
   node  interface  address        mac                udp         switch port  mac                udp
   rr1   underlay   10.250.0.2/24  02:4c:00:01:00:00  20000  <->  sw1 p0       02:4c:00:01:00:01  20001

internal/lab/topology/roles_test.go

@@ -15,7 +15,7 @@ segments:
 nodes:
   sw1: { role: switch, image: deb, cpus: 2, memory: 1024, secondary: { underlay: [169.254.0.1/28] }, frr: frr/sw1.conf }
   rr1: { role: rr, image: deb, cpus: 1, memory: 1024, segments: [underlay], secondary: { underlay: [169.254.0.3/28] }, loopback: 10.255.255.1/32, frr: frr/rr1.conf }
-  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], frr: /etc/lab/hv1.conf }
+  hv1: { role: hypervisor, image: deb, cpus: 4, memory: 16384, segments: [underlay], frr: /etc/lab/hv1.conf, release: 0.2.0rc002 }
 `
 
 func TestCompute_CarriesTheRoleFields(t *testing.T) {
@@ -99,7 +99,7 @@ segments:
   blue: { switch: sw, cidr: 10.2.0.0/24 }
 nodes:
   sw: { role: switch, image: deb, cpus: 1, memory: 512 }
-  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red, blue] }
+  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red, blue], release: 0.2.0rc002 }
   ` + c.node + `
 `
 			requireContains(t, validationError(t, doc), c.want)
@@ -115,7 +115,7 @@ segments:
 nodes:
   sw:    { role: switch, image: deb, cpus: 1, memory: 512, secondary: { red: [169.254.0.1/28], blue: [169.254.1.1/28] } }
   other: { role: switch, image: deb, cpus: 1, memory: 512 }
-  hv:    { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red, blue] }
+  hv:    { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red, blue], release: 0.2.0rc002 }
 `
 	msg := validationError(t, doc)
 	requireContains(t, msg, "node sw: secondary address given for segment blue it is not attached to")
@@ -130,7 +130,7 @@ segments:
   lo1: { switch: sw, cidr: 10.1.0.0/24 }
 nodes:
   sw: { role: switch, image: deb, cpus: 1, memory: 512 }
-  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [lo1] }
+  hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [lo1], release: 0.2.0rc002 }
 `
 	requireContains(t, validationError(t, doc), "segment lo1: name is reserved for the loopback interface")
 }
@@ -142,10 +142,10 @@ func TestWrite_ShowsTheRoles(t *testing.T) {
 	}
 	want := `
 roles
-  name  loopback             secondary                frr
+  name  loopback             secondary                frr       release
-  sw1   -                    underlay 169.254.0.1/28  sw1.conf
+  sw1   -                    underlay 169.254.0.1/28  sw1.conf  -
-  rr1   lo1 10.255.255.1/32  underlay 169.254.0.3/28  rr1.conf
+  rr1   lo1 10.255.255.1/32  underlay 169.254.0.3/28  rr1.conf  -
-  hv1   -                    -                        hv1.conf
+  hv1   -                    -                        hv1.conf  0.2.0rc002
 `
 	if !bytes.Contains(buf.Bytes(), []byte(want)) {
 		t.Errorf("plan:\n%s\ndoes not contain:\n%s", buf.String(), want)
@@ -154,10 +154,63 @@ roles
 
 func TestWrite_NoRolesSectionWithoutRoleFields(t *testing.T) {
 	var buf bytes.Buffer
-	if err := compute(t, twoHypervisors).Write(&buf); err != nil {
+	doc := header + `
+segments:
+  underlay: { switch: sw1, cidr: 10.250.0.0/24 }
+nodes:
+  sw1: { role: switch, image: deb, cpus: 1, memory: 512 }
+  rr1: { role: rr, image: deb, cpus: 1, memory: 512, segments: [underlay] }
+`
+	if err := compute(t, doc).Write(&buf); err != nil {
 		t.Fatalf("Write: %v", err)
 	}
 	if bytes.Contains(buf.Bytes(), []byte("roles")) {
 		t.Errorf("plan shows a roles section:\n%s", buf.String())
 	}
 }
+
+func TestValidate_ReleaseRejections(t *testing.T) {
+	cases := map[string]struct {
+		node string
+		want string
+	}{
+		"hypervisor without release": {
+			`hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red] }`,
+			"node hv: a hypervisor needs the release of two to deploy (release: <tag>)",
+		},
+		"release with shell characters": {
+			`hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red], release: "0.2.0; reboot" }`,
+			`node hv: release "0.2.0; reboot" must match`,
+		},
+		"release on a route reflector": {
+			`hv: { role: rr, image: deb, cpus: 1, memory: 512, segments: [red], release: 0.2.0rc002 }`,
+			"node hv: release is only for hypervisors",
+		},
+		"release on a switch": {
+			`hv: { role: hypervisor, image: deb, cpus: 1, memory: 512, segments: [red], release: 0.2.0rc002 }
+  sw2: { role: switch, image: deb, cpus: 1, memory: 512, release: 0.2.0rc002 }`,
+			"node sw2: release is only for hypervisors",
+		},
+	}
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			doc := header + `
+segments:
+  red: { switch: sw, cidr: 10.1.0.0/24 }
+nodes:
+  sw: { role: switch, image: deb, cpus: 1, memory: 512 }
+  ` + c.node + `
+`
+			requireContains(t, validationError(t, doc), c.want)
+		})
+	}
+}
+
+func TestCompute_CarriesTheRelease(t *testing.T) {
+	if got := nodeOf(t, compute(t, withRoles), "hv1").Release; got != "0.2.0rc002" {
+		t.Errorf("hv1 release = %q", got)
+	}
+	if got := nodeOf(t, compute(t, withRoles), "rr1").Release; got != "" {
+		t.Errorf("rr1 release = %q", got)
+	}
+}

internal/lab/topology/topology.go

@@ -49,6 +49,7 @@ type Node struct {
 	Secondary map[string][]string
 	Loopback  string
 	FRR       string
+	Release   string
 }
 
 type fileImage struct {
@@ -72,6 +73,7 @@ type fileNode struct {
 	Secondary map[string][]string `yaml:"secondary"`
 	Loopback  string              `yaml:"loopback"`
 	FRR       string              `yaml:"frr"`
+	Release   string              `yaml:"release"`
 }
 
 type file struct {
@@ -141,6 +143,7 @@ func Parse(data []byte) (*Topology, error) {
 			Secondary: n.Secondary,
 			Loopback:  n.Loopback,
 			FRR:       n.FRR,
+			Release:   n.Release,
 		})
 	}
 	return t, nil

internal/lab/topology/validate.go

@@ -23,6 +23,7 @@ const (
 var (
 	namePattern    = regexp.MustCompile(`^[a-z][a-z0-9-]{0,14}$`)
 	segmentPattern = regexp.MustCompile(`^[a-z][a-z0-9]{0,11}$`)
+	releasePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
 )
 
 func (t *Topology) Validate() error {
@@ -206,6 +207,14 @@ func validateExtras(n Node, segments map[string]Segment, add func(string, ...any
 			}
 		}
 	}
+	switch {
+	case n.Role == RoleHypervisor && n.Release == "":
+		add("node %s: a hypervisor needs the release of two to deploy (release: <tag>)", n.Name)
+	case n.Role == RoleHypervisor && !releasePattern.MatchString(n.Release):
+		add("node %s: release %q must match %s", n.Name, n.Release, releasePattern)
+	case n.Role != RoleHypervisor && n.Release != "":
+		add("node %s: release is only for hypervisors", n.Name)
+	}
 	if n.Loopback != "" {
 		prefix, err := netip.ParsePrefix(n.Loopback)
 		switch {

scripts/embed.go

@@ -0,0 +1,9 @@
+package scripts
+
+import _ "embed"
+
+//go:embed deploy.sh
+var Deploy []byte
+
+//go:embed bootstrap_kvm.sh
+var BootstrapKVM []byte