diff --git a/api/agent.yaml b/api/agent.yaml index 9fcd5d0..fb78148 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -448,6 +448,12 @@ components: interfaces: type: array minItems: 1 + description: > + Network interfaces, in order. The position determines the PCI slot + (0x03 + index) and therefore the interface name inside the guest. + **Exactly one** interface must be marked primary: it carries the + default route and the metadata server. All subnets must belong to + the same VPC. items: $ref: "#/components/schemas/VMInterface" storage: diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 1cea7ec..b7ecc32 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -53,6 +53,11 @@ func main() { return } + if err := migration.MigrateVMNICs(db, log.With(slog.String("component", "migration"))); err != nil { + log.Error("vm nic migration failed", "error", err) + return + } + q := worker.New(cfg.Worker.BufferSize) q.Start(cfg.Worker.Count) diff --git a/internal/api/agent/vm.go b/internal/api/agent/vm.go index d9f6bf5..a1ab6bc 100644 --- a/internal/api/agent/vm.go +++ b/internal/api/agent/vm.go @@ -3,6 +3,7 @@ package agentapi import ( "encoding/json" "net/http" + "sort" "strconv" "strings" @@ -71,6 +72,52 @@ func (s *Server) stopVM(w http.ResponseWriter, _ *http.Request, name string) { json.NewEncoder(w).Encode(vm) } +// interfacesFromDB reconstruit les interfaces depuis vm//nic//…, +// triées par index — celui-ci détermine le slot PCI, donc le nom de l'interface +// dans le guest. +func interfacesFromDB(prefix string, entries map[string]string) []VMInterface { + nicPrefix := prefix + "nic/" + byIndex := make(map[int]*VMInterface) + + for key, value := range entries { + rest := strings.TrimPrefix(key, nicPrefix) + if rest == key { + continue + } + parts := strings.Split(rest, "/") + if len(parts) != 2 { + continue + } + idx, err := strconv.Atoi(parts[0]) + if err != nil { + continue + } + if byIndex[idx] == nil { + byIndex[idx] = &VMInterface{} + } + switch parts[1] { + case "subnet": + byIndex[idx].Subnet = value + case "ip": + byIndex[idx].IP = value + case "primary": + byIndex[idx].Primary = value == "true" + } + } + + indexes := make([]int, 0, len(byIndex)) + for idx := range byIndex { + indexes = append(indexes, idx) + } + sort.Ints(indexes) + + ifaces := make([]VMInterface, 0, len(indexes)) + for _, idx := range indexes { + ifaces = append(ifaces, *byIndex[idx]) + } + return ifaces +} + func vmFromDB(name string, entries map[string]string) (VM, error) { prefix := "vm/" + name + "/" vm := VM{Name: name} @@ -81,11 +128,7 @@ func vmFromDB(name string, entries map[string]string) (VM, error) { vm.CPUs, _ = strconv.Atoi(entries[prefix+"cpus"]) vm.UEFI = entries[prefix+"uefi"] == "true" - subnet := entries[prefix+"subnet"] - ip := entries[prefix+"ip"] - if subnet != "" || ip != "" { - vm.Interfaces = []VMInterface{{Subnet: subnet, IP: ip, Primary: true}} - } + vm.Interfaces = interfacesFromDB(prefix, entries) diskPrefix := prefix + "disk/" for key, path := range entries { diff --git a/internal/api/agent/vm_test.go b/internal/api/agent/vm_test.go index 629c315..e4892f2 100644 --- a/internal/api/agent/vm_test.go +++ b/internal/api/agent/vm_test.go @@ -262,3 +262,89 @@ func TestStartVM_EmptyBase64MeansNoDocument(t *testing.T) { t.Errorf("un base64 vide est indiscernable d'un champ absent : %v", entries) } } + +// --- interfaces multiples --- + +func TestVmFromDB_MultipleInterfacesSortedByIndex(t *testing.T) { + vm, err := vmFromDB("vm-multi", map[string]string{ + "vm/vm-multi/state": "running", + "vm/vm-multi/nic/1/subnet": "sn-2", + "vm/vm-multi/nic/1/ip": "10.2.0.5", + "vm/vm-multi/nic/0/subnet": "sn-1", + "vm/vm-multi/nic/0/ip": "10.1.0.5", + "vm/vm-multi/nic/0/primary": "true", + "vm/vm-multi/disk/vda": "/data/root.qcow2", + }) + if err != nil { + t.Fatalf("vmFromDB : %v", err) + } + if len(vm.Interfaces) != 2 { + t.Fatalf("2 interfaces attendues, obtenu %d : %+v", len(vm.Interfaces), vm.Interfaces) + } + if vm.Interfaces[0].Subnet != "sn-1" || !vm.Interfaces[0].Primary { + t.Errorf("la première doit être l'index 0, primaire : %+v", vm.Interfaces[0]) + } + if vm.Interfaces[1].Subnet != "sn-2" || vm.Interfaces[1].Primary { + t.Errorf("la seconde doit être l'index 1, non primaire : %+v", vm.Interfaces[1]) + } +} + +func TestStartVM_StoresAllInterfaces(t *testing.T) { + s, db := newTestServer(t) + for _, sn := range []string{"sn-1", "sn-2"} { + kv.AddInDB(db, "subnet/"+sn+"/state", "running") + kv.AddInDB(db, "subnet/"+sn+"/vpc", "vpc-1") + } + + body, _ := json.Marshal(VMCreateRequest{ + Name: "vm-multi", + Interfaces: []VMInterface{ + {Subnet: "sn-1", IP: "10.1.0.5", Primary: true}, + {Subnet: "sn-2", IP: "10.2.0.5"}, + }, + Storage: []VMStorage{{Path: "/data/root.qcow2", Dev: "vda"}}, + }) + + w := httptest.NewRecorder() + s.VmsHandler(w, httptest.NewRequest(http.MethodPost, "/vms", bytes.NewReader(body))) + if w.Code != http.StatusAccepted { + t.Fatalf("attendu 202, obtenu %d : %s", w.Code, w.Body.String()) + } + + if got, _ := kv.GetFromDB(db, "vm/vm-multi/nic/1/subnet"); got != "sn-2" { + t.Errorf("seconde interface non stockée : %q", got) + } + if got, _ := kv.GetFromDB(db, "vm/vm-multi/nic/0/primary"); got != "true" { + t.Errorf("primaire non marquée : %q", got) + } + if _, err := kv.GetFromDB(db, "vm/vm-multi/nic/1/primary"); err == nil { + t.Error("une interface non primaire ne doit pas porter la clé primary") + } +} + +func TestStartVM_RejectsZeroOrTwoPrimaries(t *testing.T) { + cases := map[string][]VMInterface{ + "aucune primaire": {{Subnet: "sn-1", IP: "10.1.0.5"}}, + "deux primaires": { + {Subnet: "sn-1", IP: "10.1.0.5", Primary: true}, + {Subnet: "sn-2", IP: "10.2.0.5", Primary: true}, + }, + } + for label, ifaces := range cases { + s, db := newTestServer(t) + for _, sn := range []string{"sn-1", "sn-2"} { + kv.AddInDB(db, "subnet/"+sn+"/state", "running") + kv.AddInDB(db, "subnet/"+sn+"/vpc", "vpc-1") + } + body, _ := json.Marshal(VMCreateRequest{ + Name: "vm-bad", + Interfaces: ifaces, + Storage: []VMStorage{{Path: "/data/root.qcow2", Dev: "vda"}}, + }) + w := httptest.NewRecorder() + s.VmsHandler(w, httptest.NewRequest(http.MethodPost, "/vms", bytes.NewReader(body))) + if w.Code != http.StatusBadRequest { + t.Errorf("%s : attendu 400, obtenu %d — %s", label, w.Code, w.Body.String()) + } + } +} diff --git a/internal/api/agent/vms.go b/internal/api/agent/vms.go index a061a00..01d7f40 100644 --- a/internal/api/agent/vms.go +++ b/internal/api/agent/vms.go @@ -67,16 +67,17 @@ func (s *Server) startVM(w http.ResponseWriter, r *http.Request) { return } - var primary *VMInterface - for i := range req.Interfaces { - if req.Interfaces[i].Primary { - primary = &req.Interfaces[i] - break + nics := make([]dispatcher.VMNIC, len(req.Interfaces)) + primaries := 0 + for i, iface := range req.Interfaces { + nics[i] = dispatcher.VMNIC{Subnet: iface.Subnet, IP: iface.IP, Primary: iface.Primary} + if iface.Primary { + primaries++ } } - if primary == nil { + if primaries != 1 { w.WriteHeader(http.StatusBadRequest) - json.NewEncoder(w).Encode(ErrorResponse{Error: "one interface must be primary"}) + json.NewEncoder(w).Encode(ErrorResponse{Error: "exactly one interface must be primary"}) return } @@ -94,8 +95,7 @@ func (s *Server) startVM(w http.ResponseWriter, r *http.Request) { cmd := dispatcher.StartVMCommand{ Name: req.Name, - Subnet: primary.Subnet, - IP: primary.IP, + NICs: nics, Disks: disks, Memory: req.Memory, CPUs: req.CPUs, diff --git a/internal/dispatcher/agent/vm_commands.go b/internal/dispatcher/agent/vm_commands.go index 9b42289..d634d5d 100644 --- a/internal/dispatcher/agent/vm_commands.go +++ b/internal/dispatcher/agent/vm_commands.go @@ -19,10 +19,15 @@ type VMDisk struct { Dev string } +type VMNIC struct { + Subnet string + IP string + Primary bool +} + type StartVMCommand struct { Name string - Subnet string - IP string + NICs []VMNIC Disks []VMDisk Memory int CPUs int @@ -38,20 +43,28 @@ func (c StartVMCommand) Prepare(db *badger.DB, _ *configuration.Config) error { if _, err := kv.GetFromDB(db, "vm/"+c.Name+"/state"); err == nil { return fmt.Errorf("vm %q already exists", c.Name) } - subnetState, err := state.Get(db, "subnet/"+c.Subnet) - if err != nil { - return fmt.Errorf("subnet %q not found", c.Subnet) - } - if subnetState != state.Creating && subnetState != state.Running { - return fmt.Errorf("subnet %q is %s", c.Subnet, subnetState) + if err := c.validateNICs(db); err != nil { + return err } port, err := allocateMetadataPort(db) if err != nil { return fmt.Errorf("allocate metadata port: %w", err) } state.Set(db, c.Key(), state.Creating) - kv.AddInDB(db, "vm/"+c.Name+"/subnet", c.Subnet) - kv.AddInDB(db, "vm/"+c.Name+"/ip", c.IP) + for i, n := range c.NICs { + prefix := fmt.Sprintf("vm/%s/nic/%d/", c.Name, i) + if err := kv.AddInDB(db, prefix+"subnet", n.Subnet); err != nil { + return fmt.Errorf("store nic %d subnet: %w", i, err) + } + if err := kv.AddInDB(db, prefix+"ip", n.IP); err != nil { + return fmt.Errorf("store nic %d ip: %w", i, err) + } + if n.Primary { + if err := kv.AddInDB(db, prefix+"primary", "true"); err != nil { + return fmt.Errorf("store nic %d primary: %w", i, err) + } + } + } kv.AddInDB(db, "vm/"+c.Name+"/metadata_port", strconv.Itoa(port)) for _, d := range c.Disks { kv.AddInDB(db, "vm/"+c.Name+"/disk/"+d.Dev, d.Path) @@ -75,6 +88,31 @@ func (c StartVMCommand) Prepare(db *badger.DB, _ *configuration.Config) error { return nil } +// validateNICs vérifie qu'il y a exactement une interface primaire et que +// chaque subnet référencé est utilisable. +func (c StartVMCommand) validateNICs(db *badger.DB) error { + if len(c.NICs) == 0 { + return fmt.Errorf("vm %q has no interface", c.Name) + } + primaries := 0 + for _, n := range c.NICs { + if n.Primary { + primaries++ + } + subnetState, err := state.Get(db, "subnet/"+n.Subnet) + if err != nil { + return fmt.Errorf("subnet %q not found", n.Subnet) + } + if subnetState != state.Creating && subnetState != state.Running { + return fmt.Errorf("subnet %q is %s", n.Subnet, subnetState) + } + } + if primaries != 1 { + return fmt.Errorf("vm %q has %d primary interfaces, expected exactly one", c.Name, primaries) + } + return nil +} + func allocateMetadataPort(db *badger.DB) (int, error) { entries, err := kv.ListByPrefix(db, "vm/") if err != nil { @@ -99,21 +137,23 @@ func allocateMetadataPort(db *badger.DB) (int, error) { func (c StartVMCommand) Execute(db *badger.DB, cfg *configuration.Config) error { timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second) - for { - subnetState, err := state.Get(db, "subnet/"+c.Subnet) - if err != nil { - return fmt.Errorf("subnet %q not found while waiting", c.Subnet) - } - if subnetState == state.Running { - break - } - if subnetState != state.Creating { - return fmt.Errorf("subnet %q is %s, cannot start vm %q", c.Subnet, subnetState, c.Name) - } - select { - case <-timeout: - return fmt.Errorf("timed out waiting for subnet %q to be running", c.Subnet) - case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): + for _, n := range c.NICs { + for { + subnetState, err := state.Get(db, "subnet/"+n.Subnet) + if err != nil { + return fmt.Errorf("subnet %q not found while waiting", n.Subnet) + } + if subnetState == state.Running { + break + } + if subnetState != state.Creating { + return fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name) + } + select { + case <-timeout: + return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet) + case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): + } } } return vm.StartVM(db, c.Name, cfg) diff --git a/internal/dispatcher/agent/vm_commands_test.go b/internal/dispatcher/agent/vm_commands_test.go index cd483f4..1a773fc 100644 --- a/internal/dispatcher/agent/vm_commands_test.go +++ b/internal/dispatcher/agent/vm_commands_test.go @@ -15,10 +15,9 @@ func TestStartVMCommand_Prepare_SingleDisk(t *testing.T) { kv.AddInDB(db, "subnet/sn-1/vpc", "vpc-1") cmd := StartVMCommand{ - Name: "vm-1", - Subnet: "sn-1", - IP: "10.0.0.5", - Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, + Name: "vm-1", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.5", Primary: true}}, + Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, } if err := cmd.Prepare(db, nil); err != nil { t.Fatalf("Prepare a échoué : %v", err) @@ -39,9 +38,8 @@ func TestStartVMCommand_Prepare_MultiDisk(t *testing.T) { kv.AddInDB(db, "subnet/sn-1/vpc", "vpc-1") cmd := StartVMCommand{ - Name: "vm-2", - Subnet: "sn-1", - IP: "10.0.0.6", + Name: "vm-2", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.6", Primary: true}}, Disks: []VMDisk{ {Path: "/data/root.qcow2", Dev: "sda"}, {Path: "/data/data.qcow2", Dev: "sdb"}, @@ -72,9 +70,8 @@ func TestStartVMCommand_Prepare_SlotGap(t *testing.T) { // sdb absent au boot — slot réservé pour hotplug cmd := StartVMCommand{ - Name: "vm-3", - Subnet: "sn-1", - IP: "10.0.0.7", + Name: "vm-3", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.7", Primary: true}}, Disks: []VMDisk{ {Path: "/data/root.qcow2", Dev: "sda"}, {Path: "/data/extra.qcow2", Dev: "sdc"}, @@ -101,10 +98,9 @@ func TestStartVMCommand_Prepare_NoVolumePath(t *testing.T) { kv.AddInDB(db, "subnet/sn-1/vpc", "vpc-1") cmd := StartVMCommand{ - Name: "vm-4", - Subnet: "sn-1", - IP: "10.0.0.8", - Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, + Name: "vm-4", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.8", Primary: true}}, + Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, } if err := cmd.Prepare(db, nil); err != nil { t.Fatalf("Prepare a échoué : %v", err) @@ -165,10 +161,9 @@ func TestStartVMCommand_Prepare_Duplicate(t *testing.T) { kv.AddInDB(db, "vm/vm-exist/state", "running") cmd := StartVMCommand{ - Name: "vm-exist", - Subnet: "sn-1", - IP: "10.0.0.9", - Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, + Name: "vm-exist", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.9", Primary: true}}, + Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "sda"}}, } if err := cmd.Prepare(db, nil); err == nil { t.Error("Prepare devrait échouer si la VM existe déjà") @@ -185,8 +180,7 @@ func prepareWithDocuments(t *testing.T, docs map[string]string) *badger.DB { cmd := StartVMCommand{ Name: "vm-doc", - Subnet: "sn-1", - IP: "10.0.0.5", + NICs: []VMNIC{{Subnet: "sn-1", IP: "10.0.0.5", Primary: true}}, Disks: []VMDisk{{Path: "/data/root.qcow2", Dev: "vda"}}, Documents: docs, } diff --git a/internal/migration/nic.go b/internal/migration/nic.go new file mode 100644 index 0000000..ae883ad --- /dev/null +++ b/internal/migration/nic.go @@ -0,0 +1,76 @@ +package migration + +import ( + "fmt" + "log/slog" + "strings" + + "git.g3e.fr/syonad/two/pkg/db/kv" + + "github.com/dgraph-io/badger/v4" +) + +// legacyNICKeys sont les clés d'interface d'avant le multi-subnet, portées +// directement par la VM. +var legacyNICKeys = []string{"subnet", "ip", "tap_id"} + +// MigrateVMNICs déplace les clés d'interface de vm// vers +// vm//nic/0/ et marque cette interface comme primaire. +// +// Idempotente : une VM possédant déjà des clés nic/ est laissée telle quelle. +// Sans cette migration, toute VM créée avant le passage au multi-subnet +// deviendrait illisible par loadVM. +func MigrateVMNICs(db *badger.DB, log *slog.Logger) error { + entries, err := kv.ListByPrefix(db, "vm/") + if err != nil { + return fmt.Errorf("list vm/: %w", err) + } + + for _, name := range vmsToMigrate(entries) { + for _, key := range legacyNICKeys { + value, ok := entries["vm/"+name+"/"+key] + if !ok { + continue + } + if err := kv.AddInDB(db, "vm/"+name+"/nic/0/"+key, value); err != nil { + return fmt.Errorf("migrate %s of vm %s: %w", key, name, err) + } + if err := kv.DeleteInDB(db, "vm/"+name+"/"+key); err != nil { + return fmt.Errorf("delete legacy %s of vm %s: %w", key, name, err) + } + } + if err := kv.AddInDB(db, "vm/"+name+"/nic/0/primary", "true"); err != nil { + return fmt.Errorf("mark nic 0 primary for vm %s: %w", name, err) + } + log.Info("vm nics migrated", "resource", "vm/"+name, "reason", "legacy single interface") + } + return nil +} + +// vmsToMigrate retourne les VM portant l'ancien schéma et aucune clé nic/. +func vmsToMigrate(entries map[string]string) []string { + legacy := make(map[string]bool) + migrated := make(map[string]bool) + + for key := range entries { + parts := strings.Split(key, "/") + if len(parts) < 3 || parts[0] != "vm" { + continue + } + name := parts[1] + switch { + case len(parts) == 3 && (parts[2] == "subnet" || parts[2] == "ip" || parts[2] == "tap_id"): + legacy[name] = true + case parts[2] == "nic": + migrated[name] = true + } + } + + var names []string + for name := range legacy { + if !migrated[name] { + names = append(names, name) + } + } + return names +} diff --git a/internal/migration/nic_test.go b/internal/migration/nic_test.go new file mode 100644 index 0000000..571b684 --- /dev/null +++ b/internal/migration/nic_test.go @@ -0,0 +1,136 @@ +package migration + +import ( + "io" + "log/slog" + "testing" + + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +func newNICDB(t *testing.T) *badger.DB { + t.Helper() + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + return db +} + +func quietLog() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func seedLegacyVM(t *testing.T, db *badger.DB, name string) { + t.Helper() + kv.AddInDB(db, "vm/"+name+"/state", "running") + kv.AddInDB(db, "vm/"+name+"/subnet", "sn-000001") + kv.AddInDB(db, "vm/"+name+"/ip", "10.1.1.2") + kv.AddInDB(db, "vm/"+name+"/tap_id", "12345678") +} + +func TestMigrateVMNICs_MovesLegacyKeys(t *testing.T) { + db := newNICDB(t) + seedLegacyVM(t, db, "i-test1") + + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("MigrateVMNICs : %v", err) + } + + want := map[string]string{ + "vm/i-test1/nic/0/subnet": "sn-000001", + "vm/i-test1/nic/0/ip": "10.1.1.2", + "vm/i-test1/nic/0/tap_id": "12345678", + "vm/i-test1/nic/0/primary": "true", + } + for key, expected := range want { + got, err := kv.GetFromDB(db, key) + if err != nil { + t.Errorf("clé %s absente : %v", key, err) + continue + } + if got != expected { + t.Errorf("%s = %q, attendu %q", key, got, expected) + } + } +} + +func TestMigrateVMNICs_RemovesLegacyKeys(t *testing.T) { + db := newNICDB(t) + seedLegacyVM(t, db, "i-test1") + + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("MigrateVMNICs : %v", err) + } + + for _, key := range []string{"vm/i-test1/subnet", "vm/i-test1/ip", "vm/i-test1/tap_id"} { + if _, err := kv.GetFromDB(db, key); err == nil { + t.Errorf("clé héritée %s toujours présente", key) + } + } +} + +func TestMigrateVMNICs_PreservesOtherKeys(t *testing.T) { + db := newNICDB(t) + seedLegacyVM(t, db, "i-test1") + kv.AddInDB(db, "vm/i-test1/memory", "2048") + kv.AddInDB(db, "vm/i-test1/disk/vda", "/data/root.qcow2") + + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("MigrateVMNICs : %v", err) + } + + if got, _ := kv.GetFromDB(db, "vm/i-test1/memory"); got != "2048" { + t.Errorf("memory altérée : %q", got) + } + if got, _ := kv.GetFromDB(db, "vm/i-test1/disk/vda"); got != "/data/root.qcow2" { + t.Errorf("disque altéré : %q", got) + } +} + +func TestMigrateVMNICs_Idempotent(t *testing.T) { + db := newNICDB(t) + seedLegacyVM(t, db, "i-test1") + + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("premier passage : %v", err) + } + before, _ := kv.ListByPrefix(db, "vm/i-test1/") + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("second passage : %v", err) + } + after, _ := kv.ListByPrefix(db, "vm/i-test1/") + + if len(before) != len(after) { + t.Errorf("le second passage a modifié la base : %d clés puis %d", len(before), len(after)) + } + for key, value := range before { + if after[key] != value { + t.Errorf("%s : %q devenu %q", key, value, after[key]) + } + } +} + +func TestMigrateVMNICs_LeavesMigratedVMsAlone(t *testing.T) { + db := newNICDB(t) + kv.AddInDB(db, "vm/i-multi/state", "running") + kv.AddInDB(db, "vm/i-multi/nic/0/subnet", "sn-000001") + kv.AddInDB(db, "vm/i-multi/nic/0/primary", "true") + kv.AddInDB(db, "vm/i-multi/nic/1/subnet", "sn-000002") + + if err := MigrateVMNICs(db, quietLog()); err != nil { + t.Fatalf("MigrateVMNICs : %v", err) + } + + if got, _ := kv.GetFromDB(db, "vm/i-multi/nic/1/subnet"); got != "sn-000002" { + t.Errorf("la seconde interface a été perdue : %q", got) + } + if _, err := kv.GetFromDB(db, "vm/i-multi/nic/0/primary"); err != nil { + t.Error("la primaire existante a été perdue") + } +} + +func TestMigrateVMNICs_EmptyDB(t *testing.T) { + if err := MigrateVMNICs(newNICDB(t), quietLog()); err != nil { + t.Errorf("une base vide ne doit pas échouer : %v", err) + } +} diff --git a/internal/vm/create.go b/internal/vm/create.go index a715851..fad9b6c 100644 --- a/internal/vm/create.go +++ b/internal/vm/create.go @@ -31,26 +31,27 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error { if err != nil { return err } + nic := d.primary() - if err := netif.CreateTap(d.tapID, d.bridge, d.vpcName); err != nil { + if err := netif.CreateTap(nic.tapID, nic.bridge, nic.vpcName); err != nil { return fmt.Errorf("create tap: %w", err) } - if err := netns.Call(d.vpcName, func() error { - return iptables.AddMetadataRedirect(d.ip, d.interfaceIP, d.metadataPort) + if err := netns.Call(nic.vpcName, func() error { + return iptables.AddMetadataRedirect(nic.ip, nic.interfaceIP, d.metadataPort) }); err != nil { return fmt.Errorf("add metadata redirect: %w", err) } - if err := dhcp.WriteReservations(dhcp.DefaultConfDir, d.vpcName+"_"+d.bridge, name, - []dhcp.Reservation{{MAC: d.mac, IP: d.ip}}); err != nil { + if err := dhcp.WriteReservations(dhcp.DefaultConfDir, nic.vpcName+"_"+nic.bridge, name, + []dhcp.Reservation{{MAC: nic.mac, IP: nic.ip}}); err != nil { return fmt.Errorf("write dhcp reservation: %w", err) } if err := metadata.StartMetadata(metadata.NoCloudConfig{ Name: name, - VpcName: d.vpcName, - BindIP: d.interfaceIP, + VpcName: nic.vpcName, + BindIP: nic.interfaceIP, BindPort: d.metadataPort, Password: d.password, SSHKEY: d.sshkey, @@ -66,8 +67,8 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error { qcfg := qemu.Config{ Name: name, - TapID: d.tapID, - Mac: d.mac, + TapID: nic.tapID, + Mac: nic.mac, Disks: qDisks, Memory: d.memory, CPUs: d.cpus, @@ -85,7 +86,7 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error { qcfg.UEFIVarsPath = varsPath } - if err := netns.Call(d.vpcName, func() error { + if err := netns.Call(nic.vpcName, func() error { return qemu.Start(qcfg) }); err != nil { return fmt.Errorf("start qemu: %w", err) diff --git a/internal/vm/data.go b/internal/vm/data.go index b4b9327..13a3900 100644 --- a/internal/vm/data.go +++ b/internal/vm/data.go @@ -3,6 +3,7 @@ package vm import ( "fmt" "math/rand" + "sort" "strconv" "strings" @@ -16,15 +17,21 @@ type diskEntry struct { dev string } +type nicData struct { + index int + subnetName string + vpcName string + bridge string + interfaceIP string + ip string + mac string + tapID int + primary bool +} + type vmData struct { - subnetName string - vpcName string - interfaceIP string - bridge string - tapID int - ip string + nics []nicData metadataPort string - mac string disks []diskEntry memory int cpus int @@ -34,47 +41,25 @@ type vmData struct { documents map[string]string } +// primary retourne l'interface portant la route par défaut et le serveur de +// métadonnées. loadVM garantit qu'il y en a exactement une. +func (d vmData) primary() nicData { + for _, n := range d.nics { + if n.primary { + return n + } + } + return nicData{} +} + func loadVM(db *badger.DB, name string) (vmData, error) { var d vmData - subnetName, err := kv.GetFromDB(db, "vm/"+name+"/subnet") + nics, err := loadNICs(db, name) if err != nil { - return d, fmt.Errorf("get subnet: %w", err) + return d, err } - d.subnetName = subnetName - d.bridge = "br-" + strings.SplitN(subnetName, "-", 2)[1] - - vpcName, err := kv.GetFromDB(db, "subnet/"+subnetName+"/vpc") - if err != nil { - return d, fmt.Errorf("get vpc: %w", err) - } - d.vpcName = vpcName - - interfaceIP, err := kv.GetFromDB(db, "subnet/"+subnetName+"/interface_ip") - if err != nil { - return d, fmt.Errorf("get interface_ip: %w", err) - } - d.interfaceIP = interfaceIP - - tapIDStr, err := kv.GetFromDB(db, "vm/"+name+"/tap_id") - if err != nil { - d.tapID = rand.Intn(90000000) + 10000000 - if err := kv.AddInDB(db, "vm/"+name+"/tap_id", strconv.Itoa(d.tapID)); err != nil { - return d, fmt.Errorf("store tap_id: %w", err) - } - } else { - tapID, err := strconv.Atoi(tapIDStr) - if err != nil { - return d, fmt.Errorf("parse tap_id: %w", err) - } - d.tapID = tapID - } - - ip, err := kv.GetFromDB(db, "vm/"+name+"/ip") - if err != nil { - return d, fmt.Errorf("get ip: %w", err) - } - d.ip = ip + d.nics = nics metadataPort, err := kv.GetFromDB(db, "vm/"+name+"/metadata_port") if err != nil { @@ -82,12 +67,6 @@ func loadVM(db *badger.DB, name string) (vmData, error) { } d.metadataPort = metadataPort - mac, err := dhcp.GetMACForIP(db, d.subnetName, d.ip) - if err != nil { - return d, fmt.Errorf("get mac for ip %s: %w", d.ip, err) - } - d.mac = mac - diskEntries, err := kv.ListByPrefix(db, "vm/"+name+"/disk/") if err != nil { return d, fmt.Errorf("list disks: %w", err) @@ -139,3 +118,101 @@ func loadVM(db *badger.DB, name string) (vmData, error) { return d, nil } + +// loadNICs lit les interfaces d'une VM sous vm//nic//. +// Le tap est alloué à la première lecture et persisté, comme avant le passage +// au multi-interfaces — mais désormais par interface. +func loadNICs(db *badger.DB, name string) ([]nicData, error) { + prefix := "vm/" + name + "/nic/" + entries, err := kv.ListByPrefix(db, prefix) + if err != nil { + return nil, fmt.Errorf("list nics: %w", err) + } + + indexes := make(map[int]bool) + for key := range entries { + parts := strings.Split(strings.TrimPrefix(key, prefix), "/") + if len(parts) != 2 { + continue + } + idx, err := strconv.Atoi(parts[0]) + if err != nil { + return nil, fmt.Errorf("invalid nic index %q for vm %s", parts[0], name) + } + indexes[idx] = true + } + if len(indexes) == 0 { + return nil, fmt.Errorf("no interface found for vm %q", name) + } + + nics := make([]nicData, 0, len(indexes)) + for idx := range indexes { + n, err := loadNIC(db, name, idx, entries) + if err != nil { + return nil, err + } + nics = append(nics, n) + } + sort.Slice(nics, func(i, j int) bool { return nics[i].index < nics[j].index }) + + primaries := 0 + for _, n := range nics { + if n.primary { + primaries++ + } + } + if primaries != 1 { + return nil, fmt.Errorf("vm %q has %d primary interfaces, expected exactly one", name, primaries) + } + return nics, nil +} + +func loadNIC(db *badger.DB, name string, idx int, entries map[string]string) (nicData, error) { + n := nicData{index: idx} + prefix := fmt.Sprintf("vm/%s/nic/%d/", name, idx) + + n.subnetName = entries[prefix+"subnet"] + if n.subnetName == "" { + return n, fmt.Errorf("nic %d of vm %s has no subnet", idx, name) + } + n.bridge = "br-" + strings.SplitN(n.subnetName, "-", 2)[1] + n.primary = entries[prefix+"primary"] == "true" + + vpcName, err := kv.GetFromDB(db, "subnet/"+n.subnetName+"/vpc") + if err != nil { + return n, fmt.Errorf("get vpc of subnet %s: %w", n.subnetName, err) + } + n.vpcName = vpcName + + interfaceIP, err := kv.GetFromDB(db, "subnet/"+n.subnetName+"/interface_ip") + if err != nil { + return n, fmt.Errorf("get interface_ip of subnet %s: %w", n.subnetName, err) + } + n.interfaceIP = interfaceIP + + n.ip = entries[prefix+"ip"] + if n.ip == "" { + return n, fmt.Errorf("nic %d of vm %s has no ip", idx, name) + } + + mac, err := dhcp.GetMACForIP(db, n.subnetName, n.ip) + if err != nil { + return n, fmt.Errorf("get mac for ip %s: %w", n.ip, err) + } + n.mac = mac + + if tapIDStr, ok := entries[prefix+"tap_id"]; ok { + tapID, err := strconv.Atoi(tapIDStr) + if err != nil { + return n, fmt.Errorf("parse tap_id of nic %d: %w", idx, err) + } + n.tapID = tapID + return n, nil + } + + n.tapID = rand.Intn(90000000) + 10000000 + if err := kv.AddInDB(db, prefix+"tap_id", strconv.Itoa(n.tapID)); err != nil { + return n, fmt.Errorf("store tap_id of nic %d: %w", idx, err) + } + return n, nil +} diff --git a/internal/vm/data_test.go b/internal/vm/data_test.go index 9c04934..74ff76b 100644 --- a/internal/vm/data_test.go +++ b/internal/vm/data_test.go @@ -1,6 +1,8 @@ package vm import ( + "fmt" + "strconv" "testing" "git.g3e.fr/syonad/two/pkg/db/kv" @@ -12,11 +14,12 @@ func newVMInDB(t *testing.T) *badger.DB { db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) t.Cleanup(func() { db.Close() }) - kv.AddInDB(db, "vm/vm-1/subnet", "sn-000001") + kv.AddInDB(db, "vm/vm-1/nic/0/subnet", "sn-000001") + kv.AddInDB(db, "vm/vm-1/nic/0/primary", "true") kv.AddInDB(db, "subnet/sn-000001/vpc", "vp-admin") kv.AddInDB(db, "subnet/sn-000001/interface_ip", "10.1.1.1") kv.AddInDB(db, "subnet/sn-000001/dhcp/10.1.1.2", "00:22:33:00:01:02") - kv.AddInDB(db, "vm/vm-1/ip", "10.1.1.2") + kv.AddInDB(db, "vm/vm-1/nic/0/ip", "10.1.1.2") kv.AddInDB(db, "vm/vm-1/metadata_port", "8081") kv.AddInDB(db, "vm/vm-1/disk/vda", "/data/root.qcow2") kv.AddInDB(db, "vm/vm-1/memory", "2048") @@ -88,3 +91,106 @@ func TestLoadVM_EmptyDocumentIsPreserved(t *testing.T) { t.Errorf("contenu attendu vide, obtenu %q", content) } } + +// --- multi-interfaces --- + +func addNIC(t *testing.T, db *badger.DB, idx int, subnet, ip, mac string, primary bool) { + t.Helper() + prefix := fmt.Sprintf("vm/vm-1/nic/%d/", idx) + kv.AddInDB(db, prefix+"subnet", subnet) + kv.AddInDB(db, prefix+"ip", ip) + if primary { + kv.AddInDB(db, prefix+"primary", "true") + } + kv.AddInDB(db, "subnet/"+subnet+"/vpc", "vp-admin") + kv.AddInDB(db, "subnet/"+subnet+"/interface_ip", "10.1.1.1") + kv.AddInDB(db, "subnet/"+subnet+"/dhcp/"+ip, mac) +} + +func TestLoadVM_SingleNIC(t *testing.T) { + d, err := loadVM(newVMInDB(t), "vm-1") + if err != nil { + t.Fatalf("loadVM : %v", err) + } + if len(d.nics) != 1 { + t.Fatalf("1 interface attendue, obtenu %d", len(d.nics)) + } + if !d.primary().primary || d.primary().ip != "10.1.1.2" { + t.Errorf("primaire inattendue : %+v", d.primary()) + } +} + +func TestLoadVM_MultipleNICsSortedByIndex(t *testing.T) { + db := newVMInDB(t) + addNIC(t, db, 2, "sn-000003", "10.3.0.9", "00:22:33:00:03:09", false) + addNIC(t, db, 1, "sn-000002", "10.2.0.5", "00:22:33:00:02:05", false) + + d, err := loadVM(db, "vm-1") + if err != nil { + t.Fatalf("loadVM : %v", err) + } + if len(d.nics) != 3 { + t.Fatalf("3 interfaces attendues, obtenu %d", len(d.nics)) + } + for i, n := range d.nics { + if n.index != i { + t.Errorf("interface en position %d porte l'index %d — l'ordre détermine le slot PCI", i, n.index) + } + } +} + +func TestLoadVM_TapAllocatedPerNIC(t *testing.T) { + db := newVMInDB(t) + addNIC(t, db, 1, "sn-000002", "10.2.0.5", "00:22:33:00:02:05", false) + + d, err := loadVM(db, "vm-1") + if err != nil { + t.Fatalf("loadVM : %v", err) + } + if d.nics[0].tapID == d.nics[1].tapID { + t.Errorf("deux interfaces partagent le tap %d", d.nics[0].tapID) + } + for _, n := range d.nics { + stored, err := kv.GetFromDB(db, fmt.Sprintf("vm/vm-1/nic/%d/tap_id", n.index)) + if err != nil { + t.Errorf("tap_id de l'interface %d non persisté : %v", n.index, err) + continue + } + if stored != strconv.Itoa(n.tapID) { + t.Errorf("tap_id de l'interface %d : %q en base, %d en mémoire", n.index, stored, n.tapID) + } + } +} + +func TestLoadVM_NoPrimaryIsAnError(t *testing.T) { + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vm/vm-1/metadata_port", "8081") + kv.AddInDB(db, "vm/vm-1/disk/vda", "/data/root.qcow2") + kv.AddInDB(db, "vm/vm-1/memory", "2048") + kv.AddInDB(db, "vm/vm-1/cpus", "2") + addNIC(t, db, 0, "sn-000001", "10.1.1.2", "00:22:33:00:01:02", false) + + if _, err := loadVM(db, "vm-1"); err == nil { + t.Error("aucune interface primaire : loadVM doit échouer plutôt que de laisser StartVM choisir au hasard") + } +} + +func TestLoadVM_TwoPrimariesIsAnError(t *testing.T) { + db := newVMInDB(t) + addNIC(t, db, 1, "sn-000002", "10.2.0.5", "00:22:33:00:02:05", true) + + if _, err := loadVM(db, "vm-1"); err == nil { + t.Error("deux interfaces primaires doivent être refusées") + } +} + +func TestLoadVM_NoNICIsAnError(t *testing.T) { + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vm/vm-1/memory", "2048") + + if _, err := loadVM(db, "vm-1"); err == nil { + t.Error("une VM sans interface doit être refusée") + } +} diff --git a/internal/vm/delete.go b/internal/vm/delete.go index b9ab7e0..d9b376a 100644 --- a/internal/vm/delete.go +++ b/internal/vm/delete.go @@ -32,6 +32,7 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { if err != nil { return err } + nic := d.primary() socketPath := filepath.Join(cfg.QEMU.QMPDir, name+".sock") @@ -47,8 +48,8 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { } // socket absent ou QEMU déjà arrêté : cleanup direct - if err := netns.Call(d.vpcName, func() error { - return iptables.DeleteMetadataRedirect(d.ip, d.interfaceIP, d.metadataPort) + if err := netns.Call(nic.vpcName, func() error { + return iptables.DeleteMetadataRedirect(nic.ip, nic.interfaceIP, d.metadataPort) }); err != nil { return fmt.Errorf("delete metadata redirect: %w", err) } @@ -57,11 +58,11 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { return fmt.Errorf("stop metadata: %w", err) } - if err := netif.DeleteTap(d.tapID, d.vpcName); err != nil { + if err := netif.DeleteTap(nic.tapID, nic.vpcName); err != nil { return fmt.Errorf("delete tap: %w", err) } - if err := removeDHCPReservation(d, name); err != nil { + if err := removeDHCPReservation(nic, name); err != nil { return err } @@ -76,8 +77,8 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { // removeDHCPReservation retire le fichier de réservation puis redémarre dnsmasq : // un fichier ajouté dans un dhcp-hostsdir est relu à chaud, un fichier retiré ne // l'est pas (vérifié sur dnsmasq 2.90). -func removeDHCPReservation(d vmData, name string) error { - confName := d.vpcName + "_" + d.bridge +func removeDHCPReservation(nic nicData, name string) error { + confName := nic.vpcName + "_" + nic.bridge if err := dhcp.RemoveReservations(dhcp.DefaultConfDir, confName, name); err != nil { return err diff --git a/internal/watchdog/check_vm.go b/internal/watchdog/check_vm.go index 3c850a3..7d9713c 100644 --- a/internal/watchdog/check_vm.go +++ b/internal/watchdog/check_vm.go @@ -4,7 +4,9 @@ import ( "errors" "fmt" "path/filepath" + "sort" "strconv" + "strings" configuration "git.g3e.fr/syonad/two/internal/config/agent" "git.g3e.fr/syonad/two/internal/netns" @@ -46,33 +48,65 @@ func CheckVMs(db *badger.DB, cfg *configuration.Config, u unitChecker, n notify. } func checkVM(db *badger.DB, cfg *configuration.Config, name string, u unitChecker, n notify.Notifier) { - subnetName, err := kv.GetFromDB(db, prefixVM+name+"/subnet") + entries, err := kv.ListByPrefix(db, prefixVM+name+"/nic/") if err != nil { - n.Notify(kindVM, name, fmt.Sprintf("subnet unreadable in database: %v", err)) + n.Notify(kindVM, name, fmt.Sprintf("interfaces unreadable in database: %v", err)) + return + } + indexes := nicIndexes(entries, prefixVM+name+"/nic/") + if len(indexes) == 0 { + n.Notify(kindVM, name, "no interface in database") return } - vpc, err := kv.GetFromDB(db, prefixSubnet+subnetName+"/vpc") - if err != nil { - n.Notify(kindVM, name, fmt.Sprintf("vpc of subnet %s unreadable in database: %v", subnetName, err)) - return + for _, idx := range indexes { + prefix := fmt.Sprintf("%s%s/nic/%d/", prefixVM, name, idx) + subnetName := entries[prefix+"subnet"] + if subnetName == "" { + n.Notify(kindVM, name, fmt.Sprintf("interface %d has no subnet in database", idx)) + continue + } + vpc, err := kv.GetFromDB(db, prefixSubnet+subnetName+"/vpc") + if err != nil { + n.Notify(kindVM, name, fmt.Sprintf("vpc of subnet %s unreadable in database: %v", subnetName, err)) + continue + } + checkVMTap(entries[prefix+"tap_id"], name, vpc, idx, n) } - checkVMTap(db, name, vpc, n) checkVMQemu(cfg, name, n) checkUnit(kindVM, name, "metadata@"+name+".service", u, n) checkUnit(kindVM, name, qemu.ScopeName(name), u, n) } -func checkVMTap(db *badger.DB, name, vpc string, n notify.Notifier) { - raw, err := kv.GetFromDB(db, prefixVM+name+"/tap_id") - if err != nil { - n.Notify(kindVM, name, fmt.Sprintf("tap_id unreadable in database: %v", err)) +// nicIndexes retourne les index d'interface présents en base, triés. +func nicIndexes(entries map[string]string, prefix string) []int { + seen := make(map[int]bool) + for key := range entries { + parts := strings.Split(strings.TrimPrefix(key, prefix), "/") + if len(parts) != 2 { + continue + } + if idx, err := strconv.Atoi(parts[0]); err == nil { + seen[idx] = true + } + } + indexes := make([]int, 0, len(seen)) + for idx := range seen { + indexes = append(indexes, idx) + } + sort.Ints(indexes) + return indexes +} + +func checkVMTap(raw, name, vpc string, idx int, n notify.Notifier) { + if raw == "" { + n.Notify(kindVM, name, fmt.Sprintf("interface %d has no tap_id in database", idx)) return } tapID, err := strconv.Atoi(raw) if err != nil { - n.Notify(kindVM, name, fmt.Sprintf("invalid tap_id %q: %v", raw, err)) + n.Notify(kindVM, name, fmt.Sprintf("interface %d has an invalid tap_id %q: %v", idx, raw, err)) return } diff --git a/internal/watchdog/check_vm_test.go b/internal/watchdog/check_vm_test.go index eb62b59..569b896 100644 --- a/internal/watchdog/check_vm_test.go +++ b/internal/watchdog/check_vm_test.go @@ -20,10 +20,11 @@ func testCfg(t *testing.T) *configuration.Config { func seedVM(t *testing.T, db *badger.DB, name, subnetName, vpc, tapID string) { t.Helper() seedResource(t, db, prefixVM, name, state.Running) - seedKV(t, db, prefixVM+name+"/subnet", subnetName) + seedKV(t, db, prefixVM+name+"/nic/0/subnet", subnetName) + seedKV(t, db, prefixVM+name+"/nic/0/primary", "true") seedKV(t, db, prefixSubnet+subnetName+"/vpc", vpc) if tapID != "" { - seedKV(t, db, prefixVM+name+"/tap_id", tapID) + seedKV(t, db, prefixVM+name+"/nic/0/tap_id", tapID) } } @@ -73,7 +74,7 @@ func TestCheckVMs_IgnoreLesEtatsNonRunning(t *testing.T) { } } -func TestCheckVMs_SubnetManquantEnBase(t *testing.T) { +func TestCheckVMs_AucuneInterfaceEnBase(t *testing.T) { db := newTestDB(t) seedResource(t, db, prefixVM, "i-test1", state.Running) r := &recorder{} @@ -86,15 +87,16 @@ func TestCheckVMs_SubnetManquantEnBase(t *testing.T) { if len(got) != 1 { t.Fatalf("attendu 1 notification, obtenu %d : %v", len(got), got) } - if !strings.Contains(got[0].problem, "subnet unreadable") { - t.Errorf("problem = %q, devrait porter sur le subnet", got[0].problem) + if !strings.Contains(got[0].problem, "no interface") { + t.Errorf("problem = %q, devrait signaler l'absence d'interface", got[0].problem) } } func TestCheckVMs_VPCDuSubnetManquant(t *testing.T) { db := newTestDB(t) seedResource(t, db, prefixVM, "i-test1", state.Running) - seedKV(t, db, prefixVM+"i-test1/subnet", "br-000042") + seedKV(t, db, prefixVM+"i-test1/nic/0/subnet", "br-000042") + seedKV(t, db, prefixVM+"i-test1/nic/0/primary", "true") r := &recorder{} if err := CheckVMs(db, testCfg(t), newFakeUnits(), r); err != nil { @@ -115,8 +117,8 @@ func TestCheckVMs_TapIDManquant(t *testing.T) { t.Fatalf("erreur inattendue: %v", err) } - if !r.hasProblemContaining("tap_id unreadable") { - t.Errorf("devrait signaler un tap_id illisible, obtenu %v", r.calls) + if !r.hasProblemContaining("has no tap_id") { + t.Errorf("devrait signaler un tap_id absent, obtenu %v", r.calls) } }