diff --git a/api/agent.yaml b/api/agent.yaml index 97d8ef6..e8e5b7c 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -281,7 +281,7 @@ paths: "404": $ref: "#/components/responses/NotFound" "409": - description: Subnet not deletable — only running or error states can be deleted + description: Subnet not deletable — only running or error states can be deleted, and no VM that is not deleted may still have an interface on it content: application/json: schema: diff --git a/docs/concepts/vpc-subnet-vm.rst b/docs/concepts/vpc-subnet-vm.rst index 1c55ec0..abf2333 100644 --- a/docs/concepts/vpc-subnet-vm.rst +++ b/docs/concepts/vpc-subnet-vm.rst @@ -63,6 +63,11 @@ hosts dont le nommage réseau diffère. Le comportement réseau dépend du :doc:`mode `. +Un subnet ne peut être supprimé que si aucune VM non supprimée n'a d'interface dessus — sinon +409, avec la liste des VM. Dans un VPC routé par ``l3vni``, un subnet ``vxlan`` annonce son +préfixe dans la VRF du VPC à la création et le retire en premier à la suppression : c'est le +filet qui couvre une VM tant que son ``/32`` n'est pas annoncé. + VM -- diff --git a/docs/exploitation/api-agent/asynchronisme.rst b/docs/exploitation/api-agent/asynchronisme.rst index a506d55..afcee1d 100644 --- a/docs/exploitation/api-agent/asynchronisme.rst +++ b/docs/exploitation/api-agent/asynchronisme.rst @@ -71,7 +71,8 @@ Codes de retour - ressource inexistante * - ``409`` - conflit d'existence ou d'état : ressource déjà créée, ou suppression depuis un état qui - ne l'autorise pas ; pour un VPC, subnets encore présents + ne l'autorise pas ; pour un VPC, subnets encore présents ; pour un subnet, VM non + supprimées qui l'utilisent encore * - ``422`` - dépendance absente ou pas prête : VPC parent d'un subnet, subnet d'une VM * - ``500`` diff --git a/internal/dispatcher/agent/subnet_commands.go b/internal/dispatcher/agent/subnet_commands.go index b2fb5f6..f6e7f2d 100644 --- a/internal/dispatcher/agent/subnet_commands.go +++ b/internal/dispatcher/agent/subnet_commands.go @@ -2,7 +2,9 @@ package dispatcher import ( "fmt" + "sort" "strconv" + "strings" "time" configuration "git.g3e.fr/syonad/two/internal/config/agent" @@ -101,9 +103,36 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err if !state.CanDelete(current) { return fmt.Errorf("subnet %q cannot be deleted while %s", c.Name, current) } + vms, err := vmsUsingSubnet(db, c.Name) + if err != nil { + return err + } + if len(vms) > 0 { + return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", ")) + } return state.Set(db, c.Key(), state.Deleting) } +func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) { + entries, err := kv.ListByPrefix(db, "vm/") + if err != nil { + return nil, fmt.Errorf("failed to list vms: %w", err) + } + var users []string + for key, value := range entries { + parts := strings.Split(key, "/") + if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || value != subnetName { + continue + } + if s, err := state.Get(db, "vm/"+parts[1]); err == nil && s == state.Deleted { + continue + } + users = append(users, parts[1]) + } + sort.Strings(users) + return users, nil +} + func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error { if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil { return err diff --git a/internal/dispatcher/agent/subnet_commands_test.go b/internal/dispatcher/agent/subnet_commands_test.go index d94f5e4..3474df1 100644 --- a/internal/dispatcher/agent/subnet_commands_test.go +++ b/internal/dispatcher/agent/subnet_commands_test.go @@ -1,6 +1,7 @@ package dispatcher import ( + "strings" "testing" configuration "git.g3e.fr/syonad/two/internal/config/agent" @@ -327,3 +328,38 @@ func TestCreateSubnetCommand_Prepare_RejectsUnknownMode(t *testing.T) { t.Error("un mode inconnu doit être refusé") } } + +func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-gone/state", "deleted") + kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-other/state", "running") + kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b") + kv.AddInDB(db, "vm/i-b/state", "deleting") + kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-b") + kv.AddInDB(db, "vm/i-b/nic/1/subnet", "sn-a") + kv.AddInDB(db, "vm/i-a/state", "error") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-c/state", "running") + kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a") + err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) { + t.Fatalf("erreur %v", err) + } + if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" { + t.Errorf("état modifié malgré le refus : %q", s) + } +} + +func TestDeleteSubnetCommand_Prepare_AllowedOnceItsVMsAreDeleted(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-gone/state", "deleted") + kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-other/state", "running") + kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b") + if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } +} diff --git a/internal/netif/route.go b/internal/netif/route.go index f10b159..d49ec02 100644 --- a/internal/netif/route.go +++ b/internal/netif/route.go @@ -1,7 +1,9 @@ package netif import ( + "errors" "net" + "syscall" "github.com/vishvananda/netlink" ) @@ -25,3 +27,10 @@ func RouteDel(table int, dst *net.IPNet) error { Table: table, }) } + +func RouteDelIfExists(table int, dst *net.IPNet) error { + if err := RouteDel(table, dst); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} diff --git a/internal/subnet/create.go b/internal/subnet/create.go index 568b64e..ae31b49 100644 --- a/internal/subnet/create.go +++ b/internal/subnet/create.go @@ -10,6 +10,7 @@ import ( "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "github.com/dgraph-io/badger/v4" ) @@ -37,6 +38,12 @@ func CreateSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e return err } + if d.mode == ModeVxlan { + if err := vpc.ExportRoute(db, d.vpc, d.cidr); err != nil { + return err + } + } + return state.Set(db, "subnet/"+subnetName, state.Running) } diff --git a/internal/subnet/delete.go b/internal/subnet/delete.go index a8f12d0..8ad0b3d 100644 --- a/internal/subnet/delete.go +++ b/internal/subnet/delete.go @@ -9,6 +9,7 @@ import ( "git.g3e.fr/syonad/two/internal/netif" "git.g3e.fr/syonad/two/internal/netns" "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/internal/vpc" "git.g3e.fr/syonad/two/pkg/db/kv" "github.com/dgraph-io/badger/v4" @@ -33,6 +34,12 @@ func DeleteSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e return err } + if d.mode == ModeVxlan { + if err := vpc.WithdrawRoute(db, d.vpc, d.cidr); err != nil { + return err + } + } + if err := stopDHCP(db, subnetName, d, backend); err != nil { return err } diff --git a/internal/vpc/host.go b/internal/vpc/host.go index 8f2351b..37a1720 100644 --- a/internal/vpc/host.go +++ b/internal/vpc/host.go @@ -22,6 +22,8 @@ type host interface { LinkUp(ns, iface string) error RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error DeleteLink(name string) error + TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error + TableRouteDel(table uint32, dst *net.IPNet) error ApplyVRF(v frr.VRF) error WithdrawVRF(v frr.VRF) error RemoveVRF(v frr.VRF) error @@ -80,6 +82,14 @@ func (system) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error func (system) DeleteLink(name string) error { return netif.DeleteLinkIfExists(name) } +func (system) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return netif.RouteReplaceVia(int(table), dst, gw, iface) +} + +func (system) TableRouteDel(table uint32, dst *net.IPNet) error { + return netif.RouteDelIfExists(int(table), dst) +} + func (s system) ApplyVRF(v frr.VRF) error { return s.frr.ApplyVRF(v) } func (s system) WithdrawVRF(v frr.VRF) error { return s.frr.WithdrawVRF(v) } diff --git a/internal/vpc/l3_test.go b/internal/vpc/l3_test.go index f06a7bf..a56fad9 100644 --- a/internal/vpc/l3_test.go +++ b/internal/vpc/l3_test.go @@ -79,6 +79,14 @@ func (f *fakeHost) DeleteLink(name string) error { return f.record("del " + name) } +func (f *fakeHost) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return f.record(fmt.Sprintf("table %d route %s via %s dev %s", table, dst, gw, iface)) +} + +func (f *fakeHost) TableRouteDel(table uint32, dst *net.IPNet) error { + return f.record(fmt.Sprintf("table %d del %s", table, dst)) +} + func (f *fakeHost) ApplyVRF(v frr.VRF) error { return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet)) } diff --git a/internal/vpc/route.go b/internal/vpc/route.go new file mode 100644 index 0000000..3b5da19 --- /dev/null +++ b/internal/vpc/route.go @@ -0,0 +1,50 @@ +package vpc + +import ( + "fmt" + "net" + "strings" + + "github.com/dgraph-io/badger/v4" +) + +func ExportRoute(db *badger.DB, vpcName string, dst *net.IPNet) error { + return exportRoute(newSystem(), db, vpcName, dst) +} + +func WithdrawRoute(db *badger.DB, vpcName string, dst *net.IPNet) error { + return withdrawRoute(newSystem(), db, vpcName, dst) +} + +func loadRouted(db *badger.DB, vpcName string) (l3VPC, bool, error) { + parts := strings.SplitN(vpcName, "-", 2) + if len(parts) != 2 { + return l3VPC{}, false, fmt.Errorf("invalid vpc name %q", vpcName) + } + return loadL3(db, vpcName, parts[1]) +} + +func exportRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return err + } + if !v.supernet.Contains(dst.IP) { + return fmt.Errorf("vpc %q: %s is outside %s", vpcName, dst, v.supernet) + } + if err := h.TableRouteVia(v.vni, dst, transitNetns.IP, v.vethHost()); err != nil { + return fmt.Errorf("export %s in %s: %w", dst, v.vrf(), err) + } + return nil +} + +func withdrawRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error { + v, ok, err := loadRouted(db, vpcName) + if err != nil || !ok { + return err + } + if err := h.TableRouteDel(v.vni, dst); err != nil { + return fmt.Errorf("withdraw %s from %s: %w", dst, v.vrf(), err) + } + return nil +} diff --git a/internal/vpc/route_test.go b/internal/vpc/route_test.go new file mode 100644 index 0000000..ded011a --- /dev/null +++ b/internal/vpc/route_test.go @@ -0,0 +1,89 @@ +package vpc + +import ( + "errors" + "net" + "reflect" + "strings" + "testing" + + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +func routedDB(t *testing.T) *badger.DB { + t.Helper() + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vpc/vp-s7/cidr", "10.241.0.0/16") + kv.AddInDB(db, "vpc/vp-s7/l3vni", "2700") + kv.AddInDB(db, "vpc/vp-flat/cidr", "10.242.0.0/16") + return db +} + +func cidr(t *testing.T, s string) *net.IPNet { + t.Helper() + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n +} + +func TestExportRouteInstallsTheRouteInTheVRFTowardsTheNetns(t *testing.T) { + f := &fakeHost{} + if err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.0/24")); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 route 10.241.1.0/24 via 169.254.0.2 dev l3-s7-o"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestExportRouteRefusesAPrefixOutsideTheVPC(t *testing.T) { + f := &fakeHost{} + err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.99.1.0/24")) + if err == nil || !strings.Contains(err.Error(), "outside 10.241.0.0/16") || len(f.calls) != 0 { + t.Errorf("erreur %v, appels %q", err, f.calls) + } +} + +func TestWithdrawRouteDeletesFromTheVRFTable(t *testing.T) { + f := &fakeHost{} + if err := withdrawRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.10/32")); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 del 10.241.1.10/32"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestRoutesAreIgnoredForAVPCWithoutL3VNI(t *testing.T) { + f := &fakeHost{} + db := routedDB(t) + if err := exportRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil { + t.Error(err) + } + if err := withdrawRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil { + t.Error(err) + } + if len(f.calls) != 0 { + t.Errorf("appels %q", f.calls) + } +} + +func TestRouteErrorsArePropagated(t *testing.T) { + db := routedDB(t) + f := &fakeHost{fail: map[string]error{"table ": errors.New("network unreachable")}} + if err := exportRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "export 10.241.1.0/24 in vrf-s7") { + t.Errorf("export : %v", err) + } + if err := withdrawRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "withdraw 10.241.1.0/24 from vrf-s7") { + t.Errorf("withdraw : %v", err) + } + if err := exportRoute(f, db, "novpc", cidr(t, "10.241.1.0/24")); err == nil { + t.Error("nom de VPC sans tiret accepté") + } +}