From 8e89865418d78db9b4d7eda5d5930bb4a33e39ab Mon Sep 17 00:00:00 2001 From: GnomeZworc Date: Sun, 25 Oct 2026 12:42:10 +0100 Subject: [PATCH] f-79: agent: section times, renvoi de system_powerdown, attentes #79 Signed-off-by: GnomeZworc --- api/agent.yaml | 4 +- conf/agent/config.exemple.yml | 22 ++- internal/config/agent/struct.go | 17 +- internal/config/agent/times.go | 64 +++++++ internal/config/agent/times_test.go | 95 ++++++++++ internal/dispatcher/agent/subnet_commands.go | 61 +++++-- .../dispatcher/agent/subnet_commands_test.go | 2 +- internal/dispatcher/agent/vm_commands.go | 26 +-- internal/dispatcher/agent/vpc_commands.go | 60 ++++--- internal/dispatcher/agent/wait.go | 26 +++ internal/dispatcher/agent/wait_test.go | 168 ++++++++++++++++++ internal/vm/delete.go | 35 ++-- internal/vm/shutdown_test.go | 102 +++++++++++ 13 files changed, 602 insertions(+), 80 deletions(-) create mode 100644 internal/config/agent/times.go create mode 100644 internal/config/agent/times_test.go create mode 100644 internal/dispatcher/agent/wait.go create mode 100644 internal/dispatcher/agent/wait_test.go create mode 100644 internal/vm/shutdown_test.go diff --git a/api/agent.yaml b/api/agent.yaml index 65c2603..c1ceeff 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -91,7 +91,7 @@ paths: "404": $ref: "#/components/responses/NotFound" "409": - description: VPC not deletable — only running or error states can be deleted, and all its subnets must be deleted first + description: VPC not deletable — only running or error states can be deleted, and all its subnets must be deleted or deleting; subnets still deleting are awaited by the asynchronous deletion content: application/json: schema: @@ -288,7 +288,7 @@ paths: "404": $ref: "#/components/responses/NotFound" "409": - description: Subnet not deletable — only running or error states can be deleted, and no VM that is not deleted may still have an interface on it + description: Subnet not deletable — only running or error states can be deleted, and every VM with an interface on it must be deleted or deleting; VMs still deleting are awaited by the asynchronous deletion content: application/json: schema: diff --git a/conf/agent/config.exemple.yml b/conf/agent/config.exemple.yml index 3d95b94..e03bc6e 100644 --- a/conf/agent/config.exemple.yml +++ b/conf/agent/config.exemple.yml @@ -19,12 +19,22 @@ worker: # Maximum number of commands queued before Dispatch blocks buffer_size: 100 -# Timing for commands that wait on resource state transitions -dispatcher: - # How long (in seconds) to wait before giving up - timeout_seconds: 300 - # Interval (in seconds) between each state check - poll_seconds: 2 +# Timing (in seconds) for commands that wait on resource state transitions +times: + # Graceful VM shutdown before QMP quit + timeout_vms_stop: 300 + # Interval between two system_powerdown during the shutdown + next_vms_shutdown: 30 + # Subnet deletion: wait for its VMs still deleting + timeout_subnet_stop: 360 + # VPC deletion: wait for its subnets still deleting + timeout_vpc_stop: 420 + # Subnet creation: wait for its VPC to be running + timeout_subnet_start: 300 + # VM creation: wait for the subnets of its interfaces to be running + timeout_vms_start: 300 + # Interval between each state check + poll_interval: 2 # Bridge interface used when the requested iface_type is not found in interfaces default_interface: br-000000 diff --git a/internal/config/agent/struct.go b/internal/config/agent/struct.go index 1381d2f..3de53cb 100644 --- a/internal/config/agent/struct.go +++ b/internal/config/agent/struct.go @@ -24,10 +24,7 @@ type Config struct { Count int `mapstructure:"count"` BufferSize int `mapstructure:"buffer_size"` } `mapstructure:"worker"` - Dispatcher struct { - TimeoutSeconds int `mapstructure:"timeout_seconds"` - PollSeconds int `mapstructure:"poll_seconds"` - } `mapstructure:"dispatcher"` + Times Times `mapstructure:"times"` Logger struct { Level string `mapstructure:"level"` Debug bool `mapstructure:"debug"` @@ -73,8 +70,13 @@ func LoadConfig(path string) (*Config, error) { v.SetDefault("prometheus.port", 9090) v.SetDefault("worker.count", 4) v.SetDefault("worker.buffer_size", 100) - v.SetDefault("dispatcher.timeout_seconds", 300) - v.SetDefault("dispatcher.poll_seconds", 2) + v.SetDefault("times.timeout_vms_stop", TimeoutVMsStop) + v.SetDefault("times.next_vms_shutdown", NextVMsShutdown) + v.SetDefault("times.timeout_subnet_stop", TimeoutSubnetStop) + v.SetDefault("times.timeout_vpc_stop", TimeoutVPCStop) + v.SetDefault("times.timeout_subnet_start", TimeoutSubnetStart) + v.SetDefault("times.timeout_vms_start", TimeoutVMsStart) + v.SetDefault("times.poll_interval", PollInterval) v.SetDefault("metadata.run_dir", "/run/two/metadata") v.SetDefault("dhcp.backend", BackendTwo) v.SetDefault("qemu.ovmf_code_path", "/usr/share/OVMF/OVMF_CODE.fd") @@ -106,6 +108,9 @@ func LoadConfig(path string) (*Config, error) { if err := cfg.Network.Validate(); err != nil { return nil, fmt.Errorf("%s: %w", path, err) } + if err := cfg.Times.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } return &cfg, nil } diff --git a/internal/config/agent/times.go b/internal/config/agent/times.go new file mode 100644 index 0000000..c482029 --- /dev/null +++ b/internal/config/agent/times.go @@ -0,0 +1,64 @@ +package configuration + +import ( + "fmt" + "time" +) + +const ( + TimeoutVMsStop = 300 + NextVMsShutdown = 30 + TimeoutSubnetStop = 360 + TimeoutVPCStop = 420 + TimeoutSubnetStart = 300 + TimeoutVMsStart = 300 + PollInterval = 2 +) + +type Times struct { + TimeoutVMsStop int `mapstructure:"timeout_vms_stop"` + NextVMsShutdown int `mapstructure:"next_vms_shutdown"` + TimeoutSubnetStop int `mapstructure:"timeout_subnet_stop"` + TimeoutVPCStop int `mapstructure:"timeout_vpc_stop"` + TimeoutSubnetStart int `mapstructure:"timeout_subnet_start"` + TimeoutVMsStart int `mapstructure:"timeout_vms_start"` + PollInterval int `mapstructure:"poll_interval"` +} + +func seconds(value, fallback int) time.Duration { + if value == 0 { + value = fallback + } + return time.Duration(value) * time.Second +} + +func (t Times) VMsStop() time.Duration { return seconds(t.TimeoutVMsStop, TimeoutVMsStop) } +func (t Times) VMsShutdown() time.Duration { return seconds(t.NextVMsShutdown, NextVMsShutdown) } +func (t Times) SubnetStop() time.Duration { return seconds(t.TimeoutSubnetStop, TimeoutSubnetStop) } +func (t Times) VPCStop() time.Duration { return seconds(t.TimeoutVPCStop, TimeoutVPCStop) } +func (t Times) SubnetStart() time.Duration { return seconds(t.TimeoutSubnetStart, TimeoutSubnetStart) } +func (t Times) VMsStart() time.Duration { return seconds(t.TimeoutVMsStart, TimeoutVMsStart) } +func (t Times) Poll() time.Duration { return seconds(t.PollInterval, PollInterval) } + +func (t Times) Validate() error { + for _, f := range []struct { + key string + value int + }{ + {"timeout_vms_stop", t.TimeoutVMsStop}, + {"next_vms_shutdown", t.NextVMsShutdown}, + {"timeout_subnet_stop", t.TimeoutSubnetStop}, + {"timeout_vpc_stop", t.TimeoutVPCStop}, + {"timeout_subnet_start", t.TimeoutSubnetStart}, + {"timeout_vms_start", t.TimeoutVMsStart}, + {"poll_interval", t.PollInterval}, + } { + if f.value <= 0 { + return fmt.Errorf("times.%s %d must be a positive number of seconds", f.key, f.value) + } + } + if t.NextVMsShutdown >= t.TimeoutVMsStop { + return fmt.Errorf("times.next_vms_shutdown %d must be below times.timeout_vms_stop %d", t.NextVMsShutdown, t.TimeoutVMsStop) + } + return nil +} diff --git a/internal/config/agent/times_test.go b/internal/config/agent/times_test.go new file mode 100644 index 0000000..b35330c --- /dev/null +++ b/internal/config/agent/times_test.go @@ -0,0 +1,95 @@ +package configuration + +import ( + "strings" + "testing" + "time" +) + +func TestLoadConfig_TimesDefauts(t *testing.T) { + cfg, err := LoadConfig(writeYAML(t, "")) + if err != nil { + t.Fatalf("LoadConfig a échoué : %v", err) + } + want := Times{ + TimeoutVMsStop: 300, + NextVMsShutdown: 30, + TimeoutSubnetStop: 360, + TimeoutVPCStop: 420, + TimeoutSubnetStart: 300, + TimeoutVMsStart: 300, + PollInterval: 2, + } + if cfg.Times != want { + t.Errorf("times par défaut : %+v, attendu %+v", cfg.Times, want) + } +} + +func TestLoadConfig_TimesValeursExplicites(t *testing.T) { + cfg, err := LoadConfig(writeYAML(t, "times:\n timeout_vms_stop: 600\n next_vms_shutdown: 10\n timeout_subnet_stop: 700\n timeout_vpc_stop: 800\n timeout_subnet_start: 50\n timeout_vms_start: 60\n poll_interval: 1\n")) + if err != nil { + t.Fatalf("LoadConfig a échoué : %v", err) + } + want := Times{600, 10, 700, 800, 50, 60, 1} + if cfg.Times != want { + t.Errorf("times : %+v, attendu %+v", cfg.Times, want) + } +} + +func TestLoadConfig_AncienneSectionDispatcherIgnoree(t *testing.T) { + cfg, err := LoadConfig(writeYAML(t, "dispatcher:\n timeout_seconds: 900\n poll_seconds: 7\n")) + if err != nil { + t.Fatalf("une section dispatcher restante ne doit pas empêcher le démarrage : %v", err) + } + if cfg.Times.TimeoutVMsStop != 300 || cfg.Times.PollInterval != 2 { + t.Errorf("dispatcher.* ne doit rien changer : %+v", cfg.Times) + } +} + +func TestLoadConfig_TimesInvalidesRefuses(t *testing.T) { + for _, c := range []struct{ yaml, msg string }{ + {"times:\n timeout_vms_stop: 0\n", "times.timeout_vms_stop 0 must be a positive number of seconds"}, + {"times:\n next_vms_shutdown: -1\n", "times.next_vms_shutdown -1 must be a positive number of seconds"}, + {"times:\n timeout_subnet_stop: 0\n", "times.timeout_subnet_stop 0"}, + {"times:\n timeout_vpc_stop: 0\n", "times.timeout_vpc_stop 0"}, + {"times:\n timeout_subnet_start: 0\n", "times.timeout_subnet_start 0"}, + {"times:\n timeout_vms_start: 0\n", "times.timeout_vms_start 0"}, + {"times:\n poll_interval: 0\n", "times.poll_interval 0"}, + {"times:\n next_vms_shutdown: 300\n", "times.next_vms_shutdown 300 must be below times.timeout_vms_stop 300"}, + } { + if _, err := LoadConfig(writeYAML(t, c.yaml)); err == nil || !strings.Contains(err.Error(), c.msg) { + t.Errorf("%q : erreur %v, attendu %q", c.yaml, err, c.msg) + } + } + if _, err := LoadConfig(writeYAML(t, "times:\n next_vms_shutdown: 299\n")); err != nil { + t.Errorf("next_vms_shutdown 299 sous timeout_vms_stop 300 : %v", err) + } +} + +func TestTimesDurees(t *testing.T) { + var zero Times + cases := []struct { + name string + got, want time.Duration + }{ + {"VMsStop", zero.VMsStop(), 300 * time.Second}, + {"VMsShutdown", zero.VMsShutdown(), 30 * time.Second}, + {"SubnetStop", zero.SubnetStop(), 360 * time.Second}, + {"VPCStop", zero.VPCStop(), 420 * time.Second}, + {"SubnetStart", zero.SubnetStart(), 300 * time.Second}, + {"VMsStart", zero.VMsStart(), 300 * time.Second}, + {"Poll", zero.Poll(), 2 * time.Second}, + } + for _, c := range cases { + if c.got != c.want { + t.Errorf("Times{}.%s() = %v, attendu %v", c.name, c.got, c.want) + } + } + set := Times{11, 12, 13, 14, 15, 16, 17} + got := []time.Duration{set.VMsStop(), set.VMsShutdown(), set.SubnetStop(), set.VPCStop(), set.SubnetStart(), set.VMsStart(), set.Poll()} + for i, d := range got { + if want := time.Duration(11+i) * time.Second; d != want { + t.Errorf("durée %d = %v, attendu %v", i, d, want) + } + } +} diff --git a/internal/dispatcher/agent/subnet_commands.go b/internal/dispatcher/agent/subnet_commands.go index d97de05..91f08bc 100644 --- a/internal/dispatcher/agent/subnet_commands.go +++ b/internal/dispatcher/agent/subnet_commands.go @@ -1,6 +1,7 @@ package dispatcher import ( + "errors" "fmt" "sort" "strconv" @@ -93,23 +94,21 @@ func (c CreateSubnetCommand) Prepare(db *badger.DB, cfg *configuration.Config) e } func (c CreateSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error { - timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second) - for { + err := waitUntil(cfg.Times.SubnetStart(), cfg.Times.Poll(), func() (bool, error) { vpcState, err := state.Get(db, "vpc/"+c.VPC) if err != nil { - return fmt.Errorf("vpc %q not found while waiting", c.VPC) + return false, fmt.Errorf("vpc %q not found while waiting", c.VPC) } - if vpcState == state.Running { - break - } - if vpcState != state.Creating { - return fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name) - } - select { - case <-timeout: - return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC) - case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): + if vpcState != state.Running && vpcState != state.Creating { + return false, fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name) } + return vpcState == state.Running, nil + }) + if errors.Is(err, errWaitTimeout) { + return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC) + } + if err != nil { + return err } return subnet.CreateSubnet(db, c.Name, cfg) } @@ -132,12 +131,41 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err if err != nil { return err } - if len(vms) > 0 { - return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", ")) + var blocking []string + for _, name := range vms { + if s, err := state.Get(db, "vm/"+name); err != nil || s != state.Deleting { + blocking = append(blocking, name) + } + } + if len(blocking) > 0 { + return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(blocking, ", ")) } return state.Set(db, c.Key(), state.Deleting) } +func waitVMsDeleted(db *badger.DB, subnetName string, timeout, poll time.Duration) error { + err := waitUntil(timeout, poll, func() (bool, error) { + vms, err := vmsUsingSubnet(db, subnetName) + if err != nil { + return false, err + } + for _, name := range vms { + s, err := state.Get(db, "vm/"+name) + if err != nil { + continue + } + if s != state.Deleting { + return false, fmt.Errorf("vm %q is %s, subnet %q cannot be deleted", name, s, subnetName) + } + } + return len(vms) == 0, nil + }) + if errors.Is(err, errWaitTimeout) { + return fmt.Errorf("timed out waiting for the vms of subnet %q to be deleted", subnetName) + } + return err +} + func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) { entries, err := kv.ListByPrefix(db, "vm/") if err != nil { @@ -159,6 +187,9 @@ func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) { } func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error { + if err := waitVMsDeleted(db, c.Name, cfg.Times.SubnetStop(), cfg.Times.Poll()); err != nil { + return err + } if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil { return err } diff --git a/internal/dispatcher/agent/subnet_commands_test.go b/internal/dispatcher/agent/subnet_commands_test.go index d9095f2..a2d6cd5 100644 --- a/internal/dispatcher/agent/subnet_commands_test.go +++ b/internal/dispatcher/agent/subnet_commands_test.go @@ -418,7 +418,7 @@ func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) { kv.AddInDB(db, "vm/i-c/state", "running") kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a") err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil) - if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) { + if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-c`) { t.Fatalf("erreur %v", err) } if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" { diff --git a/internal/dispatcher/agent/vm_commands.go b/internal/dispatcher/agent/vm_commands.go index 3bdd7ee..bcd8b61 100644 --- a/internal/dispatcher/agent/vm_commands.go +++ b/internal/dispatcher/agent/vm_commands.go @@ -1,6 +1,7 @@ package dispatcher import ( + "errors" "fmt" "math/rand" "strconv" @@ -151,24 +152,23 @@ func allocateMetadataPort(db *badger.DB) (int, error) { } func (c StartVMCommand) Execute(db *badger.DB, cfg *configuration.Config) error { - timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second) + timeout := time.Now().Add(cfg.Times.VMsStart()) for _, n := range c.NICs { - for { + err := waitUntil(time.Until(timeout), cfg.Times.Poll(), func() (bool, error) { subnetState, err := state.Get(db, "subnet/"+n.Subnet) if err != nil { - return fmt.Errorf("subnet %q not found while waiting", n.Subnet) + return false, fmt.Errorf("subnet %q not found while waiting", n.Subnet) } - if subnetState == state.Running { - break - } - if subnetState != state.Creating { - return fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name) - } - select { - case <-timeout: - return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet) - case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): + if subnetState != state.Running && subnetState != state.Creating { + return false, fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name) } + return subnetState == state.Running, nil + }) + if errors.Is(err, errWaitTimeout) { + return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet) + } + if err != nil { + return err } } return vm.StartVM(db, c.Name, cfg) diff --git a/internal/dispatcher/agent/vpc_commands.go b/internal/dispatcher/agent/vpc_commands.go index df87004..7e73164 100644 --- a/internal/dispatcher/agent/vpc_commands.go +++ b/internal/dispatcher/agent/vpc_commands.go @@ -1,6 +1,7 @@ package dispatcher import ( + "errors" "fmt" "net" "strconv" @@ -129,31 +130,8 @@ func (c DeleteVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error } func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error { - timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second) - for { - entries, err := kv.ListByPrefix(db, "subnet/") - if err != nil { - return fmt.Errorf("failed to list subnets: %w", err) - } - pending := false - for key, value := range entries { - if strings.HasSuffix(key, "/vpc") && value == c.Name { - subnetName := strings.Split(key, "/")[1] - s, _ := state.Get(db, "subnet/"+subnetName) - if s == state.Deleting { - pending = true - break - } - } - } - if !pending { - break - } - select { - case <-timeout: - return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", c.Name) - case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second): - } + if err := waitSubnetsDeleted(db, c.Name, cfg.Times.VPCStop(), cfg.Times.Poll()); err != nil { + return err } if err := vpc.DeleteVPC(db, c.Name); err != nil { return err @@ -167,3 +145,35 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro } return nil } + +func waitSubnetsDeleted(db *badger.DB, vpcName string, timeout, poll time.Duration) error { + err := waitUntil(timeout, poll, func() (bool, error) { + entries, err := kv.ListByPrefix(db, "subnet/") + if err != nil { + return false, fmt.Errorf("failed to list subnets: %w", err) + } + pending := false + for key, value := range entries { + if !strings.HasSuffix(key, "/vpc") || value != vpcName { + continue + } + subnetName := strings.Split(key, "/")[1] + s, err := state.Get(db, "subnet/"+subnetName) + if err != nil { + continue + } + switch s { + case state.Deleted: + case state.Deleting: + pending = true + default: + return false, fmt.Errorf("subnet %q is %s, vpc %q cannot be deleted", subnetName, s, vpcName) + } + } + return !pending, nil + }) + if errors.Is(err, errWaitTimeout) { + return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", vpcName) + } + return err +} diff --git a/internal/dispatcher/agent/wait.go b/internal/dispatcher/agent/wait.go new file mode 100644 index 0000000..2274f33 --- /dev/null +++ b/internal/dispatcher/agent/wait.go @@ -0,0 +1,26 @@ +package dispatcher + +import ( + "errors" + "time" +) + +var errWaitTimeout = errors.New("timed out") + +func waitUntil(timeout, poll time.Duration, done func() (bool, error)) error { + deadline := time.After(timeout) + for { + ok, err := done() + if err != nil { + return err + } + if ok { + return nil + } + select { + case <-deadline: + return errWaitTimeout + case <-time.After(poll): + } + } +} diff --git a/internal/dispatcher/agent/wait_test.go b/internal/dispatcher/agent/wait_test.go new file mode 100644 index 0000000..f3dd7cb --- /dev/null +++ b/internal/dispatcher/agent/wait_test.go @@ -0,0 +1,168 @@ +package dispatcher + +import ( + "errors" + "strings" + "testing" + "time" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/pkg/db/kv" +) + +func TestWaitUntil(t *testing.T) { + calls := 0 + err := waitUntil(time.Second, time.Millisecond, func() (bool, error) { + calls++ + return calls == 3, nil + }) + if err != nil || calls != 3 { + t.Errorf("err %v, %d appels, attendu fin au troisième", err, calls) + } + + if err := waitUntil(30*time.Millisecond, 5*time.Millisecond, func() (bool, error) { return false, nil }); !errors.Is(err, errWaitTimeout) { + t.Errorf("délai dépassé : %v", err) + } + + boom := errors.New("boom") + calls = 0 + err = waitUntil(time.Second, time.Millisecond, func() (bool, error) { + calls++ + return false, boom + }) + if !errors.Is(err, boom) || calls != 1 { + t.Errorf("erreur %v après %d appels, attendu boom au premier", err, calls) + } +} + +func TestDeleteSubnetCommand_Prepare_AcceptsVMsBeingDeleted(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-gone/state", "deleted") + kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a") + if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil { + t.Fatalf("VM en deleting : %v", err) + } + if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "deleting" { + t.Errorf("état %q, attendu deleting", s) + } +} + +func TestDeleteSubnetCommand_Prepare_ListsOnlyTheVMsNotBeingDeleted(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "running") + kv.AddInDB(db, "vm/i-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-b/state", "running") + kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-a") + err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-b`) || strings.Contains(err.Error(), "i-a") { + t.Fatalf("erreur %v", err) + } +} + +func TestWaitVMsDeleted_ReturnsOnceTheVMsAreGone(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vm/i-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + kv.AddInDB(db, "vm/i-other/state", "running") + kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b") + go func() { + time.Sleep(30 * time.Millisecond) + kv.DeleteInDB(db, "vm/i-a") + }() + if err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond); err != nil { + t.Fatalf("attente : %v", err) + } +} + +func TestWaitVMsDeleted_FailsWhenAVMEndsInError(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vm/i-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + go func() { + time.Sleep(20 * time.Millisecond) + kv.AddInDB(db, "vm/i-a/state", "error") + }() + err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), `vm "i-a" is error, subnet "sn-a" cannot be deleted`) { + t.Fatalf("erreur %v", err) + } +} + +func TestWaitVMsDeleted_TimesOut(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vm/i-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + err := waitVMsDeleted(db, "sn-a", 30*time.Millisecond, 5*time.Millisecond) + if err == nil || err.Error() != `timed out waiting for the vms of subnet "sn-a" to be deleted` { + t.Fatalf("erreur %v", err) + } +} + +func TestWaitSubnetsDeleted_ReturnsOnceTheSubnetsAreGone(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "deleting") + kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a") + kv.AddInDB(db, "subnet/sn-b/state", "deleted") + kv.AddInDB(db, "subnet/sn-b/vpc", "vp-a") + kv.AddInDB(db, "subnet/sn-x/state", "running") + kv.AddInDB(db, "subnet/sn-x/vpc", "vp-x") + go func() { + time.Sleep(30 * time.Millisecond) + kv.DeleteInDB(db, "subnet/sn-a") + }() + if err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond); err != nil { + t.Fatalf("attente : %v", err) + } +} + +func TestWaitSubnetsDeleted_FailsWhenASubnetEndsInError(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "deleting") + kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a") + go func() { + time.Sleep(20 * time.Millisecond) + kv.AddInDB(db, "subnet/sn-a/state", "error") + }() + err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is error, vpc "vp-a" cannot be deleted`) { + t.Fatalf("erreur %v", err) + } +} + +func TestWaitSubnetsDeleted_TimesOut(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "deleting") + kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a") + err := waitSubnetsDeleted(db, "vp-a", 30*time.Millisecond, 5*time.Millisecond) + if err == nil || err.Error() != `timed out waiting for subnets of vpc "vp-a" to be deleted` { + t.Fatalf("erreur %v", err) + } +} + +func TestDeleteSubnetCommand_Execute_FailsBeforeTouchingTheHostWhenAVMIsInError(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "subnet/sn-a/state", "deleting") + kv.AddInDB(db, "vm/i-a/state", "error") + kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a") + cfg := &configuration.Config{Times: configuration.Times{TimeoutSubnetStop: 1, PollInterval: 1}} + err := DeleteSubnetCommand{Name: "sn-a"}.Execute(db, cfg) + if err == nil || err.Error() != `vm "i-a" is error, subnet "sn-a" cannot be deleted` { + t.Fatalf("erreur %v", err) + } +} + +func TestDeleteVPCCommand_Execute_FailsBeforeTouchingTheHostWhenASubnetIsInError(t *testing.T) { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vpc/vp-a/state", "deleting") + kv.AddInDB(db, "subnet/sn-a/state", "error") + kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a") + cfg := &configuration.Config{Times: configuration.Times{TimeoutVPCStop: 1, PollInterval: 1}} + err := DeleteVPCCommand{Name: "vp-a"}.Execute(db, cfg) + if err == nil || err.Error() != `subnet "sn-a" is error, vpc "vp-a" cannot be deleted` { + t.Fatalf("erreur %v", err) + } +} diff --git a/internal/vm/delete.go b/internal/vm/delete.go index 304a75d..a19fe4e 100644 --- a/internal/vm/delete.go +++ b/internal/vm/delete.go @@ -1,6 +1,7 @@ package vm import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -47,13 +48,7 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error { if _, err := os.Stat(socketPath); err == nil { // socket présent : tenter l'arrêt gracieux - if _, err := qmp.Send(socketPath, []string{`{"execute":"system_powerdown"}`}); err == nil { - waitQMPDead(socketPath, - time.Duration(cfg.Dispatcher.TimeoutSeconds)*time.Second, - time.Duration(cfg.Dispatcher.PollSeconds)*time.Second, - ) - } - // connexion QMP échouée : QEMU déjà mort + shutdownQEMU(qmp.Send, socketPath, cfg.Times.VMsStop(), cfg.Times.VMsShutdown(), cfg.Times.Poll()) } // socket absent ou QEMU déjà arrêté : cleanup direct @@ -99,15 +94,31 @@ func removeDHCPFiles(d vmData, name string, backend dhcpbackend.Backend) error { return nil } -func waitQMPDead(socketPath string, timeout, poll time.Duration) { - timer := time.After(timeout) +type qmpSender func(socketPath string, commands []string) ([]json.RawMessage, error) + +const ( + qmpPowerdown = `{"execute":"system_powerdown"}` + qmpQuit = `{"execute":"quit"}` +) + +func shutdownQEMU(send qmpSender, socketPath string, timeout, resend, poll time.Duration) { + if _, err := send(socketPath, []string{qmpPowerdown}); err != nil { + return + } + deadline := time.After(timeout) + again := time.NewTicker(resend) + defer again.Stop() for { select { - case <-timer: - qmp.Send(socketPath, []string{`{"execute":"quit"}`}) + case <-deadline: + send(socketPath, []string{qmpQuit}) return + case <-again.C: + if _, err := send(socketPath, []string{qmpPowerdown}); err != nil { + return + } case <-time.After(poll): - if _, err := qmp.Send(socketPath, nil); err != nil { + if _, err := send(socketPath, nil); err != nil { return } } diff --git a/internal/vm/shutdown_test.go b/internal/vm/shutdown_test.go new file mode 100644 index 0000000..1e5cd6c --- /dev/null +++ b/internal/vm/shutdown_test.go @@ -0,0 +1,102 @@ +package vm + +import ( + "encoding/json" + "errors" + "sync" + "testing" + "time" +) + +type fakeQEMU struct { + mu sync.Mutex + calls []string + dieAfter int + powerdown int + dead bool +} + +func (f *fakeQEMU) send(_ string, commands []string) ([]json.RawMessage, error) { + f.mu.Lock() + defer f.mu.Unlock() + call := "ping" + if len(commands) > 0 { + call = commands[0] + } + f.calls = append(f.calls, call) + if f.dead { + return nil, errors.New("qmp dial: connection refused") + } + switch call { + case qmpPowerdown: + f.powerdown++ + if f.dieAfter > 0 && f.powerdown >= f.dieAfter { + f.dead = true + } + case qmpQuit: + f.dead = true + } + return nil, nil +} + +func (f *fakeQEMU) count(call string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, c := range f.calls { + if c == call { + n++ + } + } + return n +} + +func TestShutdownQEMU_ResendsPowerdownUntilTheGuestListens(t *testing.T) { + f := &fakeQEMU{dieAfter: 3} + start := time.Now() + shutdownQEMU(f.send, "qmp.sock", 5*time.Second, 50*time.Millisecond, 10*time.Millisecond) + if d := time.Since(start); d > 2*time.Second { + t.Errorf("arrêt en %v : le renvoi aurait dû l'obtenir bien avant le délai", d) + } + if n := f.count(qmpPowerdown); n != 3 { + t.Errorf("%d system_powerdown envoyés, attendu 3 (le troisième est entendu)", n) + } + if n := f.count(qmpQuit); n != 0 { + t.Errorf("quit envoyé alors que l'invité s'est arrêté : %q", f.calls) + } +} + +func TestShutdownQEMU_QuitsAtTheDeadline(t *testing.T) { + f := &fakeQEMU{} + start := time.Now() + shutdownQEMU(f.send, "qmp.sock", 200*time.Millisecond, 60*time.Millisecond, 10*time.Millisecond) + d := time.Since(start) + if d < 200*time.Millisecond || d > 2*time.Second { + t.Errorf("arrêt en %v, attendu au délai de 200ms", d) + } + if n := f.count(qmpPowerdown); n < 3 { + t.Errorf("%d system_powerdown envoyés en 200ms avec un renvoi toutes les 60ms", n) + } + if n := f.count(qmpQuit); n != 1 { + t.Errorf("%d quit envoyés, attendu 1 : %q", n, f.calls) + } +} + +func TestShutdownQEMU_StopsWhenTheGuestShutsDownOnItsOwn(t *testing.T) { + f := &fakeQEMU{dieAfter: 1} + shutdownQEMU(f.send, "qmp.sock", 5*time.Second, time.Second, 10*time.Millisecond) + if n := f.count(qmpPowerdown); n != 1 { + t.Errorf("%d system_powerdown envoyés, attendu 1", n) + } + if n := f.count(qmpQuit); n != 0 { + t.Errorf("quit envoyé : %q", f.calls) + } +} + +func TestShutdownQEMU_NothingMoreWhenQEMUIsAlreadyDead(t *testing.T) { + f := &fakeQEMU{dead: true} + shutdownQEMU(f.send, "qmp.sock", 5*time.Second, time.Second, 10*time.Millisecond) + if len(f.calls) != 1 { + t.Errorf("appels QMP : %q, attendu un seul essai", f.calls) + } +}