f-79: agent: section times, renvoi de system_powerdown, attentes #79
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
8429dfd7e1
commit
8e89865418
13 changed files with 602 additions and 80 deletions
|
|
@ -91,7 +91,7 @@ paths:
|
|||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"409":
|
||||
description: VPC not deletable — only running or error states can be deleted, and all its subnets must be deleted first
|
||||
description: VPC not deletable — only running or error states can be deleted, and all its subnets must be deleted or deleting; subnets still deleting are awaited by the asynchronous deletion
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
|
|
@ -288,7 +288,7 @@ paths:
|
|||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"409":
|
||||
description: Subnet not deletable — only running or error states can be deleted, and no VM that is not deleted may still have an interface on it
|
||||
description: Subnet not deletable — only running or error states can be deleted, and every VM with an interface on it must be deleted or deleting; VMs still deleting are awaited by the asynchronous deletion
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
|
|
|
|||
|
|
@ -19,12 +19,22 @@ worker:
|
|||
# Maximum number of commands queued before Dispatch blocks
|
||||
buffer_size: 100
|
||||
|
||||
# Timing for commands that wait on resource state transitions
|
||||
dispatcher:
|
||||
# How long (in seconds) to wait before giving up
|
||||
timeout_seconds: 300
|
||||
# Interval (in seconds) between each state check
|
||||
poll_seconds: 2
|
||||
# Timing (in seconds) for commands that wait on resource state transitions
|
||||
times:
|
||||
# Graceful VM shutdown before QMP quit
|
||||
timeout_vms_stop: 300
|
||||
# Interval between two system_powerdown during the shutdown
|
||||
next_vms_shutdown: 30
|
||||
# Subnet deletion: wait for its VMs still deleting
|
||||
timeout_subnet_stop: 360
|
||||
# VPC deletion: wait for its subnets still deleting
|
||||
timeout_vpc_stop: 420
|
||||
# Subnet creation: wait for its VPC to be running
|
||||
timeout_subnet_start: 300
|
||||
# VM creation: wait for the subnets of its interfaces to be running
|
||||
timeout_vms_start: 300
|
||||
# Interval between each state check
|
||||
poll_interval: 2
|
||||
|
||||
# Bridge interface used when the requested iface_type is not found in interfaces
|
||||
default_interface: br-000000
|
||||
|
|
|
|||
|
|
@ -24,10 +24,7 @@ type Config struct {
|
|||
Count int `mapstructure:"count"`
|
||||
BufferSize int `mapstructure:"buffer_size"`
|
||||
} `mapstructure:"worker"`
|
||||
Dispatcher struct {
|
||||
TimeoutSeconds int `mapstructure:"timeout_seconds"`
|
||||
PollSeconds int `mapstructure:"poll_seconds"`
|
||||
} `mapstructure:"dispatcher"`
|
||||
Times Times `mapstructure:"times"`
|
||||
Logger struct {
|
||||
Level string `mapstructure:"level"`
|
||||
Debug bool `mapstructure:"debug"`
|
||||
|
|
@ -73,8 +70,13 @@ func LoadConfig(path string) (*Config, error) {
|
|||
v.SetDefault("prometheus.port", 9090)
|
||||
v.SetDefault("worker.count", 4)
|
||||
v.SetDefault("worker.buffer_size", 100)
|
||||
v.SetDefault("dispatcher.timeout_seconds", 300)
|
||||
v.SetDefault("dispatcher.poll_seconds", 2)
|
||||
v.SetDefault("times.timeout_vms_stop", TimeoutVMsStop)
|
||||
v.SetDefault("times.next_vms_shutdown", NextVMsShutdown)
|
||||
v.SetDefault("times.timeout_subnet_stop", TimeoutSubnetStop)
|
||||
v.SetDefault("times.timeout_vpc_stop", TimeoutVPCStop)
|
||||
v.SetDefault("times.timeout_subnet_start", TimeoutSubnetStart)
|
||||
v.SetDefault("times.timeout_vms_start", TimeoutVMsStart)
|
||||
v.SetDefault("times.poll_interval", PollInterval)
|
||||
v.SetDefault("metadata.run_dir", "/run/two/metadata")
|
||||
v.SetDefault("dhcp.backend", BackendTwo)
|
||||
v.SetDefault("qemu.ovmf_code_path", "/usr/share/OVMF/OVMF_CODE.fd")
|
||||
|
|
@ -106,6 +108,9 @@ func LoadConfig(path string) (*Config, error) {
|
|||
if err := cfg.Network.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if err := cfg.Times.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
|
|
|||
64
internal/config/agent/times.go
Normal file
64
internal/config/agent/times.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
TimeoutVMsStop = 300
|
||||
NextVMsShutdown = 30
|
||||
TimeoutSubnetStop = 360
|
||||
TimeoutVPCStop = 420
|
||||
TimeoutSubnetStart = 300
|
||||
TimeoutVMsStart = 300
|
||||
PollInterval = 2
|
||||
)
|
||||
|
||||
type Times struct {
|
||||
TimeoutVMsStop int `mapstructure:"timeout_vms_stop"`
|
||||
NextVMsShutdown int `mapstructure:"next_vms_shutdown"`
|
||||
TimeoutSubnetStop int `mapstructure:"timeout_subnet_stop"`
|
||||
TimeoutVPCStop int `mapstructure:"timeout_vpc_stop"`
|
||||
TimeoutSubnetStart int `mapstructure:"timeout_subnet_start"`
|
||||
TimeoutVMsStart int `mapstructure:"timeout_vms_start"`
|
||||
PollInterval int `mapstructure:"poll_interval"`
|
||||
}
|
||||
|
||||
func seconds(value, fallback int) time.Duration {
|
||||
if value == 0 {
|
||||
value = fallback
|
||||
}
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
|
||||
func (t Times) VMsStop() time.Duration { return seconds(t.TimeoutVMsStop, TimeoutVMsStop) }
|
||||
func (t Times) VMsShutdown() time.Duration { return seconds(t.NextVMsShutdown, NextVMsShutdown) }
|
||||
func (t Times) SubnetStop() time.Duration { return seconds(t.TimeoutSubnetStop, TimeoutSubnetStop) }
|
||||
func (t Times) VPCStop() time.Duration { return seconds(t.TimeoutVPCStop, TimeoutVPCStop) }
|
||||
func (t Times) SubnetStart() time.Duration { return seconds(t.TimeoutSubnetStart, TimeoutSubnetStart) }
|
||||
func (t Times) VMsStart() time.Duration { return seconds(t.TimeoutVMsStart, TimeoutVMsStart) }
|
||||
func (t Times) Poll() time.Duration { return seconds(t.PollInterval, PollInterval) }
|
||||
|
||||
func (t Times) Validate() error {
|
||||
for _, f := range []struct {
|
||||
key string
|
||||
value int
|
||||
}{
|
||||
{"timeout_vms_stop", t.TimeoutVMsStop},
|
||||
{"next_vms_shutdown", t.NextVMsShutdown},
|
||||
{"timeout_subnet_stop", t.TimeoutSubnetStop},
|
||||
{"timeout_vpc_stop", t.TimeoutVPCStop},
|
||||
{"timeout_subnet_start", t.TimeoutSubnetStart},
|
||||
{"timeout_vms_start", t.TimeoutVMsStart},
|
||||
{"poll_interval", t.PollInterval},
|
||||
} {
|
||||
if f.value <= 0 {
|
||||
return fmt.Errorf("times.%s %d must be a positive number of seconds", f.key, f.value)
|
||||
}
|
||||
}
|
||||
if t.NextVMsShutdown >= t.TimeoutVMsStop {
|
||||
return fmt.Errorf("times.next_vms_shutdown %d must be below times.timeout_vms_stop %d", t.NextVMsShutdown, t.TimeoutVMsStop)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
95
internal/config/agent/times_test.go
Normal file
95
internal/config/agent/times_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadConfig_TimesDefauts(t *testing.T) {
|
||||
cfg, err := LoadConfig(writeYAML(t, ""))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig a échoué : %v", err)
|
||||
}
|
||||
want := Times{
|
||||
TimeoutVMsStop: 300,
|
||||
NextVMsShutdown: 30,
|
||||
TimeoutSubnetStop: 360,
|
||||
TimeoutVPCStop: 420,
|
||||
TimeoutSubnetStart: 300,
|
||||
TimeoutVMsStart: 300,
|
||||
PollInterval: 2,
|
||||
}
|
||||
if cfg.Times != want {
|
||||
t.Errorf("times par défaut : %+v, attendu %+v", cfg.Times, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_TimesValeursExplicites(t *testing.T) {
|
||||
cfg, err := LoadConfig(writeYAML(t, "times:\n timeout_vms_stop: 600\n next_vms_shutdown: 10\n timeout_subnet_stop: 700\n timeout_vpc_stop: 800\n timeout_subnet_start: 50\n timeout_vms_start: 60\n poll_interval: 1\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig a échoué : %v", err)
|
||||
}
|
||||
want := Times{600, 10, 700, 800, 50, 60, 1}
|
||||
if cfg.Times != want {
|
||||
t.Errorf("times : %+v, attendu %+v", cfg.Times, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_AncienneSectionDispatcherIgnoree(t *testing.T) {
|
||||
cfg, err := LoadConfig(writeYAML(t, "dispatcher:\n timeout_seconds: 900\n poll_seconds: 7\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("une section dispatcher restante ne doit pas empêcher le démarrage : %v", err)
|
||||
}
|
||||
if cfg.Times.TimeoutVMsStop != 300 || cfg.Times.PollInterval != 2 {
|
||||
t.Errorf("dispatcher.* ne doit rien changer : %+v", cfg.Times)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_TimesInvalidesRefuses(t *testing.T) {
|
||||
for _, c := range []struct{ yaml, msg string }{
|
||||
{"times:\n timeout_vms_stop: 0\n", "times.timeout_vms_stop 0 must be a positive number of seconds"},
|
||||
{"times:\n next_vms_shutdown: -1\n", "times.next_vms_shutdown -1 must be a positive number of seconds"},
|
||||
{"times:\n timeout_subnet_stop: 0\n", "times.timeout_subnet_stop 0"},
|
||||
{"times:\n timeout_vpc_stop: 0\n", "times.timeout_vpc_stop 0"},
|
||||
{"times:\n timeout_subnet_start: 0\n", "times.timeout_subnet_start 0"},
|
||||
{"times:\n timeout_vms_start: 0\n", "times.timeout_vms_start 0"},
|
||||
{"times:\n poll_interval: 0\n", "times.poll_interval 0"},
|
||||
{"times:\n next_vms_shutdown: 300\n", "times.next_vms_shutdown 300 must be below times.timeout_vms_stop 300"},
|
||||
} {
|
||||
if _, err := LoadConfig(writeYAML(t, c.yaml)); err == nil || !strings.Contains(err.Error(), c.msg) {
|
||||
t.Errorf("%q : erreur %v, attendu %q", c.yaml, err, c.msg)
|
||||
}
|
||||
}
|
||||
if _, err := LoadConfig(writeYAML(t, "times:\n next_vms_shutdown: 299\n")); err != nil {
|
||||
t.Errorf("next_vms_shutdown 299 sous timeout_vms_stop 300 : %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimesDurees(t *testing.T) {
|
||||
var zero Times
|
||||
cases := []struct {
|
||||
name string
|
||||
got, want time.Duration
|
||||
}{
|
||||
{"VMsStop", zero.VMsStop(), 300 * time.Second},
|
||||
{"VMsShutdown", zero.VMsShutdown(), 30 * time.Second},
|
||||
{"SubnetStop", zero.SubnetStop(), 360 * time.Second},
|
||||
{"VPCStop", zero.VPCStop(), 420 * time.Second},
|
||||
{"SubnetStart", zero.SubnetStart(), 300 * time.Second},
|
||||
{"VMsStart", zero.VMsStart(), 300 * time.Second},
|
||||
{"Poll", zero.Poll(), 2 * time.Second},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if c.got != c.want {
|
||||
t.Errorf("Times{}.%s() = %v, attendu %v", c.name, c.got, c.want)
|
||||
}
|
||||
}
|
||||
set := Times{11, 12, 13, 14, 15, 16, 17}
|
||||
got := []time.Duration{set.VMsStop(), set.VMsShutdown(), set.SubnetStop(), set.VPCStop(), set.SubnetStart(), set.VMsStart(), set.Poll()}
|
||||
for i, d := range got {
|
||||
if want := time.Duration(11+i) * time.Second; d != want {
|
||||
t.Errorf("durée %d = %v, attendu %v", i, d, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
package dispatcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
|
@ -93,23 +94,21 @@ func (c CreateSubnetCommand) Prepare(db *badger.DB, cfg *configuration.Config) e
|
|||
}
|
||||
|
||||
func (c CreateSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
|
||||
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
|
||||
for {
|
||||
err := waitUntil(cfg.Times.SubnetStart(), cfg.Times.Poll(), func() (bool, error) {
|
||||
vpcState, err := state.Get(db, "vpc/"+c.VPC)
|
||||
if err != nil {
|
||||
return fmt.Errorf("vpc %q not found while waiting", c.VPC)
|
||||
return false, fmt.Errorf("vpc %q not found while waiting", c.VPC)
|
||||
}
|
||||
if vpcState == state.Running {
|
||||
break
|
||||
}
|
||||
if vpcState != state.Creating {
|
||||
return fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name)
|
||||
}
|
||||
select {
|
||||
case <-timeout:
|
||||
return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC)
|
||||
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
|
||||
if vpcState != state.Running && vpcState != state.Creating {
|
||||
return false, fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name)
|
||||
}
|
||||
return vpcState == state.Running, nil
|
||||
})
|
||||
if errors.Is(err, errWaitTimeout) {
|
||||
return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return subnet.CreateSubnet(db, c.Name, cfg)
|
||||
}
|
||||
|
|
@ -132,12 +131,41 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(vms) > 0 {
|
||||
return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", "))
|
||||
var blocking []string
|
||||
for _, name := range vms {
|
||||
if s, err := state.Get(db, "vm/"+name); err != nil || s != state.Deleting {
|
||||
blocking = append(blocking, name)
|
||||
}
|
||||
}
|
||||
if len(blocking) > 0 {
|
||||
return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(blocking, ", "))
|
||||
}
|
||||
return state.Set(db, c.Key(), state.Deleting)
|
||||
}
|
||||
|
||||
func waitVMsDeleted(db *badger.DB, subnetName string, timeout, poll time.Duration) error {
|
||||
err := waitUntil(timeout, poll, func() (bool, error) {
|
||||
vms, err := vmsUsingSubnet(db, subnetName)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, name := range vms {
|
||||
s, err := state.Get(db, "vm/"+name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if s != state.Deleting {
|
||||
return false, fmt.Errorf("vm %q is %s, subnet %q cannot be deleted", name, s, subnetName)
|
||||
}
|
||||
}
|
||||
return len(vms) == 0, nil
|
||||
})
|
||||
if errors.Is(err, errWaitTimeout) {
|
||||
return fmt.Errorf("timed out waiting for the vms of subnet %q to be deleted", subnetName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) {
|
||||
entries, err := kv.ListByPrefix(db, "vm/")
|
||||
if err != nil {
|
||||
|
|
@ -159,6 +187,9 @@ func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) {
|
|||
}
|
||||
|
||||
func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
|
||||
if err := waitVMsDeleted(db, c.Name, cfg.Times.SubnetStop(), cfg.Times.Poll()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -418,7 +418,7 @@ func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) {
|
|||
kv.AddInDB(db, "vm/i-c/state", "running")
|
||||
kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a")
|
||||
err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) {
|
||||
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-c`) {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package dispatcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
|
|
@ -151,24 +152,23 @@ func allocateMetadataPort(db *badger.DB) (int, error) {
|
|||
}
|
||||
|
||||
func (c StartVMCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
|
||||
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
|
||||
timeout := time.Now().Add(cfg.Times.VMsStart())
|
||||
for _, n := range c.NICs {
|
||||
for {
|
||||
err := waitUntil(time.Until(timeout), cfg.Times.Poll(), func() (bool, error) {
|
||||
subnetState, err := state.Get(db, "subnet/"+n.Subnet)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subnet %q not found while waiting", n.Subnet)
|
||||
return false, fmt.Errorf("subnet %q not found while waiting", n.Subnet)
|
||||
}
|
||||
if subnetState == state.Running {
|
||||
break
|
||||
}
|
||||
if subnetState != state.Creating {
|
||||
return fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name)
|
||||
}
|
||||
select {
|
||||
case <-timeout:
|
||||
return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet)
|
||||
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
|
||||
if subnetState != state.Running && subnetState != state.Creating {
|
||||
return false, fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name)
|
||||
}
|
||||
return subnetState == state.Running, nil
|
||||
})
|
||||
if errors.Is(err, errWaitTimeout) {
|
||||
return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return vm.StartVM(db, c.Name, cfg)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package dispatcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
|
|
@ -129,31 +130,8 @@ func (c DeleteVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error
|
|||
}
|
||||
|
||||
func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
|
||||
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
|
||||
for {
|
||||
entries, err := kv.ListByPrefix(db, "subnet/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list subnets: %w", err)
|
||||
}
|
||||
pending := false
|
||||
for key, value := range entries {
|
||||
if strings.HasSuffix(key, "/vpc") && value == c.Name {
|
||||
subnetName := strings.Split(key, "/")[1]
|
||||
s, _ := state.Get(db, "subnet/"+subnetName)
|
||||
if s == state.Deleting {
|
||||
pending = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !pending {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-timeout:
|
||||
return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", c.Name)
|
||||
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
|
||||
}
|
||||
if err := waitSubnetsDeleted(db, c.Name, cfg.Times.VPCStop(), cfg.Times.Poll()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vpc.DeleteVPC(db, c.Name); err != nil {
|
||||
return err
|
||||
|
|
@ -167,3 +145,35 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitSubnetsDeleted(db *badger.DB, vpcName string, timeout, poll time.Duration) error {
|
||||
err := waitUntil(timeout, poll, func() (bool, error) {
|
||||
entries, err := kv.ListByPrefix(db, "subnet/")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to list subnets: %w", err)
|
||||
}
|
||||
pending := false
|
||||
for key, value := range entries {
|
||||
if !strings.HasSuffix(key, "/vpc") || value != vpcName {
|
||||
continue
|
||||
}
|
||||
subnetName := strings.Split(key, "/")[1]
|
||||
s, err := state.Get(db, "subnet/"+subnetName)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
switch s {
|
||||
case state.Deleted:
|
||||
case state.Deleting:
|
||||
pending = true
|
||||
default:
|
||||
return false, fmt.Errorf("subnet %q is %s, vpc %q cannot be deleted", subnetName, s, vpcName)
|
||||
}
|
||||
}
|
||||
return !pending, nil
|
||||
})
|
||||
if errors.Is(err, errWaitTimeout) {
|
||||
return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", vpcName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
26
internal/dispatcher/agent/wait.go
Normal file
26
internal/dispatcher/agent/wait.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package dispatcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errWaitTimeout = errors.New("timed out")
|
||||
|
||||
func waitUntil(timeout, poll time.Duration, done func() (bool, error)) error {
|
||||
deadline := time.After(timeout)
|
||||
for {
|
||||
ok, err := done()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
return errWaitTimeout
|
||||
case <-time.After(poll):
|
||||
}
|
||||
}
|
||||
}
|
||||
168
internal/dispatcher/agent/wait_test.go
Normal file
168
internal/dispatcher/agent/wait_test.go
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
package dispatcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/pkg/db/kv"
|
||||
)
|
||||
|
||||
func TestWaitUntil(t *testing.T) {
|
||||
calls := 0
|
||||
err := waitUntil(time.Second, time.Millisecond, func() (bool, error) {
|
||||
calls++
|
||||
return calls == 3, nil
|
||||
})
|
||||
if err != nil || calls != 3 {
|
||||
t.Errorf("err %v, %d appels, attendu fin au troisième", err, calls)
|
||||
}
|
||||
|
||||
if err := waitUntil(30*time.Millisecond, 5*time.Millisecond, func() (bool, error) { return false, nil }); !errors.Is(err, errWaitTimeout) {
|
||||
t.Errorf("délai dépassé : %v", err)
|
||||
}
|
||||
|
||||
boom := errors.New("boom")
|
||||
calls = 0
|
||||
err = waitUntil(time.Second, time.Millisecond, func() (bool, error) {
|
||||
calls++
|
||||
return false, boom
|
||||
})
|
||||
if !errors.Is(err, boom) || calls != 1 {
|
||||
t.Errorf("erreur %v après %d appels, attendu boom au premier", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSubnetCommand_Prepare_AcceptsVMsBeingDeleted(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "running")
|
||||
kv.AddInDB(db, "vm/i-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
kv.AddInDB(db, "vm/i-gone/state", "deleted")
|
||||
kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a")
|
||||
if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil {
|
||||
t.Fatalf("VM en deleting : %v", err)
|
||||
}
|
||||
if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "deleting" {
|
||||
t.Errorf("état %q, attendu deleting", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSubnetCommand_Prepare_ListsOnlyTheVMsNotBeingDeleted(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "running")
|
||||
kv.AddInDB(db, "vm/i-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
kv.AddInDB(db, "vm/i-b/state", "running")
|
||||
kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-a")
|
||||
err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-b`) || strings.Contains(err.Error(), "i-a") {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitVMsDeleted_ReturnsOnceTheVMsAreGone(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "vm/i-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
kv.AddInDB(db, "vm/i-other/state", "running")
|
||||
kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b")
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
kv.DeleteInDB(db, "vm/i-a")
|
||||
}()
|
||||
if err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond); err != nil {
|
||||
t.Fatalf("attente : %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitVMsDeleted_FailsWhenAVMEndsInError(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "vm/i-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
kv.AddInDB(db, "vm/i-a/state", "error")
|
||||
}()
|
||||
err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond)
|
||||
if err == nil || !strings.Contains(err.Error(), `vm "i-a" is error, subnet "sn-a" cannot be deleted`) {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitVMsDeleted_TimesOut(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "vm/i-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
err := waitVMsDeleted(db, "sn-a", 30*time.Millisecond, 5*time.Millisecond)
|
||||
if err == nil || err.Error() != `timed out waiting for the vms of subnet "sn-a" to be deleted` {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitSubnetsDeleted_ReturnsOnceTheSubnetsAreGone(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
|
||||
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
|
||||
kv.AddInDB(db, "subnet/sn-b/state", "deleted")
|
||||
kv.AddInDB(db, "subnet/sn-b/vpc", "vp-a")
|
||||
kv.AddInDB(db, "subnet/sn-x/state", "running")
|
||||
kv.AddInDB(db, "subnet/sn-x/vpc", "vp-x")
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
kv.DeleteInDB(db, "subnet/sn-a")
|
||||
}()
|
||||
if err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond); err != nil {
|
||||
t.Fatalf("attente : %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitSubnetsDeleted_FailsWhenASubnetEndsInError(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
|
||||
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "error")
|
||||
}()
|
||||
err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond)
|
||||
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is error, vpc "vp-a" cannot be deleted`) {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitSubnetsDeleted_TimesOut(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
|
||||
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
|
||||
err := waitSubnetsDeleted(db, "vp-a", 30*time.Millisecond, 5*time.Millisecond)
|
||||
if err == nil || err.Error() != `timed out waiting for subnets of vpc "vp-a" to be deleted` {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSubnetCommand_Execute_FailsBeforeTouchingTheHostWhenAVMIsInError(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
|
||||
kv.AddInDB(db, "vm/i-a/state", "error")
|
||||
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
|
||||
cfg := &configuration.Config{Times: configuration.Times{TimeoutSubnetStop: 1, PollInterval: 1}}
|
||||
err := DeleteSubnetCommand{Name: "sn-a"}.Execute(db, cfg)
|
||||
if err == nil || err.Error() != `vm "i-a" is error, subnet "sn-a" cannot be deleted` {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteVPCCommand_Execute_FailsBeforeTouchingTheHostWhenASubnetIsInError(t *testing.T) {
|
||||
_, db := newTestDispatcher(t)
|
||||
kv.AddInDB(db, "vpc/vp-a/state", "deleting")
|
||||
kv.AddInDB(db, "subnet/sn-a/state", "error")
|
||||
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
|
||||
cfg := &configuration.Config{Times: configuration.Times{TimeoutVPCStop: 1, PollInterval: 1}}
|
||||
err := DeleteVPCCommand{Name: "vp-a"}.Execute(db, cfg)
|
||||
if err == nil || err.Error() != `subnet "sn-a" is error, vpc "vp-a" cannot be deleted` {
|
||||
t.Fatalf("erreur %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
package vm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -47,13 +48,7 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error {
|
|||
|
||||
if _, err := os.Stat(socketPath); err == nil {
|
||||
// socket présent : tenter l'arrêt gracieux
|
||||
if _, err := qmp.Send(socketPath, []string{`{"execute":"system_powerdown"}`}); err == nil {
|
||||
waitQMPDead(socketPath,
|
||||
time.Duration(cfg.Dispatcher.TimeoutSeconds)*time.Second,
|
||||
time.Duration(cfg.Dispatcher.PollSeconds)*time.Second,
|
||||
)
|
||||
}
|
||||
// connexion QMP échouée : QEMU déjà mort
|
||||
shutdownQEMU(qmp.Send, socketPath, cfg.Times.VMsStop(), cfg.Times.VMsShutdown(), cfg.Times.Poll())
|
||||
}
|
||||
// socket absent ou QEMU déjà arrêté : cleanup direct
|
||||
|
||||
|
|
@ -99,15 +94,31 @@ func removeDHCPFiles(d vmData, name string, backend dhcpbackend.Backend) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func waitQMPDead(socketPath string, timeout, poll time.Duration) {
|
||||
timer := time.After(timeout)
|
||||
type qmpSender func(socketPath string, commands []string) ([]json.RawMessage, error)
|
||||
|
||||
const (
|
||||
qmpPowerdown = `{"execute":"system_powerdown"}`
|
||||
qmpQuit = `{"execute":"quit"}`
|
||||
)
|
||||
|
||||
func shutdownQEMU(send qmpSender, socketPath string, timeout, resend, poll time.Duration) {
|
||||
if _, err := send(socketPath, []string{qmpPowerdown}); err != nil {
|
||||
return
|
||||
}
|
||||
deadline := time.After(timeout)
|
||||
again := time.NewTicker(resend)
|
||||
defer again.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-timer:
|
||||
qmp.Send(socketPath, []string{`{"execute":"quit"}`})
|
||||
case <-deadline:
|
||||
send(socketPath, []string{qmpQuit})
|
||||
return
|
||||
case <-again.C:
|
||||
if _, err := send(socketPath, []string{qmpPowerdown}); err != nil {
|
||||
return
|
||||
}
|
||||
case <-time.After(poll):
|
||||
if _, err := qmp.Send(socketPath, nil); err != nil {
|
||||
if _, err := send(socketPath, nil); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
102
internal/vm/shutdown_test.go
Normal file
102
internal/vm/shutdown_test.go
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
package vm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeQEMU struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
dieAfter int
|
||||
powerdown int
|
||||
dead bool
|
||||
}
|
||||
|
||||
func (f *fakeQEMU) send(_ string, commands []string) ([]json.RawMessage, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
call := "ping"
|
||||
if len(commands) > 0 {
|
||||
call = commands[0]
|
||||
}
|
||||
f.calls = append(f.calls, call)
|
||||
if f.dead {
|
||||
return nil, errors.New("qmp dial: connection refused")
|
||||
}
|
||||
switch call {
|
||||
case qmpPowerdown:
|
||||
f.powerdown++
|
||||
if f.dieAfter > 0 && f.powerdown >= f.dieAfter {
|
||||
f.dead = true
|
||||
}
|
||||
case qmpQuit:
|
||||
f.dead = true
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeQEMU) count(call string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range f.calls {
|
||||
if c == call {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestShutdownQEMU_ResendsPowerdownUntilTheGuestListens(t *testing.T) {
|
||||
f := &fakeQEMU{dieAfter: 3}
|
||||
start := time.Now()
|
||||
shutdownQEMU(f.send, "qmp.sock", 5*time.Second, 50*time.Millisecond, 10*time.Millisecond)
|
||||
if d := time.Since(start); d > 2*time.Second {
|
||||
t.Errorf("arrêt en %v : le renvoi aurait dû l'obtenir bien avant le délai", d)
|
||||
}
|
||||
if n := f.count(qmpPowerdown); n != 3 {
|
||||
t.Errorf("%d system_powerdown envoyés, attendu 3 (le troisième est entendu)", n)
|
||||
}
|
||||
if n := f.count(qmpQuit); n != 0 {
|
||||
t.Errorf("quit envoyé alors que l'invité s'est arrêté : %q", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownQEMU_QuitsAtTheDeadline(t *testing.T) {
|
||||
f := &fakeQEMU{}
|
||||
start := time.Now()
|
||||
shutdownQEMU(f.send, "qmp.sock", 200*time.Millisecond, 60*time.Millisecond, 10*time.Millisecond)
|
||||
d := time.Since(start)
|
||||
if d < 200*time.Millisecond || d > 2*time.Second {
|
||||
t.Errorf("arrêt en %v, attendu au délai de 200ms", d)
|
||||
}
|
||||
if n := f.count(qmpPowerdown); n < 3 {
|
||||
t.Errorf("%d system_powerdown envoyés en 200ms avec un renvoi toutes les 60ms", n)
|
||||
}
|
||||
if n := f.count(qmpQuit); n != 1 {
|
||||
t.Errorf("%d quit envoyés, attendu 1 : %q", n, f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownQEMU_StopsWhenTheGuestShutsDownOnItsOwn(t *testing.T) {
|
||||
f := &fakeQEMU{dieAfter: 1}
|
||||
shutdownQEMU(f.send, "qmp.sock", 5*time.Second, time.Second, 10*time.Millisecond)
|
||||
if n := f.count(qmpPowerdown); n != 1 {
|
||||
t.Errorf("%d system_powerdown envoyés, attendu 1", n)
|
||||
}
|
||||
if n := f.count(qmpQuit); n != 0 {
|
||||
t.Errorf("quit envoyé : %q", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownQEMU_NothingMoreWhenQEMUIsAlreadyDead(t *testing.T) {
|
||||
f := &fakeQEMU{dead: true}
|
||||
shutdownQEMU(f.send, "qmp.sock", 5*time.Second, time.Second, 10*time.Millisecond)
|
||||
if len(f.calls) != 1 {
|
||||
t.Errorf("appels QMP : %q, attendu un seul essai", f.calls)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue