f-79: agent: section times, renvoi de system_powerdown, attentes #79

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-25 12:42:10 +01:00
commit 8e89865418
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
13 changed files with 602 additions and 80 deletions

View file

@ -1,6 +1,7 @@
package dispatcher
import (
"errors"
"fmt"
"sort"
"strconv"
@ -93,23 +94,21 @@ func (c CreateSubnetCommand) Prepare(db *badger.DB, cfg *configuration.Config) e
}
func (c CreateSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
for {
err := waitUntil(cfg.Times.SubnetStart(), cfg.Times.Poll(), func() (bool, error) {
vpcState, err := state.Get(db, "vpc/"+c.VPC)
if err != nil {
return fmt.Errorf("vpc %q not found while waiting", c.VPC)
return false, fmt.Errorf("vpc %q not found while waiting", c.VPC)
}
if vpcState == state.Running {
break
}
if vpcState != state.Creating {
return fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name)
}
select {
case <-timeout:
return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC)
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
if vpcState != state.Running && vpcState != state.Creating {
return false, fmt.Errorf("vpc %q is %s, cannot create subnet %q", c.VPC, vpcState, c.Name)
}
return vpcState == state.Running, nil
})
if errors.Is(err, errWaitTimeout) {
return fmt.Errorf("timed out waiting for vpc %q to be running", c.VPC)
}
if err != nil {
return err
}
return subnet.CreateSubnet(db, c.Name, cfg)
}
@ -132,12 +131,41 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err
if err != nil {
return err
}
if len(vms) > 0 {
return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", "))
var blocking []string
for _, name := range vms {
if s, err := state.Get(db, "vm/"+name); err != nil || s != state.Deleting {
blocking = append(blocking, name)
}
}
if len(blocking) > 0 {
return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(blocking, ", "))
}
return state.Set(db, c.Key(), state.Deleting)
}
func waitVMsDeleted(db *badger.DB, subnetName string, timeout, poll time.Duration) error {
err := waitUntil(timeout, poll, func() (bool, error) {
vms, err := vmsUsingSubnet(db, subnetName)
if err != nil {
return false, err
}
for _, name := range vms {
s, err := state.Get(db, "vm/"+name)
if err != nil {
continue
}
if s != state.Deleting {
return false, fmt.Errorf("vm %q is %s, subnet %q cannot be deleted", name, s, subnetName)
}
}
return len(vms) == 0, nil
})
if errors.Is(err, errWaitTimeout) {
return fmt.Errorf("timed out waiting for the vms of subnet %q to be deleted", subnetName)
}
return err
}
func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) {
entries, err := kv.ListByPrefix(db, "vm/")
if err != nil {
@ -159,6 +187,9 @@ func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) {
}
func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
if err := waitVMsDeleted(db, c.Name, cfg.Times.SubnetStop(), cfg.Times.Poll()); err != nil {
return err
}
if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil {
return err
}

View file

@ -418,7 +418,7 @@ func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) {
kv.AddInDB(db, "vm/i-c/state", "running")
kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a")
err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil)
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) {
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-c`) {
t.Fatalf("erreur %v", err)
}
if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" {

View file

@ -1,6 +1,7 @@
package dispatcher
import (
"errors"
"fmt"
"math/rand"
"strconv"
@ -151,24 +152,23 @@ func allocateMetadataPort(db *badger.DB) (int, error) {
}
func (c StartVMCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
timeout := time.Now().Add(cfg.Times.VMsStart())
for _, n := range c.NICs {
for {
err := waitUntil(time.Until(timeout), cfg.Times.Poll(), func() (bool, error) {
subnetState, err := state.Get(db, "subnet/"+n.Subnet)
if err != nil {
return fmt.Errorf("subnet %q not found while waiting", n.Subnet)
return false, fmt.Errorf("subnet %q not found while waiting", n.Subnet)
}
if subnetState == state.Running {
break
}
if subnetState != state.Creating {
return fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name)
}
select {
case <-timeout:
return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet)
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
if subnetState != state.Running && subnetState != state.Creating {
return false, fmt.Errorf("subnet %q is %s, cannot start vm %q", n.Subnet, subnetState, c.Name)
}
return subnetState == state.Running, nil
})
if errors.Is(err, errWaitTimeout) {
return fmt.Errorf("timed out waiting for subnet %q to be running", n.Subnet)
}
if err != nil {
return err
}
}
return vm.StartVM(db, c.Name, cfg)

View file

@ -1,6 +1,7 @@
package dispatcher
import (
"errors"
"fmt"
"net"
"strconv"
@ -129,31 +130,8 @@ func (c DeleteVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error
}
func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
timeout := time.After(time.Duration(cfg.Dispatcher.TimeoutSeconds) * time.Second)
for {
entries, err := kv.ListByPrefix(db, "subnet/")
if err != nil {
return fmt.Errorf("failed to list subnets: %w", err)
}
pending := false
for key, value := range entries {
if strings.HasSuffix(key, "/vpc") && value == c.Name {
subnetName := strings.Split(key, "/")[1]
s, _ := state.Get(db, "subnet/"+subnetName)
if s == state.Deleting {
pending = true
break
}
}
}
if !pending {
break
}
select {
case <-timeout:
return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", c.Name)
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
}
if err := waitSubnetsDeleted(db, c.Name, cfg.Times.VPCStop(), cfg.Times.Poll()); err != nil {
return err
}
if err := vpc.DeleteVPC(db, c.Name); err != nil {
return err
@ -167,3 +145,35 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro
}
return nil
}
func waitSubnetsDeleted(db *badger.DB, vpcName string, timeout, poll time.Duration) error {
err := waitUntil(timeout, poll, func() (bool, error) {
entries, err := kv.ListByPrefix(db, "subnet/")
if err != nil {
return false, fmt.Errorf("failed to list subnets: %w", err)
}
pending := false
for key, value := range entries {
if !strings.HasSuffix(key, "/vpc") || value != vpcName {
continue
}
subnetName := strings.Split(key, "/")[1]
s, err := state.Get(db, "subnet/"+subnetName)
if err != nil {
continue
}
switch s {
case state.Deleted:
case state.Deleting:
pending = true
default:
return false, fmt.Errorf("subnet %q is %s, vpc %q cannot be deleted", subnetName, s, vpcName)
}
}
return !pending, nil
})
if errors.Is(err, errWaitTimeout) {
return fmt.Errorf("timed out waiting for subnets of vpc %q to be deleted", vpcName)
}
return err
}

View file

@ -0,0 +1,26 @@
package dispatcher
import (
"errors"
"time"
)
var errWaitTimeout = errors.New("timed out")
func waitUntil(timeout, poll time.Duration, done func() (bool, error)) error {
deadline := time.After(timeout)
for {
ok, err := done()
if err != nil {
return err
}
if ok {
return nil
}
select {
case <-deadline:
return errWaitTimeout
case <-time.After(poll):
}
}
}

View file

@ -0,0 +1,168 @@
package dispatcher
import (
"errors"
"strings"
"testing"
"time"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/pkg/db/kv"
)
func TestWaitUntil(t *testing.T) {
calls := 0
err := waitUntil(time.Second, time.Millisecond, func() (bool, error) {
calls++
return calls == 3, nil
})
if err != nil || calls != 3 {
t.Errorf("err %v, %d appels, attendu fin au troisième", err, calls)
}
if err := waitUntil(30*time.Millisecond, 5*time.Millisecond, func() (bool, error) { return false, nil }); !errors.Is(err, errWaitTimeout) {
t.Errorf("délai dépassé : %v", err)
}
boom := errors.New("boom")
calls = 0
err = waitUntil(time.Second, time.Millisecond, func() (bool, error) {
calls++
return false, boom
})
if !errors.Is(err, boom) || calls != 1 {
t.Errorf("erreur %v après %d appels, attendu boom au premier", err, calls)
}
}
func TestDeleteSubnetCommand_Prepare_AcceptsVMsBeingDeleted(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "running")
kv.AddInDB(db, "vm/i-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-gone/state", "deleted")
kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a")
if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil {
t.Fatalf("VM en deleting : %v", err)
}
if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "deleting" {
t.Errorf("état %q, attendu deleting", s)
}
}
func TestDeleteSubnetCommand_Prepare_ListsOnlyTheVMsNotBeingDeleted(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "running")
kv.AddInDB(db, "vm/i-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-b/state", "running")
kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-a")
err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil)
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-b`) || strings.Contains(err.Error(), "i-a") {
t.Fatalf("erreur %v", err)
}
}
func TestWaitVMsDeleted_ReturnsOnceTheVMsAreGone(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vm/i-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-other/state", "running")
kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b")
go func() {
time.Sleep(30 * time.Millisecond)
kv.DeleteInDB(db, "vm/i-a")
}()
if err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond); err != nil {
t.Fatalf("attente : %v", err)
}
}
func TestWaitVMsDeleted_FailsWhenAVMEndsInError(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vm/i-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
go func() {
time.Sleep(20 * time.Millisecond)
kv.AddInDB(db, "vm/i-a/state", "error")
}()
err := waitVMsDeleted(db, "sn-a", 2*time.Second, 5*time.Millisecond)
if err == nil || !strings.Contains(err.Error(), `vm "i-a" is error, subnet "sn-a" cannot be deleted`) {
t.Fatalf("erreur %v", err)
}
}
func TestWaitVMsDeleted_TimesOut(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vm/i-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
err := waitVMsDeleted(db, "sn-a", 30*time.Millisecond, 5*time.Millisecond)
if err == nil || err.Error() != `timed out waiting for the vms of subnet "sn-a" to be deleted` {
t.Fatalf("erreur %v", err)
}
}
func TestWaitSubnetsDeleted_ReturnsOnceTheSubnetsAreGone(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
kv.AddInDB(db, "subnet/sn-b/state", "deleted")
kv.AddInDB(db, "subnet/sn-b/vpc", "vp-a")
kv.AddInDB(db, "subnet/sn-x/state", "running")
kv.AddInDB(db, "subnet/sn-x/vpc", "vp-x")
go func() {
time.Sleep(30 * time.Millisecond)
kv.DeleteInDB(db, "subnet/sn-a")
}()
if err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond); err != nil {
t.Fatalf("attente : %v", err)
}
}
func TestWaitSubnetsDeleted_FailsWhenASubnetEndsInError(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
go func() {
time.Sleep(20 * time.Millisecond)
kv.AddInDB(db, "subnet/sn-a/state", "error")
}()
err := waitSubnetsDeleted(db, "vp-a", 2*time.Second, 5*time.Millisecond)
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is error, vpc "vp-a" cannot be deleted`) {
t.Fatalf("erreur %v", err)
}
}
func TestWaitSubnetsDeleted_TimesOut(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
err := waitSubnetsDeleted(db, "vp-a", 30*time.Millisecond, 5*time.Millisecond)
if err == nil || err.Error() != `timed out waiting for subnets of vpc "vp-a" to be deleted` {
t.Fatalf("erreur %v", err)
}
}
func TestDeleteSubnetCommand_Execute_FailsBeforeTouchingTheHostWhenAVMIsInError(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "deleting")
kv.AddInDB(db, "vm/i-a/state", "error")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
cfg := &configuration.Config{Times: configuration.Times{TimeoutSubnetStop: 1, PollInterval: 1}}
err := DeleteSubnetCommand{Name: "sn-a"}.Execute(db, cfg)
if err == nil || err.Error() != `vm "i-a" is error, subnet "sn-a" cannot be deleted` {
t.Fatalf("erreur %v", err)
}
}
func TestDeleteVPCCommand_Execute_FailsBeforeTouchingTheHostWhenASubnetIsInError(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vpc/vp-a/state", "deleting")
kv.AddInDB(db, "subnet/sn-a/state", "error")
kv.AddInDB(db, "subnet/sn-a/vpc", "vp-a")
cfg := &configuration.Config{Times: configuration.Times{TimeoutVPCStop: 1, PollInterval: 1}}
err := DeleteVPCCommand{Name: "vp-a"}.Execute(db, cfg)
if err == nil || err.Error() != `subnet "sn-a" is error, vpc "vp-a" cannot be deleted` {
t.Fatalf("erreur %v", err)
}
}