f-29: watchdog: add checksubnet #29

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>

internal/dhcp/struct.go

@@ -4,6 +4,8 @@ import (
 	"net"
 )
 
+const DefaultConfDir = "/etc/dnsmasq.d"
+
 type Config struct {
 	Network        *net.IPNet
 	VPCGateway     net.IP     // next-hop for VPCRoute (option 121)

internal/subnet/create.go

@@ -138,7 +138,7 @@ func startDHCP(db *badger.DB, subnetName string, d subnetData) error {
 	conf := dhcp.Config{
 		Network: d.cidr,
 		Name:    d.vpc + "_" + d.bridge,
-		ConfDir: "/etc/dnsmasq.d",
+		ConfDir: dhcp.DefaultConfDir,
 	}
 	switch d.mode {
 	case "vxlan":

internal/subnet/delete.go

@@ -3,7 +3,9 @@ package subnet
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
+	"git.g3e.fr/syonad/two/internal/dhcp"
 	"git.g3e.fr/syonad/two/internal/ebtables"
 	"git.g3e.fr/syonad/two/internal/netif"
 	"git.g3e.fr/syonad/two/internal/netns"
@@ -62,7 +64,7 @@ func stopDHCP(db *badger.DB, subnetName string, d subnetData) error {
 		}
 	}
 
-	if err := os.Remove("/etc/dnsmasq.d/" + d.vpc + "_" + d.bridge + ".conf"); err != nil && !os.IsNotExist(err) {
+	if err := os.Remove(filepath.Join(dhcp.DefaultConfDir, d.vpc+"_"+d.bridge+".conf")); err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("remove dnsmasq config: %w", err)
 	}
 

internal/watchdog/check_subnet.go

@@ -0,0 +1,134 @@
+package watchdog
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"git.g3e.fr/syonad/two/internal/dhcp"
+	"git.g3e.fr/syonad/two/internal/netns"
+	"git.g3e.fr/syonad/two/internal/state"
+	"git.g3e.fr/syonad/two/internal/watchdog/notify"
+	"git.g3e.fr/syonad/two/pkg/db/kv"
+
+	"github.com/dgraph-io/badger/v4"
+)
+
+const (
+	modeVxlan  = "vxlan"
+	modeBridge = "bridge"
+)
+
+func subnetIfaceNames(subnetName string) (hostVeth, nsVeth, bridge string, err error) {
+	parts := strings.SplitN(subnetName, "-", 2)
+	if len(parts) < 2 || parts[1] == "" {
+		return "", "", "", fmt.Errorf("nom de subnet %q sans identifiant après le tiret, interfaces indéductibles", subnetName)
+	}
+	id := parts[1]
+	return "v-" + id + "-e", "v-" + id + "-i", "br-" + id, nil
+}
+
+func dnsmasqName(vpc, bridge string) string {
+	return vpc + "_" + bridge
+}
+
+func CheckSubnets(db *badger.DB, u unitChecker, n notify.Notifier) error {
+	pairs, err := kv.ListByPrefix(db, prefixSubnet)
+	if err != nil {
+		return fmt.Errorf("watchdog: lecture des subnets: %w", err)
+	}
+
+	for _, name := range resourceNames(pairs, prefixSubnet) {
+		st, err := state.Get(db, prefixSubnet+name)
+		if err != nil {
+			n.Notify(kindSubnet, name, fmt.Sprintf("état illisible en base: %v", err))
+			continue
+		}
+		if st != state.Running {
+			continue
+		}
+		checkSubnet(db, name, u, n)
+	}
+	return nil
+}
+
+func checkSubnet(db *badger.DB, name string, u unitChecker, n notify.Notifier) {
+	hostVeth, nsVeth, bridge, err := subnetIfaceNames(name)
+	if err != nil {
+		n.Notify(kindSubnet, name, err.Error())
+		return
+	}
+
+	vpc, err := kv.GetFromDB(db, prefixSubnet+name+"/vpc")
+	if err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("vpc illisible en base: %v", err))
+		return
+	}
+
+	mode, err := kv.GetFromDB(db, prefixSubnet+name+"/mode")
+	if err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("mode illisible en base: %v", err))
+		return
+	}
+
+	if p := linkProblem(hostVeth); p != "" {
+		n.Notify(kindSubnet, name, p)
+	}
+
+	switch mode {
+	case modeVxlan:
+		if p := linkProblem(bridge); p != "" {
+			n.Notify(kindSubnet, name, p+" (host)")
+		}
+		checkVxlanIface(db, name, n)
+	case modeBridge:
+	default:
+		n.Notify(kindSubnet, name, fmt.Sprintf("mode inconnu %q", mode))
+	}
+
+	checkSubnetNetns(name, vpc, nsVeth, bridge, n)
+
+	dnsName := dnsmasqName(vpc, bridge)
+	conf := filepath.Join(dhcp.DefaultConfDir, dnsName+".conf")
+	if _, err := os.Stat(conf); err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("config dnsmasq absente (%s): %v", conf, err))
+	}
+
+	checkUnit(kindSubnet, name, "dnsmasq@"+dnsName+".service", u, n)
+}
+
+func checkVxlanIface(db *badger.DB, name string, n notify.Notifier) {
+	raw, err := kv.GetFromDB(db, prefixSubnet+name+"/vxlan_id")
+	if err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("vxlan_id illisible en base: %v", err))
+		return
+	}
+	id, err := strconv.Atoi(raw)
+	if err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("vxlan_id invalide %q: %v", raw, err))
+		return
+	}
+	if p := linkProblem(fmt.Sprintf("vxlan-%d", id)); p != "" {
+		n.Notify(kindSubnet, name, p)
+	}
+}
+
+func checkSubnetNetns(name, vpc, nsVeth, bridge string, n notify.Notifier) {
+	if !netns.Exist(vpc) {
+		n.Notify(kindSubnet, name, "netns "+vpc+" absent (/var/run/netns/"+vpc+")")
+		return
+	}
+
+	if err := netns.Call(vpc, func() error {
+		for _, iface := range []string{nsVeth, bridge} {
+			if p := linkProblem(iface); p != "" {
+				n.Notify(kindSubnet, name, p+" (dans le netns "+vpc+")")
+			}
+		}
+		return nil
+	}); err != nil {
+		n.Notify(kindSubnet, name, fmt.Sprintf("entrée dans le netns %s impossible: %v", vpc, err))
+	}
+}

internal/watchdog/check_subnet_test.go

@@ -0,0 +1,288 @@
+package watchdog
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"git.g3e.fr/syonad/two/internal/state"
+
+	"github.com/dgraph-io/badger/v4"
+)
+
+func seedSubnet(t *testing.T, db *badger.DB, name, vpc, mode string) {
+	t.Helper()
+	seedResource(t, db, prefixSubnet, name, state.Running)
+	seedKV(t, db, prefixSubnet+name+"/vpc", vpc)
+	seedKV(t, db, prefixSubnet+name+"/mode", mode)
+}
+
+func TestSubnetIfaceNames_NomStandard(t *testing.T) {
+	hostVeth, nsVeth, bridge, err := subnetIfaceNames("br-000042")
+	if err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+	if hostVeth != "v-000042-e" || nsVeth != "v-000042-i" || bridge != "br-000042" {
+		t.Errorf("obtenu (%q, %q, %q), attendu (v-000042-e, v-000042-i, br-000042)", hostVeth, nsVeth, bridge)
+	}
+}
+
+func TestSubnetIfaceNames_NomSansTiret(t *testing.T) {
+	if _, _, _, err := subnetIfaceNames("subnet"); err == nil {
+		t.Fatal("un nom sans tiret devrait produire une erreur, pas une panique")
+	}
+}
+
+func TestSubnetIfaceNames_TiretFinal(t *testing.T) {
+	if _, _, _, err := subnetIfaceNames("br-"); err == nil {
+		t.Fatal("un identifiant vide devrait produire une erreur")
+	}
+}
+
+func TestDnsmasqName(t *testing.T) {
+	if got := dnsmasqName("vp-admin", "br-000000"); got != "vp-admin_br-000000" {
+		t.Errorf("dnsmasqName = %q, attendu vp-admin_br-000000", got)
+	}
+}
+
+func TestCheckSubnets_BaseVide(t *testing.T) {
+	db := newTestDB(t)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+	if len(r.calls) != 0 {
+		t.Errorf("aucune notification attendue, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_IgnoreLesEtatsNonRunning(t *testing.T) {
+	db := newTestDB(t)
+	for _, s := range []state.State{state.Creating, state.Deleting, state.Error, state.Deleted} {
+		seedResource(t, db, prefixSubnet, "br-"+string(s), s)
+	}
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+	if len(r.calls) != 0 {
+		t.Errorf("aucune notification attendue, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_VPCManquantEnBase(t *testing.T) {
+	db := newTestDB(t)
+	seedResource(t, db, prefixSubnet, "br-000042", state.Running)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	got := r.forName("br-000042")
+	if len(got) != 1 {
+		t.Fatalf("attendu 1 notification, obtenu %d : %v", len(got), got)
+	}
+	if !strings.Contains(got[0].problem, "vpc illisible") {
+		t.Errorf("problem = %q, devrait porter sur le vpc", got[0].problem)
+	}
+}
+
+func TestCheckSubnets_ModeManquantEnBase(t *testing.T) {
+	db := newTestDB(t)
+	seedResource(t, db, prefixSubnet, "br-000042", state.Running)
+	seedKV(t, db, prefixSubnet+"br-000042/vpc", "vp-admin")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("mode illisible") {
+		t.Errorf("devrait signaler un mode illisible, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_ModeInconnu(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", "macvlan")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining(`mode inconnu "macvlan"`) {
+		t.Errorf("devrait signaler un mode inconnu, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_ModeBridgeNeVerifiePasDeVxlan(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	for _, c := range r.calls {
+		if strings.Contains(c.problem, "vxlan") {
+			t.Errorf("un subnet en mode bridge ne doit rien signaler sur vxlan: %q", c.problem)
+		}
+		if strings.Contains(c.problem, "(host)") {
+			t.Errorf("un subnet en mode bridge n'a pas de bridge host: %q", c.problem)
+		}
+	}
+}
+
+func TestCheckSubnets_ModeVxlanSansVxlanID(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeVxlan)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("vxlan_id illisible") {
+		t.Errorf("devrait signaler un vxlan_id illisible, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_ModeVxlanVxlanIDInvalide(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeVxlan)
+	seedKV(t, db, prefixSubnet+"br-000042/vxlan_id", "pas-un-nombre")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("vxlan_id invalide") {
+		t.Errorf("devrait signaler un vxlan_id invalide, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_ModeVxlanVerifieLInterfaceVxlan(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeVxlan)
+	seedKV(t, db, prefixSubnet+"br-000042/vxlan_id", "42")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("vxlan-42") {
+		t.Errorf("devrait signaler l'interface vxlan-42, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_ConfigDnsmasqAbsente(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("config dnsmasq absente") {
+		t.Errorf("devrait signaler la config dnsmasq absente, obtenu %v", r.calls)
+	}
+	if !r.hasProblemContaining("vp-admin_br-000042.conf") {
+		t.Errorf("devrait nommer le fichier attendu, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_UnitDnsmasqInterrogee(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	u := newFakeUnits().active("dnsmasq@vp-admin_br-000042.service")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, u, r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if len(u.asked) != 1 || u.asked[0] != "dnsmasq@vp-admin_br-000042.service" {
+		t.Errorf("units interrogées = %v, attendu [dnsmasq@vp-admin_br-000042.service]", u.asked)
+	}
+	if r.hasProblemContaining("unit ") {
+		t.Errorf("une unit active ne doit rien signaler, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_UnitDnsmasqInactive(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	u := newFakeUnits().inactive("dnsmasq@vp-admin_br-000042.service", "failed")
+	r := &recorder{}
+
+	if err := CheckSubnets(db, u, r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("unit dnsmasq@vp-admin_br-000042.service inactive (failed)") {
+		t.Errorf("devrait signaler l'unit inactive, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_UnitIllisible(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	u := newFakeUnits().failing("dnsmasq@vp-admin_br-000042.service", errors.New("dbus indisponible"))
+	r := &recorder{}
+
+	if err := CheckSubnets(db, u, r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if !r.hasProblemContaining("unit dnsmasq@vp-admin_br-000042.service illisible") {
+		t.Errorf("devrait signaler l'unit illisible, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_SansUnitCheckerPasDeVerificationDUnit(t *testing.T) {
+	db := newTestDB(t)
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, nil, r); err != nil {
+		t.Fatalf("erreur inattendue: %v", err)
+	}
+
+	if r.hasProblemContaining("unit ") {
+		t.Errorf("sans unitChecker, aucune notification d'unit attendue, obtenu %v", r.calls)
+	}
+}
+
+func TestCheckSubnets_EtatCorrompuNInterrompPasLaBoucle(t *testing.T) {
+	db := newTestDB(t)
+	seedKV(t, db, prefixSubnet+"br-corrompu"+stateSuffix, "n_importe_quoi")
+	seedSubnet(t, db, "br-000042", "vp-admin", modeBridge)
+	r := &recorder{}
+
+	if err := CheckSubnets(db, newFakeUnits(), r); err != nil {
+		t.Fatalf("un état corrompu ne doit pas faire échouer CheckSubnets: %v", err)
+	}
+
+	if !strings.Contains(strings.Join(problems(r.forName("br-corrompu")), " "), "état illisible") {
+		t.Errorf("devrait signaler l'état corrompu, obtenu %v", r.calls)
+	}
+	if len(r.forName("br-000042")) == 0 {
+		t.Error("une clé corrompue ne doit pas empêcher l'examen des subnets suivants")
+	}
+}
+
+func problems(ns []notification) []string {
+	out := make([]string, 0, len(ns))
+	for _, n := range ns {
+		out = append(out, n.problem)
+	}
+	return out
+}

internal/watchdog/helpers_test.go

@@ -7,10 +7,18 @@ import (
 	"git.g3e.fr/syonad/two/internal/state"
 	"git.g3e.fr/syonad/two/internal/watchdog/notify"
 	"git.g3e.fr/syonad/two/pkg/db/kv"
+	"git.g3e.fr/syonad/two/pkg/systemd"
 
 	"github.com/dgraph-io/badger/v4"
 )
 
+func seedKV(t *testing.T, db *badger.DB, key, value string) {
+	t.Helper()
+	if err := kv.AddInDB(db, key, value); err != nil {
+		t.Fatalf("seedKV %s: %v", key, err)
+	}
+}
+
 type notification struct {
 	kind    string
 	name    string
@@ -46,6 +54,47 @@ func (r *recorder) hasProblemContaining(substr string) bool {
 	return false
 }
 
+type fakeUnits struct {
+	status map[string]*systemd.ServiceStatus
+	err    map[string]error
+	asked  []string
+}
+
+var _ unitChecker = (*fakeUnits)(nil)
+
+func newFakeUnits() *fakeUnits {
+	return &fakeUnits{
+		status: map[string]*systemd.ServiceStatus{},
+		err:    map[string]error{},
+	}
+}
+
+func (f *fakeUnits) active(unit string) *fakeUnits {
+	f.status[unit] = &systemd.ServiceStatus{Name: unit, LoadState: "loaded", ActiveState: "active", SubState: "running"}
+	return f
+}
+
+func (f *fakeUnits) inactive(unit, sub string) *fakeUnits {
+	f.status[unit] = &systemd.ServiceStatus{Name: unit, LoadState: "loaded", ActiveState: "inactive", SubState: sub}
+	return f
+}
+
+func (f *fakeUnits) failing(unit string, err error) *fakeUnits {
+	f.err[unit] = err
+	return f
+}
+
+func (f *fakeUnits) Status(unit string) (*systemd.ServiceStatus, error) {
+	f.asked = append(f.asked, unit)
+	if err, ok := f.err[unit]; ok {
+		return nil, err
+	}
+	if st, ok := f.status[unit]; ok {
+		return st, nil
+	}
+	return &systemd.ServiceStatus{Name: unit, LoadState: "not-found", ActiveState: "inactive", SubState: "dead"}, nil
+}
+
 func newTestDB(t *testing.T) *badger.DB {
 	t.Helper()
 	db := kv.InitDB(kv.Config{Path: t.TempDir()}, false)

internal/watchdog/resources.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 
 	"git.g3e.fr/syonad/two/internal/netif"
+	"git.g3e.fr/syonad/two/internal/watchdog/notify"
+	"git.g3e.fr/syonad/two/pkg/systemd"
 )
 
 const (
@@ -40,6 +42,24 @@ func resourceNames(pairs map[string]string, prefix string) []string {
 	return names
 }
 
+type unitChecker interface {
+	Status(unit string) (*systemd.ServiceStatus, error)
+}
+
+func checkUnit(kind, name, unit string, u unitChecker, n notify.Notifier) {
+	if u == nil {
+		return
+	}
+	st, err := u.Status(unit)
+	if err != nil {
+		n.Notify(kind, name, fmt.Sprintf("unit %s illisible: %v", unit, err))
+		return
+	}
+	if st.ActiveState != "active" {
+		n.Notify(kind, name, fmt.Sprintf("unit %s %s (%s)", unit, st.ActiveState, st.SubState))
+	}
+}
+
 func linkProblem(iface string) string {
 	up, err := netif.LinkIsUp(iface)
 	switch {