f-86: agent: dummy, DNAT et open par VPC, DHCP 6 et 42, watchdog #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
d4b83bdb17
commit
2e24a5da0a
16 changed files with 565 additions and 13 deletions
110
internal/watchdog/check_hostsvc.go
Normal file
110
internal/watchdog/check_hostsvc.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
|
||||
hostsvcclient "git.g3e.fr/syonad/two/internal/client/hostsvc"
|
||||
"git.g3e.fr/syonad/two/internal/hostsvc"
|
||||
"git.g3e.fr/syonad/two/internal/iptables"
|
||||
"git.g3e.fr/syonad/two/internal/netif"
|
||||
"git.g3e.fr/syonad/two/internal/netns"
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
"git.g3e.fr/syonad/two/internal/vpc"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
"git.g3e.fr/syonad/two/pkg/db/kv"
|
||||
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
const kindHostSvc = "hostsvc"
|
||||
|
||||
type hostSvcProbe interface {
|
||||
Served() ([]string, error)
|
||||
ServiceAddress(vpc string) (bool, error)
|
||||
Redirect(vpc string) (bool, error)
|
||||
}
|
||||
|
||||
type systemHostSvcProbe struct {
|
||||
client *hostsvcclient.Client
|
||||
}
|
||||
|
||||
func newSystemHostSvcProbe() systemHostSvcProbe {
|
||||
return systemHostSvcProbe{client: hostsvcclient.New(hostsvcapi.SocketPath(hostsvcapi.DefaultRunDir))}
|
||||
}
|
||||
|
||||
func (p systemHostSvcProbe) Served() ([]string, error) { return p.client.List() }
|
||||
|
||||
func (systemHostSvcProbe) ServiceAddress(name string) (bool, error) {
|
||||
want := net.ParseIP(hostsvc.ServiceIP)
|
||||
found := false
|
||||
err := netns.Call(name, func() error {
|
||||
ips, err := netif.LinkIPv4(vpc.HostSvcLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip.Equal(want) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
|
||||
func (systemHostSvcProbe) Redirect(name string) (bool, error) {
|
||||
var ok bool
|
||||
err := netns.Call(name, func() error {
|
||||
var err error
|
||||
ok, err = iptables.HasHostSvcRedirect(hostsvc.ServiceIP)
|
||||
return err
|
||||
})
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func CheckHostSvc(db *badger.DB, p hostSvcProbe, n notify.Notifier) error {
|
||||
pairs, err := kv.ListByPrefix(db, prefixVPC)
|
||||
if err != nil {
|
||||
return fmt.Errorf("watchdog: listing vpcs: %w", err)
|
||||
}
|
||||
var running []string
|
||||
for _, name := range resourceNames(pairs, prefixVPC) {
|
||||
if st, err := state.Get(db, prefixVPC+name); err == nil && st == state.Running {
|
||||
running = append(running, name)
|
||||
}
|
||||
}
|
||||
if len(running) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
served, err := p.Served()
|
||||
if err != nil {
|
||||
n.Notify(kindHostSvc, hostsvcapi.Unit, fmt.Sprintf("unreachable, no vpc of this host gets dns or ntp: %v", err))
|
||||
} else {
|
||||
open := map[string]bool{}
|
||||
for _, name := range served {
|
||||
open[name] = true
|
||||
}
|
||||
for _, name := range running {
|
||||
if !open[name] {
|
||||
n.Notify(kindVPC, name, "not served by two-hostsvc: its vms get no dns nor ntp")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range running {
|
||||
if ok, err := p.ServiceAddress(name); err != nil {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("%s on %s unreadable: %v", hostsvc.ServiceIP, vpc.HostSvcLink, err))
|
||||
} else if !ok {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("%s missing on %s (in netns)", hostsvc.ServiceIP, vpc.HostSvcLink))
|
||||
}
|
||||
if ok, err := p.Redirect(name); err != nil {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection unreadable: %v", err))
|
||||
} else if !ok {
|
||||
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection to %s missing or incomplete (chain %s)", hostsvc.ServiceIP, iptables.HostSvcChain))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
129
internal/watchdog/check_hostsvc_test.go
Normal file
129
internal/watchdog/check_hostsvc_test.go
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.g3e.fr/syonad/two/internal/state"
|
||||
)
|
||||
|
||||
type fakeHostSvcProbe struct {
|
||||
served []string
|
||||
servedErr error
|
||||
address map[string]bool
|
||||
redirect map[string]bool
|
||||
readErr error
|
||||
asked int
|
||||
}
|
||||
|
||||
func (f *fakeHostSvcProbe) Served() ([]string, error) {
|
||||
f.asked++
|
||||
return f.served, f.servedErr
|
||||
}
|
||||
|
||||
func (f *fakeHostSvcProbe) ServiceAddress(vpc string) (bool, error) {
|
||||
return f.address[vpc], f.readErr
|
||||
}
|
||||
|
||||
func (f *fakeHostSvcProbe) Redirect(vpc string) (bool, error) {
|
||||
return f.redirect[vpc], f.readErr
|
||||
}
|
||||
|
||||
func healthyHostSvc(vpcs ...string) *fakeHostSvcProbe {
|
||||
f := &fakeHostSvcProbe{served: vpcs, address: map[string]bool{}, redirect: map[string]bool{}}
|
||||
for _, v := range vpcs {
|
||||
f.address[v] = true
|
||||
f.redirect[v] = true
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func TestCheckHostSvc_SilentWhenEverythingIsInPlace(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-a", state.Running)
|
||||
r := &recorder{}
|
||||
if err := CheckHostSvc(db, healthyHostSvc("vp-a"), r); err != nil {
|
||||
t.Fatalf("CheckHostSvc: %v", err)
|
||||
}
|
||||
if len(r.calls) != 0 {
|
||||
t.Errorf("notifications = %+v, want none", r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSvc_NoRunningVPCDoesNotAskTheRelay(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-a", state.Creating)
|
||||
p := &fakeHostSvcProbe{servedErr: errors.New("down")}
|
||||
r := &recorder{}
|
||||
if err := CheckHostSvc(db, p, r); err != nil {
|
||||
t.Fatalf("CheckHostSvc: %v", err)
|
||||
}
|
||||
if p.asked != 0 || len(r.calls) != 0 {
|
||||
t.Errorf("asked %d times, notifications %+v: a host without running vpc needs no relay", p.asked, r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSvc_ReportsAnUnreachableRelay(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-a", state.Running)
|
||||
p := healthyHostSvc("vp-a")
|
||||
p.servedErr = errors.New("dial /run/two/hostsvc/hostsvc.sock: connection refused")
|
||||
r := &recorder{}
|
||||
if err := CheckHostSvc(db, p, r); err != nil {
|
||||
t.Fatalf("CheckHostSvc: %v", err)
|
||||
}
|
||||
got := r.forName("two-hostsvc.service")
|
||||
if len(got) != 1 || got[0].kind != "hostsvc" {
|
||||
t.Fatalf("notifications = %+v, want one for two-hostsvc.service", r.calls)
|
||||
}
|
||||
if !r.hasProblemContaining("unreachable, no vpc of this host gets dns or ntp: dial /run/two/hostsvc/hostsvc.sock: connection refused") {
|
||||
t.Errorf("problem = %q", got[0].problem)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSvc_ReportsEachMissingPiece(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
for _, v := range []string{"vp-ok", "vp-unserved", "vp-noaddr", "vp-noredirect"} {
|
||||
seedResource(t, db, prefixVPC, v, state.Running)
|
||||
}
|
||||
p := healthyHostSvc("vp-ok", "vp-unserved", "vp-noaddr", "vp-noredirect")
|
||||
p.served = []string{"vp-ok", "vp-noaddr", "vp-noredirect"}
|
||||
p.address["vp-noaddr"] = false
|
||||
p.redirect["vp-noredirect"] = false
|
||||
r := &recorder{}
|
||||
if err := CheckHostSvc(db, p, r); err != nil {
|
||||
t.Fatalf("CheckHostSvc: %v", err)
|
||||
}
|
||||
for name, want := range map[string]string{
|
||||
"vp-unserved": "not served by two-hostsvc: its vms get no dns nor ntp",
|
||||
"vp-noaddr": "169.254.169.253 missing on hostsvc (in netns)",
|
||||
"vp-noredirect": "dns and ntp redirection to 169.254.169.253 missing or incomplete (chain TWO-HOSTSVC)",
|
||||
} {
|
||||
got := r.forName(name)
|
||||
if len(got) != 1 || got[0].kind != "vpc" || got[0].problem != want {
|
||||
t.Errorf("%s: notifications %+v, want one vpc problem %q", name, got, want)
|
||||
}
|
||||
}
|
||||
if got := r.forName("vp-ok"); len(got) != 0 {
|
||||
t.Errorf("vp-ok: notifications %+v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSvc_ReportsUnreadableChecks(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
seedResource(t, db, prefixVPC, "vp-a", state.Running)
|
||||
p := healthyHostSvc("vp-a")
|
||||
p.readErr = errors.New("cannot enter netns")
|
||||
r := &recorder{}
|
||||
if err := CheckHostSvc(db, p, r); err != nil {
|
||||
t.Fatalf("CheckHostSvc: %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"169.254.169.253 on hostsvc unreadable: cannot enter netns",
|
||||
"dns and ntp redirection unreadable: cannot enter netns",
|
||||
} {
|
||||
if !r.hasProblemContaining(want) {
|
||||
t.Errorf("missing %q in %+v", want, r.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ type Watchdog struct {
|
|||
interval time.Duration
|
||||
dbusDown bool
|
||||
l3 l3Probe
|
||||
hostsvc hostSvcProbe
|
||||
public publicProbe
|
||||
}
|
||||
|
||||
|
|
@ -42,6 +43,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl
|
|||
logger: logger,
|
||||
interval: interval,
|
||||
l3: systemProbe{frr: frr.NewClient()},
|
||||
hostsvc: newSystemHostSvcProbe(),
|
||||
public: systemPublicProbe{},
|
||||
}
|
||||
}
|
||||
|
|
@ -70,6 +72,9 @@ func (w *Watchdog) tick() {
|
|||
if err := CheckVPCs(w.db, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: vpc check failed", "err", err)
|
||||
}
|
||||
if err := CheckHostSvc(w.db, w.hostsvc, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: hostsvc check failed", "err", err)
|
||||
}
|
||||
if err := CheckSubnets(w.db, w.cfg, u, w.notifier); err != nil {
|
||||
w.logger.Error("watchdog: subnet check failed", "err", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue