90 lines
2 KiB
Go
90 lines
2 KiB
Go
package watchdog
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"time"
|
|
|
|
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
|
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
|
"git.g3e.fr/syonad/two/pkg/systemd"
|
|
|
|
"github.com/dgraph-io/badger/v4"
|
|
)
|
|
|
|
const defaultInterval = 60 * time.Second
|
|
|
|
type Watchdog struct {
|
|
db *badger.DB
|
|
cfg *configuration.Config
|
|
notifier notify.Notifier
|
|
logger *slog.Logger
|
|
interval time.Duration
|
|
dbusDown bool
|
|
}
|
|
|
|
func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog {
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
if interval <= 0 {
|
|
logger.Warn("watchdog: invalid interval, default applied",
|
|
"interval", interval, "default", defaultInterval)
|
|
interval = defaultInterval
|
|
}
|
|
return &Watchdog{
|
|
db: db,
|
|
cfg: cfg,
|
|
notifier: n,
|
|
logger: logger,
|
|
interval: interval,
|
|
}
|
|
}
|
|
|
|
func (w *Watchdog) Run(ctx context.Context) {
|
|
ticker := time.NewTicker(w.interval)
|
|
defer ticker.Stop()
|
|
|
|
w.logger.Info("watchdog: starting", "interval", w.interval)
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
w.logger.Info("watchdog: stopping")
|
|
return
|
|
case <-ticker.C:
|
|
w.tick()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Watchdog) tick() {
|
|
u, closeUnits := w.units()
|
|
defer closeUnits()
|
|
|
|
if err := CheckVPCs(w.db, w.notifier); err != nil {
|
|
w.logger.Error("watchdog: vpc check failed", "err", err)
|
|
}
|
|
if err := CheckSubnets(w.db, w.cfg, u, w.notifier); err != nil {
|
|
w.logger.Error("watchdog: subnet check failed", "err", err)
|
|
}
|
|
if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil {
|
|
w.logger.Error("watchdog: vm check failed", "err", err)
|
|
}
|
|
}
|
|
|
|
func (w *Watchdog) units() (unitChecker, func()) {
|
|
m, err := systemd.New()
|
|
if err != nil {
|
|
if !w.dbusDown {
|
|
w.logger.Warn("watchdog: systemd unreachable, unit checks disabled", "err", err)
|
|
w.dbusDown = true
|
|
}
|
|
return nil, func() {}
|
|
}
|
|
if w.dbusDown {
|
|
w.logger.Info("watchdog: systemd connection restored")
|
|
w.dbusDown = false
|
|
}
|
|
return m, m.Close
|
|
}
|