f-86: hostsvc: relais DNS et NTP par hôte, socket de contrôle #86

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-25 20:32:06 +01:00
commit d4b83bdb17
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
20 changed files with 2235 additions and 0 deletions

78
cmd/two-hostsvc/main.go Normal file
View file

@ -0,0 +1,78 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"net"
"os"
"os/signal"
"syscall"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/pkg/logger"
)
var (
confFile = flag.String("conf", "/etc/two/agent.yml", "configuration file")
runDir = flag.String("run-dir", hostsvcapi.DefaultRunDir, "directory of the control socket and the state file")
)
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "two-hostsvc: %v\n", err)
os.Exit(1)
}
}
func run() error {
cfg, err := configuration.LoadConfig(*confFile)
if err != nil {
return fmt.Errorf("load config: %w", err)
}
log := logger.New(cfg.Logger.Level, cfg.Logger.Debug)
svc, err := hostsvc.New(hostsvc.Options{
DNS: cfg.HostSvc.DNS(),
NTP: cfg.HostSvc.NTP(),
Rate: cfg.HostSvc.Rate(),
Logger: log,
}, hostsvcapi.StatePath(*runDir))
if err != nil {
return err
}
if err := svc.Restore(); err != nil {
log.Error("some vpcs could not be restored", "error", err)
}
control, err := hostsvcapi.Listen(svc, hostsvcapi.SocketPath(*runDir), log)
if err != nil {
return fmt.Errorf("listen on the control socket: %w", err)
}
defer control.Close()
log.Info("two-hostsvc started", "socket", control.Addr(), "vpcs", svc.List())
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
go func() {
if err := control.Serve(); err != nil && !errors.Is(err, net.ErrClosed) {
log.Error("control socket stopped", "error", err)
stop()
}
}()
<-ctx.Done()
log.Info("two-hostsvc stopping")
if err := control.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
log.Error("close the control socket", "error", err)
}
return svc.Shutdown()
}

View file

@ -82,6 +82,20 @@ metadata:
dhcp:
backend: two
# DNS and NTP relay of the VMs (two-hostsvc, one process per hypervisor).
# It listens on 169.254.169.253 in every VPC netns of the host; the VMs reach it
# through their gateway (interface_ip, announced by DHCP options 6 and 42).
# dns_upstream : resolvers queried first (ip or ip:port, port 53 by default);
# the host resolver (/etc/resolv.conf) comes after them
# ntp_upstream : NTP servers relayed first (ip or ip:port, port 123 by
# default); without an answer, the host clock replies
# rate_limit_per_vpc : requests per second, DNS and NTP together, per VPC
# Its control socket and state file live in /run/two/hostsvc.
hostsvc:
dns_upstream: []
ntp_upstream: []
rate_limit_per_vpc: 100
# QEMU runtime paths
qemu:
# UEFI firmware (requires apt install ovmf on Debian/Ubuntu)

View file

@ -0,0 +1,31 @@
package hostsvcapi
import "path/filepath"
type Verb string
const (
VerbOpen Verb = "open"
VerbClose Verb = "close"
VerbList Verb = "list"
)
const (
MaxMessageBytes = 64 * 1024
DefaultRunDir = "/run/two/hostsvc"
Unit = "two-hostsvc.service"
)
func SocketPath(runDir string) string { return filepath.Join(runDir, "hostsvc.sock") }
func StatePath(runDir string) string { return filepath.Join(runDir, "hostsvc.state") }
type Request struct {
Verb Verb `json:"verb"`
VPC string `json:"vpc,omitempty"`
}
type Response struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
VPCs []string `json:"vpcs,omitempty"`
}

View file

@ -0,0 +1,128 @@
package hostsvcapi
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"os"
"path/filepath"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
const SocketMode = 0o600
type Service interface {
Open(vpc string) error
Close(vpc string) error
List() []string
}
type Server struct {
svc Service
listener net.Listener
logger *slog.Logger
}
func Listen(svc Service, path string, logger *slog.Logger) (*Server, error) {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, statefile.DirMode); err != nil {
return nil, fmt.Errorf("create %s: %w", dir, err)
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("remove stale socket %s: %w", path, err)
}
listener, err := net.Listen("unix", path)
if err != nil {
return nil, fmt.Errorf("listen on %s: %w", path, err)
}
if err := os.Chmod(path, SocketMode); err != nil {
listener.Close()
return nil, fmt.Errorf("chmod %s: %w", path, err)
}
return &Server{svc: svc, listener: listener, logger: logger}, nil
}
func (s *Server) Addr() string {
return s.listener.Addr().String()
}
func (s *Server) Close() error {
return s.listener.Close()
}
func (s *Server) Serve() error {
for {
conn, err := s.listener.Accept()
if err != nil {
return err
}
go s.handleConn(conn)
}
}
func (s *Server) handleConn(conn net.Conn) {
defer conn.Close()
defer func() {
if r := recover(); r != nil {
s.logger.Error("control connection panicked", "panic", r)
}
}()
scanner := bufio.NewScanner(conn)
scanner.Buffer(make([]byte, 0, 4096), MaxMessageBytes)
encoder := json.NewEncoder(conn)
for scanner.Scan() {
line := scanner.Bytes()
if len(line) == 0 {
continue
}
var req Request
if err := json.Unmarshal(line, &req); err != nil {
if err := encoder.Encode(failure(fmt.Errorf("malformed request: %w", err))); err != nil {
return
}
continue
}
if err := encoder.Encode(s.dispatch(req)); err != nil {
return
}
}
if err := scanner.Err(); err != nil {
s.logger.Error("control connection read failed", "error", err)
}
}
func failure(err error) Response {
return Response{OK: false, Error: err.Error()}
}
func (s *Server) dispatch(req Request) Response {
switch req.Verb {
case VerbOpen:
if err := s.svc.Open(req.VPC); err != nil {
return failure(err)
}
return Response{OK: true}
case VerbClose:
if err := s.svc.Close(req.VPC); err != nil {
return failure(err)
}
return Response{OK: true}
case VerbList:
return Response{OK: true, VPCs: s.svc.List()}
default:
return failure(fmt.Errorf("unknown verb %q", req.Verb))
}
}

View file

@ -0,0 +1,82 @@
package hostsvcclient
import (
"bufio"
"encoding/json"
"fmt"
"net"
"time"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
)
const DefaultTimeout = 5 * time.Second
type Client struct {
path string
timeout time.Duration
}
func New(path string) *Client {
return &Client{path: path, timeout: DefaultTimeout}
}
func (c *Client) WithTimeout(d time.Duration) *Client {
return &Client{path: c.path, timeout: d}
}
func (c *Client) call(req hostsvcapi.Request) (hostsvcapi.Response, error) {
conn, err := net.DialTimeout("unix", c.path, c.timeout)
if err != nil {
return hostsvcapi.Response{}, fmt.Errorf("dial %s: %w", c.path, err)
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(c.timeout)); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("set deadline on %s: %w", c.path, err)
}
raw, err := json.Marshal(req)
if err != nil {
return hostsvcapi.Response{}, fmt.Errorf("encode %s: %w", req.Verb, err)
}
if _, err := conn.Write(append(raw, '\n')); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("send %s: %w", req.Verb, err)
}
scanner := bufio.NewScanner(conn)
scanner.Buffer(make([]byte, 0, 4096), hostsvcapi.MaxMessageBytes)
if !scanner.Scan() {
if err := scanner.Err(); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("read reply to %s: %w", req.Verb, err)
}
return hostsvcapi.Response{}, fmt.Errorf("no reply to %s", req.Verb)
}
var resp hostsvcapi.Response
if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("parse reply to %s: %w", req.Verb, err)
}
if !resp.OK {
return resp, fmt.Errorf("%s refused: %s", req.Verb, resp.Error)
}
return resp, nil
}
func (c *Client) Open(vpc string) error {
_, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbOpen, VPC: vpc})
return err
}
func (c *Client) Close(vpc string) error {
_, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbClose, VPC: vpc})
return err
}
func (c *Client) List() ([]string, error) {
resp, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbList})
if err != nil {
return nil, err
}
return resp.VPCs, nil
}

View file

@ -0,0 +1,163 @@
package hostsvcclient
import (
"errors"
"io"
"log/slog"
"net"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
)
type fakeService struct {
mu sync.Mutex
open map[string]bool
failOn string
}
func (f *fakeService) Open(vpc string) error {
f.mu.Lock()
defer f.mu.Unlock()
if vpc == f.failOn {
return errors.New("netns not found")
}
f.open[vpc] = true
return nil
}
func (f *fakeService) Close(vpc string) error {
f.mu.Lock()
defer f.mu.Unlock()
if vpc == f.failOn {
return errors.New("close failed")
}
delete(f.open, vpc)
return nil
}
func (f *fakeService) List() []string {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for vpc := range f.open {
out = append(out, vpc)
}
return out
}
func shortTempDir(t *testing.T) string {
t.Helper()
dir, err := os.MkdirTemp("", "hostsvc")
if err != nil {
t.Fatalf("MkdirTemp: %v", err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
return dir
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
func serve(t *testing.T) (*Client, *fakeService, string) {
t.Helper()
svc := &fakeService{open: map[string]bool{}, failOn: "vpc-broken"}
socketPath := hostsvcapi.SocketPath(shortTempDir(t))
server, err := hostsvcapi.Listen(svc, socketPath, discardLogger())
if err != nil {
t.Fatalf("Listen: %v", err)
}
go server.Serve()
t.Cleanup(func() { server.Close() })
return New(socketPath), svc, socketPath
}
func TestListen_SocketIsOwnerOnly(t *testing.T) {
_, _, socketPath := serve(t)
info, err := os.Stat(socketPath)
if err != nil {
t.Fatalf("Stat: %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Errorf("mode = %o, want 600: whoever reaches it opens sockets in any vpc", got)
}
}
func TestListen_ReplacesAStaleSocketFile(t *testing.T) {
socketPath := filepath.Join(shortTempDir(t), "stale.sock")
if err := os.WriteFile(socketPath, []byte("leftover"), 0o600); err != nil {
t.Fatalf("WriteFile: %v", err)
}
server, err := hostsvcapi.Listen(&fakeService{open: map[string]bool{}}, socketPath, discardLogger())
if err != nil {
t.Fatalf("a socket left by an unclean stop must not block startup: %v", err)
}
server.Close()
}
func TestOpenCloseList(t *testing.T) {
client, svc, _ := serve(t)
if err := client.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
got, err := client.List()
if err != nil {
t.Fatalf("List: %v", err)
}
if !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("List = %v", got)
}
if err := client.Close("vpc-00001"); err != nil {
t.Fatalf("Close: %v", err)
}
if len(svc.List()) != 0 {
t.Errorf("the vpc is still open on the service: %v", svc.List())
}
}
func TestServiceErrorsAreReported(t *testing.T) {
client, _, _ := serve(t)
if err := client.Open("vpc-broken"); err == nil || !strings.Contains(err.Error(), "open refused: netns not found") {
t.Errorf("Open: error %v", err)
}
if err := client.Close("vpc-broken"); err == nil || !strings.Contains(err.Error(), "close refused: close failed") {
t.Errorf("Close: error %v", err)
}
}
func TestUnknownVerbAndMalformedRequest(t *testing.T) {
_, _, socketPath := serve(t)
conn, err := net.Dial("unix", socketPath)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
if _, err := conn.Write([]byte("{\"verb\":\"reboot\"}\nnot json\n")); err != nil {
t.Fatalf("Write: %v", err)
}
buf := make([]byte, 4096)
var all string
for !strings.Contains(all, "malformed request") {
n, err := conn.Read(buf)
if err != nil {
t.Fatalf("Read: %v (got %q)", err, all)
}
all += string(buf[:n])
}
if !strings.Contains(all, `unknown verb \"reboot\"`) {
t.Errorf("replies = %q, want the unknown verb refused", all)
}
}
func TestClient_NoServer(t *testing.T) {
client := New(filepath.Join(shortTempDir(t), "absent.sock"))
if err := client.Open("vpc-00001"); err == nil || !strings.Contains(err.Error(), "dial") {
t.Errorf("error %v, want a dial failure", err)
}
}

View file

@ -0,0 +1,79 @@
package configuration
import (
"fmt"
"net"
"strconv"
)
const (
RateLimitPerVPC = 100
DNSPort = 53
NTPPort = 123
)
type HostSvc struct {
DNSUpstream []string `mapstructure:"dns_upstream"`
NTPUpstream []string `mapstructure:"ntp_upstream"`
RateLimitPerVPC int `mapstructure:"rate_limit_per_vpc"`
}
func (h HostSvc) DNS() []string { return withPort(h.DNSUpstream, DNSPort) }
func (h HostSvc) NTP() []string { return withPort(h.NTPUpstream, NTPPort) }
func (h HostSvc) Rate() int {
if h.RateLimitPerVPC == 0 {
return RateLimitPerVPC
}
return h.RateLimitPerVPC
}
func withPort(servers []string, port int) []string {
out := make([]string, 0, len(servers))
for _, s := range servers {
if net.ParseIP(s) != nil {
out = append(out, net.JoinHostPort(s, strconv.Itoa(port)))
continue
}
out = append(out, s)
}
return out
}
func (h HostSvc) Validate() error {
if h.RateLimitPerVPC <= 0 {
return fmt.Errorf("hostsvc.rate_limit_per_vpc %d must be a positive number of requests per second", h.RateLimitPerVPC)
}
for _, list := range []struct {
key string
servers []string
}{
{"dns_upstream", h.DNSUpstream},
{"ntp_upstream", h.NTPUpstream},
} {
for _, s := range list.servers {
if err := validServer(s); err != nil {
return fmt.Errorf("hostsvc.%s: %w", list.key, err)
}
}
}
return nil
}
func validServer(s string) error {
if net.ParseIP(s) != nil {
return nil
}
host, port, err := net.SplitHostPort(s)
if err != nil {
return fmt.Errorf("%q is neither an ip nor ip:port", s)
}
if net.ParseIP(host) == nil {
return fmt.Errorf("%q: %q is not an ip address", s, host)
}
n, err := strconv.Atoi(port)
if err != nil || n < 1 || n > 65535 {
return fmt.Errorf("%q: invalid port %q", s, port)
}
return nil
}

View file

@ -0,0 +1,51 @@
package configuration
import (
"reflect"
"strings"
"testing"
)
func TestLoadConfig_HostSvcDefaults(t *testing.T) {
cfg, err := LoadConfig(writeYAML(t, ""))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if len(cfg.HostSvc.DNS()) != 0 || len(cfg.HostSvc.NTP()) != 0 {
t.Errorf("without upstreams the lists must stay empty, the host serves: %+v", cfg.HostSvc)
}
if cfg.HostSvc.Rate() != 100 {
t.Errorf("default rate per vpc = %d, want 100", cfg.HostSvc.Rate())
}
}
func TestLoadConfig_HostSvcDefaultPorts(t *testing.T) {
cfg, err := LoadConfig(writeYAML(t, "hostsvc:\n dns_upstream: [192.0.2.53, \"192.0.2.54:5353\"]\n ntp_upstream: [192.0.2.123]\n rate_limit_per_vpc: 20\n"))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if got, want := cfg.HostSvc.DNS(), []string{"192.0.2.53:53", "192.0.2.54:5353"}; !reflect.DeepEqual(got, want) {
t.Errorf("dns = %v, want %v", got, want)
}
if got, want := cfg.HostSvc.NTP(), []string{"192.0.2.123:123"}; !reflect.DeepEqual(got, want) {
t.Errorf("ntp = %v, want %v", got, want)
}
if cfg.HostSvc.Rate() != 20 {
t.Errorf("rate = %d, want 20", cfg.HostSvc.Rate())
}
}
func TestLoadConfig_HostSvcInvalidIsRefused(t *testing.T) {
for _, c := range []struct{ yaml, msg string }{
{"hostsvc:\n rate_limit_per_vpc: 0\n", "hostsvc.rate_limit_per_vpc 0 must be a positive number"},
{"hostsvc:\n rate_limit_per_vpc: -5\n", "hostsvc.rate_limit_per_vpc -5"},
{"hostsvc:\n dns_upstream: [dns.example]\n", `hostsvc.dns_upstream: "dns.example" is neither an ip nor ip:port`},
{"hostsvc:\n dns_upstream: [\"dns.example:53\"]\n", `"dns.example" is not an ip address`},
{"hostsvc:\n ntp_upstream: [\"192.0.2.1:0\"]\n", `hostsvc.ntp_upstream: "192.0.2.1:0": invalid port "0"`},
{"hostsvc:\n ntp_upstream: [\"192.0.2.1:70000\"]\n", `invalid port "70000"`},
} {
if _, err := LoadConfig(writeYAML(t, c.yaml)); err == nil || !strings.Contains(err.Error(), c.msg) {
t.Errorf("%q: error %v, want %q", c.yaml, err, c.msg)
}
}
}

View file

@ -54,6 +54,7 @@ type Config struct {
} `mapstructure:"watchdog"`
Network Network `mapstructure:"network"`
BGP BGP `mapstructure:"bgp"`
HostSvc HostSvc `mapstructure:"hostsvc"`
DefaultInterface string `mapstructure:"default_interface"`
Interfaces map[string]string `mapstructure:"interfaces"`
}
@ -79,6 +80,9 @@ func LoadConfig(path string) (*Config, error) {
v.SetDefault("times.poll_interval", PollInterval)
v.SetDefault("metadata.run_dir", "/run/two/metadata")
v.SetDefault("dhcp.backend", BackendTwo)
v.SetDefault("hostsvc.dns_upstream", []string{})
v.SetDefault("hostsvc.ntp_upstream", []string{})
v.SetDefault("hostsvc.rate_limit_per_vpc", RateLimitPerVPC)
v.SetDefault("qemu.ovmf_code_path", "/usr/share/OVMF/OVMF_CODE.fd")
v.SetDefault("qemu.ovmf_vars_template", "/usr/share/OVMF/OVMF_VARS.fd")
v.SetDefault("qemu.uefi_vars_dir", "/run/two/vms/uefi")
@ -111,6 +115,9 @@ func LoadConfig(path string) (*Config, error) {
if err := cfg.Times.Validate(); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
if err := cfg.HostSvc.Validate(); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
return &cfg, nil
}

View file

@ -0,0 +1,23 @@
//go:build linux
package hostsvc
import (
"time"
"golang.org/x/sys/unix"
)
type SystemClock struct{}
func (SystemClock) Now() time.Time { return time.Now() }
func (SystemClock) Sync() (bool, time.Duration, error) {
var tx unix.Timex
state, err := unix.Adjtimex(&tx)
if err != nil {
return false, 0, err
}
synced := state != unix.TIME_ERROR && tx.Status&unix.STA_UNSYNC == 0
return synced, time.Duration(tx.Maxerror) * time.Microsecond, nil
}

View file

@ -0,0 +1,11 @@
//go:build !linux
package hostsvc
import "time"
type SystemClock struct{}
func (SystemClock) Now() time.Time { return time.Now() }
func (SystemClock) Sync() (bool, time.Duration, error) { return false, 0, nil }

162
internal/hostsvc/dns.go Normal file
View file

@ -0,0 +1,162 @@
package hostsvc
import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"os"
"strings"
"time"
)
const (
dnsHeaderLen = 12
maxDNSMessage = 65535
)
func validQuery(b []byte) error {
if len(b) < dnsHeaderLen {
return fmt.Errorf("dns message of %d bytes is shorter than its header", len(b))
}
if b[2]&0x80 != 0 {
return errors.New("dns message is a response, not a query")
}
if op := (b[2] >> 3) & 0x0f; op != 0 {
return fmt.Errorf("dns opcode %d is not a standard query", op)
}
if n := binary.BigEndian.Uint16(b[4:6]); n != 1 {
return fmt.Errorf("dns query carries %d questions, want 1", n)
}
return nil
}
func validAnswer(query, answer []byte) error {
if len(answer) < dnsHeaderLen {
return fmt.Errorf("dns answer of %d bytes is shorter than its header", len(answer))
}
if answer[0] != query[0] || answer[1] != query[1] {
return errors.New("dns answer id does not match the query")
}
if answer[2]&0x80 == 0 {
return errors.New("dns answer is not flagged as a response")
}
return nil
}
func resolvers(path string) ([]string, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open %s: %w", path, err)
}
defer f.Close()
var out []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 2 || fields[0] != "nameserver" || net.ParseIP(fields[1]) == nil {
continue
}
out = append(out, net.JoinHostPort(fields[1], "53"))
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("read %s: %w", path, err)
}
return out, nil
}
func exchangeUDP(servers []string, query []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeUDPOne(server, query, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no dns upstream answered: %w", errors.Join(errs...))
}
func exchangeUDPOne(server string, query []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("udp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if _, err := conn.Write(query); err != nil {
return nil, err
}
buf := make([]byte, maxDNSMessage)
for {
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
if validAnswer(query, buf[:n]) == nil {
return buf[:n], nil
}
}
}
func exchangeTCP(servers []string, query []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeTCPOne(server, query, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no dns upstream answered: %w", errors.Join(errs...))
}
func exchangeTCPOne(server string, query []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("tcp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if err := writeTCPMessage(conn, query); err != nil {
return nil, err
}
answer, err := readTCPMessage(conn)
if err != nil {
return nil, err
}
if err := validAnswer(query, answer); err != nil {
return nil, err
}
return answer, nil
}
func readTCPMessage(r io.Reader) ([]byte, error) {
var size [2]byte
if _, err := io.ReadFull(r, size[:]); err != nil {
return nil, err
}
msg := make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := io.ReadFull(r, msg); err != nil {
return nil, err
}
return msg, nil
}
func writeTCPMessage(w io.Writer, msg []byte) error {
if len(msg) > maxDNSMessage {
return fmt.Errorf("dns message of %d bytes does not fit a tcp frame", len(msg))
}
frame := make([]byte, 2+len(msg))
binary.BigEndian.PutUint16(frame, uint16(len(msg)))
copy(frame[2:], msg)
_, err := w.Write(frame)
return err
}

View file

@ -0,0 +1,49 @@
//go:build linux
package hostsvc
import (
"fmt"
"os"
"runtime"
"golang.org/x/sys/unix"
)
const netnsDir = "/var/run/netns"
func netnsExists(vpc string) bool {
_, err := os.Stat(netnsDir + "/" + vpc)
return err == nil
}
func enterNetns(vpc string, fn func() error) error {
runtime.LockOSThread()
orig, err := os.Open("/proc/thread-self/ns/net")
if err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("open the current netns: %w", err)
}
defer orig.Close()
target, err := os.Open(netnsDir + "/" + vpc)
if err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("open netns %s: %w", vpc, err)
}
defer target.Close()
if err := unix.Setns(int(target.Fd()), unix.CLONE_NEWNET); err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("enter netns %s: %w", vpc, err)
}
err = fn()
if restore := unix.Setns(int(orig.Fd()), unix.CLONE_NEWNET); restore != nil {
return fmt.Errorf("leave netns %s, thread discarded: %w", vpc, restore)
}
runtime.UnlockOSThread()
return err
}

View file

@ -0,0 +1,7 @@
//go:build !linux
package hostsvc
func netnsExists(string) bool { return false }
func enterNetns(_ string, fn func() error) error { return fn() }

View file

@ -0,0 +1,56 @@
package hostsvc
import (
"bytes"
"testing"
"time"
)
func FuzzValidQuery(f *testing.F) {
f.Add(query(1))
f.Add(answerTo(query(1)))
f.Add([]byte{})
f.Fuzz(func(t *testing.T, b []byte) {
if validQuery(b) == nil && len(b) < dnsHeaderLen {
t.Fatalf("a %d-byte message passed", len(b))
}
})
}
func FuzzValidAnswer(f *testing.F) {
f.Add(query(1), answerTo(query(1)))
f.Add(query(1), []byte{1})
f.Fuzz(func(t *testing.T, q, a []byte) {
if validQuery(q) != nil {
return
}
if validAnswer(q, a) == nil && (len(a) < dnsHeaderLen || a[0] != q[0] || a[1] != q[1]) {
t.Fatalf("answer %x passed for query %x", a, q)
}
})
}
func FuzzReadTCPMessage(f *testing.F) {
f.Add([]byte{0, 3, 'a', 'b', 'c'})
f.Add([]byte{0xff, 0xff})
f.Fuzz(func(t *testing.T, b []byte) {
msg, err := readTCPMessage(bytes.NewReader(b))
if err == nil && len(msg) != int(b[0])<<8|int(b[1]) {
t.Fatalf("message of %d bytes for prefix %x", len(msg), b[:2])
}
})
}
func FuzzHostReply(f *testing.F) {
f.Add(ntpRequest(4, 1))
f.Add([]byte("short"))
f.Fuzz(func(t *testing.T, b []byte) {
if validNTPRequest(b) != nil {
return
}
r := hostReply(b, time.Unix(0, 0), time.Unix(1, 0), true, 0)
if err := validNTPAnswer(b, r); err != nil {
t.Fatalf("own reply rejected: %v", err)
}
})
}

View file

@ -0,0 +1,34 @@
package hostsvc
import (
"sync"
"time"
)
type limiter struct {
mu sync.Mutex
rate float64
tokens float64
last time.Time
now func() time.Time
}
func newLimiter(rate int, now func() time.Time) *limiter {
return &limiter{rate: float64(rate), tokens: float64(rate), last: now(), now: now}
}
func (l *limiter) allow() bool {
l.mu.Lock()
defer l.mu.Unlock()
t := l.now()
l.tokens += t.Sub(l.last).Seconds() * l.rate
if l.tokens > l.rate {
l.tokens = l.rate
}
l.last = t
if l.tokens < 1 {
return false
}
l.tokens--
return true
}

126
internal/hostsvc/ntp.go Normal file
View file

@ -0,0 +1,126 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"net"
"time"
)
const (
ntpPacketLen = 48
ntpModeClient = 3
ntpModeServer = 4
ntpUnsynced = 3
ntpStratumHost = 3
ntpStratumNone = 16
ntpPrecision = -20
ntpEpochOffset = 2208988800
)
var ntpRefID = [4]byte{'L', 'O', 'C', 'L'}
type Clock interface {
Now() time.Time
Sync() (synced bool, maxError time.Duration, err error)
}
func validNTPRequest(b []byte) error {
if len(b) < ntpPacketLen {
return fmt.Errorf("ntp packet of %d bytes is shorter than %d", len(b), ntpPacketLen)
}
if mode := b[0] & 0x07; mode != ntpModeClient {
return fmt.Errorf("ntp mode %d is not a client request", mode)
}
return nil
}
func validNTPAnswer(request, answer []byte) error {
if len(answer) < ntpPacketLen {
return fmt.Errorf("ntp answer of %d bytes is shorter than %d", len(answer), ntpPacketLen)
}
if mode := answer[0] & 0x07; mode != ntpModeServer {
return fmt.Errorf("ntp answer mode %d is not a server reply", mode)
}
if !bytes.Equal(answer[24:32], request[40:48]) {
return errors.New("ntp answer origin timestamp does not match the request")
}
return nil
}
func ntpTimestamp(t time.Time) uint64 {
secs := uint64(t.Unix() + ntpEpochOffset)
frac := (uint64(t.Nanosecond()) << 32) / uint64(time.Second)
return secs<<32 | frac
}
func ntpShort(d time.Duration) uint32 {
if d < 0 {
d = 0
}
secs := uint64(d / time.Second)
if secs > 0xffff {
return 0xffffffff
}
frac := (uint64(d%time.Second) << 16) / uint64(time.Second)
return uint32(secs<<16 | frac)
}
func hostReply(request []byte, received, sent time.Time, synced bool, maxError time.Duration) []byte {
reply := make([]byte, ntpPacketLen)
version := (request[0] >> 3) & 0x07
leap, stratum := byte(0), byte(ntpStratumHost)
if !synced {
leap, stratum = ntpUnsynced, ntpStratumNone
}
reply[0] = leap<<6 | version<<3 | ntpModeServer
reply[1] = stratum
reply[2] = request[2]
precision := int8(ntpPrecision)
reply[3] = byte(precision)
binary.BigEndian.PutUint32(reply[8:12], ntpShort(maxError))
copy(reply[12:16], ntpRefID[:])
binary.BigEndian.PutUint64(reply[16:24], ntpTimestamp(received))
copy(reply[24:32], request[40:48])
binary.BigEndian.PutUint64(reply[32:40], ntpTimestamp(received))
binary.BigEndian.PutUint64(reply[40:48], ntpTimestamp(sent))
return reply
}
func exchangeNTP(servers []string, request []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeNTPOne(server, request, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no ntp upstream answered: %w", errors.Join(errs...))
}
func exchangeNTPOne(server string, request []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("udp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if _, err := conn.Write(request[:ntpPacketLen]); err != nil {
return nil, err
}
buf := make([]byte, 1024)
for {
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
if validNTPAnswer(request, buf[:n]) == nil {
return buf[:n], nil
}
}
}

View file

@ -0,0 +1,234 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func query(id uint16) []byte {
q := make([]byte, dnsHeaderLen, dnsHeaderLen+17)
binary.BigEndian.PutUint16(q[0:2], id)
q[2] = 0x01
binary.BigEndian.PutUint16(q[4:6], 1)
q = append(q, 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', 3, 'o', 'r', 'g', 0, 0, 1, 0, 1)
return q
}
func answerTo(q []byte) []byte {
a := append([]byte{}, q...)
a[2] |= 0x80
return a
}
func ntpRequest(version byte, transmit uint64) []byte {
r := make([]byte, ntpPacketLen)
r[0] = version<<3 | ntpModeClient
r[2] = 6
binary.BigEndian.PutUint64(r[40:48], transmit)
return r
}
func TestValidQuery(t *testing.T) {
if err := validQuery(query(7)); err != nil {
t.Fatalf("a standard query must pass: %v", err)
}
response := answerTo(query(7))
opcode := query(7)
opcode[2] |= 2 << 3
twoQuestions := query(7)
binary.BigEndian.PutUint16(twoQuestions[4:6], 2)
noQuestion := query(7)
binary.BigEndian.PutUint16(noQuestion[4:6], 0)
for _, c := range []struct {
name string
msg []byte
want string
}{
{"short", query(7)[:11], "11 bytes is shorter than its header"},
{"response", response, "is a response, not a query"},
{"opcode", opcode, "opcode 2 is not a standard query"},
{"two questions", twoQuestions, "carries 2 questions, want 1"},
{"no question", noQuestion, "carries 0 questions, want 1"},
} {
if err := validQuery(c.msg); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: error %v, want %q", c.name, err, c.want)
}
}
}
func TestValidAnswer(t *testing.T) {
q := query(0x1234)
if err := validAnswer(q, answerTo(q)); err != nil {
t.Fatalf("a matching answer must pass: %v", err)
}
if err := validAnswer(q, answerTo(query(0x1235))); err == nil || !strings.Contains(err.Error(), "id does not match") {
t.Errorf("another id: error %v", err)
}
if err := validAnswer(q, q); err == nil || !strings.Contains(err.Error(), "not flagged as a response") {
t.Errorf("a query echoed back: error %v", err)
}
if err := validAnswer(q, q[:5]); err == nil || !strings.Contains(err.Error(), "shorter than its header") {
t.Errorf("short: error %v", err)
}
}
func TestResolvers(t *testing.T) {
path := filepath.Join(t.TempDir(), "resolv.conf")
content := "# comment\nsearch example.org\nnameserver 192.0.2.1\nnameserver 2001:db8::1\nnameserver not-an-ip\nnameserver\noptions edns0\n"
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
got, err := resolvers(path)
if err != nil {
t.Fatalf("resolvers: %v", err)
}
if want := []string{"192.0.2.1:53", "[2001:db8::1]:53"}; !reflect.DeepEqual(got, want) {
t.Errorf("resolvers = %v, want %v", got, want)
}
if _, err := resolvers(filepath.Join(t.TempDir(), "absent")); err == nil {
t.Error("a missing resolv.conf must be reported")
}
}
func TestTCPMessageRoundTrip(t *testing.T) {
var buf bytes.Buffer
if err := writeTCPMessage(&buf, query(9)); err != nil {
t.Fatalf("writeTCPMessage: %v", err)
}
if got := binary.BigEndian.Uint16(buf.Bytes()[:2]); int(got) != len(query(9)) {
t.Errorf("length prefix = %d, want %d", got, len(query(9)))
}
msg, err := readTCPMessage(&buf)
if err != nil {
t.Fatalf("readTCPMessage: %v", err)
}
if !bytes.Equal(msg, query(9)) {
t.Errorf("message changed on the way: %x", msg)
}
if err := writeTCPMessage(&buf, make([]byte, maxDNSMessage+1)); err == nil {
t.Error("a message larger than a tcp frame must be refused")
}
}
func TestValidNTPRequest(t *testing.T) {
if err := validNTPRequest(ntpRequest(4, 1)); err != nil {
t.Fatalf("a client request must pass: %v", err)
}
server := ntpRequest(4, 1)
server[0] = 4<<3 | ntpModeServer
if err := validNTPRequest(server); err == nil || !strings.Contains(err.Error(), "mode 4 is not a client request") {
t.Errorf("server mode: error %v", err)
}
if err := validNTPRequest(ntpRequest(4, 1)[:47]); err == nil || !strings.Contains(err.Error(), "47 bytes is shorter than 48") {
t.Errorf("short: error %v", err)
}
}
func TestValidNTPAnswer(t *testing.T) {
req := ntpRequest(4, 0xdeadbeef)
ans := hostReply(req, time.Unix(0, 0), time.Unix(0, 0), true, 0)
if err := validNTPAnswer(req, ans); err != nil {
t.Fatalf("a matching answer must pass: %v", err)
}
if err := validNTPAnswer(ntpRequest(4, 0xdeadbeee), ans); err == nil || !strings.Contains(err.Error(), "origin timestamp") {
t.Errorf("another request: error %v", err)
}
if err := validNTPAnswer(req, req); err == nil || !strings.Contains(err.Error(), "mode 3 is not a server reply") {
t.Errorf("a request echoed back: error %v", err)
}
}
func TestNTPTimestamp(t *testing.T) {
if got := ntpTimestamp(time.Unix(0, 0)); got != 2208988800<<32 {
t.Errorf("unix epoch = %#x, want %#x", got, uint64(2208988800)<<32)
}
if got := ntpTimestamp(time.Unix(1, 500_000_000)); got != (2208988801<<32 | 0x80000000) {
t.Errorf("half a second = %#x", got)
}
if got := ntpShort(1500 * time.Millisecond); got != 0x00018000 {
t.Errorf("ntpShort(1.5 s) = %#x, want 0x18000", got)
}
if got := ntpShort(-time.Second); got != 0 {
t.Errorf("ntpShort of a negative duration = %#x, want 0", got)
}
}
func TestHostReply_Synced(t *testing.T) {
req := ntpRequest(3, 0x0102030405060708)
received := time.Unix(1000, 0)
sent := time.Unix(1001, 0)
r := hostReply(req, received, sent, true, 500*time.Millisecond)
if r[0] != 0<<6|3<<3|4 {
t.Errorf("li/vn/mode = %#08b, want li 0, the client version 3, mode 4", r[0])
}
if r[1] != 3 {
t.Errorf("stratum = %d, want 3", r[1])
}
if r[2] != 6 {
t.Errorf("poll = %d, want the client's 6", r[2])
}
if int8(r[3]) != -20 {
t.Errorf("precision = %d, want -20", int8(r[3]))
}
if got := binary.BigEndian.Uint32(r[8:12]); got != 0x8000 {
t.Errorf("root dispersion = %#x, want 0.5 s", got)
}
if string(r[12:16]) != "LOCL" {
t.Errorf("reference id = %q, want LOCL", r[12:16])
}
if got := binary.BigEndian.Uint64(r[24:32]); got != 0x0102030405060708 {
t.Errorf("origin = %#x, want the client's transmit timestamp", got)
}
if got := binary.BigEndian.Uint64(r[32:40]); got != ntpTimestamp(received) {
t.Errorf("receive = %#x, want %#x", got, ntpTimestamp(received))
}
if got := binary.BigEndian.Uint64(r[40:48]); got != ntpTimestamp(sent) {
t.Errorf("transmit = %#x, want %#x", got, ntpTimestamp(sent))
}
}
func TestHostReply_UnsyncedRaisesTheAlarm(t *testing.T) {
r := hostReply(ntpRequest(4, 1), time.Unix(0, 0), time.Unix(0, 0), false, 0)
if li := r[0] >> 6; li != 3 {
t.Errorf("leap indicator = %d, want 3: an unsynchronized host must not be trusted", li)
}
if r[1] != 16 {
t.Errorf("stratum = %d, want 16 (unsynchronized)", r[1])
}
}
func TestLimiter(t *testing.T) {
now := time.Unix(0, 0)
l := newLimiter(3, func() time.Time { return now })
for i := 0; i < 3; i++ {
if !l.allow() {
t.Fatalf("request %d refused inside the burst", i+1)
}
}
if l.allow() {
t.Fatal("the burst is exhausted, the next request must be refused")
}
now = now.Add(400 * time.Millisecond)
if !l.allow() {
t.Fatal("400 ms refill 1.2 tokens at 3 per second, one request must pass")
}
if l.allow() {
t.Fatal("only one token was refilled")
}
now = now.Add(time.Hour)
for i := 0; i < 3; i++ {
if !l.allow() {
t.Fatalf("after a long pause, request %d refused: the bucket holds 3", i+1)
}
}
if l.allow() {
t.Fatal("the bucket must not grow beyond its rate")
}
}

362
internal/hostsvc/service.go Normal file
View file

@ -0,0 +1,362 @@
package hostsvc
import (
"errors"
"fmt"
"io"
"log/slog"
"net"
"sort"
"strings"
"sync"
"time"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
const (
ServiceIP = "169.254.169.253"
DefaultResolv = "/etc/resolv.conf"
DefaultTimeout = 2 * time.Second
tcpIdle = 10 * time.Second
maxTCPConns = 16
)
type Options struct {
DNS []string
NTP []string
Rate int
Resolv string
DNSListen string
NTPListen string
Timeout time.Duration
Clock Clock
Enter func(vpc string, fn func() error) error
Exists func(vpc string) bool
Logger *slog.Logger
}
func (o Options) withDefaults() Options {
if o.Resolv == "" {
o.Resolv = DefaultResolv
}
if o.DNSListen == "" {
o.DNSListen = net.JoinHostPort(ServiceIP, "53")
}
if o.NTPListen == "" {
o.NTPListen = net.JoinHostPort(ServiceIP, "123")
}
if o.Timeout == 0 {
o.Timeout = DefaultTimeout
}
if o.Clock == nil {
o.Clock = SystemClock{}
}
if o.Enter == nil {
o.Enter = enterNetns
}
if o.Exists == nil {
o.Exists = netnsExists
}
if o.Logger == nil {
o.Logger = slog.New(slog.NewTextHandler(io.Discard, nil))
}
return o
}
type State struct {
VPCs []string `json:"vpcs"`
}
type Service struct {
opts Options
state *statefile.File[State]
mu sync.Mutex
vpcs map[string]*instance
}
func New(opts Options, statePath string) (*Service, error) {
if opts.Rate <= 0 {
return nil, fmt.Errorf("rate limit %d must be a positive number of requests per second", opts.Rate)
}
return &Service{
opts: opts.withDefaults(),
state: statefile.New[State](statePath),
vpcs: map[string]*instance{},
}, nil
}
func validVPC(vpc string) error {
if vpc == "" || vpc == "." || vpc == ".." || strings.ContainsAny(vpc, "/\x00") {
return fmt.Errorf("invalid vpc name %q", vpc)
}
return nil
}
func (s *Service) Restore() error {
saved, err := s.state.Load()
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
var errs []error
for _, vpc := range saved.VPCs {
if !s.opts.Exists(vpc) {
s.opts.Logger.Info("netns gone, vpc dropped from the state", "vpc", vpc)
continue
}
if err := s.open(vpc); err != nil {
errs = append(errs, err)
}
}
if err := s.save(); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func (s *Service) Open(vpc string) error {
if err := validVPC(vpc); err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if _, open := s.vpcs[vpc]; open {
return nil
}
if err := s.open(vpc); err != nil {
return err
}
return s.save()
}
func (s *Service) Close(vpc string) error {
if err := validVPC(vpc); err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
inst, open := s.vpcs[vpc]
if !open {
return nil
}
delete(s.vpcs, vpc)
err := inst.close()
return errors.Join(err, s.save())
}
func (s *Service) List() []string {
s.mu.Lock()
defer s.mu.Unlock()
return s.names()
}
func (s *Service) Shutdown() error {
s.mu.Lock()
defer s.mu.Unlock()
var errs []error
for vpc, inst := range s.vpcs {
errs = append(errs, inst.close())
delete(s.vpcs, vpc)
}
return errors.Join(errs...)
}
func (s *Service) names() []string {
out := make([]string, 0, len(s.vpcs))
for vpc := range s.vpcs {
out = append(out, vpc)
}
sort.Strings(out)
return out
}
func (s *Service) save() error {
return s.state.Save(State{VPCs: s.names()})
}
func (s *Service) open(vpc string) error {
inst := &instance{
vpc: vpc,
svc: s,
limit: newLimiter(s.opts.Rate, time.Now),
conns: make(chan struct{}, maxTCPConns),
log: s.opts.Logger.With("vpc", vpc),
}
dns, ntp := s.opts.DNSListen, s.opts.NTPListen
err := s.opts.Enter(vpc, func() error {
var err error
if inst.udpDNS, err = net.ListenPacket("udp4", dns); err != nil {
return fmt.Errorf("listen udp %s: %w", dns, err)
}
if inst.tcpDNS, err = net.Listen("tcp4", dns); err != nil {
return fmt.Errorf("listen tcp %s: %w", dns, err)
}
if inst.udpNTP, err = net.ListenPacket("udp4", ntp); err != nil {
return fmt.Errorf("listen udp %s: %w", ntp, err)
}
return nil
})
if err != nil {
return errors.Join(fmt.Errorf("open vpc %s: %w", vpc, err), inst.close())
}
inst.serve()
s.vpcs[vpc] = inst
inst.log.Info("vpc served", "dns", dns, "ntp", ntp)
return nil
}
func (s *Service) dnsServers() []string {
servers := append([]string{}, s.opts.DNS...)
host, err := resolvers(s.opts.Resolv)
if err != nil {
s.opts.Logger.Warn("host resolver unreadable", "error", err)
}
return append(servers, host...)
}
func (s *Service) ntpAnswer(request []byte, received time.Time) []byte {
if len(s.opts.NTP) > 0 {
answer, err := exchangeNTP(s.opts.NTP, request, s.opts.Timeout)
if err == nil {
return answer
}
s.opts.Logger.Warn("ntp upstreams failed, answering from the host clock", "error", err)
}
synced, maxError, err := s.opts.Clock.Sync()
if err != nil {
s.opts.Logger.Warn("host clock status unreadable, answering as unsynchronized", "error", err)
synced = false
}
return hostReply(request, received, s.opts.Clock.Now(), synced, maxError)
}
type instance struct {
vpc string
svc *Service
limit *limiter
conns chan struct{}
log *slog.Logger
udpDNS net.PacketConn
tcpDNS net.Listener
udpNTP net.PacketConn
wg sync.WaitGroup
}
func (i *instance) serve() {
i.wg.Add(3)
go i.loopUDP(i.udpDNS, i.answerDNS)
go i.loopUDP(i.udpNTP, i.answerNTP)
go i.loopTCP()
}
func (i *instance) close() error {
var errs []error
for _, c := range []interface{ Close() error }{i.udpDNS, i.tcpDNS, i.udpNTP} {
if c == nil {
continue
}
if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
errs = append(errs, err)
}
}
i.wg.Wait()
return errors.Join(errs...)
}
func (i *instance) loopUDP(conn net.PacketConn, answer func([]byte, time.Time) ([]byte, error)) {
defer i.wg.Done()
buf := make([]byte, maxDNSMessage)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
if !errors.Is(err, net.ErrClosed) {
i.log.Error("udp socket stopped", "error", err)
}
return
}
received := i.svc.opts.Clock.Now()
if !i.limit.allow() {
i.log.Debug("rate limit reached, request dropped", "from", from)
continue
}
request := append([]byte{}, buf[:n]...)
go func() {
reply, err := answer(request, received)
if err != nil {
i.log.Debug("request dropped", "from", from, "error", err)
return
}
if _, err := conn.WriteTo(reply, from); err != nil && !errors.Is(err, net.ErrClosed) {
i.log.Debug("reply not sent", "to", from, "error", err)
}
}()
}
}
func (i *instance) answerDNS(query []byte, _ time.Time) ([]byte, error) {
if err := validQuery(query); err != nil {
return nil, err
}
return exchangeUDP(i.svc.dnsServers(), query, i.svc.opts.Timeout)
}
func (i *instance) answerNTP(request []byte, received time.Time) ([]byte, error) {
if err := validNTPRequest(request); err != nil {
return nil, err
}
return i.svc.ntpAnswer(request, received), nil
}
func (i *instance) loopTCP() {
defer i.wg.Done()
for {
conn, err := i.tcpDNS.Accept()
if err != nil {
if !errors.Is(err, net.ErrClosed) {
i.log.Error("tcp socket stopped", "error", err)
}
return
}
select {
case i.conns <- struct{}{}:
go func() {
defer func() { <-i.conns }()
i.serveTCP(conn)
}()
default:
i.log.Debug("too many tcp connections, connection refused", "from", conn.RemoteAddr())
conn.Close()
}
}
}
func (i *instance) serveTCP(conn net.Conn) {
defer conn.Close()
for {
if err := conn.SetDeadline(time.Now().Add(tcpIdle)); err != nil {
return
}
query, err := readTCPMessage(conn)
if err != nil {
return
}
if !i.limit.allow() {
i.log.Debug("rate limit reached, connection closed", "from", conn.RemoteAddr())
return
}
if err := validQuery(query); err != nil {
i.log.Debug("request dropped", "from", conn.RemoteAddr(), "error", err)
return
}
answer, err := exchangeTCP(i.svc.dnsServers(), query, i.svc.opts.Timeout)
if err != nil {
i.log.Debug("request dropped", "from", conn.RemoteAddr(), "error", err)
return
}
if err := writeTCPMessage(conn, answer); err != nil {
return
}
}
}

View file

@ -0,0 +1,538 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"errors"
"net"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
type fakeClock struct {
now time.Time
synced bool
maxError time.Duration
err error
}
func (c fakeClock) Now() time.Time { return c.now }
func (c fakeClock) Sync() (bool, time.Duration, error) {
return c.synced, c.maxError, c.err
}
type netnsRecorder struct {
mu sync.Mutex
entered []string
fail error
}
func (r *netnsRecorder) enter(vpc string, fn func() error) error {
r.mu.Lock()
r.entered = append(r.entered, vpc)
r.mu.Unlock()
if err := fn(); err != nil {
return err
}
return r.fail
}
func (r *netnsRecorder) calls() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string{}, r.entered...)
}
func emptyResolv(t *testing.T) string {
t.Helper()
path := filepath.Join(t.TempDir(), "resolv.conf")
if err := os.WriteFile(path, nil, 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
return path
}
func newService(t *testing.T, opts Options) (*Service, *netnsRecorder, string) {
t.Helper()
rec := &netnsRecorder{}
opts.DNSListen = "127.0.0.1:0"
opts.NTPListen = "127.0.0.1:0"
if opts.Rate == 0 {
opts.Rate = 100
}
if opts.Resolv == "" {
opts.Resolv = emptyResolv(t)
}
if opts.Clock == nil {
opts.Clock = fakeClock{now: time.Unix(2000, 0), synced: true}
}
if opts.Timeout == 0 {
opts.Timeout = 300 * time.Millisecond
}
opts.Enter = rec.enter
if opts.Exists == nil {
opts.Exists = func(string) bool { return true }
}
statePath := filepath.Join(t.TempDir(), "hostsvc.state")
s, err := New(opts, statePath)
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(func() { s.Shutdown() })
return s, rec, statePath
}
func (s *Service) addrs(t *testing.T, vpc string) (dnsUDP, dnsTCP, ntp string) {
t.Helper()
s.mu.Lock()
defer s.mu.Unlock()
inst, ok := s.vpcs[vpc]
if !ok {
t.Fatalf("vpc %s is not open", vpc)
}
return inst.udpDNS.LocalAddr().String(), inst.tcpDNS.Addr().String(), inst.udpNTP.LocalAddr().String()
}
func savedVPCs(t *testing.T, path string) []string {
t.Helper()
st, err := statefile.New[State](path).Load()
if err != nil {
t.Fatalf("load state: %v", err)
}
return st.VPCs
}
func udpUpstream(t *testing.T, reply func(req []byte) [][]byte) (string, func() int) {
t.Helper()
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("ListenPacket: %v", err)
}
t.Cleanup(func() { conn.Close() })
var mu sync.Mutex
count := 0
go func() {
buf := make([]byte, 65535)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
mu.Lock()
count++
mu.Unlock()
for _, r := range reply(append([]byte{}, buf[:n]...)) {
conn.WriteTo(r, from)
}
}
}()
return conn.LocalAddr().String(), func() int {
mu.Lock()
defer mu.Unlock()
return count
}
}
func tcpUpstream(t *testing.T) string {
t.Helper()
l, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("Listen: %v", err)
}
t.Cleanup(func() { l.Close() })
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
q, err := readTCPMessage(conn)
if err != nil {
return
}
writeTCPMessage(conn, answerTo(q))
}()
}
}()
return l.Addr().String()
}
func deadAddr(t *testing.T) string {
t.Helper()
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("ListenPacket: %v", err)
}
addr := conn.LocalAddr().String()
conn.Close()
return addr
}
func ask(t *testing.T, addr string, msg []byte, wait time.Duration) ([]byte, error) {
t.Helper()
conn, err := net.Dial("udp4", addr)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(wait))
if _, err := conn.Write(msg); err != nil {
t.Fatalf("Write: %v", err)
}
buf := make([]byte, 65535)
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
return buf[:n], nil
}
func TestNew_RefusesANullRate(t *testing.T) {
if _, err := New(Options{Rate: 0}, filepath.Join(t.TempDir(), "s")); err == nil || !strings.Contains(err.Error(), "rate limit 0") {
t.Errorf("error %v: a null rate would silently drop every request", err)
}
}
func TestOpen_EntersTheVPCNetnsAndSavesIt(t *testing.T) {
s, rec, statePath := newService(t, Options{})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
if got := rec.calls(); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("netns entered = %v, want the vpc's", got)
}
if got := savedVPCs(t, statePath); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("state = %v", got)
}
if got := s.List(); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("List = %v", got)
}
}
func TestOpen_IsIdempotent(t *testing.T) {
s, rec, _ := newService(t, Options{})
for i := 0; i < 2; i++ {
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open %d: %v", i+1, err)
}
}
if got := rec.calls(); len(got) != 1 {
t.Errorf("netns entered %d times, want once: a second open must not bind again", len(got))
}
}
func TestOpen_InvalidNamesAreRefusedWithoutEnteringAnything(t *testing.T) {
s, rec, statePath := newService(t, Options{})
for _, name := range []string{"", ".", "..", "../etc", "a/b", "x\x00y"} {
if err := s.Open(name); err == nil || !strings.Contains(err.Error(), "invalid vpc name") {
t.Errorf("Open(%q): error %v", name, err)
}
if err := s.Close(name); err == nil || !strings.Contains(err.Error(), "invalid vpc name") {
t.Errorf("Close(%q): error %v", name, err)
}
}
if got := rec.calls(); len(got) != 0 {
t.Errorf("netns entered for an invalid name: %v", got)
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
}
func TestOpen_FailureLeavesNothingBehind(t *testing.T) {
s, rec, statePath := newService(t, Options{})
rec.fail = errors.New("boom")
if err := s.Open("vpc-00001"); err == nil || !strings.Contains(err.Error(), "open vpc vpc-00001: boom") {
t.Fatalf("error %v", err)
}
if got := s.List(); len(got) != 0 {
t.Errorf("List = %v, want empty after a failed open", got)
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
}
func TestClose_StopsServingAndForgetsTheVPC(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, statePath := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, query(1), time.Second); err != nil {
t.Fatalf("witness: the open vpc must answer: %v", err)
}
if err := s.Close("vpc-00001"); err != nil {
t.Fatalf("Close: %v", err)
}
if _, err := ask(t, dns, query(2), 300*time.Millisecond); err == nil {
t.Error("a closed vpc still answers")
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
if err := s.Close("vpc-00001"); err != nil {
t.Errorf("closing twice must succeed: %v", err)
}
}
func TestRestore_ReopensOnlyTheVPCsWhoseNetnsExists(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "hostsvc.state")
if err := statefile.New[State](statePath).Save(State{VPCs: []string{"vpc-gone", "vpc-here"}}); err != nil {
t.Fatalf("Save: %v", err)
}
rec := &netnsRecorder{}
s, err := New(Options{
Rate: 10, DNSListen: "127.0.0.1:0", NTPListen: "127.0.0.1:0", Resolv: emptyResolv(t),
Enter: rec.enter,
Exists: func(vpc string) bool { return vpc == "vpc-here" },
}, statePath)
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(func() { s.Shutdown() })
if err := s.Restore(); err != nil {
t.Fatalf("Restore: %v", err)
}
if got := rec.calls(); !reflect.DeepEqual(got, []string{"vpc-here"}) {
t.Errorf("netns entered = %v, want only vpc-here", got)
}
if got := savedVPCs(t, statePath); !reflect.DeepEqual(got, []string{"vpc-here"}) {
t.Errorf("state = %v, the vanished vpc must be dropped", got)
}
}
func TestDNS_UDPQueryIsRelayedToTheUpstream(t *testing.T) {
var got []byte
var mu sync.Mutex
up, _ := udpUpstream(t, func(q []byte) [][]byte {
mu.Lock()
got = q
mu.Unlock()
return [][]byte{answerTo(q)}
})
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
reply, err := ask(t, dns, query(0x4242), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if !bytes.Equal(reply, answerTo(query(0x4242))) {
t.Errorf("reply = %x, want the upstream's answer untouched", reply)
}
mu.Lock()
defer mu.Unlock()
if !bytes.Equal(got, query(0x4242)) {
t.Errorf("upstream received %x, want the query untouched", got)
}
}
func TestDNS_SpoofedAnswerIsIgnored(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte {
bad := answerTo(q)
bad[0] ^= 0xff
return [][]byte{bad, answerTo(q)}
})
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
reply, err := ask(t, dns, query(0x1000), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if binary.BigEndian.Uint16(reply[0:2]) != 0x1000 {
t.Errorf("reply id = %#x, the answer with a foreign id must be skipped", binary.BigEndian.Uint16(reply[0:2]))
}
}
func TestDNS_NextUpstreamWhenTheFirstIsDown(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{deadAddr(t), up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, query(3), 2*time.Second); err != nil {
t.Fatalf("the second upstream must answer: %v", err)
}
}
func TestDNS_HostResolverComesAfterTheConfiguredUpstreams(t *testing.T) {
path := filepath.Join(t.TempDir(), "resolv.conf")
if err := os.WriteFile(path, []byte("nameserver 192.0.2.1\n"), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
s, _, _ := newService(t, Options{DNS: []string{"198.51.100.1:53"}, Resolv: path})
if got, want := s.dnsServers(), []string{"198.51.100.1:53", "192.0.2.1:53"}; !reflect.DeepEqual(got, want) {
t.Errorf("dnsServers = %v, want %v", got, want)
}
}
func TestDNS_InvalidQueryGetsNoAnswerAndIsNotRelayed(t *testing.T) {
up, count := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, answerTo(query(5)), 300*time.Millisecond); err == nil {
t.Error("a response sent to the relay must not be answered")
}
if _, err := ask(t, dns, query(6), time.Second); err != nil {
t.Fatalf("witness: a valid query must be answered: %v", err)
}
if n := count(); n != 1 {
t.Errorf("upstream saw %d messages, want only the valid one", n)
}
}
func TestDNS_TCPQueryIsRelayed(t *testing.T) {
s, _, _ := newService(t, Options{DNS: []string{tcpUpstream(t)}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, tcp, _ := s.addrs(t, "vpc-00001")
conn, err := net.Dial("tcp4", tcp)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(2 * time.Second))
for id := uint16(1); id <= 2; id++ {
if err := writeTCPMessage(conn, query(id)); err != nil {
t.Fatalf("write: %v", err)
}
reply, err := readTCPMessage(conn)
if err != nil {
t.Fatalf("query %d: no answer: %v", id, err)
}
if !bytes.Equal(reply, answerTo(query(id))) {
t.Errorf("query %d: reply %x", id, reply)
}
}
}
func TestDNS_RateLimitPerVPC(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{up}, Rate: 2})
for _, vpc := range []string{"vpc-00001", "vpc-00002"} {
if err := s.Open(vpc); err != nil {
t.Fatalf("Open %s: %v", vpc, err)
}
}
dns1, _, _ := s.addrs(t, "vpc-00001")
dns2, _, _ := s.addrs(t, "vpc-00002")
for i := 0; i < 2; i++ {
if _, err := ask(t, dns1, query(uint16(i)), time.Second); err != nil {
t.Fatalf("query %d within the burst: %v", i+1, err)
}
}
if _, err := ask(t, dns1, query(9), 200*time.Millisecond); err == nil {
t.Error("the third query in a row must be dropped at 2 per second")
}
if _, err := ask(t, dns2, query(10), time.Second); err != nil {
t.Errorf("another vpc has its own bucket: %v", err)
}
}
func TestNTP_HostClockWithoutUpstream(t *testing.T) {
clock := fakeClock{now: time.Unix(5000, 0), synced: true, maxError: time.Second}
s, _, _ := newService(t, Options{Clock: clock})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
req := ntpRequest(4, 0xabcdef)
reply, err := ask(t, ntp, req, time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if err := validNTPAnswer(req, reply); err != nil {
t.Fatalf("invalid answer: %v", err)
}
if got := binary.BigEndian.Uint64(reply[40:48]); got != ntpTimestamp(clock.now) {
t.Errorf("transmit = %#x, want the host clock %#x", got, ntpTimestamp(clock.now))
}
if reply[0]>>6 != 0 || reply[1] != 3 {
t.Errorf("li %d stratum %d, want 0 and 3 for a synced host", reply[0]>>6, reply[1])
}
}
func TestNTP_UnreadableClockStatusIsAnsweredAsUnsynced(t *testing.T) {
s, _, _ := newService(t, Options{Clock: fakeClock{now: time.Unix(5000, 0), synced: true, err: errors.New("eperm")}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
reply, err := ask(t, ntp, ntpRequest(4, 1), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if reply[0]>>6 != 3 {
t.Errorf("li = %d, want 3 when the clock status cannot be read", reply[0]>>6)
}
}
func TestNTP_UpstreamIsPreferredAndChecked(t *testing.T) {
up, _ := udpUpstream(t, func(req []byte) [][]byte {
spoofed := hostReply(ntpRequest(4, 0), time.Unix(0, 0), time.Unix(0, 0), true, 0)
real := hostReply(req, time.Unix(7000, 0), time.Unix(7000, 0), true, 0)
real[1] = 1
return [][]byte{spoofed, real}
})
s, _, _ := newService(t, Options{NTP: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
req := ntpRequest(4, 0x55)
reply, err := ask(t, ntp, req, time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if reply[1] != 1 {
t.Errorf("stratum = %d, want the upstream's 1 (spoofed answer skipped, host clock unused)", reply[1])
}
}
func TestNTP_HostClockWhenUpstreamsFail(t *testing.T) {
clock := fakeClock{now: time.Unix(6000, 0), synced: true}
s, _, _ := newService(t, Options{NTP: []string{deadAddr(t)}, Clock: clock})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
reply, err := ask(t, ntp, ntpRequest(4, 2), 2*time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if got := binary.BigEndian.Uint64(reply[40:48]); got != ntpTimestamp(clock.now) {
t.Errorf("transmit = %#x, want the host clock as fallback", got)
}
}
func TestNTP_InvalidRequestGetsNoAnswer(t *testing.T) {
s, _, _ := newService(t, Options{})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
if _, err := ask(t, ntp, []byte("not ntp"), 300*time.Millisecond); err == nil {
t.Error("a short packet must not be answered")
}
}