Compare commits

...

5 commits

Author SHA1 Message Date
76b6657dbf
f-86: doc: DNS et NTP des VM, two-hostsvc, notes 0.3.0 #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-25 21:11:20 +01:00
ef23f51125
f-86: e2e: 29-dns-ntp, DNS et NTP des VM en vxlan et en bridge #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-25 21:03:55 +01:00
0ed7f0dd1d
f-86: deploy: unité two-hostsvc, release et provision du lab #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-25 20:58:39 +01:00
27dda545cf
f-86: agent: dummy, DNAT et open par VPC, DHCP 6 et 42, watchdog #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-25 20:45:24 +01:00
aa1cc58ae3
f-86: hostsvc: relais DNS et NTP par hôte, socket de contrôle #86
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-25 20:32:06 +01:00
54 changed files with 3204 additions and 37 deletions

View file

@ -52,6 +52,7 @@ jobs:
- metadata
- agent
- dhcp
- two-hostsvc
uses: ./.forgejo/workflows/build.yml
with:
tag: ${{ needs.set-release-target.outputs.release_cible }}
@ -80,6 +81,8 @@ jobs:
name: dhcp@.service
- path: systemd/metadata@.service
name: metadata@.service
- path: systemd/two-hostsvc.service
name: two-hostsvc.service
steps:
- uses: actions/checkout@v3
- name: Move asset

78
cmd/two-hostsvc/main.go Normal file
View file

@ -0,0 +1,78 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"net"
"os"
"os/signal"
"syscall"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/pkg/logger"
)
var (
confFile = flag.String("conf", "/etc/two/agent.yml", "configuration file")
runDir = flag.String("run-dir", hostsvcapi.DefaultRunDir, "directory of the control socket and the state file")
)
func main() {
flag.Parse()
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "two-hostsvc: %v\n", err)
os.Exit(1)
}
}
func run() error {
cfg, err := configuration.LoadConfig(*confFile)
if err != nil {
return fmt.Errorf("load config: %w", err)
}
log := logger.New(cfg.Logger.Level, cfg.Logger.Debug)
svc, err := hostsvc.New(hostsvc.Options{
DNS: cfg.HostSvc.DNS(),
NTP: cfg.HostSvc.NTP(),
Rate: cfg.HostSvc.Rate(),
Logger: log,
}, hostsvcapi.StatePath(*runDir))
if err != nil {
return err
}
if err := svc.Restore(); err != nil {
log.Error("some vpcs could not be restored", "error", err)
}
control, err := hostsvcapi.Listen(svc, hostsvcapi.SocketPath(*runDir), log)
if err != nil {
return fmt.Errorf("listen on the control socket: %w", err)
}
defer control.Close()
log.Info("two-hostsvc started", "socket", control.Addr(), "vpcs", svc.List())
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
go func() {
if err := control.Serve(); err != nil && !errors.Is(err, net.ErrClosed) {
log.Error("control socket stopped", "error", err)
stop()
}
}()
<-ctx.Done()
log.Info("two-hostsvc stopping")
if err := control.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
log.Error("close the control socket", "error", err)
}
return svc.Shutdown()
}

View file

@ -80,6 +80,20 @@ metadata:
dhcp:
backend: two
# DNS and NTP relay of the VMs (two-hostsvc, one process per hypervisor).
# It listens on 169.254.169.253 in every VPC netns of the host; the VMs reach it
# through their gateway (interface_ip, announced by DHCP options 6 and 42).
# dns_upstream : resolvers queried first (ip or ip:port, port 53 by default);
# the host resolver (/etc/resolv.conf) comes after them
# ntp_upstream : NTP servers relayed first (ip or ip:port, port 123 by
# default); without an answer, the host clock replies
# rate_limit_per_vpc : requests per second, DNS and NTP together, per VPC
# Its control socket and state file live in /run/two/hostsvc.
hostsvc:
dns_upstream: []
ntp_upstream: []
rate_limit_per_vpc: 100
# QEMU runtime paths
qemu:
# UEFI firmware (requires apt install ovmf on Debian/Ubuntu)

85
docs/concepts/dns-ntp.rst Normal file
View file

@ -0,0 +1,85 @@
DNS et NTP des VM
=================
Les VM résolvent les noms et lisent l'heure par leur gateway : le DHCP annonce l'``interface_ip``
du subnet comme serveur DNS (option 6) et NTP (option 42), dans les deux backends. Un relais
unique par hyperviseur, ``two-hostsvc``, répond derrière cette adresse dans le netns de chaque
VPC. Une VM sans sortie Internet garde ainsi DNS et NTP, et aucune route n'est ouverte entre la
VRF du VPC et l'hôte.
Chemin d'une requête
--------------------
.. mermaid::
graph LR
V["VM<br/>vers interface_ip:53 ou :123"] --> N["netns du VPC<br/>chaîne TWO-HOSTSVC<br/>DNAT vers 169.254.169.253"]
N --> H["two-hostsvc<br/>socket dans le netns<br/>169.254.169.253 sur la dummy hostsvc"]
H -->|"depuis l'hôte"| U["serveurs amont<br/>puis résolveur ou horloge de l'hôte"]
À la création d'un VPC sur l'hyperviseur, l'agent pose dans son netns :
* une interface ``dummy`` ``hostsvc`` portant ``169.254.169.253/32`` ;
* une chaîne iptables ``TWO-HOSTSVC`` (table ``nat``), appelée depuis ``PREROUTING`` pour le seul
trafic à destination d'une adresse locale du netns (``addrtype LOCAL`` : les gateways des
subnets, anycast compris). Elle renvoie d'abord ce qui arrive par ``br-public`` (le segment
public de l'hôte) et par ``l3-+`` (le lien vers la VRF), puis redirige 53 en UDP et TCP et 123
en UDP vers ``169.254.169.253`` ;
puis demande à ``two-hostsvc`` d'ouvrir ses sockets dans ce netns. La réponse repart par
conntrack, avec l'adresse de la gateway pour source. Une VM qui interroge un résolveur extérieur
(par une passerelle NAT) n'est pas redirigée : seules les adresses locales du netns le sont.
À la suppression du VPC, l'agent demande la fermeture des sockets ; la ``dummy`` et la chaîne
disparaissent avec le netns.
**Si ``two-hostsvc`` ne répond pas, la création du VPC échoue** (``error``) plutôt que de créer
un VPC sans DNS ni NTP ; la suppression échoue de même si la fermeture échoue, et se relance
depuis ``error``. ``two-hostsvc`` doit donc tourner sur tout hyperviseur avant la création d'un
VPC.
Le relais
---------
``two-hostsvc`` est un seul processus par hyperviseur (``two-hostsvc.service``). Il entre dans le
netns de chaque VPC le temps d'y ouvrir trois sockets sur ``169.254.169.253`` (53/UDP, 53/TCP,
123/UDP), qui restent attachées à ce netns ; les requêtes vers l'amont partent de l'espace
réseau de l'hôte.
DNS
La requête est transmise telle quelle aux serveurs de ``hostsvc.dns_upstream``, puis à ceux de
``/etc/resolv.conf`` de l'hôte. Seule une question standard est acceptée ; une réponse amont
dont l'identifiant ne correspond pas est ignorée. Pas de cache : rien de ce qu'un locataire
demande n'est gardé pour un autre.
NTP
La requête est relayée aux serveurs de ``hostsvc.ntp_upstream`` ; une réponse dont
l'horodatage d'origine ne correspond pas est ignorée. Sans serveur amont, ou sans réponse,
``two-hostsvc`` répond depuis l'horloge de l'hôte (SNTP, stratum 3) ; si l'hôte n'est pas
synchronisé, la réponse porte l'indicateur d'alarme (LI = 3, stratum 16) et les clients
l'ignorent.
Limite de débit
``hostsvc.rate_limit_per_vpc`` requêtes par seconde, DNS et NTP confondus, par VPC ; au-delà,
les requêtes sont abandonnées. Au plus 16 connexions TCP simultanées par VPC.
Le relais garde la liste des VPC servis dans ``/run/two/hostsvc/hostsvc.state`` : à son
redémarrage, il rouvre seul les sockets des VPC dont le netns existe encore. Sa socket de
contrôle, ``/run/two/hostsvc/hostsvc.sock`` (mode 600), n'est utilisée que par l'agent.
Isolation
---------
* Le relais n'écoute que sur une adresse link-local, et seulement dans les netns des VPC ; il ne
crée aucune route.
* Une réponse ne repart que vers l'adresse qui a posé la question, par la socket du VPC.
* Le trafic du segment public (``br-public``) et du lien vers la VRF n'atteint pas le relais.
* L'unité systemd ne garde que trois capacités : ``CAP_SYS_ADMIN`` (entrer dans un netns),
``CAP_NET_BIND_SERVICE`` (ports 53 et 123) et ``CAP_DAC_READ_SEARCH`` (les fichiers de
``/run/netns`` sont en mode 000). Le système de fichiers lui est en lecture seule, hors
``/run/two/hostsvc``.
* Les netns créés après le démarrage du service ne lui sont visibles que si ``/`` est en
propagation ``shared``, le réglage par défaut de systemd.
Si ``two-hostsvc`` s'arrête, DNS et NTP s'arrêtent pour tous les VPC de l'hyperviseur ;
systemd le relance (``Restart=on-failure``) et le watchdog le signale.

View file

@ -12,3 +12,4 @@ fonctionnent entre eux. Le contrat HTTP correspondant est dans
modes-reseau
cycle-de-vie
metadata-cloud-init
dns-ntp

View file

@ -24,6 +24,11 @@ netns). Ce nom de bridge est **en dur** : l'host doit porter un bridge ``br-publ
crée ``deploy.sh -i`` avec la valeur par défaut de ``-P`` (:doc:`/demarrage/installation`) ;
sans lui, toute création de VPC échoue.
Dans le netns, l'agent pose aussi la ``dummy`` ``hostsvc`` (``169.254.169.253/32``) et la chaîne
iptables ``TWO-HOSTSVC``, puis demande à ``two-hostsvc`` de servir le VPC : c'est le chemin du
DNS et du NTP des VM (voir :doc:`dns-ntp`). Si ``two-hostsvc`` ne répond pas, la création du VPC
échoue.
Un VPC ne peut être supprimé que si tous ses subnets le sont déjà — sinon 409.
Routage entre hyperviseurs : ``l3vni``

View file

@ -155,11 +155,12 @@ Vérifier l'installation
.. code-block:: bash
systemctl status agent
systemctl status two-hostsvc agent
curl -s http://127.0.0.1:8080/vpcs
Une liste JSON — vide au premier démarrage — signifie que l'API répond. Passez à
:doc:`/demarrage/premier-vpc`.
Une liste JSON — vide au premier démarrage — signifie que l'API répond. ``two-hostsvc``, le relais
DNS et NTP des VM, doit être actif : sans lui, la création d'un VPC échoue (voir
:doc:`/concepts/dns-ntp`). Passez à :doc:`/demarrage/premier-vpc`.
.. important::

View file

@ -128,11 +128,13 @@ subnets ``vxlan`` (``iface_type: vms``, clé absente) restent sur ``default_inte
ne change pas. Le bridge n'est créé qu'au premier démarrage, comme le reste du provisionnement.
**Tester l'arbre courant plutôt que la release.** ``test/e2e/build.sh`` compile ``agent``,
``dhcp`` et ``metadata`` pour linux/amd64 dans ``test/e2e/topologies/bin/`` (ignoré par git),
que les hyperviseurs reçoivent par ``files: {bin: bin}``. Après ``deploy.sh``,
``dhcp``, ``metadata`` et ``two-hostsvc`` pour linux/amd64 dans ``test/e2e/topologies/bin/``
(ignoré par git), et y copie l'unit ``two-hostsvc.service``, absente de la release 0.2.0 ; les
hyperviseurs reçoivent le répertoire par ``files: {bin: bin}``. Après ``deploy.sh``,
``hypervisor.sh`` installe chaque binaire présent dans ``/opt/two/local/bin/`` et y fait pointer
``/opt/two/bin/<nom>`` ; le reste (units, scripts) vient de la release. ``build.sh --clean`` vide
le répertoire : le lab suivant teste la release seule.
``/opt/two/bin/<nom>``, puis installe, active et redémarre chaque unit ``*.service`` présente ; le
reste (units, scripts) vient de la release. ``build.sh --clean`` vide le répertoire : le lab
suivant teste la release seule.
.. code-block:: text
@ -333,6 +335,10 @@ Générée par ``test/e2e/run.sh --matrice`` à partir des en-têtes ; à régé
- MTU des VM (network.mtu_vm, 1500 par défaut) — annoncé par le DHCP (option 26, serveur intégré sur hv1, dnsmasq sur hv2) et par QEMU (host_mtu), porté par le tap, le bridge, le veth et le VXLAN du subnet ; subnet vxlan refusé sur un underlay qui n'est pas à 9000
- hv1 hv2
- #56
* - ``29-dns-ntp``
- DNS et NTP des VM par two-hostsvc — options DHCP 6 et 42 = interface_ip, redirection vers 169.254.169.253 dans le VPC (vxlan et bridge), seuls 53 et 123 servis, alertes du watchdog, VPC en error si two-hostsvc ne répond pas
- hv1
- #86
* - ``31-gateway``
- gateway d'un subnet vxlan — gateway anycast et adresse d'émission calculées, default_route et gateway explicite, routes DHCP, macvlan gw-, FDB locale, sysctls ARP, règles de confinement, suppression et recréation
- hv1

View file

@ -277,6 +277,31 @@ La bascule est **manuelle** et suppose un hyperviseur vide :
Le retour arrière suit la même procédure. Il n'y a pas de bascule à chaud, dans un sens ni dans
l'autre.
Relais DNS et NTP
-----------------
.. code-block:: yaml
hostsvc:
dns_upstream: []
ntp_upstream: []
rate_limit_per_vpc: 100
Lue par ``two-hostsvc`` (voir :doc:`../concepts/dns-ntp`), au démarrage du service.
``dns_upstream``
résolveurs interrogés en premier, sous la forme ``ip`` ou ``ip:port`` (port 53 par défaut).
Ceux de ``/etc/resolv.conf`` de l'hôte viennent ensuite ; liste vide : l'hôte seul.
``ntp_upstream``
serveurs NTP relayés en premier, ``ip`` ou ``ip:port`` (port 123 par défaut). Sans serveur ou
sans réponse, l'horloge de l'hôte répond.
``rate_limit_per_vpc``
requêtes par seconde, DNS et NTP confondus, pour chaque VPC.
Un nom d'hôte, un port hors de 1–65535 ou une limite nulle ou négative font échouer le
démarrage de l'agent comme de ``two-hostsvc``, qui lisent le même fichier. Une modification prend
effet au redémarrage de ``two-hostsvc``.
Watchdog
--------

View file

@ -145,6 +145,41 @@ réellement écrit :
Un base64 invalide, lui, aurait été rejeté en 400 à la création.
La VM ne résout pas les noms ou n'a pas l'heure
-----------------------------------------------
La VM interroge sa gateway (``interface_ip``), que le DHCP annonce en DNS et en NTP ; la requête
est redirigée vers ``two-hostsvc`` dans le netns du VPC (voir :doc:`../concepts/dns-ntp`).
.. code-block:: bash
# Dans la VM : DNS et NTP annoncés par le bail (systemd-networkd)
sudo cat /run/systemd/netif/leases/$(cat /sys/class/net/ens3/ifindex) | grep -E '^(DNS|NTP)='
# Sur l'hyperviseur
systemctl status two-hostsvc
cat /run/two/hostsvc/hostsvc.state # le VPC doit y figurer
ip -n <vpc> -4 addr show dev hostsvc # 169.254.169.253/32
ip netns exec <vpc> iptables -t nat -S TWO-HOSTSVC # 2 RETURN puis 3 DNAT
ip netns exec <vpc> iptables -t nat -L TWO-HOSTSVC -v -n # compteurs des redirections
ip netns exec <vpc> ss -lntu | grep 169.254.169.253 # 53/udp, 53/tcp, 123/udp
journalctl -u two-hostsvc -n 50
Causes rencontrées :
* ``two-hostsvc`` arrêté : plus de DNS ni de NTP pour aucun VPC de l'hyperviseur, et un VPC créé
pendant ce temps part en ``error`` ;
* adresse ``169.254.169.253`` ou règle de la chaîne retirée à la main : la requête n'arrive plus
au relais ; le watchdog le signale ;
* VPC créé par une version antérieure à la 0.3.0 : ni ``dummy`` ni chaîne ni socket. Le recréer ;
* ``open`` refusé par ``two-hostsvc`` avec ``permission denied`` ou ``invalid argument`` sur
``/var/run/netns/<vpc>`` : l'unit n'a pas ``CAP_DAC_READ_SEARCH``, ou le netns, créé après le
démarrage du service, ne lui est pas visible parce que ``/`` n'est pas en propagation
``shared`` (``findmnt -no PROPAGATION /``). Rencontré en conteneur, pas sur un hyperviseur.
Sans serveur ``ntp_upstream`` qui réponde, l'heure vient de l'horloge de l'hyperviseur : si
celle-ci n'est pas synchronisée, la réponse porte l'indicateur d'alarme et la VM l'ignore.
La VM ne démarre pas (UEFI)
---------------------------

View file

@ -39,6 +39,16 @@ défaut** (``enabled: false``) ; l'activer explicitement :
Il contrôle notamment l'existence des network namespaces, des liens réseau des subnets, des taps
de VM et la réponse des units systemd associées.
Dès qu'un VPC est ``running`` sur l'hyperviseur, il contrôle aussi le relais DNS et NTP (voir
:doc:`../concepts/dns-ntp`) :
* ``two-hostsvc`` injoignable sur sa socket de contrôle : une notification, aucun VPC de
l'hyperviseur n'a plus de DNS ni de NTP ;
* pour chaque VPC ``running`` : VPC absent de la liste servie par ``two-hostsvc``,
``169.254.169.253`` absente de la ``dummy`` ``hostsvc``, ou chaîne ``TWO-HOSTSVC`` absente ou
incomplète dans son netns. Un VPC créé par une version antérieure à la 0.3.0 n'a rien de tout
cela : il est signalé à chaque tick jusqu'à sa recréation.
Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de contrôle :
* **Sessions BGP** : une notification par route reflector dont la session n'est pas

View file

@ -1,7 +1,7 @@
Services systemd
================
Quatre units, installées dans ``/etc/systemd/system`` par ``deploy.sh`` ; les binaires et scripts
Cinq units, installées dans ``/etc/systemd/system`` par ``deploy.sh`` ; les binaires et scripts
qu'elles lancent sont sous ``/opt/two/bin``. Les deux units DHCP
s'excluent : celle qui tourne dépend de ``dhcp.backend`` (voir :doc:`configuration`).
@ -24,6 +24,10 @@ s'excluent : celle qui tourne dépend de ``dhcp.backend`` (voir :doc:`configurat
* - ``metadata@.service``
- ``<nom de la VM>``
- serveur de metadata cloud-init, un par VM
* - ``two-hostsvc.service``
- —
- relais DNS et NTP des VM, un par hyperviseur, démarré avant l'agent (voir
:doc:`../concepts/dns-ntp`)
Les instances sont créées et pilotées par l'agent au fil des créations de subnets et de VM : il
n'y a pas à les démarrer à la main en fonctionnement normal.
@ -34,6 +38,7 @@ n'y a pas à les démarrer à la main en fonctionnement normal.
systemctl status 'dnsmasq@vp-admin_br-000001' # backend dnsmasq
systemctl status 'dhcp@vp-admin_br-000001' # backend two
systemctl status 'metadata@i-web'
systemctl status two-hostsvc
dnsmasq
-------
@ -96,6 +101,27 @@ processus serait mort sans nettoyer (``SIGKILL``, plantage).
Diagnostic : voir :doc:`diagnostic`, qui montre comment interroger la socket.
Relais DNS et NTP
-----------------
``two-hostsvc.service`` lance ``/opt/two/bin/two-hostsvc``, un seul processus par hyperviseur,
qui sert DNS et NTP aux VM de tous les VPC (voir :doc:`../concepts/dns-ntp`). Il lit la section
``hostsvc`` de ``/etc/two/agent.yml`` (voir :doc:`configuration`).
.. list-table::
:widths: 40 60
* - Socket de contrôle
- ``/run/two/hostsvc/hostsvc.sock``
* - État
- ``/run/two/hostsvc/hostsvc.state`` (VPC servis)
* - Journal
- ``journalctl -u two-hostsvc``
L'unit démarre avant ``agent.service`` et se relance en cas d'échec. Son répertoire
``/run/two/hostsvc`` survit à un redémarrage du service : le processus y relit la liste des VPC
servis et rouvre leurs sockets. Un VPC ne peut pas être créé tant que le service ne répond pas.
QEMU n'est pas une unit
-----------------------

View file

@ -0,0 +1,31 @@
package hostsvcapi
import "path/filepath"
type Verb string
const (
VerbOpen Verb = "open"
VerbClose Verb = "close"
VerbList Verb = "list"
)
const (
MaxMessageBytes = 64 * 1024
DefaultRunDir = "/run/two/hostsvc"
Unit = "two-hostsvc.service"
)
func SocketPath(runDir string) string { return filepath.Join(runDir, "hostsvc.sock") }
func StatePath(runDir string) string { return filepath.Join(runDir, "hostsvc.state") }
type Request struct {
Verb Verb `json:"verb"`
VPC string `json:"vpc,omitempty"`
}
type Response struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
VPCs []string `json:"vpcs,omitempty"`
}

View file

@ -0,0 +1,128 @@
package hostsvcapi
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"os"
"path/filepath"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
const SocketMode = 0o600
type Service interface {
Open(vpc string) error
Close(vpc string) error
List() []string
}
type Server struct {
svc Service
listener net.Listener
logger *slog.Logger
}
func Listen(svc Service, path string, logger *slog.Logger) (*Server, error) {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, statefile.DirMode); err != nil {
return nil, fmt.Errorf("create %s: %w", dir, err)
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("remove stale socket %s: %w", path, err)
}
listener, err := net.Listen("unix", path)
if err != nil {
return nil, fmt.Errorf("listen on %s: %w", path, err)
}
if err := os.Chmod(path, SocketMode); err != nil {
listener.Close()
return nil, fmt.Errorf("chmod %s: %w", path, err)
}
return &Server{svc: svc, listener: listener, logger: logger}, nil
}
func (s *Server) Addr() string {
return s.listener.Addr().String()
}
func (s *Server) Close() error {
return s.listener.Close()
}
func (s *Server) Serve() error {
for {
conn, err := s.listener.Accept()
if err != nil {
return err
}
go s.handleConn(conn)
}
}
func (s *Server) handleConn(conn net.Conn) {
defer conn.Close()
defer func() {
if r := recover(); r != nil {
s.logger.Error("control connection panicked", "panic", r)
}
}()
scanner := bufio.NewScanner(conn)
scanner.Buffer(make([]byte, 0, 4096), MaxMessageBytes)
encoder := json.NewEncoder(conn)
for scanner.Scan() {
line := scanner.Bytes()
if len(line) == 0 {
continue
}
var req Request
if err := json.Unmarshal(line, &req); err != nil {
if err := encoder.Encode(failure(fmt.Errorf("malformed request: %w", err))); err != nil {
return
}
continue
}
if err := encoder.Encode(s.dispatch(req)); err != nil {
return
}
}
if err := scanner.Err(); err != nil {
s.logger.Error("control connection read failed", "error", err)
}
}
func failure(err error) Response {
return Response{OK: false, Error: err.Error()}
}
func (s *Server) dispatch(req Request) Response {
switch req.Verb {
case VerbOpen:
if err := s.svc.Open(req.VPC); err != nil {
return failure(err)
}
return Response{OK: true}
case VerbClose:
if err := s.svc.Close(req.VPC); err != nil {
return failure(err)
}
return Response{OK: true}
case VerbList:
return Response{OK: true, VPCs: s.svc.List()}
default:
return failure(fmt.Errorf("unknown verb %q", req.Verb))
}
}

View file

@ -248,8 +248,8 @@ func TestProbe_DescribesWhatWouldBeSent(t *testing.T) {
if lease.Router != "10.0.5.254" {
t.Errorf("router = %s, want 10.0.5.254", lease.Router)
}
if len(lease.DNS) != 2 {
t.Errorf("dns = %v, want two servers", lease.DNS)
if len(lease.DNS) != 1 || lease.DNS[0] != "10.0.5.1" {
t.Errorf("dns = %v, want only the interface ip 10.0.5.1", lease.DNS)
}
if len(lease.Routes) != 3 {
t.Errorf("routes = %v, want metadata, vpc and default", lease.Routes)

View file

@ -0,0 +1,82 @@
package hostsvcclient
import (
"bufio"
"encoding/json"
"fmt"
"net"
"time"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
)
const DefaultTimeout = 5 * time.Second
type Client struct {
path string
timeout time.Duration
}
func New(path string) *Client {
return &Client{path: path, timeout: DefaultTimeout}
}
func (c *Client) WithTimeout(d time.Duration) *Client {
return &Client{path: c.path, timeout: d}
}
func (c *Client) call(req hostsvcapi.Request) (hostsvcapi.Response, error) {
conn, err := net.DialTimeout("unix", c.path, c.timeout)
if err != nil {
return hostsvcapi.Response{}, fmt.Errorf("dial %s: %w", c.path, err)
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(c.timeout)); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("set deadline on %s: %w", c.path, err)
}
raw, err := json.Marshal(req)
if err != nil {
return hostsvcapi.Response{}, fmt.Errorf("encode %s: %w", req.Verb, err)
}
if _, err := conn.Write(append(raw, '\n')); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("send %s: %w", req.Verb, err)
}
scanner := bufio.NewScanner(conn)
scanner.Buffer(make([]byte, 0, 4096), hostsvcapi.MaxMessageBytes)
if !scanner.Scan() {
if err := scanner.Err(); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("read reply to %s: %w", req.Verb, err)
}
return hostsvcapi.Response{}, fmt.Errorf("no reply to %s", req.Verb)
}
var resp hostsvcapi.Response
if err := json.Unmarshal(scanner.Bytes(), &resp); err != nil {
return hostsvcapi.Response{}, fmt.Errorf("parse reply to %s: %w", req.Verb, err)
}
if !resp.OK {
return resp, fmt.Errorf("%s refused: %s", req.Verb, resp.Error)
}
return resp, nil
}
func (c *Client) Open(vpc string) error {
_, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbOpen, VPC: vpc})
return err
}
func (c *Client) Close(vpc string) error {
_, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbClose, VPC: vpc})
return err
}
func (c *Client) List() ([]string, error) {
resp, err := c.call(hostsvcapi.Request{Verb: hostsvcapi.VerbList})
if err != nil {
return nil, err
}
return resp.VPCs, nil
}

View file

@ -0,0 +1,163 @@
package hostsvcclient
import (
"errors"
"io"
"log/slog"
"net"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
)
type fakeService struct {
mu sync.Mutex
open map[string]bool
failOn string
}
func (f *fakeService) Open(vpc string) error {
f.mu.Lock()
defer f.mu.Unlock()
if vpc == f.failOn {
return errors.New("netns not found")
}
f.open[vpc] = true
return nil
}
func (f *fakeService) Close(vpc string) error {
f.mu.Lock()
defer f.mu.Unlock()
if vpc == f.failOn {
return errors.New("close failed")
}
delete(f.open, vpc)
return nil
}
func (f *fakeService) List() []string {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for vpc := range f.open {
out = append(out, vpc)
}
return out
}
func shortTempDir(t *testing.T) string {
t.Helper()
dir, err := os.MkdirTemp("", "hostsvc")
if err != nil {
t.Fatalf("MkdirTemp: %v", err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
return dir
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
func serve(t *testing.T) (*Client, *fakeService, string) {
t.Helper()
svc := &fakeService{open: map[string]bool{}, failOn: "vpc-broken"}
socketPath := hostsvcapi.SocketPath(shortTempDir(t))
server, err := hostsvcapi.Listen(svc, socketPath, discardLogger())
if err != nil {
t.Fatalf("Listen: %v", err)
}
go server.Serve()
t.Cleanup(func() { server.Close() })
return New(socketPath), svc, socketPath
}
func TestListen_SocketIsOwnerOnly(t *testing.T) {
_, _, socketPath := serve(t)
info, err := os.Stat(socketPath)
if err != nil {
t.Fatalf("Stat: %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Errorf("mode = %o, want 600: whoever reaches it opens sockets in any vpc", got)
}
}
func TestListen_ReplacesAStaleSocketFile(t *testing.T) {
socketPath := filepath.Join(shortTempDir(t), "stale.sock")
if err := os.WriteFile(socketPath, []byte("leftover"), 0o600); err != nil {
t.Fatalf("WriteFile: %v", err)
}
server, err := hostsvcapi.Listen(&fakeService{open: map[string]bool{}}, socketPath, discardLogger())
if err != nil {
t.Fatalf("a socket left by an unclean stop must not block startup: %v", err)
}
server.Close()
}
func TestOpenCloseList(t *testing.T) {
client, svc, _ := serve(t)
if err := client.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
got, err := client.List()
if err != nil {
t.Fatalf("List: %v", err)
}
if !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("List = %v", got)
}
if err := client.Close("vpc-00001"); err != nil {
t.Fatalf("Close: %v", err)
}
if len(svc.List()) != 0 {
t.Errorf("the vpc is still open on the service: %v", svc.List())
}
}
func TestServiceErrorsAreReported(t *testing.T) {
client, _, _ := serve(t)
if err := client.Open("vpc-broken"); err == nil || !strings.Contains(err.Error(), "open refused: netns not found") {
t.Errorf("Open: error %v", err)
}
if err := client.Close("vpc-broken"); err == nil || !strings.Contains(err.Error(), "close refused: close failed") {
t.Errorf("Close: error %v", err)
}
}
func TestUnknownVerbAndMalformedRequest(t *testing.T) {
_, _, socketPath := serve(t)
conn, err := net.Dial("unix", socketPath)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
if _, err := conn.Write([]byte("{\"verb\":\"reboot\"}\nnot json\n")); err != nil {
t.Fatalf("Write: %v", err)
}
buf := make([]byte, 4096)
var all string
for !strings.Contains(all, "malformed request") {
n, err := conn.Read(buf)
if err != nil {
t.Fatalf("Read: %v (got %q)", err, all)
}
all += string(buf[:n])
}
if !strings.Contains(all, `unknown verb \"reboot\"`) {
t.Errorf("replies = %q, want the unknown verb refused", all)
}
}
func TestClient_NoServer(t *testing.T) {
client := New(filepath.Join(shortTempDir(t), "absent.sock"))
if err := client.Open("vpc-00001"); err == nil || !strings.Contains(err.Error(), "dial") {
t.Errorf("error %v, want a dial failure", err)
}
}

View file

@ -0,0 +1,79 @@
package configuration
import (
"fmt"
"net"
"strconv"
)
const (
RateLimitPerVPC = 100
DNSPort = 53
NTPPort = 123
)
type HostSvc struct {
DNSUpstream []string `mapstructure:"dns_upstream"`
NTPUpstream []string `mapstructure:"ntp_upstream"`
RateLimitPerVPC int `mapstructure:"rate_limit_per_vpc"`
}
func (h HostSvc) DNS() []string { return withPort(h.DNSUpstream, DNSPort) }
func (h HostSvc) NTP() []string { return withPort(h.NTPUpstream, NTPPort) }
func (h HostSvc) Rate() int {
if h.RateLimitPerVPC == 0 {
return RateLimitPerVPC
}
return h.RateLimitPerVPC
}
func withPort(servers []string, port int) []string {
out := make([]string, 0, len(servers))
for _, s := range servers {
if net.ParseIP(s) != nil {
out = append(out, net.JoinHostPort(s, strconv.Itoa(port)))
continue
}
out = append(out, s)
}
return out
}
func (h HostSvc) Validate() error {
if h.RateLimitPerVPC <= 0 {
return fmt.Errorf("hostsvc.rate_limit_per_vpc %d must be a positive number of requests per second", h.RateLimitPerVPC)
}
for _, list := range []struct {
key string
servers []string
}{
{"dns_upstream", h.DNSUpstream},
{"ntp_upstream", h.NTPUpstream},
} {
for _, s := range list.servers {
if err := validServer(s); err != nil {
return fmt.Errorf("hostsvc.%s: %w", list.key, err)
}
}
}
return nil
}
func validServer(s string) error {
if net.ParseIP(s) != nil {
return nil
}
host, port, err := net.SplitHostPort(s)
if err != nil {
return fmt.Errorf("%q is neither an ip nor ip:port", s)
}
if net.ParseIP(host) == nil {
return fmt.Errorf("%q: %q is not an ip address", s, host)
}
n, err := strconv.Atoi(port)
if err != nil || n < 1 || n > 65535 {
return fmt.Errorf("%q: invalid port %q", s, port)
}
return nil
}

View file

@ -0,0 +1,51 @@
package configuration
import (
"reflect"
"strings"
"testing"
)
func TestLoadConfig_HostSvcDefaults(t *testing.T) {
cfg, err := LoadConfig(writeYAML(t, ""))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if len(cfg.HostSvc.DNS()) != 0 || len(cfg.HostSvc.NTP()) != 0 {
t.Errorf("without upstreams the lists must stay empty, the host serves: %+v", cfg.HostSvc)
}
if cfg.HostSvc.Rate() != 100 {
t.Errorf("default rate per vpc = %d, want 100", cfg.HostSvc.Rate())
}
}
func TestLoadConfig_HostSvcDefaultPorts(t *testing.T) {
cfg, err := LoadConfig(writeYAML(t, "hostsvc:\n dns_upstream: [192.0.2.53, \"192.0.2.54:5353\"]\n ntp_upstream: [192.0.2.123]\n rate_limit_per_vpc: 20\n"))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if got, want := cfg.HostSvc.DNS(), []string{"192.0.2.53:53", "192.0.2.54:5353"}; !reflect.DeepEqual(got, want) {
t.Errorf("dns = %v, want %v", got, want)
}
if got, want := cfg.HostSvc.NTP(), []string{"192.0.2.123:123"}; !reflect.DeepEqual(got, want) {
t.Errorf("ntp = %v, want %v", got, want)
}
if cfg.HostSvc.Rate() != 20 {
t.Errorf("rate = %d, want 20", cfg.HostSvc.Rate())
}
}
func TestLoadConfig_HostSvcInvalidIsRefused(t *testing.T) {
for _, c := range []struct{ yaml, msg string }{
{"hostsvc:\n rate_limit_per_vpc: 0\n", "hostsvc.rate_limit_per_vpc 0 must be a positive number"},
{"hostsvc:\n rate_limit_per_vpc: -5\n", "hostsvc.rate_limit_per_vpc -5"},
{"hostsvc:\n dns_upstream: [dns.example]\n", `hostsvc.dns_upstream: "dns.example" is neither an ip nor ip:port`},
{"hostsvc:\n dns_upstream: [\"dns.example:53\"]\n", `"dns.example" is not an ip address`},
{"hostsvc:\n ntp_upstream: [\"192.0.2.1:0\"]\n", `hostsvc.ntp_upstream: "192.0.2.1:0": invalid port "0"`},
{"hostsvc:\n ntp_upstream: [\"192.0.2.1:70000\"]\n", `invalid port "70000"`},
} {
if _, err := LoadConfig(writeYAML(t, c.yaml)); err == nil || !strings.Contains(err.Error(), c.msg) {
t.Errorf("%q: error %v, want %q", c.yaml, err, c.msg)
}
}
}

View file

@ -54,6 +54,7 @@ type Config struct {
} `mapstructure:"watchdog"`
Network Network `mapstructure:"network"`
BGP BGP `mapstructure:"bgp"`
HostSvc HostSvc `mapstructure:"hostsvc"`
DefaultInterface string `mapstructure:"default_interface"`
Interfaces map[string]string `mapstructure:"interfaces"`
}
@ -79,6 +80,9 @@ func LoadConfig(path string) (*Config, error) {
v.SetDefault("times.poll_interval", PollInterval)
v.SetDefault("metadata.run_dir", "/run/two/metadata")
v.SetDefault("dhcp.backend", BackendTwo)
v.SetDefault("hostsvc.dns_upstream", []string{})
v.SetDefault("hostsvc.ntp_upstream", []string{})
v.SetDefault("hostsvc.rate_limit_per_vpc", RateLimitPerVPC)
v.SetDefault("qemu.ovmf_code_path", "/usr/share/OVMF/OVMF_CODE.fd")
v.SetDefault("qemu.ovmf_vars_template", "/usr/share/OVMF/OVMF_VARS.fd")
v.SetDefault("qemu.uefi_vars_dir", "/run/two/vms/uefi")
@ -111,6 +115,9 @@ func LoadConfig(path string) (*Config, error) {
if err := cfg.Times.Validate(); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
if err := cfg.HostSvc.Validate(); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
return &cfg, nil
}

View file

@ -69,6 +69,24 @@ func TestGenerateConfig_ContainsDefaultGateway(t *testing.T) {
}
}
func TestGenerateConfig_DNSAndNTPAreTheInterfaceIP(t *testing.T) {
conf := newConf(t, "192.168.1.0/24")
path, err := GenerateConfig(conf)
if err != nil {
t.Fatalf("GenerateConfig : %v", err)
}
content, _ := os.ReadFile(path)
for _, want := range []string{"\ndhcp-option=6,192.168.1.254\n", "\ndhcp-option=42,192.168.1.254\n"} {
if !strings.Contains(string(content), want) {
t.Errorf("%q absente : DNS et NTP passent par la gateway, relayés par two-hostsvc\n%s", strings.TrimSpace(want), content)
}
}
if strings.Contains(string(content), "1.1.1.1") || strings.Contains(string(content), "8.8.8.8") {
t.Errorf("résolveurs publics encore annoncés :\n%s", content)
}
}
func TestGenerateConfig_NoDefaultGateway(t *testing.T) {
conf := newConf(t, "192.168.1.0/29")
conf.DefaultGateway = nil

View file

@ -30,7 +30,8 @@ func GenerateConfig(c Config) (string, error) {
} else {
fmt.Fprintf(&sb, "dhcp-option=3\n")
}
fmt.Fprintf(&sb, "dhcp-option=6,1.1.1.1,8.8.8.8\n")
fmt.Fprintf(&sb, "dhcp-option=6,%s\n", c.InterfaceIP.String())
fmt.Fprintf(&sb, "dhcp-option=42,%s\n", c.InterfaceIP.String())
fmt.Fprintf(&sb, "dhcp-option=26,%d\n", c.interfaceMTU())
fmt.Fprintf(&sb, "dhcp-hostsdir=%s\n", HostsDir(c.ConfDir, c.Name))
fmt.Fprintf(&sb, "dhcp-optsdir=%s\n", OptsDir(c.ConfDir, c.Name))

View file

@ -7,13 +7,6 @@ import (
const LeaseTime = 12 * time.Hour
func DNSServers() []net.IP {
return []net.IP{
net.IPv4(1, 1, 1, 1),
net.IPv4(8, 8, 8, 8),
}
}
type SubnetConfig struct {
Network *net.IPNet
InterfaceIP net.IP

View file

@ -86,7 +86,8 @@ func BuildReply(c SubnetConfig, h Host, req *dhcpv4.DHCPv4) (*dhcpv4.DHCPv4, err
dhcpv4.WithNetmask(c.Network.Mask),
dhcpv4.WithLeaseTime(uint32(LeaseTime.Seconds())),
dhcpv4.WithOption(dhcpv4.OptServerIdentifier(c.InterfaceIP)),
dhcpv4.WithOption(dhcpv4.OptDNS(DNSServers()...)),
dhcpv4.WithOption(dhcpv4.OptDNS(c.InterfaceIP)),
dhcpv4.WithOption(dhcpv4.OptNTPServers(c.InterfaceIP)),
dhcpv4.WithOption(dhcpv4.OptClasslessStaticRoute(routes...)),
dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, dhcpv4.Uint16(c.InterfaceMTU()).ToBytes())),
}

View file

@ -244,8 +244,11 @@ func TestBuildReply_CarriesAddressMaskLeaseAndServerIdentifier(t *testing.T) {
if got := reply.ServerIdentifier(); !got.Equal(c.InterfaceIP) {
t.Errorf("server identifier = %s, want %s", got, c.InterfaceIP)
}
if got := reply.DNS(); len(got) != 2 || !got[0].Equal(net.IPv4(1, 1, 1, 1)) || !got[1].Equal(net.IPv4(8, 8, 8, 8)) {
t.Errorf("dns = %v, want 1.1.1.1 and 8.8.8.8", got)
if got := reply.DNS(); len(got) != 1 || !got[0].Equal(net.ParseIP("10.0.5.1")) {
t.Errorf("dns = %v, want only the interface ip 10.0.5.1, relayed by two-hostsvc", got)
}
if got := reply.NTPServers(); len(got) != 1 || !got[0].Equal(net.ParseIP("10.0.5.1")) {
t.Errorf("ntp = %v, want only the interface ip 10.0.5.1, relayed by two-hostsvc", got)
}
}

View file

@ -0,0 +1,23 @@
//go:build linux
package hostsvc
import (
"time"
"golang.org/x/sys/unix"
)
type SystemClock struct{}
func (SystemClock) Now() time.Time { return time.Now() }
func (SystemClock) Sync() (bool, time.Duration, error) {
var tx unix.Timex
state, err := unix.Adjtimex(&tx)
if err != nil {
return false, 0, err
}
synced := state != unix.TIME_ERROR && tx.Status&unix.STA_UNSYNC == 0
return synced, time.Duration(tx.Maxerror) * time.Microsecond, nil
}

View file

@ -0,0 +1,11 @@
//go:build !linux
package hostsvc
import "time"
type SystemClock struct{}
func (SystemClock) Now() time.Time { return time.Now() }
func (SystemClock) Sync() (bool, time.Duration, error) { return false, 0, nil }

162
internal/hostsvc/dns.go Normal file
View file

@ -0,0 +1,162 @@
package hostsvc
import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"os"
"strings"
"time"
)
const (
dnsHeaderLen = 12
maxDNSMessage = 65535
)
func validQuery(b []byte) error {
if len(b) < dnsHeaderLen {
return fmt.Errorf("dns message of %d bytes is shorter than its header", len(b))
}
if b[2]&0x80 != 0 {
return errors.New("dns message is a response, not a query")
}
if op := (b[2] >> 3) & 0x0f; op != 0 {
return fmt.Errorf("dns opcode %d is not a standard query", op)
}
if n := binary.BigEndian.Uint16(b[4:6]); n != 1 {
return fmt.Errorf("dns query carries %d questions, want 1", n)
}
return nil
}
func validAnswer(query, answer []byte) error {
if len(answer) < dnsHeaderLen {
return fmt.Errorf("dns answer of %d bytes is shorter than its header", len(answer))
}
if answer[0] != query[0] || answer[1] != query[1] {
return errors.New("dns answer id does not match the query")
}
if answer[2]&0x80 == 0 {
return errors.New("dns answer is not flagged as a response")
}
return nil
}
func resolvers(path string) ([]string, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open %s: %w", path, err)
}
defer f.Close()
var out []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 2 || fields[0] != "nameserver" || net.ParseIP(fields[1]) == nil {
continue
}
out = append(out, net.JoinHostPort(fields[1], "53"))
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("read %s: %w", path, err)
}
return out, nil
}
func exchangeUDP(servers []string, query []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeUDPOne(server, query, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no dns upstream answered: %w", errors.Join(errs...))
}
func exchangeUDPOne(server string, query []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("udp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if _, err := conn.Write(query); err != nil {
return nil, err
}
buf := make([]byte, maxDNSMessage)
for {
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
if validAnswer(query, buf[:n]) == nil {
return buf[:n], nil
}
}
}
func exchangeTCP(servers []string, query []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeTCPOne(server, query, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no dns upstream answered: %w", errors.Join(errs...))
}
func exchangeTCPOne(server string, query []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("tcp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if err := writeTCPMessage(conn, query); err != nil {
return nil, err
}
answer, err := readTCPMessage(conn)
if err != nil {
return nil, err
}
if err := validAnswer(query, answer); err != nil {
return nil, err
}
return answer, nil
}
func readTCPMessage(r io.Reader) ([]byte, error) {
var size [2]byte
if _, err := io.ReadFull(r, size[:]); err != nil {
return nil, err
}
msg := make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := io.ReadFull(r, msg); err != nil {
return nil, err
}
return msg, nil
}
func writeTCPMessage(w io.Writer, msg []byte) error {
if len(msg) > maxDNSMessage {
return fmt.Errorf("dns message of %d bytes does not fit a tcp frame", len(msg))
}
frame := make([]byte, 2+len(msg))
binary.BigEndian.PutUint16(frame, uint16(len(msg)))
copy(frame[2:], msg)
_, err := w.Write(frame)
return err
}

View file

@ -0,0 +1,49 @@
//go:build linux
package hostsvc
import (
"fmt"
"os"
"runtime"
"golang.org/x/sys/unix"
)
const netnsDir = "/var/run/netns"
func netnsExists(vpc string) bool {
_, err := os.Stat(netnsDir + "/" + vpc)
return err == nil
}
func enterNetns(vpc string, fn func() error) error {
runtime.LockOSThread()
orig, err := os.Open("/proc/thread-self/ns/net")
if err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("open the current netns: %w", err)
}
defer orig.Close()
target, err := os.Open(netnsDir + "/" + vpc)
if err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("open netns %s: %w", vpc, err)
}
defer target.Close()
if err := unix.Setns(int(target.Fd()), unix.CLONE_NEWNET); err != nil {
runtime.UnlockOSThread()
return fmt.Errorf("enter netns %s: %w", vpc, err)
}
err = fn()
if restore := unix.Setns(int(orig.Fd()), unix.CLONE_NEWNET); restore != nil {
return fmt.Errorf("leave netns %s, thread discarded: %w", vpc, restore)
}
runtime.UnlockOSThread()
return err
}

View file

@ -0,0 +1,7 @@
//go:build !linux
package hostsvc
func netnsExists(string) bool { return false }
func enterNetns(_ string, fn func() error) error { return fn() }

View file

@ -0,0 +1,56 @@
package hostsvc
import (
"bytes"
"testing"
"time"
)
func FuzzValidQuery(f *testing.F) {
f.Add(query(1))
f.Add(answerTo(query(1)))
f.Add([]byte{})
f.Fuzz(func(t *testing.T, b []byte) {
if validQuery(b) == nil && len(b) < dnsHeaderLen {
t.Fatalf("a %d-byte message passed", len(b))
}
})
}
func FuzzValidAnswer(f *testing.F) {
f.Add(query(1), answerTo(query(1)))
f.Add(query(1), []byte{1})
f.Fuzz(func(t *testing.T, q, a []byte) {
if validQuery(q) != nil {
return
}
if validAnswer(q, a) == nil && (len(a) < dnsHeaderLen || a[0] != q[0] || a[1] != q[1]) {
t.Fatalf("answer %x passed for query %x", a, q)
}
})
}
func FuzzReadTCPMessage(f *testing.F) {
f.Add([]byte{0, 3, 'a', 'b', 'c'})
f.Add([]byte{0xff, 0xff})
f.Fuzz(func(t *testing.T, b []byte) {
msg, err := readTCPMessage(bytes.NewReader(b))
if err == nil && len(msg) != int(b[0])<<8|int(b[1]) {
t.Fatalf("message of %d bytes for prefix %x", len(msg), b[:2])
}
})
}
func FuzzHostReply(f *testing.F) {
f.Add(ntpRequest(4, 1))
f.Add([]byte("short"))
f.Fuzz(func(t *testing.T, b []byte) {
if validNTPRequest(b) != nil {
return
}
r := hostReply(b, time.Unix(0, 0), time.Unix(1, 0), true, 0)
if err := validNTPAnswer(b, r); err != nil {
t.Fatalf("own reply rejected: %v", err)
}
})
}

View file

@ -0,0 +1,34 @@
package hostsvc
import (
"sync"
"time"
)
type limiter struct {
mu sync.Mutex
rate float64
tokens float64
last time.Time
now func() time.Time
}
func newLimiter(rate int, now func() time.Time) *limiter {
return &limiter{rate: float64(rate), tokens: float64(rate), last: now(), now: now}
}
func (l *limiter) allow() bool {
l.mu.Lock()
defer l.mu.Unlock()
t := l.now()
l.tokens += t.Sub(l.last).Seconds() * l.rate
if l.tokens > l.rate {
l.tokens = l.rate
}
l.last = t
if l.tokens < 1 {
return false
}
l.tokens--
return true
}

126
internal/hostsvc/ntp.go Normal file
View file

@ -0,0 +1,126 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"net"
"time"
)
const (
ntpPacketLen = 48
ntpModeClient = 3
ntpModeServer = 4
ntpUnsynced = 3
ntpStratumHost = 3
ntpStratumNone = 16
ntpPrecision = -20
ntpEpochOffset = 2208988800
)
var ntpRefID = [4]byte{'L', 'O', 'C', 'L'}
type Clock interface {
Now() time.Time
Sync() (synced bool, maxError time.Duration, err error)
}
func validNTPRequest(b []byte) error {
if len(b) < ntpPacketLen {
return fmt.Errorf("ntp packet of %d bytes is shorter than %d", len(b), ntpPacketLen)
}
if mode := b[0] & 0x07; mode != ntpModeClient {
return fmt.Errorf("ntp mode %d is not a client request", mode)
}
return nil
}
func validNTPAnswer(request, answer []byte) error {
if len(answer) < ntpPacketLen {
return fmt.Errorf("ntp answer of %d bytes is shorter than %d", len(answer), ntpPacketLen)
}
if mode := answer[0] & 0x07; mode != ntpModeServer {
return fmt.Errorf("ntp answer mode %d is not a server reply", mode)
}
if !bytes.Equal(answer[24:32], request[40:48]) {
return errors.New("ntp answer origin timestamp does not match the request")
}
return nil
}
func ntpTimestamp(t time.Time) uint64 {
secs := uint64(t.Unix() + ntpEpochOffset)
frac := (uint64(t.Nanosecond()) << 32) / uint64(time.Second)
return secs<<32 | frac
}
func ntpShort(d time.Duration) uint32 {
if d < 0 {
d = 0
}
secs := uint64(d / time.Second)
if secs > 0xffff {
return 0xffffffff
}
frac := (uint64(d%time.Second) << 16) / uint64(time.Second)
return uint32(secs<<16 | frac)
}
func hostReply(request []byte, received, sent time.Time, synced bool, maxError time.Duration) []byte {
reply := make([]byte, ntpPacketLen)
version := (request[0] >> 3) & 0x07
leap, stratum := byte(0), byte(ntpStratumHost)
if !synced {
leap, stratum = ntpUnsynced, ntpStratumNone
}
reply[0] = leap<<6 | version<<3 | ntpModeServer
reply[1] = stratum
reply[2] = request[2]
precision := int8(ntpPrecision)
reply[3] = byte(precision)
binary.BigEndian.PutUint32(reply[8:12], ntpShort(maxError))
copy(reply[12:16], ntpRefID[:])
binary.BigEndian.PutUint64(reply[16:24], ntpTimestamp(received))
copy(reply[24:32], request[40:48])
binary.BigEndian.PutUint64(reply[32:40], ntpTimestamp(received))
binary.BigEndian.PutUint64(reply[40:48], ntpTimestamp(sent))
return reply
}
func exchangeNTP(servers []string, request []byte, timeout time.Duration) ([]byte, error) {
var errs []error
for _, server := range servers {
answer, err := exchangeNTPOne(server, request, timeout)
if err == nil {
return answer, nil
}
errs = append(errs, fmt.Errorf("%s: %w", server, err))
}
return nil, fmt.Errorf("no ntp upstream answered: %w", errors.Join(errs...))
}
func exchangeNTPOne(server string, request []byte, timeout time.Duration) ([]byte, error) {
conn, err := net.DialTimeout("udp", server, timeout)
if err != nil {
return nil, err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, err
}
if _, err := conn.Write(request[:ntpPacketLen]); err != nil {
return nil, err
}
buf := make([]byte, 1024)
for {
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
if validNTPAnswer(request, buf[:n]) == nil {
return buf[:n], nil
}
}
}

View file

@ -0,0 +1,234 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func query(id uint16) []byte {
q := make([]byte, dnsHeaderLen, dnsHeaderLen+17)
binary.BigEndian.PutUint16(q[0:2], id)
q[2] = 0x01
binary.BigEndian.PutUint16(q[4:6], 1)
q = append(q, 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', 3, 'o', 'r', 'g', 0, 0, 1, 0, 1)
return q
}
func answerTo(q []byte) []byte {
a := append([]byte{}, q...)
a[2] |= 0x80
return a
}
func ntpRequest(version byte, transmit uint64) []byte {
r := make([]byte, ntpPacketLen)
r[0] = version<<3 | ntpModeClient
r[2] = 6
binary.BigEndian.PutUint64(r[40:48], transmit)
return r
}
func TestValidQuery(t *testing.T) {
if err := validQuery(query(7)); err != nil {
t.Fatalf("a standard query must pass: %v", err)
}
response := answerTo(query(7))
opcode := query(7)
opcode[2] |= 2 << 3
twoQuestions := query(7)
binary.BigEndian.PutUint16(twoQuestions[4:6], 2)
noQuestion := query(7)
binary.BigEndian.PutUint16(noQuestion[4:6], 0)
for _, c := range []struct {
name string
msg []byte
want string
}{
{"short", query(7)[:11], "11 bytes is shorter than its header"},
{"response", response, "is a response, not a query"},
{"opcode", opcode, "opcode 2 is not a standard query"},
{"two questions", twoQuestions, "carries 2 questions, want 1"},
{"no question", noQuestion, "carries 0 questions, want 1"},
} {
if err := validQuery(c.msg); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: error %v, want %q", c.name, err, c.want)
}
}
}
func TestValidAnswer(t *testing.T) {
q := query(0x1234)
if err := validAnswer(q, answerTo(q)); err != nil {
t.Fatalf("a matching answer must pass: %v", err)
}
if err := validAnswer(q, answerTo(query(0x1235))); err == nil || !strings.Contains(err.Error(), "id does not match") {
t.Errorf("another id: error %v", err)
}
if err := validAnswer(q, q); err == nil || !strings.Contains(err.Error(), "not flagged as a response") {
t.Errorf("a query echoed back: error %v", err)
}
if err := validAnswer(q, q[:5]); err == nil || !strings.Contains(err.Error(), "shorter than its header") {
t.Errorf("short: error %v", err)
}
}
func TestResolvers(t *testing.T) {
path := filepath.Join(t.TempDir(), "resolv.conf")
content := "# comment\nsearch example.org\nnameserver 192.0.2.1\nnameserver 2001:db8::1\nnameserver not-an-ip\nnameserver\noptions edns0\n"
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
got, err := resolvers(path)
if err != nil {
t.Fatalf("resolvers: %v", err)
}
if want := []string{"192.0.2.1:53", "[2001:db8::1]:53"}; !reflect.DeepEqual(got, want) {
t.Errorf("resolvers = %v, want %v", got, want)
}
if _, err := resolvers(filepath.Join(t.TempDir(), "absent")); err == nil {
t.Error("a missing resolv.conf must be reported")
}
}
func TestTCPMessageRoundTrip(t *testing.T) {
var buf bytes.Buffer
if err := writeTCPMessage(&buf, query(9)); err != nil {
t.Fatalf("writeTCPMessage: %v", err)
}
if got := binary.BigEndian.Uint16(buf.Bytes()[:2]); int(got) != len(query(9)) {
t.Errorf("length prefix = %d, want %d", got, len(query(9)))
}
msg, err := readTCPMessage(&buf)
if err != nil {
t.Fatalf("readTCPMessage: %v", err)
}
if !bytes.Equal(msg, query(9)) {
t.Errorf("message changed on the way: %x", msg)
}
if err := writeTCPMessage(&buf, make([]byte, maxDNSMessage+1)); err == nil {
t.Error("a message larger than a tcp frame must be refused")
}
}
func TestValidNTPRequest(t *testing.T) {
if err := validNTPRequest(ntpRequest(4, 1)); err != nil {
t.Fatalf("a client request must pass: %v", err)
}
server := ntpRequest(4, 1)
server[0] = 4<<3 | ntpModeServer
if err := validNTPRequest(server); err == nil || !strings.Contains(err.Error(), "mode 4 is not a client request") {
t.Errorf("server mode: error %v", err)
}
if err := validNTPRequest(ntpRequest(4, 1)[:47]); err == nil || !strings.Contains(err.Error(), "47 bytes is shorter than 48") {
t.Errorf("short: error %v", err)
}
}
func TestValidNTPAnswer(t *testing.T) {
req := ntpRequest(4, 0xdeadbeef)
ans := hostReply(req, time.Unix(0, 0), time.Unix(0, 0), true, 0)
if err := validNTPAnswer(req, ans); err != nil {
t.Fatalf("a matching answer must pass: %v", err)
}
if err := validNTPAnswer(ntpRequest(4, 0xdeadbeee), ans); err == nil || !strings.Contains(err.Error(), "origin timestamp") {
t.Errorf("another request: error %v", err)
}
if err := validNTPAnswer(req, req); err == nil || !strings.Contains(err.Error(), "mode 3 is not a server reply") {
t.Errorf("a request echoed back: error %v", err)
}
}
func TestNTPTimestamp(t *testing.T) {
if got := ntpTimestamp(time.Unix(0, 0)); got != 2208988800<<32 {
t.Errorf("unix epoch = %#x, want %#x", got, uint64(2208988800)<<32)
}
if got := ntpTimestamp(time.Unix(1, 500_000_000)); got != (2208988801<<32 | 0x80000000) {
t.Errorf("half a second = %#x", got)
}
if got := ntpShort(1500 * time.Millisecond); got != 0x00018000 {
t.Errorf("ntpShort(1.5 s) = %#x, want 0x18000", got)
}
if got := ntpShort(-time.Second); got != 0 {
t.Errorf("ntpShort of a negative duration = %#x, want 0", got)
}
}
func TestHostReply_Synced(t *testing.T) {
req := ntpRequest(3, 0x0102030405060708)
received := time.Unix(1000, 0)
sent := time.Unix(1001, 0)
r := hostReply(req, received, sent, true, 500*time.Millisecond)
if r[0] != 0<<6|3<<3|4 {
t.Errorf("li/vn/mode = %#08b, want li 0, the client version 3, mode 4", r[0])
}
if r[1] != 3 {
t.Errorf("stratum = %d, want 3", r[1])
}
if r[2] != 6 {
t.Errorf("poll = %d, want the client's 6", r[2])
}
if int8(r[3]) != -20 {
t.Errorf("precision = %d, want -20", int8(r[3]))
}
if got := binary.BigEndian.Uint32(r[8:12]); got != 0x8000 {
t.Errorf("root dispersion = %#x, want 0.5 s", got)
}
if string(r[12:16]) != "LOCL" {
t.Errorf("reference id = %q, want LOCL", r[12:16])
}
if got := binary.BigEndian.Uint64(r[24:32]); got != 0x0102030405060708 {
t.Errorf("origin = %#x, want the client's transmit timestamp", got)
}
if got := binary.BigEndian.Uint64(r[32:40]); got != ntpTimestamp(received) {
t.Errorf("receive = %#x, want %#x", got, ntpTimestamp(received))
}
if got := binary.BigEndian.Uint64(r[40:48]); got != ntpTimestamp(sent) {
t.Errorf("transmit = %#x, want %#x", got, ntpTimestamp(sent))
}
}
func TestHostReply_UnsyncedRaisesTheAlarm(t *testing.T) {
r := hostReply(ntpRequest(4, 1), time.Unix(0, 0), time.Unix(0, 0), false, 0)
if li := r[0] >> 6; li != 3 {
t.Errorf("leap indicator = %d, want 3: an unsynchronized host must not be trusted", li)
}
if r[1] != 16 {
t.Errorf("stratum = %d, want 16 (unsynchronized)", r[1])
}
}
func TestLimiter(t *testing.T) {
now := time.Unix(0, 0)
l := newLimiter(3, func() time.Time { return now })
for i := 0; i < 3; i++ {
if !l.allow() {
t.Fatalf("request %d refused inside the burst", i+1)
}
}
if l.allow() {
t.Fatal("the burst is exhausted, the next request must be refused")
}
now = now.Add(400 * time.Millisecond)
if !l.allow() {
t.Fatal("400 ms refill 1.2 tokens at 3 per second, one request must pass")
}
if l.allow() {
t.Fatal("only one token was refilled")
}
now = now.Add(time.Hour)
for i := 0; i < 3; i++ {
if !l.allow() {
t.Fatalf("after a long pause, request %d refused: the bucket holds 3", i+1)
}
}
if l.allow() {
t.Fatal("the bucket must not grow beyond its rate")
}
}

362
internal/hostsvc/service.go Normal file
View file

@ -0,0 +1,362 @@
package hostsvc
import (
"errors"
"fmt"
"io"
"log/slog"
"net"
"sort"
"strings"
"sync"
"time"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
const (
ServiceIP = "169.254.169.253"
DefaultResolv = "/etc/resolv.conf"
DefaultTimeout = 2 * time.Second
tcpIdle = 10 * time.Second
maxTCPConns = 16
)
type Options struct {
DNS []string
NTP []string
Rate int
Resolv string
DNSListen string
NTPListen string
Timeout time.Duration
Clock Clock
Enter func(vpc string, fn func() error) error
Exists func(vpc string) bool
Logger *slog.Logger
}
func (o Options) withDefaults() Options {
if o.Resolv == "" {
o.Resolv = DefaultResolv
}
if o.DNSListen == "" {
o.DNSListen = net.JoinHostPort(ServiceIP, "53")
}
if o.NTPListen == "" {
o.NTPListen = net.JoinHostPort(ServiceIP, "123")
}
if o.Timeout == 0 {
o.Timeout = DefaultTimeout
}
if o.Clock == nil {
o.Clock = SystemClock{}
}
if o.Enter == nil {
o.Enter = enterNetns
}
if o.Exists == nil {
o.Exists = netnsExists
}
if o.Logger == nil {
o.Logger = slog.New(slog.NewTextHandler(io.Discard, nil))
}
return o
}
type State struct {
VPCs []string `json:"vpcs"`
}
type Service struct {
opts Options
state *statefile.File[State]
mu sync.Mutex
vpcs map[string]*instance
}
func New(opts Options, statePath string) (*Service, error) {
if opts.Rate <= 0 {
return nil, fmt.Errorf("rate limit %d must be a positive number of requests per second", opts.Rate)
}
return &Service{
opts: opts.withDefaults(),
state: statefile.New[State](statePath),
vpcs: map[string]*instance{},
}, nil
}
func validVPC(vpc string) error {
if vpc == "" || vpc == "." || vpc == ".." || strings.ContainsAny(vpc, "/\x00") {
return fmt.Errorf("invalid vpc name %q", vpc)
}
return nil
}
func (s *Service) Restore() error {
saved, err := s.state.Load()
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
var errs []error
for _, vpc := range saved.VPCs {
if !s.opts.Exists(vpc) {
s.opts.Logger.Info("netns gone, vpc dropped from the state", "vpc", vpc)
continue
}
if err := s.open(vpc); err != nil {
errs = append(errs, err)
}
}
if err := s.save(); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func (s *Service) Open(vpc string) error {
if err := validVPC(vpc); err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if _, open := s.vpcs[vpc]; open {
return nil
}
if err := s.open(vpc); err != nil {
return err
}
return s.save()
}
func (s *Service) Close(vpc string) error {
if err := validVPC(vpc); err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
inst, open := s.vpcs[vpc]
if !open {
return nil
}
delete(s.vpcs, vpc)
err := inst.close()
return errors.Join(err, s.save())
}
func (s *Service) List() []string {
s.mu.Lock()
defer s.mu.Unlock()
return s.names()
}
func (s *Service) Shutdown() error {
s.mu.Lock()
defer s.mu.Unlock()
var errs []error
for vpc, inst := range s.vpcs {
errs = append(errs, inst.close())
delete(s.vpcs, vpc)
}
return errors.Join(errs...)
}
func (s *Service) names() []string {
out := make([]string, 0, len(s.vpcs))
for vpc := range s.vpcs {
out = append(out, vpc)
}
sort.Strings(out)
return out
}
func (s *Service) save() error {
return s.state.Save(State{VPCs: s.names()})
}
func (s *Service) open(vpc string) error {
inst := &instance{
vpc: vpc,
svc: s,
limit: newLimiter(s.opts.Rate, time.Now),
conns: make(chan struct{}, maxTCPConns),
log: s.opts.Logger.With("vpc", vpc),
}
dns, ntp := s.opts.DNSListen, s.opts.NTPListen
err := s.opts.Enter(vpc, func() error {
var err error
if inst.udpDNS, err = net.ListenPacket("udp4", dns); err != nil {
return fmt.Errorf("listen udp %s: %w", dns, err)
}
if inst.tcpDNS, err = net.Listen("tcp4", dns); err != nil {
return fmt.Errorf("listen tcp %s: %w", dns, err)
}
if inst.udpNTP, err = net.ListenPacket("udp4", ntp); err != nil {
return fmt.Errorf("listen udp %s: %w", ntp, err)
}
return nil
})
if err != nil {
return errors.Join(fmt.Errorf("open vpc %s: %w", vpc, err), inst.close())
}
inst.serve()
s.vpcs[vpc] = inst
inst.log.Info("vpc served", "dns", dns, "ntp", ntp)
return nil
}
func (s *Service) dnsServers() []string {
servers := append([]string{}, s.opts.DNS...)
host, err := resolvers(s.opts.Resolv)
if err != nil {
s.opts.Logger.Warn("host resolver unreadable", "error", err)
}
return append(servers, host...)
}
func (s *Service) ntpAnswer(request []byte, received time.Time) []byte {
if len(s.opts.NTP) > 0 {
answer, err := exchangeNTP(s.opts.NTP, request, s.opts.Timeout)
if err == nil {
return answer
}
s.opts.Logger.Warn("ntp upstreams failed, answering from the host clock", "error", err)
}
synced, maxError, err := s.opts.Clock.Sync()
if err != nil {
s.opts.Logger.Warn("host clock status unreadable, answering as unsynchronized", "error", err)
synced = false
}
return hostReply(request, received, s.opts.Clock.Now(), synced, maxError)
}
type instance struct {
vpc string
svc *Service
limit *limiter
conns chan struct{}
log *slog.Logger
udpDNS net.PacketConn
tcpDNS net.Listener
udpNTP net.PacketConn
wg sync.WaitGroup
}
func (i *instance) serve() {
i.wg.Add(3)
go i.loopUDP(i.udpDNS, i.answerDNS)
go i.loopUDP(i.udpNTP, i.answerNTP)
go i.loopTCP()
}
func (i *instance) close() error {
var errs []error
for _, c := range []interface{ Close() error }{i.udpDNS, i.tcpDNS, i.udpNTP} {
if c == nil {
continue
}
if err := c.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
errs = append(errs, err)
}
}
i.wg.Wait()
return errors.Join(errs...)
}
func (i *instance) loopUDP(conn net.PacketConn, answer func([]byte, time.Time) ([]byte, error)) {
defer i.wg.Done()
buf := make([]byte, maxDNSMessage)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
if !errors.Is(err, net.ErrClosed) {
i.log.Error("udp socket stopped", "error", err)
}
return
}
received := i.svc.opts.Clock.Now()
if !i.limit.allow() {
i.log.Debug("rate limit reached, request dropped", "from", from)
continue
}
request := append([]byte{}, buf[:n]...)
go func() {
reply, err := answer(request, received)
if err != nil {
i.log.Debug("request dropped", "from", from, "error", err)
return
}
if _, err := conn.WriteTo(reply, from); err != nil && !errors.Is(err, net.ErrClosed) {
i.log.Debug("reply not sent", "to", from, "error", err)
}
}()
}
}
func (i *instance) answerDNS(query []byte, _ time.Time) ([]byte, error) {
if err := validQuery(query); err != nil {
return nil, err
}
return exchangeUDP(i.svc.dnsServers(), query, i.svc.opts.Timeout)
}
func (i *instance) answerNTP(request []byte, received time.Time) ([]byte, error) {
if err := validNTPRequest(request); err != nil {
return nil, err
}
return i.svc.ntpAnswer(request, received), nil
}
func (i *instance) loopTCP() {
defer i.wg.Done()
for {
conn, err := i.tcpDNS.Accept()
if err != nil {
if !errors.Is(err, net.ErrClosed) {
i.log.Error("tcp socket stopped", "error", err)
}
return
}
select {
case i.conns <- struct{}{}:
go func() {
defer func() { <-i.conns }()
i.serveTCP(conn)
}()
default:
i.log.Debug("too many tcp connections, connection refused", "from", conn.RemoteAddr())
conn.Close()
}
}
}
func (i *instance) serveTCP(conn net.Conn) {
defer conn.Close()
for {
if err := conn.SetDeadline(time.Now().Add(tcpIdle)); err != nil {
return
}
query, err := readTCPMessage(conn)
if err != nil {
return
}
if !i.limit.allow() {
i.log.Debug("rate limit reached, connection closed", "from", conn.RemoteAddr())
return
}
if err := validQuery(query); err != nil {
i.log.Debug("request dropped", "from", conn.RemoteAddr(), "error", err)
return
}
answer, err := exchangeTCP(i.svc.dnsServers(), query, i.svc.opts.Timeout)
if err != nil {
i.log.Debug("request dropped", "from", conn.RemoteAddr(), "error", err)
return
}
if err := writeTCPMessage(conn, answer); err != nil {
return
}
}
}

View file

@ -0,0 +1,538 @@
package hostsvc
import (
"bytes"
"encoding/binary"
"errors"
"net"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"git.g3e.fr/syonad/two/pkg/db/statefile"
)
type fakeClock struct {
now time.Time
synced bool
maxError time.Duration
err error
}
func (c fakeClock) Now() time.Time { return c.now }
func (c fakeClock) Sync() (bool, time.Duration, error) {
return c.synced, c.maxError, c.err
}
type netnsRecorder struct {
mu sync.Mutex
entered []string
fail error
}
func (r *netnsRecorder) enter(vpc string, fn func() error) error {
r.mu.Lock()
r.entered = append(r.entered, vpc)
r.mu.Unlock()
if err := fn(); err != nil {
return err
}
return r.fail
}
func (r *netnsRecorder) calls() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string{}, r.entered...)
}
func emptyResolv(t *testing.T) string {
t.Helper()
path := filepath.Join(t.TempDir(), "resolv.conf")
if err := os.WriteFile(path, nil, 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
return path
}
func newService(t *testing.T, opts Options) (*Service, *netnsRecorder, string) {
t.Helper()
rec := &netnsRecorder{}
opts.DNSListen = "127.0.0.1:0"
opts.NTPListen = "127.0.0.1:0"
if opts.Rate == 0 {
opts.Rate = 100
}
if opts.Resolv == "" {
opts.Resolv = emptyResolv(t)
}
if opts.Clock == nil {
opts.Clock = fakeClock{now: time.Unix(2000, 0), synced: true}
}
if opts.Timeout == 0 {
opts.Timeout = 300 * time.Millisecond
}
opts.Enter = rec.enter
if opts.Exists == nil {
opts.Exists = func(string) bool { return true }
}
statePath := filepath.Join(t.TempDir(), "hostsvc.state")
s, err := New(opts, statePath)
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(func() { s.Shutdown() })
return s, rec, statePath
}
func (s *Service) addrs(t *testing.T, vpc string) (dnsUDP, dnsTCP, ntp string) {
t.Helper()
s.mu.Lock()
defer s.mu.Unlock()
inst, ok := s.vpcs[vpc]
if !ok {
t.Fatalf("vpc %s is not open", vpc)
}
return inst.udpDNS.LocalAddr().String(), inst.tcpDNS.Addr().String(), inst.udpNTP.LocalAddr().String()
}
func savedVPCs(t *testing.T, path string) []string {
t.Helper()
st, err := statefile.New[State](path).Load()
if err != nil {
t.Fatalf("load state: %v", err)
}
return st.VPCs
}
func udpUpstream(t *testing.T, reply func(req []byte) [][]byte) (string, func() int) {
t.Helper()
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("ListenPacket: %v", err)
}
t.Cleanup(func() { conn.Close() })
var mu sync.Mutex
count := 0
go func() {
buf := make([]byte, 65535)
for {
n, from, err := conn.ReadFrom(buf)
if err != nil {
return
}
mu.Lock()
count++
mu.Unlock()
for _, r := range reply(append([]byte{}, buf[:n]...)) {
conn.WriteTo(r, from)
}
}
}()
return conn.LocalAddr().String(), func() int {
mu.Lock()
defer mu.Unlock()
return count
}
}
func tcpUpstream(t *testing.T) string {
t.Helper()
l, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("Listen: %v", err)
}
t.Cleanup(func() { l.Close() })
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
q, err := readTCPMessage(conn)
if err != nil {
return
}
writeTCPMessage(conn, answerTo(q))
}()
}
}()
return l.Addr().String()
}
func deadAddr(t *testing.T) string {
t.Helper()
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("ListenPacket: %v", err)
}
addr := conn.LocalAddr().String()
conn.Close()
return addr
}
func ask(t *testing.T, addr string, msg []byte, wait time.Duration) ([]byte, error) {
t.Helper()
conn, err := net.Dial("udp4", addr)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(wait))
if _, err := conn.Write(msg); err != nil {
t.Fatalf("Write: %v", err)
}
buf := make([]byte, 65535)
n, err := conn.Read(buf)
if err != nil {
return nil, err
}
return buf[:n], nil
}
func TestNew_RefusesANullRate(t *testing.T) {
if _, err := New(Options{Rate: 0}, filepath.Join(t.TempDir(), "s")); err == nil || !strings.Contains(err.Error(), "rate limit 0") {
t.Errorf("error %v: a null rate would silently drop every request", err)
}
}
func TestOpen_EntersTheVPCNetnsAndSavesIt(t *testing.T) {
s, rec, statePath := newService(t, Options{})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
if got := rec.calls(); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("netns entered = %v, want the vpc's", got)
}
if got := savedVPCs(t, statePath); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("state = %v", got)
}
if got := s.List(); !reflect.DeepEqual(got, []string{"vpc-00001"}) {
t.Errorf("List = %v", got)
}
}
func TestOpen_IsIdempotent(t *testing.T) {
s, rec, _ := newService(t, Options{})
for i := 0; i < 2; i++ {
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open %d: %v", i+1, err)
}
}
if got := rec.calls(); len(got) != 1 {
t.Errorf("netns entered %d times, want once: a second open must not bind again", len(got))
}
}
func TestOpen_InvalidNamesAreRefusedWithoutEnteringAnything(t *testing.T) {
s, rec, statePath := newService(t, Options{})
for _, name := range []string{"", ".", "..", "../etc", "a/b", "x\x00y"} {
if err := s.Open(name); err == nil || !strings.Contains(err.Error(), "invalid vpc name") {
t.Errorf("Open(%q): error %v", name, err)
}
if err := s.Close(name); err == nil || !strings.Contains(err.Error(), "invalid vpc name") {
t.Errorf("Close(%q): error %v", name, err)
}
}
if got := rec.calls(); len(got) != 0 {
t.Errorf("netns entered for an invalid name: %v", got)
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
}
func TestOpen_FailureLeavesNothingBehind(t *testing.T) {
s, rec, statePath := newService(t, Options{})
rec.fail = errors.New("boom")
if err := s.Open("vpc-00001"); err == nil || !strings.Contains(err.Error(), "open vpc vpc-00001: boom") {
t.Fatalf("error %v", err)
}
if got := s.List(); len(got) != 0 {
t.Errorf("List = %v, want empty after a failed open", got)
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
}
func TestClose_StopsServingAndForgetsTheVPC(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, statePath := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, query(1), time.Second); err != nil {
t.Fatalf("witness: the open vpc must answer: %v", err)
}
if err := s.Close("vpc-00001"); err != nil {
t.Fatalf("Close: %v", err)
}
if _, err := ask(t, dns, query(2), 300*time.Millisecond); err == nil {
t.Error("a closed vpc still answers")
}
if got := savedVPCs(t, statePath); len(got) != 0 {
t.Errorf("state = %v, want empty", got)
}
if err := s.Close("vpc-00001"); err != nil {
t.Errorf("closing twice must succeed: %v", err)
}
}
func TestRestore_ReopensOnlyTheVPCsWhoseNetnsExists(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "hostsvc.state")
if err := statefile.New[State](statePath).Save(State{VPCs: []string{"vpc-gone", "vpc-here"}}); err != nil {
t.Fatalf("Save: %v", err)
}
rec := &netnsRecorder{}
s, err := New(Options{
Rate: 10, DNSListen: "127.0.0.1:0", NTPListen: "127.0.0.1:0", Resolv: emptyResolv(t),
Enter: rec.enter,
Exists: func(vpc string) bool { return vpc == "vpc-here" },
}, statePath)
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(func() { s.Shutdown() })
if err := s.Restore(); err != nil {
t.Fatalf("Restore: %v", err)
}
if got := rec.calls(); !reflect.DeepEqual(got, []string{"vpc-here"}) {
t.Errorf("netns entered = %v, want only vpc-here", got)
}
if got := savedVPCs(t, statePath); !reflect.DeepEqual(got, []string{"vpc-here"}) {
t.Errorf("state = %v, the vanished vpc must be dropped", got)
}
}
func TestDNS_UDPQueryIsRelayedToTheUpstream(t *testing.T) {
var got []byte
var mu sync.Mutex
up, _ := udpUpstream(t, func(q []byte) [][]byte {
mu.Lock()
got = q
mu.Unlock()
return [][]byte{answerTo(q)}
})
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
reply, err := ask(t, dns, query(0x4242), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if !bytes.Equal(reply, answerTo(query(0x4242))) {
t.Errorf("reply = %x, want the upstream's answer untouched", reply)
}
mu.Lock()
defer mu.Unlock()
if !bytes.Equal(got, query(0x4242)) {
t.Errorf("upstream received %x, want the query untouched", got)
}
}
func TestDNS_SpoofedAnswerIsIgnored(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte {
bad := answerTo(q)
bad[0] ^= 0xff
return [][]byte{bad, answerTo(q)}
})
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
reply, err := ask(t, dns, query(0x1000), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if binary.BigEndian.Uint16(reply[0:2]) != 0x1000 {
t.Errorf("reply id = %#x, the answer with a foreign id must be skipped", binary.BigEndian.Uint16(reply[0:2]))
}
}
func TestDNS_NextUpstreamWhenTheFirstIsDown(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{deadAddr(t), up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, query(3), 2*time.Second); err != nil {
t.Fatalf("the second upstream must answer: %v", err)
}
}
func TestDNS_HostResolverComesAfterTheConfiguredUpstreams(t *testing.T) {
path := filepath.Join(t.TempDir(), "resolv.conf")
if err := os.WriteFile(path, []byte("nameserver 192.0.2.1\n"), 0o644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
s, _, _ := newService(t, Options{DNS: []string{"198.51.100.1:53"}, Resolv: path})
if got, want := s.dnsServers(), []string{"198.51.100.1:53", "192.0.2.1:53"}; !reflect.DeepEqual(got, want) {
t.Errorf("dnsServers = %v, want %v", got, want)
}
}
func TestDNS_InvalidQueryGetsNoAnswerAndIsNotRelayed(t *testing.T) {
up, count := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
dns, _, _ := s.addrs(t, "vpc-00001")
if _, err := ask(t, dns, answerTo(query(5)), 300*time.Millisecond); err == nil {
t.Error("a response sent to the relay must not be answered")
}
if _, err := ask(t, dns, query(6), time.Second); err != nil {
t.Fatalf("witness: a valid query must be answered: %v", err)
}
if n := count(); n != 1 {
t.Errorf("upstream saw %d messages, want only the valid one", n)
}
}
func TestDNS_TCPQueryIsRelayed(t *testing.T) {
s, _, _ := newService(t, Options{DNS: []string{tcpUpstream(t)}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, tcp, _ := s.addrs(t, "vpc-00001")
conn, err := net.Dial("tcp4", tcp)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
conn.SetDeadline(time.Now().Add(2 * time.Second))
for id := uint16(1); id <= 2; id++ {
if err := writeTCPMessage(conn, query(id)); err != nil {
t.Fatalf("write: %v", err)
}
reply, err := readTCPMessage(conn)
if err != nil {
t.Fatalf("query %d: no answer: %v", id, err)
}
if !bytes.Equal(reply, answerTo(query(id))) {
t.Errorf("query %d: reply %x", id, reply)
}
}
}
func TestDNS_RateLimitPerVPC(t *testing.T) {
up, _ := udpUpstream(t, func(q []byte) [][]byte { return [][]byte{answerTo(q)} })
s, _, _ := newService(t, Options{DNS: []string{up}, Rate: 2})
for _, vpc := range []string{"vpc-00001", "vpc-00002"} {
if err := s.Open(vpc); err != nil {
t.Fatalf("Open %s: %v", vpc, err)
}
}
dns1, _, _ := s.addrs(t, "vpc-00001")
dns2, _, _ := s.addrs(t, "vpc-00002")
for i := 0; i < 2; i++ {
if _, err := ask(t, dns1, query(uint16(i)), time.Second); err != nil {
t.Fatalf("query %d within the burst: %v", i+1, err)
}
}
if _, err := ask(t, dns1, query(9), 200*time.Millisecond); err == nil {
t.Error("the third query in a row must be dropped at 2 per second")
}
if _, err := ask(t, dns2, query(10), time.Second); err != nil {
t.Errorf("another vpc has its own bucket: %v", err)
}
}
func TestNTP_HostClockWithoutUpstream(t *testing.T) {
clock := fakeClock{now: time.Unix(5000, 0), synced: true, maxError: time.Second}
s, _, _ := newService(t, Options{Clock: clock})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
req := ntpRequest(4, 0xabcdef)
reply, err := ask(t, ntp, req, time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if err := validNTPAnswer(req, reply); err != nil {
t.Fatalf("invalid answer: %v", err)
}
if got := binary.BigEndian.Uint64(reply[40:48]); got != ntpTimestamp(clock.now) {
t.Errorf("transmit = %#x, want the host clock %#x", got, ntpTimestamp(clock.now))
}
if reply[0]>>6 != 0 || reply[1] != 3 {
t.Errorf("li %d stratum %d, want 0 and 3 for a synced host", reply[0]>>6, reply[1])
}
}
func TestNTP_UnreadableClockStatusIsAnsweredAsUnsynced(t *testing.T) {
s, _, _ := newService(t, Options{Clock: fakeClock{now: time.Unix(5000, 0), synced: true, err: errors.New("eperm")}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
reply, err := ask(t, ntp, ntpRequest(4, 1), time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if reply[0]>>6 != 3 {
t.Errorf("li = %d, want 3 when the clock status cannot be read", reply[0]>>6)
}
}
func TestNTP_UpstreamIsPreferredAndChecked(t *testing.T) {
up, _ := udpUpstream(t, func(req []byte) [][]byte {
spoofed := hostReply(ntpRequest(4, 0), time.Unix(0, 0), time.Unix(0, 0), true, 0)
real := hostReply(req, time.Unix(7000, 0), time.Unix(7000, 0), true, 0)
real[1] = 1
return [][]byte{spoofed, real}
})
s, _, _ := newService(t, Options{NTP: []string{up}})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
req := ntpRequest(4, 0x55)
reply, err := ask(t, ntp, req, time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if reply[1] != 1 {
t.Errorf("stratum = %d, want the upstream's 1 (spoofed answer skipped, host clock unused)", reply[1])
}
}
func TestNTP_HostClockWhenUpstreamsFail(t *testing.T) {
clock := fakeClock{now: time.Unix(6000, 0), synced: true}
s, _, _ := newService(t, Options{NTP: []string{deadAddr(t)}, Clock: clock})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
reply, err := ask(t, ntp, ntpRequest(4, 2), 2*time.Second)
if err != nil {
t.Fatalf("no answer: %v", err)
}
if got := binary.BigEndian.Uint64(reply[40:48]); got != ntpTimestamp(clock.now) {
t.Errorf("transmit = %#x, want the host clock as fallback", got)
}
}
func TestNTP_InvalidRequestGetsNoAnswer(t *testing.T) {
s, _, _ := newService(t, Options{})
if err := s.Open("vpc-00001"); err != nil {
t.Fatalf("Open: %v", err)
}
_, _, ntp := s.addrs(t, "vpc-00001")
if _, err := ask(t, ntp, []byte("not ntp"), 300*time.Millisecond); err == nil {
t.Error("a short packet must not be answered")
}
}

View file

@ -0,0 +1,60 @@
package iptables
import (
"errors"
"fmt"
"os/exec"
)
const HostSvcChain = "TWO-HOSTSVC"
var hostSvcExcluded = []string{"br-public", "l3-+"}
func hostSvcJump() []string {
return []string{"PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", HostSvcChain}
}
func hostSvcRules(target string) [][]string {
var rules [][]string
for _, iface := range hostSvcExcluded {
rules = append(rules, []string{HostSvcChain, "-i", iface, "-j", "RETURN"})
}
for _, p := range []struct{ proto, port string }{{"udp", "53"}, {"tcp", "53"}, {"udp", "123"}} {
rules = append(rules, []string{HostSvcChain,
"-p", p.proto, "-m", p.proto, "--dport", p.port,
"-j", "DNAT", "--to-destination", target,
})
}
return rules
}
func nat(args ...string) *exec.Cmd {
return exec.Command("iptables", append([]string{"-t", "nat"}, args...)...)
}
func AddHostSvcRedirect(target string) error {
if out, err := nat("-N", HostSvcChain).CombinedOutput(); err != nil {
return fmt.Errorf("iptables create chain %s: %w: %s", HostSvcChain, err, out)
}
for _, rule := range append(hostSvcRules(target), hostSvcJump()) {
if out, err := nat(append([]string{"-A"}, rule...)...).CombinedOutput(); err != nil {
return fmt.Errorf("iptables hostsvc redirect %v: %w: %s", rule, err, out)
}
}
return nil
}
func HasHostSvcRedirect(target string) (bool, error) {
for _, rule := range append(hostSvcRules(target), hostSvcJump()) {
err := nat(append([]string{"-C"}, rule...)...).Run()
var exit *exec.ExitError
switch {
case err == nil:
case errors.As(err, &exit) && exit.ExitCode() == 1:
return false, nil
default:
return false, fmt.Errorf("iptables check %v: %w", rule, err)
}
}
return true, nil
}

View file

@ -0,0 +1,48 @@
package iptables
import (
"reflect"
"strings"
"testing"
)
func TestHostSvcJump_OnlyLocalDestinations(t *testing.T) {
want := []string{"PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "TWO-HOSTSVC"}
if got := hostSvcJump(); !reflect.DeepEqual(got, want) {
t.Errorf("jump = %v, want %v: traffic to an outside resolver must not be hijacked", got, want)
}
}
func TestHostSvcRules_ExclusionsComeFirst(t *testing.T) {
rules := hostSvcRules("169.254.169.253")
if len(rules) != 5 {
t.Fatalf("%d rules, want 2 exclusions and 3 redirections: %v", len(rules), rules)
}
for i, iface := range []string{"br-public", "l3-+"} {
want := []string{"TWO-HOSTSVC", "-i", iface, "-j", "RETURN"}
if !reflect.DeepEqual(rules[i], want) {
t.Errorf("rule %d = %v, want %v: the public segment and the vrf link must never reach the relay", i, rules[i], want)
}
}
}
func TestHostSvcRules_RedirectDNSAndNTPOnly(t *testing.T) {
var got []string
for _, rule := range hostSvcRules("169.254.169.253") {
if rule[len(rule)-3] != "DNAT" {
continue
}
if rule[len(rule)-1] != "169.254.169.253" {
t.Errorf("rule %v does not target 169.254.169.253", rule)
}
got = append(got, strings.Join(rule[1:7], " "))
}
want := []string{
"-p udp -m udp --dport 53",
"-p tcp -m tcp --dport 53",
"-p udp -m udp --dport 123",
}
if !reflect.DeepEqual(got, want) {
t.Errorf("redirections = %v, want %v", got, want)
}
}

13
internal/netif/dummy.go Normal file
View file

@ -0,0 +1,13 @@
package netif
import (
"github.com/vishvananda/netlink"
)
func CreateDummy(name string) error {
dummy := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name}}
if err := netlink.LinkAdd(dummy); err != nil {
return err
}
return netlink.LinkSetUp(dummy)
}

View file

@ -53,6 +53,9 @@ func CreateVPC(db *badger.DB, name string, cfg *configuration.Config) error {
if err := l3vni.Create(db, name, cfg); err != nil {
return err
}
if err := setupHostSvc(name, newSystemHostSvc()); err != nil {
return err
}
return state.Set(db, "vpc/"+name, state.Running)
}
return nil

View file

@ -17,6 +17,10 @@ func DeleteVPC(db *badger.DB, name string) error {
} else if current == state.Deleting {
vpcID := strings.SplitN(name, "-", 2)[1]
if err := teardownHostSvc(name, newSystemHostSvc()); err != nil {
return err
}
if err := l3vni.Delete(db, name); err != nil {
return err
}

72
internal/vpc/hostsvc.go Normal file
View file

@ -0,0 +1,72 @@
package vpc
import (
"fmt"
"net"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
hostsvcclient "git.g3e.fr/syonad/two/internal/client/hostsvc"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
)
const HostSvcLink = "hostsvc"
type hostSvcHost interface {
InNetns(vpc string, fn func() error) error
CreateDummy(name string) error
AddrAdd(iface string, ip net.IP) error
AddRedirect(target string) error
Open(vpc string) error
Close(vpc string) error
}
type systemHostSvc struct {
client *hostsvcclient.Client
}
func newSystemHostSvc() systemHostSvc {
return systemHostSvc{client: hostsvcclient.New(hostsvcapi.SocketPath(hostsvcapi.DefaultRunDir))}
}
func (systemHostSvc) InNetns(vpc string, fn func() error) error { return netns.Call(vpc, fn) }
func (systemHostSvc) CreateDummy(name string) error { return netif.CreateDummy(name) }
func (systemHostSvc) AddrAdd(iface string, ip net.IP) error { return netif.AddrAdd(iface, ip) }
func (systemHostSvc) AddRedirect(target string) error { return iptables.AddHostSvcRedirect(target) }
func (s systemHostSvc) Open(vpc string) error { return s.client.Open(vpc) }
func (s systemHostSvc) Close(vpc string) error { return s.client.Close(vpc) }
func setupHostSvc(vpc string, h hostSvcHost) error {
ip := net.ParseIP(hostsvc.ServiceIP)
if err := h.InNetns(vpc, func() error {
steps := []struct {
name string
fn func() error
}{
{"create " + HostSvcLink, func() error { return h.CreateDummy(HostSvcLink) }},
{"address " + hostsvc.ServiceIP + " on " + HostSvcLink, func() error { return h.AddrAdd(HostSvcLink, ip) }},
{"redirect dns and ntp to " + hostsvc.ServiceIP, func() error { return h.AddRedirect(hostsvc.ServiceIP) }},
}
for _, step := range steps {
if err := step.fn(); err != nil {
return fmt.Errorf("%s: %w", step.name, err)
}
}
return nil
}); err != nil {
return fmt.Errorf("hostsvc in %s: %w", vpc, err)
}
if err := h.Open(vpc); err != nil {
return fmt.Errorf("hostsvc open %s: %w", vpc, err)
}
return nil
}
func teardownHostSvc(vpc string, h hostSvcHost) error {
if err := h.Close(vpc); err != nil {
return fmt.Errorf("hostsvc close %s: %w", vpc, err)
}
return nil
}

View file

@ -0,0 +1,92 @@
package vpc
import (
"errors"
"net"
"reflect"
"testing"
)
type fakeHostSvc struct {
calls []string
inside string
failOn string
}
func (f *fakeHostSvc) record(kind, detail string) error {
call := kind + " " + detail
if f.inside != "" {
call = f.inside + ": " + call
}
f.calls = append(f.calls, call)
if kind == f.failOn {
return errors.New("boom")
}
return nil
}
func (f *fakeHostSvc) InNetns(vpc string, fn func() error) error {
f.inside = vpc
defer func() { f.inside = "" }()
return fn()
}
func (f *fakeHostSvc) CreateDummy(name string) error { return f.record("dummy", name) }
func (f *fakeHostSvc) AddrAdd(iface string, ip net.IP) error {
return f.record("addr", ip.String()+" "+iface)
}
func (f *fakeHostSvc) AddRedirect(target string) error { return f.record("redirect", target) }
func (f *fakeHostSvc) Open(vpc string) error { return f.record("open", vpc) }
func (f *fakeHostSvc) Close(vpc string) error { return f.record("close", vpc) }
func TestSetupHostSvc_OrderAndPlace(t *testing.T) {
f := &fakeHostSvc{}
if err := setupHostSvc("vpc-00001", f); err != nil {
t.Fatalf("setupHostSvc: %v", err)
}
want := []string{
"vpc-00001: dummy hostsvc",
"vpc-00001: addr 169.254.169.253 hostsvc",
"vpc-00001: redirect 169.254.169.253",
"open vpc-00001",
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("calls = %v, want %v: the address and the redirection live in the vpc netns, the open is asked from the host once they exist", f.calls, want)
}
}
func TestSetupHostSvc_StopsAtTheFirstFailure(t *testing.T) {
for _, c := range []struct {
failOn string
want string
last string
}{
{"dummy", "hostsvc in vpc-00001: create hostsvc: boom", "vpc-00001: dummy hostsvc"},
{"addr", "hostsvc in vpc-00001: address 169.254.169.253 on hostsvc: boom", "vpc-00001: addr 169.254.169.253 hostsvc"},
{"redirect", "hostsvc in vpc-00001: redirect dns and ntp to 169.254.169.253: boom", "vpc-00001: redirect 169.254.169.253"},
{"open", "hostsvc open vpc-00001: boom", "open vpc-00001"},
} {
f := &fakeHostSvc{failOn: c.failOn}
err := setupHostSvc("vpc-00001", f)
if err == nil || err.Error() != c.want {
t.Errorf("fail on %s: error %v, want %q", c.failOn, err, c.want)
}
if got := f.calls[len(f.calls)-1]; got != c.last {
t.Errorf("fail on %s: last call %q, want %q: nothing must run after a failure", c.failOn, got, c.last)
}
}
}
func TestTeardownHostSvc(t *testing.T) {
f := &fakeHostSvc{}
if err := teardownHostSvc("vpc-00001", f); err != nil {
t.Fatalf("teardownHostSvc: %v", err)
}
if !reflect.DeepEqual(f.calls, []string{"close vpc-00001"}) {
t.Errorf("calls = %v: only the close is needed, the dummy and the rules go away with the netns", f.calls)
}
f = &fakeHostSvc{failOn: "close"}
if err := teardownHostSvc("vpc-00001", f); err == nil || err.Error() != "hostsvc close vpc-00001: boom" {
t.Errorf("error %v: a failed close must stop the deletion", err)
}
}

View file

@ -0,0 +1,110 @@
package watchdog
import (
"fmt"
"net"
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
hostsvcclient "git.g3e.fr/syonad/two/internal/client/hostsvc"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"git.g3e.fr/syonad/two/internal/watchdog/notify"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
const kindHostSvc = "hostsvc"
type hostSvcProbe interface {
Served() ([]string, error)
ServiceAddress(vpc string) (bool, error)
Redirect(vpc string) (bool, error)
}
type systemHostSvcProbe struct {
client *hostsvcclient.Client
}
func newSystemHostSvcProbe() systemHostSvcProbe {
return systemHostSvcProbe{client: hostsvcclient.New(hostsvcapi.SocketPath(hostsvcapi.DefaultRunDir))}
}
func (p systemHostSvcProbe) Served() ([]string, error) { return p.client.List() }
func (systemHostSvcProbe) ServiceAddress(name string) (bool, error) {
want := net.ParseIP(hostsvc.ServiceIP)
found := false
err := netns.Call(name, func() error {
ips, err := netif.LinkIPv4(vpc.HostSvcLink)
if err != nil {
return err
}
for _, ip := range ips {
if ip.Equal(want) {
found = true
}
}
return nil
})
return found, err
}
func (systemHostSvcProbe) Redirect(name string) (bool, error) {
var ok bool
err := netns.Call(name, func() error {
var err error
ok, err = iptables.HasHostSvcRedirect(hostsvc.ServiceIP)
return err
})
return ok, err
}
func CheckHostSvc(db *badger.DB, p hostSvcProbe, n notify.Notifier) error {
pairs, err := kv.ListByPrefix(db, prefixVPC)
if err != nil {
return fmt.Errorf("watchdog: listing vpcs: %w", err)
}
var running []string
for _, name := range resourceNames(pairs, prefixVPC) {
if st, err := state.Get(db, prefixVPC+name); err == nil && st == state.Running {
running = append(running, name)
}
}
if len(running) == 0 {
return nil
}
served, err := p.Served()
if err != nil {
n.Notify(kindHostSvc, hostsvcapi.Unit, fmt.Sprintf("unreachable, no vpc of this host gets dns or ntp: %v", err))
} else {
open := map[string]bool{}
for _, name := range served {
open[name] = true
}
for _, name := range running {
if !open[name] {
n.Notify(kindVPC, name, "not served by two-hostsvc: its vms get no dns nor ntp")
}
}
}
for _, name := range running {
if ok, err := p.ServiceAddress(name); err != nil {
n.Notify(kindVPC, name, fmt.Sprintf("%s on %s unreadable: %v", hostsvc.ServiceIP, vpc.HostSvcLink, err))
} else if !ok {
n.Notify(kindVPC, name, fmt.Sprintf("%s missing on %s (in netns)", hostsvc.ServiceIP, vpc.HostSvcLink))
}
if ok, err := p.Redirect(name); err != nil {
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection unreadable: %v", err))
} else if !ok {
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection to %s missing or incomplete (chain %s)", hostsvc.ServiceIP, iptables.HostSvcChain))
}
}
return nil
}

View file

@ -0,0 +1,129 @@
package watchdog
import (
"errors"
"testing"
"git.g3e.fr/syonad/two/internal/state"
)
type fakeHostSvcProbe struct {
served []string
servedErr error
address map[string]bool
redirect map[string]bool
readErr error
asked int
}
func (f *fakeHostSvcProbe) Served() ([]string, error) {
f.asked++
return f.served, f.servedErr
}
func (f *fakeHostSvcProbe) ServiceAddress(vpc string) (bool, error) {
return f.address[vpc], f.readErr
}
func (f *fakeHostSvcProbe) Redirect(vpc string) (bool, error) {
return f.redirect[vpc], f.readErr
}
func healthyHostSvc(vpcs ...string) *fakeHostSvcProbe {
f := &fakeHostSvcProbe{served: vpcs, address: map[string]bool{}, redirect: map[string]bool{}}
for _, v := range vpcs {
f.address[v] = true
f.redirect[v] = true
}
return f
}
func TestCheckHostSvc_SilentWhenEverythingIsInPlace(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-a", state.Running)
r := &recorder{}
if err := CheckHostSvc(db, healthyHostSvc("vp-a"), r); err != nil {
t.Fatalf("CheckHostSvc: %v", err)
}
if len(r.calls) != 0 {
t.Errorf("notifications = %+v, want none", r.calls)
}
}
func TestCheckHostSvc_NoRunningVPCDoesNotAskTheRelay(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-a", state.Creating)
p := &fakeHostSvcProbe{servedErr: errors.New("down")}
r := &recorder{}
if err := CheckHostSvc(db, p, r); err != nil {
t.Fatalf("CheckHostSvc: %v", err)
}
if p.asked != 0 || len(r.calls) != 0 {
t.Errorf("asked %d times, notifications %+v: a host without running vpc needs no relay", p.asked, r.calls)
}
}
func TestCheckHostSvc_ReportsAnUnreachableRelay(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-a", state.Running)
p := healthyHostSvc("vp-a")
p.servedErr = errors.New("dial /run/two/hostsvc/hostsvc.sock: connection refused")
r := &recorder{}
if err := CheckHostSvc(db, p, r); err != nil {
t.Fatalf("CheckHostSvc: %v", err)
}
got := r.forName("two-hostsvc.service")
if len(got) != 1 || got[0].kind != "hostsvc" {
t.Fatalf("notifications = %+v, want one for two-hostsvc.service", r.calls)
}
if !r.hasProblemContaining("unreachable, no vpc of this host gets dns or ntp: dial /run/two/hostsvc/hostsvc.sock: connection refused") {
t.Errorf("problem = %q", got[0].problem)
}
}
func TestCheckHostSvc_ReportsEachMissingPiece(t *testing.T) {
db := newTestDB(t)
for _, v := range []string{"vp-ok", "vp-unserved", "vp-noaddr", "vp-noredirect"} {
seedResource(t, db, prefixVPC, v, state.Running)
}
p := healthyHostSvc("vp-ok", "vp-unserved", "vp-noaddr", "vp-noredirect")
p.served = []string{"vp-ok", "vp-noaddr", "vp-noredirect"}
p.address["vp-noaddr"] = false
p.redirect["vp-noredirect"] = false
r := &recorder{}
if err := CheckHostSvc(db, p, r); err != nil {
t.Fatalf("CheckHostSvc: %v", err)
}
for name, want := range map[string]string{
"vp-unserved": "not served by two-hostsvc: its vms get no dns nor ntp",
"vp-noaddr": "169.254.169.253 missing on hostsvc (in netns)",
"vp-noredirect": "dns and ntp redirection to 169.254.169.253 missing or incomplete (chain TWO-HOSTSVC)",
} {
got := r.forName(name)
if len(got) != 1 || got[0].kind != "vpc" || got[0].problem != want {
t.Errorf("%s: notifications %+v, want one vpc problem %q", name, got, want)
}
}
if got := r.forName("vp-ok"); len(got) != 0 {
t.Errorf("vp-ok: notifications %+v, want none", got)
}
}
func TestCheckHostSvc_ReportsUnreadableChecks(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-a", state.Running)
p := healthyHostSvc("vp-a")
p.readErr = errors.New("cannot enter netns")
r := &recorder{}
if err := CheckHostSvc(db, p, r); err != nil {
t.Fatalf("CheckHostSvc: %v", err)
}
for _, want := range []string{
"169.254.169.253 on hostsvc unreadable: cannot enter netns",
"dns and ntp redirection unreadable: cannot enter netns",
} {
if !r.hasProblemContaining(want) {
t.Errorf("missing %q in %+v", want, r.calls)
}
}
}

View file

@ -23,6 +23,7 @@ type Watchdog struct {
interval time.Duration
dbusDown bool
l3 l3Probe
hostsvc hostSvcProbe
}
func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog {
@ -41,6 +42,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl
logger: logger,
interval: interval,
l3: systemProbe{frr: frr.NewClient()},
hostsvc: newSystemHostSvcProbe(),
}
}
@ -68,6 +70,9 @@ func (w *Watchdog) tick() {
if err := CheckVPCs(w.db, w.notifier); err != nil {
w.logger.Error("watchdog: vpc check failed", "err", err)
}
if err := CheckHostSvc(w.db, w.hostsvc, w.notifier); err != nil {
w.logger.Error("watchdog: hostsvc check failed", "err", err)
}
if err := CheckSubnets(w.db, w.cfg, u, w.notifier); err != nil {
w.logger.Error("watchdog: subnet check failed", "err", err)
}

View file

@ -14,6 +14,23 @@
quand ses VM sont en `deleting`, et attend leur suppression jusqu'à `timeout_subnet_stop`. Le
409 reste pour une VM ni supprimée ni en `deleting`
**DNS et NTP des VM par l'hyperviseur : `two-hostsvc` (#86)**
- Nouveau service `two-hostsvc.service`, un par hyperviseur, démarré avant l'agent : il relaie
DNS (UDP et TCP) et NTP pour les VM de tous les VPC, depuis l'adresse `169.254.169.253` posée
dans le netns de chaque VPC. Une VM sans sortie Internet garde DNS et NTP, sans route entre la
VRF du VPC et l'hôte
- **Le DHCP annonce l'`interface_ip` du subnet comme DNS (option 6) et NTP (option 42)**, dans les
deux backends ; `1.1.1.1` et `8.8.8.8` ne sont plus annoncés. Les requêtes des VM vers leur
gateway sur 53 et 123 sont redirigées vers `two-hostsvc` ; celles vers un résolveur extérieur
ne le sont pas
- Nouvelle section `hostsvc` de la configuration : `dns_upstream` et `ntp_upstream` (serveurs
interrogés en premier ; à défaut, le résolveur et l'horloge de l'hôte) et `rate_limit_per_vpc`
(100 requêtes par seconde par défaut, DNS et NTP confondus)
- **La création d'un VPC échoue si `two-hostsvc` ne répond pas**, plutôt que de créer un VPC sans
DNS ni NTP. Le watchdog signale un `two-hostsvc` injoignable et, pour chaque VPC, un élément
manquant (adresse, redirection, VPC non servi)
## Correctifs
- **Chaque VRF de VPC porte une route `unreachable default` (#81).** Une recherche qui échouait
@ -52,3 +69,12 @@
et les valeurs par défaut de `times` s'appliquent. Une valeur personnalisée de
`dispatcher.timeout_seconds` est à reporter dans `times.timeout_vms_stop` (et dans les délais
qui en dépendent)
- **DNS et NTP (#86) : vider l'hyperviseur avant la mise à jour, puis recréer les VPC.** Les VPC
créés par la 0.2.0 n'ont ni l'adresse `169.254.169.253`, ni la redirection, ni de place dans
`two-hostsvc`, et l'agent ne les reprend pas au démarrage. Or le serveur DHCP intégré (backend
`two`) annonce la gateway comme DNS dès son redémarrage par `deploy.sh` : au renouvellement
de leur bail (au plus 6 h), les VM de ces VPC perdraient leur DNS. Avec dnsmasq, le fichier de
configuration n'est pas régénéré et les VM gardent `1.1.1.1`. Dans les deux cas, le watchdog
signale ces VPC à chaque tick. Procédure : supprimer VM, subnets et VPC de l'hyperviseur,
mettre à jour avec `deploy.sh` (qui installe et démarre `two-hostsvc`), vérifier
`systemctl status two-hostsvc`, puis recréer VPC, subnets et VM

View file

@ -116,7 +116,7 @@ start_services () {
profile_units () {
case "${1}" in
kvm) echo "agent.service dnsmasq@.service dhcp@.service metadata@.service" ;;
kvm) echo "two-hostsvc.service agent.service dnsmasq@.service dhcp@.service metadata@.service" ;;
intel) echo "" ;;
*) return 1 ;;
esac
@ -124,7 +124,7 @@ profile_units () {
profile_binaries () {
case "${1}" in
kvm) echo "agent metadata dhcp run-dnsmasq-in-netns.sh run-dhcp-in-netns.sh" ;;
kvm) echo "agent metadata dhcp two-hostsvc run-dnsmasq-in-netns.sh run-dhcp-in-netns.sh" ;;
intel) echo "" ;;
*) return 1 ;;
esac

View file

@ -0,0 +1,25 @@
[Unit]
Description=two DNS and NTP relay of the VPCs
After=network.target
Before=agent.service
[Service]
Type=simple
ExecStart=/opt/two/bin/two-hostsvc
Restart=on-failure
RuntimeDirectory=two/hostsvc
RuntimeDirectoryMode=0700
RuntimeDirectoryPreserve=yes
CapabilityBoundingSet=CAP_SYS_ADMIN CAP_NET_BIND_SERVICE CAP_DAC_READ_SEARCH
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
LockPersonality=yes
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
[Install]
WantedBy=multi-user.target

View file

@ -3,34 +3,38 @@
E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR="$(cd "${E2E_DIR}/../.." && pwd)"
BIN_DIR="${BIN_DIR:-${E2E_DIR}/topologies/bin}"
BINARIES="agent dhcp metadata"
BINARIES="agent dhcp metadata two-hostsvc"
UNITS="two-hostsvc.service"
usage () {
cat >&2 <<USAGE
usage: ${0##*/} [--clean]
sans option : compile ${BINARIES} pour linux/amd64 depuis l'arbre courant dans ${BIN_DIR} ;
le prochain lab les installe par-dessus la release épinglée dans
topologies/provision/hypervisor.sh
sans option : compile ${BINARIES} pour linux/amd64 depuis l'arbre courant dans ${BIN_DIR}
et y copie ${UNITS} ; le prochain lab les installe par-dessus la release
épinglée dans topologies/provision/hypervisor.sh
--clean : vide ${BIN_DIR} ; le prochain lab teste la release seule
USAGE
exit 2
}
clean () {
local binary
for binary in ${BINARIES}; do
rm -f "${BIN_DIR}/${binary}" || return 1
local file
for file in ${BINARIES} ${UNITS}; do
rm -f "${BIN_DIR}/${file}" || return 1
done
}
build () {
local binary
local binary unit
for binary in ${BINARIES}; do
echo "== ${binary}" >&2
(cd "${REPO_DIR}" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "${BIN_DIR}/${binary}" "./cmd/${binary}") || return 1
done
(cd "${BIN_DIR}" && sha256sum ${BINARIES} 2>/dev/null || shasum -a 256 ${BINARIES})
for unit in ${UNITS}; do
cp "${REPO_DIR}/systemd/${unit}" "${BIN_DIR}/${unit}" || return 1
done
(cd "${BIN_DIR}" && sha256sum ${BINARIES} ${UNITS} 2>/dev/null || shasum -a 256 ${BINARIES} ${UNITS})
}
main () {

View file

@ -0,0 +1,118 @@
# fonctionnalité: DNS et NTP des VM par two-hostsvc — options DHCP 6 et 42 = interface_ip, redirection vers 169.254.169.253 dans le VPC (vxlan et bridge), seuls 53 et 123 servis, alertes du watchdog, VPC en error si two-hostsvc ne répond pas
# hôtes: hv1
# tickets: #86
on hv1 <<'NODE'
KEY=$(vm_key)
dns_py () {
cat <<'PY'
import socket, struct, sys
server, proto, name = sys.argv[1], sys.argv[2], sys.argv[3]
q = struct.pack('!HHHHHH', 0x2929, 0x0100, 1, 0, 0, 0)
q += b''.join(bytes([len(p)]) + p.encode() for p in name.split('.')) + b'\0' + struct.pack('!HH', 1, 1)
if proto == 'tcp':
s = socket.create_connection((server, 53), timeout=5)
s.sendall(struct.pack('!H', len(q)) + q)
size = struct.unpack('!H', s.recv(2))[0]
a = b''
while len(a) < size:
a += s.recv(size - len(a))
else:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(5)
s.sendto(q, (server, int(sys.argv[4]) if len(sys.argv) > 4 else 53))
a = s.recv(65535)
print('id=%d rcode=%d answers=%d' % (struct.unpack('!H', a[:2])[0], a[3] & 15, struct.unpack('!H', a[6:8])[0]))
PY
}
ntp_py () {
cat <<'PY'
import socket, struct, sys, time
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(5)
req = b'\x23' + b'\0' * 39 + struct.pack('!Q', 0x2929292929292929)
s.sendto(req, (sys.argv[1], 123))
r = s.recv(1024)
t = struct.unpack('!I', r[40:44])[0] - 2208988800
print('mode=%d origin=%s li=%d stratum=%d offset=%d' % (r[0] & 7, 'ok' if r[24:32] == req[40:48] else 'ko', r[0] >> 6, r[1], abs(t - time.time())))
PY
}
vm_py () {
local vpc="${1}" ip="${2}" code="${3}"
shift 3
vm_ssh "${vpc}" "${ip}" "echo $(${code} | base64 | tr -d '\n') | base64 -d | timeout 10 python3 - $*"
}
dns_ok () { vm_py "${1}" "${2}" dns_py "${3}" "${4}" deb.debian.org | grep -qE '^id=10537 rcode=0 answers=[1-9]'; }
ntp_ok () {
local out
out=$(vm_py "${1}" "${2}" ntp_py "${3}") || { echo "pas de réponse NTP de ${3}"; return 1; }
echo "${out}" | grep -qE '^mode=4 origin=ok li=[0-3] stratum=[0-9]+ offset=[0-4]$' || { echo "${out}"; return 1; }
}
no_answer () { ! vm_py "${1}" "${2}" dns_py "${3}" udp deb.debian.org "${4}" >/dev/null 2>&1; }
served_ports () { ip netns exec "${1}" ss -Hlntu | awk '$5 ~ /^169\.254\.169\.253:/ {print $1, $5}' | sort | tr '\n' ' '; }
rules () { ip netns exec "${1}" iptables -t nat -S TWO-HOSTSVC; }
state_has () { grep -q "\"${1}\"" /run/two/hostsvc/hostsvc.state; }
excluded_first () { [ "$(rules "${1}" | sed -n '2p;3p' | tr '\n' ' ')" = "-A TWO-HOSTSVC -i br-public -j RETURN -A TWO-HOSTSVC -i l3-+ -j RETURN " ]; }
no_dns () { ! dns_ok "$@" >/dev/null 2>&1; }
no_ntp () { ! ntp_ok "$@" >/dev/null 2>&1; }
vpc_in_error () {
local out
out=$(wait_state "vpcs/${1}" running 2>&1) && { echo "${1} est running"; return 1; }
grep -q "vpcs/${1}: error" <<<"${out}" || { echo "${out}"; return 1; }
}
restored () { state_has vp-t29 && state_has vp-t29b && ! state_has vp-t29c; }
vpc_delete () { api_delete "vpcs/${1}" && wait_state "vpcs/${1}" deleted; }
check "two-hostsvc actif" systemctl is-active --quiet two-hostsvc.service
check "VPC vp-t29 (vxlan)" vpc_create vp-t29 10.29.0.0/16
check "subnet sn-t29 (vxlan)" subnet_create sn-t29 vp-t29 2901 10.29.1.0/24
check "VPC vp-t29b (bridge)" vpc_create vp-t29b 10.33.0.0/16
check "subnet sn-t29b en mode bridge sur br-lan, interface_ip 10.33.0.29" subnet_create_bridge sn-t29b vp-t29b 10.33.0.0/24 10.33.0.29
for vpc in vp-t29 vp-t29b; do
check "${vpc} : 169.254.169.253 sur la dummy hostsvc" sh -c "ip -n ${vpc} -4 -o addr show dev hostsvc | grep -q ' 169.254.169.253/32 '"
check "${vpc} : two-hostsvc n'écoute que 53/udp, 53/tcp et 123/udp sur 169.254.169.253" test "$(served_ports "${vpc}")" = "tcp 169.254.169.253:53 udp 169.254.169.253:123 udp 169.254.169.253:53 "
check "${vpc} : chaîne TWO-HOSTSVC, br-public et l3-+ exclus avant toute redirection" excluded_first "${vpc}"
check "${vpc} : redirection de 53/udp, 53/tcp et 123/udp vers 169.254.169.253" test "$(rules "${vpc}" | grep -c 'DNAT --to-destination 169.254.169.253')" -eq 3
check "${vpc} : appel depuis PREROUTING pour les destinations locales seulement" sh -c "ip netns exec ${vpc} iptables -t nat -S PREROUTING | grep -qx -- '-A PREROUTING -m addrtype --dst-type LOCAL -j TWO-HOSTSVC'"
check "${vpc} : inscrit dans l'état de two-hostsvc" state_has "${vpc}"
done
check "VM t29-a 10.29.1.10 (vxlan)" vm_create t29-a sn-t29 10.29.1.10 "${KEY}"
check "VM t29-b 10.33.0.129 (bridge)" vm_launch t29-b --nic sn-t29b=10.33.0.129 --sshkey "${KEY}"
for vm in vp-t29:10.29.1.10 vp-t29b:10.33.0.129; do
vpc=${vm%%:*}; ip=${vm#*:}
check "${ip} joignable" vm_wait "${vpc}" "${ip}"
GW=$(vm_lease "${vpc}" "${ip}" SERVER_ADDRESS)
check "${ip} : gateway lue dans le bail (${GW:-absente})" test -n "${GW}"
check "${ip} : option 6, DNS = la gateway ${GW}" test "$(vm_lease "${vpc}" "${ip}" DNS)" = "${GW}"
check "${ip} : option 42, NTP = la gateway ${GW}" test "$(vm_lease "${vpc}" "${ip}" NTP)" = "${GW}"
check "${ip} : DNS en UDP via ${GW}" dns_ok "${vpc}" "${ip}" "${GW}" udp
check "${ip} : DNS en TCP via ${GW}" dns_ok "${vpc}" "${ip}" "${GW}" tcp
check "${ip} : NTP via ${GW}, réponse serveur, origine vérifiée, écart < 5 s" ntp_ok "${vpc}" "${ip}" "${GW}"
info "${ip} : NTP $(vm_py "${vpc}" "${ip}" ntp_py "${GW}" 2>&1)"
info "${ip} : résolveur système : $(vm_ssh "${vpc}" "${ip}" 'getent hosts deb.debian.org' 2>&1 | head -1)"
check "${ip} : port 5353 de la gateway non redirigé (aucune réponse)" no_answer "${vpc}" "${ip}" "${GW}" 5353
done
ip -n vp-t29 addr del 169.254.169.253/32 dev hostsvc
check "watchdog : adresse 169.254.169.253 retirée signalée" wait_watchdog "169.254.169.253 missing on hostsvc (in netns)"
ip -n vp-t29 addr add 169.254.169.253/32 dev hostsvc
check "10.29.1.10 : DNS de nouveau servi une fois l'adresse remise" dns_ok vp-t29 10.29.1.10 10.29.1.254 udp
ip netns exec vp-t29 iptables -t nat -D TWO-HOSTSVC -p udp -m udp --dport 123 -j DNAT --to-destination 169.254.169.253
check "watchdog : redirection incomplète signalée" wait_watchdog "dns and ntp redirection to 169.254.169.253 missing or incomplete (chain TWO-HOSTSVC)"
check "10.29.1.10 : sans la règle 123, plus de NTP (témoin)" no_ntp vp-t29 10.29.1.10 10.29.1.254
ip netns exec vp-t29 iptables -t nat -A TWO-HOSTSVC -p udp -m udp --dport 123 -j DNAT --to-destination 169.254.169.253
check "10.29.1.10 : NTP de nouveau servi une fois la règle remise" ntp_ok vp-t29 10.29.1.10 10.29.1.254
systemctl stop two-hostsvc.service
check "watchdog : two-hostsvc arrêté signalé" wait_watchdog "unreachable, no vpc of this host gets dns or ntp"
check "10.29.1.10 : plus de DNS pendant l'arrêt (témoin)" no_dns vp-t29 10.29.1.10 10.29.1.254 udp
api_post vpcs '{"name":"vp-t29c","cidr":"10.129.0.0/16"}'
check "VPC vp-t29c créé pendant l'arrêt : en error" vpc_in_error vp-t29c
systemctl start two-hostsvc.service
check "two-hostsvc redémarré" systemctl is-active --quiet two-hostsvc.service
check "redémarrage : vp-t29 et vp-t29b rouverts depuis le fichier d'état" restored
check "10.29.1.10 : DNS de nouveau servi après le redémarrage" dns_ok vp-t29 10.29.1.10 10.29.1.254 udp
check "10.33.0.129 : DNS de nouveau servi après le redémarrage" dns_ok vp-t29b 10.33.0.129 10.33.0.29 udp
check "VPC vp-t29c supprimé depuis error" vpc_delete vp-t29c
NODE

View file

@ -36,10 +36,10 @@ nodes:
segments: [underlay, lan], addresses: { underlay: 192.168.14.11, lan: 10.33.0.11 },
frr: frr/hv1.conf, provision: provision/hypervisor.sh,
post_provision: provision/hypervisor-post.sh, files: { agent.yml: agent/two.yml, bin: bin },
services: [agent.service] }
services: [two-hostsvc.service, agent.service] }
hv2:
{ role: host, image: debian12, cpus: 4, memory: 16384,
segments: [underlay, lan], addresses: { underlay: 192.168.14.12, lan: 10.33.0.12 },
frr: frr/hv2.conf, provision: provision/hypervisor.sh,
post_provision: provision/hypervisor-post.sh, files: { agent.yml: agent/dnsmasq.yml, bin: bin },
services: [agent.service] }
services: [two-hostsvc.service, agent.service] }

View file

@ -36,15 +36,15 @@ nodes:
segments: [underlay, lan], addresses: { underlay: 192.168.14.11, lan: 10.33.0.11 },
frr: frr/hv1.conf, provision: provision/hypervisor.sh,
post_provision: provision/hypervisor-post.sh, files: { agent.yml: agent/two.yml, bin: bin },
services: [agent.service] }
services: [two-hostsvc.service, agent.service] }
hv2:
{ role: host, image: debian12, cpus: 4, memory: 16384,
segments: [underlay, lan], addresses: { underlay: 192.168.14.12, lan: 10.33.0.12 },
frr: frr/hv2.conf, provision: provision/hypervisor.sh,
post_provision: provision/hypervisor-post.sh, files: { agent.yml: agent/dnsmasq.yml, bin: bin },
services: [agent.service] }
services: [two-hostsvc.service, agent.service] }
hv3:
{ role: host, image: debian12, cpus: 4, memory: 16384, segments: [underlay],
addresses: { underlay: 192.168.14.13 }, frr: frr/hv3.conf, provision: provision/hypervisor.sh,
post_provision: provision/hypervisor-post.sh, files: { agent.yml: agent/dnsmasq.yml, bin: bin },
services: [agent.service] }
services: [two-hostsvc.service, agent.service] }

View file

@ -31,7 +31,17 @@ bash ./deploy.sh --noup_script -i -u "${LAB_UPLINK}" -t "${RELEASE}"
for binary in "${LAB_FILES}"/bin/*; do
[ -f "${binary}" ] || continue
name=$(basename "${binary}")
case "${name}" in *.service) continue ;; esac
install -D -m 0755 "${binary}" "/opt/two/local/bin/${name}"
ln -sfn "/opt/two/local/bin/${name}" "/opt/two/bin/${name}"
echo "lab: ${name} compilé en local remplace celui de la release ${RELEASE}"
done
for unit in "${LAB_FILES}"/bin/*.service; do
[ -f "${unit}" ] || continue
name=$(basename "${unit}")
install -m 0644 "${unit}" "/etc/systemd/system/${name}"
systemctl daemon-reload
systemctl enable "${name}"
systemctl restart "${name}"
echo "lab: unité ${name} de l'arbre local installée par-dessus la release ${RELEASE}"
done