Compare commits

..

9 commits

Author SHA1 Message Date
9477dc91d2
f-41: watchdog: sessions BGP, L3VNI et routes exportées des VPC routés #41
All checks were successful
Documentation / publish (push) Successful in 1m27s
Release Pipeline / set-release-target (push) Successful in 2s
Release Pipeline / upload-assets (agent.service, systemd/agent.service) (push) Successful in 8s
Release Pipeline / upload-assets (dhcp@.service, systemd/dhcp@.service) (push) Successful in 7s
Release Pipeline / upload-assets (dnsmasq@.service, systemd/dnsmasq@.service) (push) Successful in 7s
Release Pipeline / upload-assets (metadata@.service, systemd/metadata@.service) (push) Successful in 7s
Release Pipeline / upload-assets (run-dhcp-in-netns.sh, scripts/run-dhcp-in-netns.sh) (push) Successful in 7s
Release Pipeline / build (dhcp, amd64, linux) (push) Successful in 0s
Release Pipeline / build (agent, amd64, linux) (push) Successful in 0s
Release Pipeline / build (metadata, amd64, linux) (push) Successful in 0s
Release Pipeline / upload-assets (run-dnsmasq-in-netns.sh, scripts/run-dnsmasq-in-netns.sh) (push) Successful in 7s
Release Pipeline / checksums (push) Successful in 8s
Release Pipeline / release (push) Successful in 28s
Release Pipeline / publish (push) Successful in 0s
Release Pipeline / build (push) Successful in 2m9s
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:42:01 +02:00
03029eabe8
f-41: vm: /32 de chaque interface annoncé dans la VRF #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:32:16 +02:00
fe27d45262
f-41: subnet: export du préfixe dans la VRF, suppression refusée tant qu'une VM l'utilise #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:28:16 +02:00
3c6852dc18
f-41: vpc: VRF, L3VNI et transit à la création des VPC routées #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:21:19 +02:00
e2faf85e96
f-41: dispatcher: arrêt des workers avant la fermeture de la base dans les tests #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:21:14 +02:00
3dd94f35dd
f-41: api, vpc: champ l3vni des VPC #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 20:05:20 +02:00
bf0e83862f
f-41: config, frr: base BGP pilotée par l'agent au démarrage #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-11 19:52:46 +02:00
94c2f82867
f-41: frr, sysctl, netif: primitives du L3VNI #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-06 14:13:08 +02:00
0f31b2faf4
f-41: subnet: MAC de gateway anycast dérivée de la VNI #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
2026-10-06 00:50:06 +02:00
62 changed files with 3200 additions and 25 deletions

View file

@ -45,13 +45,13 @@ paths:
schema:
$ref: "#/components/schemas/VPC"
"400":
description: Missing required field or invalid request body
description: Missing required field, invalid request body, or l3vni out of range
content:
application/json:
schema:
$ref: "#/components/schemas/Error"
"409":
description: VPC already exists
description: VPC already exists, l3vni already used by another VPC, or l3vni requested on an agent without a bgp section
content:
application/json:
schema:
@ -281,7 +281,7 @@ paths:
"404":
$ref: "#/components/responses/NotFound"
"409":
description: Subnet not deletable — only running or error states can be deleted
description: Subnet not deletable — only running or error states can be deleted, and no VM that is not deleted may still have an interface on it
content:
application/json:
schema:
@ -317,6 +317,17 @@ components:
type: string
description: CIDR block for the entire VPC address space
example: "10.0.0.0/16"
l3vni:
type: integer
format: int32
minimum: 1
maximum: 16777215
description: >-
Optional. L3VNI of the VPC, used for inter-subnet routing between hypervisors
(symmetric IRB). Also the id of the routing table of the VPC VRF, so 253-255 are
rejected. Requires the bgp section in the agent configuration, and must not be used
by another VPC of the agent. Absent: the VPC has no L3VNI.
example: 4001
VPC:
type: object
@ -331,6 +342,10 @@ components:
cidr:
type: string
example: "10.0.0.0/16"
l3vni:
type: integer
description: L3VNI of the VPC, absent when the VPC has none
example: 4001
SubnetCreateRequest:
type: object

View file

@ -13,7 +13,9 @@ import (
agentapi "git.g3e.fr/syonad/two/internal/api/agent"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/migration"
"git.g3e.fr/syonad/two/internal/netif"
agentmetrics "git.g3e.fr/syonad/two/internal/prometheus/agent"
"git.g3e.fr/syonad/two/internal/watchdog"
"git.g3e.fr/syonad/two/internal/watchdog/notify"
@ -58,6 +60,14 @@ func main() {
return
}
if cfg.BGP.Enabled() {
if err := applyBGPBase(cfg, frr.NewClient()); err != nil {
log.Error("failed to apply the bgp base configuration", "error", err)
return
}
log.Info("bgp base configuration applied", "asn", cfg.BGP.ASN, "route_reflectors", cfg.BGP.RouteReflectors)
}
q := worker.New(cfg.Worker.BufferSize)
q.Start(cfg.Worker.Count)
@ -110,6 +120,22 @@ func main() {
closeDB = shutdown(log, q, servers, shutdownTimeout)
}
type baseApplier interface {
ApplyBase(frr.Base) error
}
func applyBGPBase(cfg *configuration.Config, frrClient baseApplier) error {
reflectors, err := cfg.BGP.Reflectors()
if err != nil {
return err
}
vtep, err := netif.VTEPAddress(cfg.DefaultInterface)
if err != nil {
return err
}
return frrClient.ApplyBase(frr.Base{ASN: cfg.BGP.ASN, RouterID: vtep, RouteReflectors: reflectors})
}
type httpShutdowner interface {
Shutdown(context.Context) error
}

View file

@ -10,6 +10,8 @@ import (
"testing"
"time"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/pkg/worker"
)
@ -96,3 +98,22 @@ func TestShutdown_SansServeur(t *testing.T) {
t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre")
}
}
type fakeBase struct{ called bool }
func (f *fakeBase) ApplyBase(frr.Base) error {
f.called = true
return nil
}
func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) {
cfg := &configuration.Config{}
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
f := &fakeBase{}
if err := applyBGPBase(cfg, f); err == nil || !strings.Contains(err.Error(), "bgp.asn must be set") {
t.Errorf("erreur %v", err)
}
if f.called {
t.Error("ApplyBase appelé avec une section bgp invalide")
}
}

View file

@ -35,6 +35,15 @@ interfaces:
internet: br-000000
admin: br-000000
# BGP control plane of the hypervisor (optional).
# Absent: the agent leaves FRR alone. Present: at every start, before serving its
# API, the agent applies the base BGP configuration through vtysh (router-id = the
# primary IPv4 of default_interface) and saves it with "write memory".
# Changing the asn requires removing the previous instance by hand.
# bgp:
# asn: 64600
# route_reflectors: [10.255.255.1]
# Metadata server runtime directory (cloud-init files per VM)
metadata:
run_dir: "/run/two/metadata"

View file

@ -8,6 +8,7 @@ de sérialisation structurée.
vpc/<name>/state → creating | running | error | deleting | deleted
vpc/<name>/cidr → <cidr>
vpc/<name>/l3vni → <id> (optionnel : VPC routé par L3VNI)
subnet/<name>/state → creating | running | error | deleting | deleted
subnet/<name>/vpc → <vpc-name>

View file

@ -42,6 +42,12 @@ vxlan
Le subnet vit dans le netns du VPC. La VM n'est donc **pas joignable depuis l'host** sans route
explicite — point à connaître avant de câbler un outil externe dessus.
Le bridge du subnet, qui porte ``interface_ip``, a une **MAC de gateway anycast** calculée depuis
le ``vxlan_id`` : ``02:01:00`` suivi des 24 bits de la VNI (``vxlan_id`` 2811 →
``02:01:00:00:0a:fb``). Elle est identique sur tous les hôtes qui portent le subnet : une VM qui
change d'hôte garde une entrée ARP valide pour sa gateway. Elle n'est posée qu'à la création du
subnet — un subnet créé avant garde la MAC choisie par le noyau jusqu'à sa recréation.
bridge
------

View file

@ -20,6 +20,35 @@ d'isolation : deux VPC ne se voient pas, et peuvent réutiliser les mêmes plage
Un VPC ne peut être supprimé que si tous ses subnets le sont déjà — sinon 409.
Routage entre hyperviseurs : ``l3vni``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Un VPC créé avec ``l3vni`` route ses subnets entre hyperviseurs par un **VNI de transit** propre
au VPC (IRB symétrique, `#41 <https://git.g3e.fr/syonad/two/issues/41>`_) : un hyperviseur n'a
plus besoin de porter un subnet pour joindre ses VM. Sans ``l3vni``, le VPC se comporte comme
avant. Le champ exige la section ``bgp`` de l'agent (:doc:`/exploitation/configuration`) et ne
se change pas après création.
À la création, l'agent pose sur l'hyperviseur, pour un VPC ``vp-<id>`` :
.. code-block:: text
vrf-<id> VRF, table de routage = l3vni
├── brl3-<id> bridge sans adresse, MAC de routeur 02:02:<IPv4 du VTEP>
│ └── vxl3-<id> VXLAN du l3vni, VTEP = IPv4 primaire de default_interface
└── l3-<id>-o 169.254.0.1/30 ── l3-<id>-i 169.254.0.2/30, dans le netns du VPC
Dans le netns, le ``cidr`` du VPC est routé vers ``169.254.0.1`` ; le transit ``/30`` est le même
sur tous les hyperviseurs et pour tous les VPC. Le forwarding IPv4 est activé dans le netns, et
sur l'hôte **seulement** sur ``brl3-<id>`` et ``l3-<id>-o`` : l'hyperviseur ne devient pas un
routeur entre ses autres interfaces. FRR reçoit la VRF, son ``vni``, une instance
``router bgp <asn> vrf vrf-<id>`` avec RD ``<VTEP>:<l3vni>`` et RT ``<asn>:<l3vni>``, et
l'export des routes du ``cidr`` présentes dans la VRF.
La suppression retire d'abord la configuration de routage de FRR, puis les interfaces, puis la
VRF de FRR. Cet ordre est imposé : supprimer la VRF du noyau sous une instance BGP encore
configurée fait planter ``bgpd`` (FRR 10.7.1).
Subnet
------
@ -34,6 +63,11 @@ hosts dont le nommage réseau diffère.
Le comportement réseau dépend du :doc:`mode </concepts/modes-reseau>`.
Un subnet ne peut être supprimé que si aucune VM non supprimée n'a d'interface dessus — sinon
409, avec la liste des VM. Dans un VPC routé par ``l3vni``, un subnet ``vxlan`` annonce son
préfixe dans la VRF du VPC à la création et le retire en premier à la suppression : c'est le
filet qui couvre une VM tant que son ``/32`` n'est pas annoncé.
VM
--
@ -44,6 +78,12 @@ le nom de l'interface dans le guest. Exactement une interface doit être ``prima
la route par défaut et le serveur de metadata. Tous les subnets d'une VM doivent appartenir au
**même VPC**.
**Routes.** Dans un VPC routé par ``l3vni``, chaque interface sur un subnet ``vxlan`` est annoncée
en ``/32`` dans la VRF du VPC une fois QEMU démarré, juste avant le passage en ``running``. À la
suppression, ces ``/32`` sont retirés **avant** l'arrêt de QEMU. Comme un subnet ne peut être
supprimé tant qu'une VM l'utilise, le ``/32`` d'une VM est toujours retiré avant le ``/24`` de son
subnet — l'ordre qui évite le trou noir de `#41 <https://git.g3e.fr/syonad/two/issues/41>`_.
**Stockage.** Un seul disque ``vdX`` (virtio-blk) par VM ; les disques supplémentaires passent
par le contrôleur SCSI (``sdX``). Cette contrainte vient de la carte PCI figée — voir
:doc:`/architecture/contraintes`.

View file

@ -27,8 +27,18 @@ l'agent désactive l'apprentissage et ne configure aucun voisin — cf.
Configuration de FRR
~~~~~~~~~~~~~~~~~~~~
Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. D'un hyperviseur à l'autre ne changent que
``hostname`` et ``router-id``, l'adresse de l'hyperviseur sur son segment.
Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. Le reste est posé par l'agent, à chaque
démarrage, à partir de la section ``bgp`` de ``agent.yml`` (:doc:`/exploitation/configuration`) :
.. code-block:: yaml
bgp:
asn: 64600
route_reflectors: [10.255.255.1]
L'agent en déduit la configuration ci-dessous ; le ``router-id`` est l'adresse IPv4 primaire de
``default_interface``, la seule valeur qui change d'un hyperviseur à l'autre. ``hostname`` et
``log`` ne sont pas gérés par l'agent.
.. literalinclude:: ../../test/e2e/topologies/frr/hv1.conf
:language: text

View file

@ -71,7 +71,8 @@ Codes de retour
- ressource inexistante
* - ``409``
- conflit d'existence ou d'état : ressource déjà créée, ou suppression depuis un état qui
ne l'autorise pas ; pour un VPC, subnets encore présents
ne l'autorise pas ; pour un VPC, subnets encore présents ; pour un subnet, VM non
supprimées qui l'utilisent encore
* - ``422``
- dépendance absente ou pas prête : VPC parent d'un subnet, subnet d'une VM
* - ``500``

View file

@ -104,6 +104,38 @@ Traduit les clés logiques ``iface_type`` de l'API vers les bridges physiques de
inconnue ou omise retombe silencieusement sur ``default_interface`` — ce n'est pas une erreur,
mais c'est une source de subnets branchés au mauvais endroit sans le dire.
Plan de contrôle BGP
--------------------
.. code-block:: yaml
bgp:
asn: 64600
route_reflectors: [10.255.255.1]
Section **facultative**. Absente, l'agent ne touche pas à FRR : sa configuration reste à poser
à la main, comme avant. Présente, l'agent **pilote la configuration BGP de l'hyperviseur** :
à chaque démarrage, avant d'ouvrir son API, il applique par ``vtysh`` la base décrite dans
:doc:`/deploiement/premier-hyperviseur`, puis l'enregistre par ``write memory``.
* ``asn`` — l'AS de l'hyperviseur, commun à tous les hyperviseurs du cluster (iBGP vers le
route reflector). Obligatoire dès que la section existe.
* ``route_reflectors`` — au moins une adresse IPv4, sans doublon. Un route reflector retiré de
la liste est retiré de FRR au démarrage suivant.
Le ``router-id`` n'est pas configurable : c'est l'adresse IPv4 primaire de
``default_interface``, celle qui sert de VTEP.
Le démarrage **échoue** si la section est incomplète, si ``default_interface`` n'a pas
d'adresse IPv4 primaire, si ``vtysh`` refuse une commande, ou si FRR tourne déjà sous un autre
AS : changer d'``asn`` suppose de retirer l'ancienne instance à la main.
.. warning::
``/etc/frr/frr.conf`` devient un fichier **généré** : ``write memory`` le réécrit à partir de
la configuration en mémoire de FRR. Une modification faite dans le fichier sans être
rechargée est perdue au prochain démarrage de l'agent.
Metadata et QEMU
----------------

View file

@ -38,6 +38,20 @@ toujours sur le système, et **notifie les écarts sans jamais réparer**.
Il contrôle notamment l'existence des network namespaces, des liens réseau des subnets, des taps
de VM et la réponse des units systemd associées.
Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de contrôle :
* **Sessions BGP** : une notification par route reflector dont la session n'est pas
``Established`` (``show bgp neighbors <ip>``).
* **VPC routés** (``l3vni``, état ``running``) : présence et état des interfaces ``vrf-<id>``,
``brl3-<id>``, ``vxl3-<id>``, ``l3-<id>-o`` et, dans le netns, ``l3-<id>-i`` ; L3VNI de type
``L3``, ``Up``, avec la MAC de routeur attendue (``show evpn vni <l3vni>``).
* **Réconciliation des routes exportées** dans la VRF : chaque ``/24`` de subnet ``vxlan`` et chaque
``/32`` d'interface de VM ``running`` doit être exporté ; à l'inverse, une route exportée dont
le propriétaire n'existe plus — ni ``running``, ni en ``creating``, ``deleting`` ou ``error`` —
est signalée comme un **trou noir** : l'annonce attire du trafic que plus rien ne reçoit.
Un VPC routé alors que la section ``bgp`` a disparu de la configuration est signalé lui aussi.
.. note::
Un écart persistant est signalé **à chaque tick**, sans déduplication. Le volume de

View file

@ -16,11 +16,15 @@ import (
// The worker queue is buffered but has no running goroutines: Dispatch enqueues
// without blocking and Execute never runs, so DB state reflects only Prepare writes.
func newTestServer(t *testing.T) (*Server, *badger.DB) {
t.Helper()
return newTestServerWith(t, &configuration.Config{DefaultInterface: "br-test"})
}
func newTestServerWith(t *testing.T, cfg *configuration.Config) (*Server, *badger.DB) {
t.Helper()
db := kv.InitDB(kv.Config{Path: t.TempDir()}, false)
t.Cleanup(func() { db.Close() })
q := worker.New(100)
cfg := &configuration.Config{DefaultInterface: "br-test"}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
d := dispatcher.New(q, db, cfg, logger)
return New(d, db, logger, "127.0.0.1:0"), db

View file

@ -1,14 +1,16 @@
package agentapi
type VPCCreateRequest struct {
Name string `json:"name"`
CIDR string `json:"cidr"`
Name string `json:"name"`
CIDR string `json:"cidr"`
L3VNI *uint32 `json:"l3vni,omitempty"`
}
type VPC struct {
Name string `json:"name"`
State string `json:"state"`
CIDR string `json:"cidr"`
L3VNI uint32 `json:"l3vni,omitempty"`
}
type SubnetCreateRequest struct {

View file

@ -36,8 +36,9 @@ func (s *Server) getVpc(w http.ResponseWriter, _ *http.Request, name string) {
return
}
cidr, _ := kv.GetFromDB(s.db, "vpc/"+name+"/cidr")
l3vni, _ := kv.GetFromDB(s.db, "vpc/"+name+"/l3vni")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(VPC{Name: name, State: state, CIDR: cidr})
json.NewEncoder(w).Encode(VPC{Name: name, State: state, CIDR: cidr, L3VNI: parseL3VNI(l3vni)})
}
func (s *Server) deleteVpc(w http.ResponseWriter, _ *http.Request, name string) {

View file

@ -5,9 +5,12 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
// --- VpcsHandler ---
@ -236,3 +239,84 @@ func TestVpcByName_InvalidMethod(t *testing.T) {
t.Errorf("attendu 405, obtenu %d", w.Code)
}
}
func newBGPTestServer(t *testing.T) (*Server, *badger.DB) {
cfg := &configuration.Config{DefaultInterface: "br-test"}
cfg.BGP.ASN = 64600
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
return newTestServerWith(t, cfg)
}
func postVpcJSON(s *Server, body string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
s.VpcsHandler(w, httptest.NewRequest(http.MethodPost, "/vpcs", strings.NewReader(body)))
return w
}
func TestPostVpc_WithL3VNI(t *testing.T) {
s, db := newBGPTestServer(t)
w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":4001}`)
if w.Code != http.StatusAccepted {
t.Fatalf("attendu 202, obtenu %d: %s", w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), `"l3vni":4001`) {
t.Errorf("réponse sans l3vni : %s", w.Body.String())
}
if got, _ := kv.GetFromDB(db, "vpc/vp-1/l3vni"); got != "4001" {
t.Errorf("l3vni en DB = %q", got)
}
}
func TestPostVpc_WithoutL3VNIOmitsTheField(t *testing.T) {
s, _ := newBGPTestServer(t)
w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16"}`)
if w.Code != http.StatusAccepted || strings.Contains(w.Body.String(), "l3vni") {
t.Errorf("%d : %s", w.Code, w.Body.String())
}
}
func TestPostVpc_RejectsAnInvalidL3VNIAsABadRequest(t *testing.T) {
s, db := newBGPTestServer(t)
for _, body := range []string{
`{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":0}`,
`{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":255}`,
`{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":16777216}`,
`{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":-1}`,
} {
if w := postVpcJSON(s, body); w.Code != http.StatusBadRequest {
t.Errorf("%s : attendu 400, obtenu %d: %s", body, w.Code, w.Body.String())
}
}
if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 {
t.Errorf("écritures malgré le refus : %v", entries)
}
}
func TestPostVpc_L3VNIWithoutBGPIsAConflict(t *testing.T) {
s, _ := newTestServer(t)
w := postVpcJSON(s, `{"name":"vp-1","cidr":"10.0.0.0/16","l3vni":4001}`)
if w.Code != http.StatusConflict || !strings.Contains(w.Body.String(), "requires the bgp section") {
t.Errorf("%d : %s", w.Code, w.Body.String())
}
}
func TestGetAndListVpc_ReturnTheL3VNI(t *testing.T) {
s, db := newTestServer(t)
kv.AddInDB(db, "vpc/vp-1/state", "running")
kv.AddInDB(db, "vpc/vp-1/cidr", "10.0.0.0/16")
kv.AddInDB(db, "vpc/vp-1/l3vni", "4001")
w := httptest.NewRecorder()
s.VpcByNameHandler(w, httptest.NewRequest(http.MethodGet, "/vpcs/vp-1", nil))
var got VPC
json.NewDecoder(w.Body).Decode(&got)
if got.L3VNI != 4001 {
t.Errorf("GET : l3vni = %d", got.L3VNI)
}
w = httptest.NewRecorder()
s.VpcsHandler(w, httptest.NewRequest(http.MethodGet, "/vpcs", nil))
var list []VPC
json.NewDecoder(w.Body).Decode(&list)
if len(list) != 1 || list[0].L3VNI != 4001 {
t.Errorf("liste : %+v", list)
}
}

View file

@ -4,9 +4,11 @@ import (
"encoding/json"
"net"
"net/http"
"strconv"
"strings"
dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent"
"git.g3e.fr/syonad/two/internal/vpc"
"git.g3e.fr/syonad/two/pkg/db/kv"
)
@ -44,6 +46,8 @@ func (s *Server) listVpcs(w http.ResponseWriter, _ *http.Request) {
vpcs[name].State = value
case "cidr":
vpcs[name].CIDR = value
case "l3vni":
vpcs[name].L3VNI = parseL3VNI(value)
}
}
result := make([]VPC, 0, len(vpcs))
@ -77,6 +81,14 @@ func (s *Server) postVpc(w http.ResponseWriter, r *http.Request) {
return
}
cmd := dispatcher.CreateVPCCommand{Name: req.Name, CIDR: req.CIDR}
if req.L3VNI != nil {
if err := vpc.ValidL3VNI(*req.L3VNI); err != nil {
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()})
return
}
cmd.L3VNI = *req.L3VNI
}
if err := s.dispatcher.Prepare(cmd); err != nil {
w.WriteHeader(http.StatusConflict)
json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()})
@ -90,5 +102,10 @@ func (s *Server) postVpc(w http.ResponseWriter, r *http.Request) {
return
}
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(VPC{Name: req.Name, State: state, CIDR: req.CIDR})
json.NewEncoder(w).Encode(VPC{Name: req.Name, State: state, CIDR: req.CIDR, L3VNI: cmd.L3VNI})
}
func parseL3VNI(value string) uint32 {
id, _ := strconv.ParseUint(value, 10, 32)
return uint32(id)
}

View file

@ -0,0 +1,38 @@
package configuration
import (
"fmt"
"net"
)
type BGP struct {
ASN uint32 `mapstructure:"asn"`
RouteReflectors []string `mapstructure:"route_reflectors"`
}
func (b BGP) Enabled() bool {
return b.ASN != 0 || len(b.RouteReflectors) > 0
}
func (b BGP) Reflectors() ([]net.IP, error) {
if b.ASN == 0 {
return nil, fmt.Errorf("bgp.asn must be set")
}
if len(b.RouteReflectors) == 0 {
return nil, fmt.Errorf("bgp.route_reflectors must list at least one address")
}
seen := make(map[string]bool, len(b.RouteReflectors))
ips := make([]net.IP, 0, len(b.RouteReflectors))
for _, rr := range b.RouteReflectors {
ip := net.ParseIP(rr).To4()
if ip == nil {
return nil, fmt.Errorf("bgp.route_reflectors: %q is not an IPv4 address", rr)
}
if seen[ip.String()] {
return nil, fmt.Errorf("bgp.route_reflectors: %s is listed twice", ip)
}
seen[ip.String()] = true
ips = append(ips, ip)
}
return ips, nil
}

View file

@ -0,0 +1,54 @@
package configuration
import (
"strings"
"testing"
)
func TestLoadConfig_BGPAbsentIsDisabled(t *testing.T) {
cfg, err := LoadConfig(writeConfig(t, "database:\n path: /tmp/two\n"))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if cfg.BGP.Enabled() {
t.Errorf("bgp enabled without a bgp section: %+v", cfg.BGP)
}
}
func TestLoadConfig_ReadsTheBGPSection(t *testing.T) {
cfg, err := LoadConfig(writeConfig(t, "bgp:\n asn: 4200000000\n route_reflectors: [10.255.255.1, 10.255.255.2]\n"))
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
if !cfg.BGP.Enabled() || cfg.BGP.ASN != 4200000000 {
t.Fatalf("bgp = %+v", cfg.BGP)
}
rrs, err := cfg.BGP.Reflectors()
if err != nil {
t.Fatalf("Reflectors: %v", err)
}
if len(rrs) != 2 || rrs[0].String() != "10.255.255.1" || rrs[1].String() != "10.255.255.2" {
t.Errorf("reflectors = %v", rrs)
}
}
func TestBGPReflectors_RejectsAnIncompleteOrInvalidSection(t *testing.T) {
cases := []struct {
bgp BGP
want string
}{
{BGP{RouteReflectors: []string{"10.255.255.1"}}, "bgp.asn must be set"},
{BGP{ASN: 64600}, "at least one address"},
{BGP{ASN: 64600, RouteReflectors: []string{"rr1"}}, `"rr1" is not an IPv4 address`},
{BGP{ASN: 64600, RouteReflectors: []string{"fd00::1"}}, `"fd00::1" is not an IPv4 address`},
{BGP{ASN: 64600, RouteReflectors: []string{"10.255.255.1", "10.255.255.1"}}, "10.255.255.1 is listed twice"},
}
for _, c := range cases {
if !c.bgp.Enabled() {
t.Errorf("%+v: Enabled() = false", c.bgp)
}
if _, err := c.bgp.Reflectors(); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%+v: erreur %v, attendu %q", c.bgp, err, c.want)
}
}
}

View file

@ -55,6 +55,7 @@ type Config struct {
Enabled bool `mapstructure:"enabled"`
IntervalSeconds int `mapstructure:"interval_seconds"`
} `mapstructure:"watchdog"`
BGP BGP `mapstructure:"bgp"`
DefaultInterface string `mapstructure:"default_interface"`
Interfaces map[string]string `mapstructure:"interfaces"`
}

View file

@ -17,6 +17,7 @@ func newTestDispatcher(t *testing.T) (*Dispatcher, *badger.DB) {
t.Cleanup(func() { db.Close() })
q := worker.New(100)
q.Start(2)
t.Cleanup(q.Stop)
cfg := &configuration.Config{DefaultInterface: "br-default"}
cfg.Interfaces = map[string]string{"vms": "br-vms"}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))

View file

@ -2,7 +2,9 @@ package dispatcher
import (
"fmt"
"sort"
"strconv"
"strings"
"time"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
@ -101,9 +103,36 @@ func (c DeleteSubnetCommand) Prepare(db *badger.DB, _ *configuration.Config) err
if !state.CanDelete(current) {
return fmt.Errorf("subnet %q cannot be deleted while %s", c.Name, current)
}
vms, err := vmsUsingSubnet(db, c.Name)
if err != nil {
return err
}
if len(vms) > 0 {
return fmt.Errorf("subnet %q is still used by vms %s", c.Name, strings.Join(vms, ", "))
}
return state.Set(db, c.Key(), state.Deleting)
}
func vmsUsingSubnet(db *badger.DB, subnetName string) ([]string, error) {
entries, err := kv.ListByPrefix(db, "vm/")
if err != nil {
return nil, fmt.Errorf("failed to list vms: %w", err)
}
var users []string
for key, value := range entries {
parts := strings.Split(key, "/")
if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || value != subnetName {
continue
}
if s, err := state.Get(db, "vm/"+parts[1]); err == nil && s == state.Deleted {
continue
}
users = append(users, parts[1])
}
sort.Strings(users)
return users, nil
}
func (c DeleteSubnetCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
if err := subnet.DeleteSubnet(db, c.Name, cfg); err != nil {
return err

View file

@ -1,6 +1,7 @@
package dispatcher
import (
"strings"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
@ -327,3 +328,38 @@ func TestCreateSubnetCommand_Prepare_RejectsUnknownMode(t *testing.T) {
t.Error("un mode inconnu doit être refusé")
}
}
func TestDeleteSubnetCommand_Prepare_RefusedWhileAVMUsesIt(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "running")
kv.AddInDB(db, "vm/i-gone/state", "deleted")
kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-other/state", "running")
kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b")
kv.AddInDB(db, "vm/i-b/state", "deleting")
kv.AddInDB(db, "vm/i-b/nic/0/subnet", "sn-b")
kv.AddInDB(db, "vm/i-b/nic/1/subnet", "sn-a")
kv.AddInDB(db, "vm/i-a/state", "error")
kv.AddInDB(db, "vm/i-a/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-c/state", "running")
kv.AddInDB(db, "vm/i-c/nic/0/subnet", "sn-a")
err := DeleteSubnetCommand{Name: "sn-a"}.Prepare(db, nil)
if err == nil || !strings.Contains(err.Error(), `subnet "sn-a" is still used by vms i-a, i-b, i-c`) {
t.Fatalf("erreur %v", err)
}
if s, _ := kv.GetFromDB(db, "subnet/sn-a/state"); s != "running" {
t.Errorf("état modifié malgré le refus : %q", s)
}
}
func TestDeleteSubnetCommand_Prepare_AllowedOnceItsVMsAreDeleted(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "subnet/sn-a/state", "running")
kv.AddInDB(db, "vm/i-gone/state", "deleted")
kv.AddInDB(db, "vm/i-gone/nic/0/subnet", "sn-a")
kv.AddInDB(db, "vm/i-other/state", "running")
kv.AddInDB(db, "vm/i-other/nic/0/subnet", "sn-b")
if err := (DeleteSubnetCommand{Name: "sn-a"}).Prepare(db, nil); err != nil {
t.Fatalf("Prepare a échoué : %v", err)
}
}

View file

@ -3,6 +3,7 @@ package dispatcher
import (
"fmt"
"net"
"strconv"
"strings"
"time"
@ -14,27 +15,63 @@ import (
)
type CreateVPCCommand struct {
Name string
CIDR string
Name string
CIDR string
L3VNI uint32
}
func (c CreateVPCCommand) Key() string { return "vpc/" + c.Name }
func (c CreateVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error {
func (c CreateVPCCommand) Prepare(db *badger.DB, cfg *configuration.Config) error {
if _, err := kv.GetFromDB(db, "vpc/"+c.Name+"/state"); err == nil {
return fmt.Errorf("vpc %q already exists", c.Name)
}
if _, _, err := net.ParseCIDR(c.CIDR); err != nil {
return fmt.Errorf("invalid cidr %q: %w", c.CIDR, err)
}
if c.L3VNI != 0 {
if err := c.checkL3VNI(db, cfg); err != nil {
return err
}
}
if err := kv.AddInDB(db, "vpc/"+c.Name+"/cidr", c.CIDR); err != nil {
return err
}
if c.L3VNI != 0 {
if err := kv.AddInDB(db, "vpc/"+c.Name+"/l3vni", strconv.FormatUint(uint64(c.L3VNI), 10)); err != nil {
return err
}
}
return state.Set(db, c.Key(), state.Creating)
}
func (c CreateVPCCommand) Execute(db *badger.DB, _ *configuration.Config) error {
return vpc.CreateVPC(db, c.Name)
func (c CreateVPCCommand) checkL3VNI(db *badger.DB, cfg *configuration.Config) error {
if err := vpc.ValidL3VNI(c.L3VNI); err != nil {
return err
}
if cfg == nil || !cfg.BGP.Enabled() {
return fmt.Errorf("l3vni requires the bgp section in the agent configuration")
}
entries, err := kv.ListByPrefix(db, "vpc/")
if err != nil {
return fmt.Errorf("failed to list vpcs: %w", err)
}
want := strconv.FormatUint(uint64(c.L3VNI), 10)
for key, value := range entries {
parts := strings.Split(key, "/")
if len(parts) != 3 || parts[2] != "l3vni" || value != want {
continue
}
if s, err := state.Get(db, "vpc/"+parts[1]); err == nil && s == state.Deleted {
continue
}
return fmt.Errorf("l3vni %d is already used by vpc %q", c.L3VNI, parts[1])
}
return nil
}
func (c CreateVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) error {
return vpc.CreateVPC(db, c.Name, cfg)
}
type DeleteVPCCommand struct {
@ -95,7 +132,7 @@ func (c DeleteVPCCommand) Execute(db *badger.DB, cfg *configuration.Config) erro
case <-time.After(time.Duration(cfg.Dispatcher.PollSeconds) * time.Second):
}
}
if err := vpc.DeleteVPC(db, c.Name); err != nil {
if err := vpc.DeleteVPC(db, c.Name, cfg); err != nil {
return err
}
current, err := state.Get(db, c.Key())

View file

@ -1,8 +1,10 @@
package dispatcher
import (
"strings"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/pkg/db/kv"
)
@ -137,3 +139,81 @@ func TestDeleteVPCCommand_Prepare_AllowedWhenSubnetDeleting(t *testing.T) {
t.Fatalf("Prepare devrait réussir si le subnet est deleting : %v", err)
}
}
func bgpConfig() *configuration.Config {
cfg := &configuration.Config{}
cfg.BGP.ASN = 64600
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
return cfg
}
func TestCreateVPCCommand_Prepare_StoresTheL3VNI(t *testing.T) {
_, db := newTestDispatcher(t)
cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 4001}
if err := cmd.Prepare(db, bgpConfig()); err != nil {
t.Fatalf("Prepare a échoué : %v", err)
}
if got, err := kv.GetFromDB(db, "vpc/vpc-1/l3vni"); err != nil || got != "4001" {
t.Errorf("l3vni en DB = %q, %v ; attendu 4001", got, err)
}
}
func TestCreateVPCCommand_Prepare_WithoutL3VNIStoresNothing(t *testing.T) {
_, db := newTestDispatcher(t)
cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16"}
if err := cmd.Prepare(db, nil); err != nil {
t.Fatalf("Prepare a échoué : %v", err)
}
if got, err := kv.GetFromDB(db, "vpc/vpc-1/l3vni"); err == nil {
t.Errorf("clé l3vni écrite : %q", got)
}
}
func TestCreateVPCCommand_Prepare_RefusesAnL3VNIWithoutBGP(t *testing.T) {
_, db := newTestDispatcher(t)
cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 4001}
err := cmd.Prepare(db, &configuration.Config{})
if err == nil || !strings.Contains(err.Error(), "requires the bgp section") {
t.Fatalf("erreur %v", err)
}
if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 {
t.Errorf("écritures malgré le refus : %v", entries)
}
}
func TestCreateVPCCommand_Prepare_RefusesAReservedL3VNI(t *testing.T) {
_, db := newTestDispatcher(t)
cmd := CreateVPCCommand{Name: "vpc-1", CIDR: "10.0.0.0/16", L3VNI: 254}
if err := cmd.Prepare(db, bgpConfig()); err == nil || !strings.Contains(err.Error(), "l3vni 254 is reserved") {
t.Fatalf("erreur %v", err)
}
if entries, _ := kv.ListByPrefix(db, "vpc/"); len(entries) != 0 {
t.Errorf("écritures malgré le refus : %v", entries)
}
}
func TestCreateVPCCommand_Prepare_RefusesAnL3VNIAlreadyUsed(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vpc/vpc-a/state", "deleting")
kv.AddInDB(db, "vpc/vpc-a/l3vni", "4001")
cmd := CreateVPCCommand{Name: "vpc-b", CIDR: "10.0.0.0/16", L3VNI: 4001}
err := cmd.Prepare(db, bgpConfig())
if err == nil || !strings.Contains(err.Error(), `l3vni 4001 is already used by vpc "vpc-a"`) {
t.Fatalf("erreur %v", err)
}
if _, err := kv.GetFromDB(db, "vpc/vpc-b/state"); err == nil {
t.Error("vpc-b créé malgré le doublon")
}
}
func TestCreateVPCCommand_Prepare_ReusesTheL3VNIOfADeletedVPC(t *testing.T) {
_, db := newTestDispatcher(t)
kv.AddInDB(db, "vpc/vpc-a/state", "deleted")
kv.AddInDB(db, "vpc/vpc-a/l3vni", "4001")
kv.AddInDB(db, "vpc/vpc-c/state", "running")
kv.AddInDB(db, "vpc/vpc-c/l3vni", "4002")
cmd := CreateVPCCommand{Name: "vpc-b", CIDR: "10.0.0.0/16", L3VNI: 4001}
if err := cmd.Prepare(db, bgpConfig()); err != nil {
t.Fatalf("Prepare a échoué : %v", err)
}
}

124
internal/frr/client.go Normal file
View file

@ -0,0 +1,124 @@
package frr
import (
"fmt"
"os/exec"
"strings"
)
type runner func(args ...string) ([]byte, error)
type Client struct {
run runner
}
func NewClient() *Client {
return &Client{run: func(args ...string) ([]byte, error) {
return exec.Command("vtysh", args...).CombinedOutput()
}}
}
func (c *Client) ApplyBase(b Base) error {
if err := b.validate(); err != nil {
return err
}
running, err := c.runningConfig()
if err != nil {
return err
}
stale, err := b.staleReflectors(running)
if err != nil {
return err
}
if err := c.configure(b.commands()...); err != nil {
return err
}
if len(stale) > 0 {
cmds := []string{fmt.Sprintf("router bgp %d", b.ASN)}
for _, rr := range stale {
cmds = append(cmds, "no neighbor "+rr)
}
if err := c.configure(append(cmds, "exit")...); err != nil {
return err
}
}
return c.save()
}
func (c *Client) ApplyVRF(v VRF) error {
if err := v.validate(); err != nil {
return err
}
if err := c.configure(v.commands()...); err != nil {
return err
}
return c.save()
}
func (c *Client) WithdrawVRF(v VRF) error {
return c.teardown(v, routingTeardown)
}
func (c *Client) RemoveVRF(v VRF) error {
if err := c.teardown(v, vrfTeardown); err != nil {
return err
}
running, err := c.runningConfig()
if err != nil {
return err
}
return remaining(routingTeardown(v, running))
}
func (c *Client) teardown(v VRF, plan func(VRF, string) []step) error {
if err := v.validateName(); err != nil {
return err
}
running, err := c.runningConfig()
if err != nil {
return err
}
for _, s := range plan(v, running) {
if err := c.configure(s...); err != nil {
return err
}
}
if err := c.save(); err != nil {
return err
}
if running, err = c.runningConfig(); err != nil {
return err
}
return remaining(plan(v, running))
}
func (c *Client) runningConfig() (string, error) {
return c.exec("show running-config")
}
func (c *Client) configure(cmds ...string) error {
_, err := c.exec(append([]string{"configure terminal"}, cmds...)...)
return err
}
func (c *Client) save() error {
_, err := c.exec("write memory")
return err
}
func (c *Client) exec(cmds ...string) (string, error) {
args := make([]string, 0, 2*len(cmds))
for _, cmd := range cmds {
args = append(args, "-c", cmd)
}
out, err := c.run(args...)
if err != nil {
return "", fmt.Errorf("vtysh %q: %w: %s", cmds, err, strings.TrimSpace(string(out)))
}
for _, l := range strings.Split(string(out), "\n") {
if strings.HasPrefix(strings.TrimSpace(l), "%") {
return "", fmt.Errorf("vtysh %q: %s", cmds, strings.TrimSpace(l))
}
}
return string(out), nil
}

251
internal/frr/client_test.go Normal file
View file

@ -0,0 +1,251 @@
package frr
import (
"errors"
"net"
"reflect"
"strings"
"testing"
)
type fakeVtysh struct {
calls [][]string
running []string
fail map[string]string
status map[string]error
}
func (f *fakeVtysh) run(args ...string) ([]byte, error) {
var cmds []string
for i := 1; i < len(args); i += 2 {
cmds = append(cmds, args[i])
}
f.calls = append(f.calls, cmds)
last := cmds[len(cmds)-1]
if last == "show running-config" {
if len(f.running) == 0 {
return nil, errors.New("show running-config inattendu")
}
out := f.running[0]
f.running = f.running[1:]
return []byte(out), nil
}
for _, c := range cmds {
if out, ok := f.fail[c]; ok {
return []byte(out), f.status[c]
}
}
return nil, nil
}
func (f *fakeVtysh) client() *Client {
return &Client{run: f.run}
}
func (f *fakeVtysh) sent(cmd string) bool {
for _, c := range f.calls {
for _, x := range c {
if x == cmd {
return true
}
}
}
return false
}
func TestApplyVRFConfiguresThenSaves(t *testing.T) {
f := &fakeVtysh{}
if err := f.client().ApplyVRF(labVRF(t)); err != nil {
t.Fatal(err)
}
want := [][]string{
append([]string{"configure terminal"}, labVRF(t).commands()...),
{"write memory"},
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
}
}
func TestApplyVRFRefusesAnInvalidVRFWithoutCallingVtysh(t *testing.T) {
f := &fakeVtysh{}
v := labVRF(t)
v.L3VNI = 0
if err := f.client().ApplyVRF(v); err == nil || !strings.Contains(err.Error(), "l3vni 0") {
t.Errorf("erreur %v", err)
}
if len(f.calls) != 0 {
t.Errorf("vtysh appelé : %q", f.calls)
}
}
func TestExecReportsAnErrorPrintedByVtysh(t *testing.T) {
f := &fakeVtysh{fail: map[string]string{"vni 4001": "% Please unconfigure l3vni 4001\n"}}
err := f.client().ApplyVRF(labVRF(t))
if err == nil || !strings.Contains(err.Error(), "% Please unconfigure l3vni 4001") {
t.Errorf("erreur %v", err)
}
if f.sent("write memory") {
t.Error("configuration sauvegardée malgré l'erreur")
}
}
func TestExecReportsANonZeroExit(t *testing.T) {
f := &fakeVtysh{
fail: map[string]string{"write memory": "Building Configuration...\n"},
status: map[string]error{"write memory": errors.New("exit status 1")},
}
err := f.client().ApplyVRF(labVRF(t))
if err == nil || !strings.Contains(err.Error(), "exit status 1") || !strings.Contains(err.Error(), "Building Configuration...") {
t.Errorf("erreur %v", err)
}
}
func TestWithdrawVRFRunsEachStepInOrderThenSaves(t *testing.T) {
f := &fakeVtysh{running: []string{fullRunning, otherOnlyRunning}}
if err := f.client().WithdrawVRF(labVRF(t)); err != nil {
t.Fatal(err)
}
want := [][]string{
{"show running-config"},
{"configure terminal", "router bgp 64600 vrf vrf-a1b2", "address-family ipv4 unicast", "no redistribute kernel"},
{"configure terminal", "vrf vrf-a1b2", "no vni 4001"},
{"configure terminal", "no router bgp 64600 vrf vrf-a1b2"},
{"configure terminal", "no route-map EXPORT-a1b2"},
{"configure terminal", "no ip prefix-list PL-a1b2"},
{"write memory"},
{"show running-config"},
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
}
}
func TestWithdrawVRFStopsAtTheFirstRefusedStep(t *testing.T) {
f := &fakeVtysh{
running: []string{fullRunning},
fail: map[string]string{"no vni 4001": "% Please unconfigure l3vni 4001\n"},
}
if err := f.client().WithdrawVRF(labVRF(t)); err == nil || !strings.Contains(err.Error(), "Please unconfigure") {
t.Errorf("erreur %v", err)
}
if f.sent("no router bgp 64600 vrf vrf-a1b2") {
t.Error("instance BGP supprimée après un refus de l'étape précédente")
}
}
func TestWithdrawVRFFailsWhenFRRKeptTheConfiguration(t *testing.T) {
f := &fakeVtysh{running: []string{fullRunning, fullRunning}}
err := f.client().WithdrawVRF(labVRF(t))
if err == nil || !strings.Contains(err.Error(), "configuration still present after teardown") || !strings.Contains(err.Error(), "no router bgp 64600 vrf vrf-a1b2") {
t.Errorf("erreur %v", err)
}
}
func TestRemoveVRFDeletesTheVRFAndChecksNothingIsLeft(t *testing.T) {
gone := "vrf vrf-a1b2c\n vni 4002\nexit-vrf\n"
f := &fakeVtysh{running: []string{otherOnlyRunning, gone, gone}}
if err := f.client().RemoveVRF(labVRF(t)); err != nil {
t.Fatal(err)
}
if !f.sent("no vrf vrf-a1b2") {
t.Errorf("appels : %q", f.calls)
}
}
func TestRemoveVRFFailsWhenRoutingConfigurationIsStillThere(t *testing.T) {
f := &fakeVtysh{running: []string{otherOnlyRunning, "", fullRunning}}
err := f.client().RemoveVRF(labVRF(t))
if err == nil || !strings.Contains(err.Error(), "configuration still present after teardown") {
t.Errorf("erreur %v", err)
}
}
func TestTeardownNeedsOnlyTheASNAndTheID(t *testing.T) {
f := &fakeVtysh{running: []string{"", ""}}
if err := f.client().WithdrawVRF(VRF{ASN: 64600, ID: "a1b2"}); err != nil {
t.Errorf("démontage refusé : %v", err)
}
f = &fakeVtysh{}
if err := f.client().WithdrawVRF(VRF{ASN: 64600}); err == nil || len(f.calls) != 0 {
t.Errorf("démontage sans id : erreur %v, appels %q", err, f.calls)
}
}
const baseRunning = `frr version 10.7.1
router bgp 64600
bgp router-id 192.168.14.13
neighbor fabric peer-group
neighbor fabric remote-as 64600
neighbor 10.255.255.1 peer-group fabric
neighbor 10.255.255.9 peer-group fabric
neighbor 10.0.0.5 peer-group monitoring
!
address-family l2vpn evpn
neighbor fabric activate
advertise-all-vni
exit-address-family
exit
!
router bgp 64600 vrf vrf-a1b2
neighbor 10.255.255.7 peer-group fabric
exit
`
func labBase() Base {
return Base{ASN: 64600, RouterID: net.ParseIP("192.168.14.13"), RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}}
}
func TestApplyBaseOnAFreshFRRConfiguresThenSaves(t *testing.T) {
f := &fakeVtysh{running: []string{""}}
if err := f.client().ApplyBase(labBase()); err != nil {
t.Fatal(err)
}
want := [][]string{
{"show running-config"},
append([]string{"configure terminal"}, labBase().commands()...),
{"write memory"},
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
}
}
func TestApplyBaseRemovesReflectorsNoLongerConfiguredAfterAddingTheNewOnes(t *testing.T) {
f := &fakeVtysh{running: []string{baseRunning}}
if err := f.client().ApplyBase(labBase()); err != nil {
t.Fatal(err)
}
want := [][]string{
{"show running-config"},
append([]string{"configure terminal"}, labBase().commands()...),
{"configure terminal", "router bgp 64600", "no neighbor 10.255.255.9", "exit"},
{"write memory"},
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
}
}
func TestApplyBaseRefusesToChangeTheASN(t *testing.T) {
f := &fakeVtysh{running: []string{baseRunning}}
b := labBase()
b.ASN = 64601
err := f.client().ApplyBase(b)
if err == nil || !strings.Contains(err.Error(), "bgp already runs as AS 64600, not 64601") {
t.Errorf("erreur %v", err)
}
if len(f.calls) != 1 {
t.Errorf("appels après le refus : %q", f.calls)
}
}
func TestApplyBaseStopsWhenTheRunningConfigIsUnreadable(t *testing.T) {
f := &fakeVtysh{}
if err := f.client().ApplyBase(labBase()); err == nil {
t.Fatal("aucune erreur")
}
if f.sent("write memory") || f.sent("router bgp 64600") {
t.Errorf("appels : %q", f.calls)
}
}

155
internal/frr/config.go Normal file
View file

@ -0,0 +1,155 @@
package frr
import (
"fmt"
"net"
"strings"
)
const maxVNI = 1<<24 - 1
type Base struct {
ASN uint32
RouterID net.IP
RouteReflectors []net.IP
}
type VRF struct {
ASN uint32
ID string
L3VNI uint32
VTEP net.IP
Supernet *net.IPNet
}
func (b Base) validate() error {
if b.ASN == 0 {
return fmt.Errorf("frr: asn must not be 0")
}
if b.RouterID.To4() == nil {
return fmt.Errorf("frr: router-id %v is not an IPv4 address", b.RouterID)
}
if len(b.RouteReflectors) == 0 {
return fmt.Errorf("frr: at least one route reflector is required")
}
for _, rr := range b.RouteReflectors {
if rr.To4() == nil {
return fmt.Errorf("frr: route reflector %v is not an IPv4 address", rr)
}
}
return nil
}
func (b Base) commands() []string {
cmds := []string{
fmt.Sprintf("router bgp %d", b.ASN),
fmt.Sprintf("bgp router-id %s", b.RouterID),
"no bgp default ipv4-unicast",
"neighbor fabric peer-group",
fmt.Sprintf("neighbor fabric remote-as %d", b.ASN),
"neighbor fabric capability extended-nexthop",
}
for _, rr := range b.RouteReflectors {
cmds = append(cmds, fmt.Sprintf("neighbor %s peer-group fabric", rr))
}
return append(cmds,
"address-family l2vpn evpn",
"neighbor fabric activate",
"advertise-all-vni",
"exit-address-family",
"exit",
)
}
func (b Base) staleReflectors(running string) ([]string, error) {
want := make(map[string]bool, len(b.RouteReflectors))
for _, rr := range b.RouteReflectors {
want[rr.String()] = true
}
var stale []string
in := false
for _, l := range strings.Split(running, "\n") {
l = strings.TrimRight(l, " \r")
f := strings.Fields(l)
switch {
case len(f) == 3 && f[0] == "router" && f[1] == "bgp":
if f[2] != fmt.Sprint(b.ASN) {
return nil, fmt.Errorf("frr: bgp already runs as AS %s, not %d: changing the asn requires removing that instance by hand", f[2], b.ASN)
}
in = true
case in && l == "exit":
in = false
case in && len(f) == 4 && f[0] == "neighbor" && f[2] == "peer-group" && f[3] == "fabric" && !want[f[1]]:
stale = append(stale, f[1])
}
}
return stale, nil
}
func (v VRF) Name() string {
return "vrf-" + v.ID
}
func (v VRF) prefixList() string {
return "PL-" + v.ID
}
func (v VRF) routeMap() string {
return "EXPORT-" + v.ID
}
func (v VRF) bgpInstance() string {
return fmt.Sprintf("router bgp %d vrf %s", v.ASN, v.Name())
}
func (v VRF) validateName() error {
if v.ASN == 0 {
return fmt.Errorf("frr: asn must not be 0")
}
if v.ID == "" {
return fmt.Errorf("frr: vrf id must not be empty")
}
return nil
}
func (v VRF) validate() error {
if err := v.validateName(); err != nil {
return err
}
if v.L3VNI == 0 || v.L3VNI > maxVNI {
return fmt.Errorf("frr: l3vni %d does not fit in 24 bits", v.L3VNI)
}
if v.VTEP.To4() == nil {
return fmt.Errorf("frr: vtep %v is not an IPv4 address", v.VTEP)
}
if v.Supernet == nil || v.Supernet.IP.To4() == nil {
return fmt.Errorf("frr: supernet %v is not an IPv4 prefix", v.Supernet)
}
return nil
}
func (v VRF) commands() []string {
return []string{
fmt.Sprintf("ip prefix-list %s seq 10 permit %s le 32", v.prefixList(), v.Supernet),
fmt.Sprintf("route-map %s permit 10", v.routeMap()),
fmt.Sprintf("match ip address prefix-list %s", v.prefixList()),
"exit",
fmt.Sprintf("route-map %s deny 20", v.routeMap()),
"exit",
fmt.Sprintf("vrf %s", v.Name()),
fmt.Sprintf("vni %d", v.L3VNI),
"exit-vrf",
v.bgpInstance(),
fmt.Sprintf("bgp router-id %s", v.VTEP),
"no bgp default ipv4-unicast",
"address-family ipv4 unicast",
fmt.Sprintf("redistribute kernel route-map %s", v.routeMap()),
"exit-address-family",
"address-family l2vpn evpn",
fmt.Sprintf("rd %s:%d", v.VTEP, v.L3VNI),
fmt.Sprintf("route-target both %d:%d", v.ASN, v.L3VNI),
"advertise ipv4 unicast",
"exit-address-family",
"exit",
}
}

142
internal/frr/config_test.go Normal file
View file

@ -0,0 +1,142 @@
package frr
import (
"net"
"reflect"
"strings"
"testing"
)
func mustCIDR(t *testing.T, s string) *net.IPNet {
t.Helper()
_, n, err := net.ParseCIDR(s)
if err != nil {
t.Fatal(err)
}
return n
}
func labVRF(t *testing.T) VRF {
return VRF{
ASN: 64600,
ID: "a1b2",
L3VNI: 4001,
VTEP: net.ParseIP("192.168.14.11"),
Supernet: mustCIDR(t, "10.10.0.0/16"),
}
}
func TestBaseCommands(t *testing.T) {
b := Base{
ASN: 64600,
RouterID: net.ParseIP("192.168.14.13"),
RouteReflectors: []net.IP{net.ParseIP("10.255.255.1"), net.ParseIP("10.255.255.2")},
}
want := []string{
"router bgp 64600",
"bgp router-id 192.168.14.13",
"no bgp default ipv4-unicast",
"neighbor fabric peer-group",
"neighbor fabric remote-as 64600",
"neighbor fabric capability extended-nexthop",
"neighbor 10.255.255.1 peer-group fabric",
"neighbor 10.255.255.2 peer-group fabric",
"address-family l2vpn evpn",
"neighbor fabric activate",
"advertise-all-vni",
"exit-address-family",
"exit",
}
if got := b.commands(); !reflect.DeepEqual(got, want) {
t.Errorf("commandes de base :\n%s\nattendu :\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
}
func TestVRFCommands(t *testing.T) {
want := []string{
"ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32",
"route-map EXPORT-a1b2 permit 10",
"match ip address prefix-list PL-a1b2",
"exit",
"route-map EXPORT-a1b2 deny 20",
"exit",
"vrf vrf-a1b2",
"vni 4001",
"exit-vrf",
"router bgp 64600 vrf vrf-a1b2",
"bgp router-id 192.168.14.11",
"no bgp default ipv4-unicast",
"address-family ipv4 unicast",
"redistribute kernel route-map EXPORT-a1b2",
"exit-address-family",
"address-family l2vpn evpn",
"rd 192.168.14.11:4001",
"route-target both 64600:4001",
"advertise ipv4 unicast",
"exit-address-family",
"exit",
}
if got := labVRF(t).commands(); !reflect.DeepEqual(got, want) {
t.Errorf("commandes de VRF :\n%s\nattendu :\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
}
func TestBaseValidateRejects(t *testing.T) {
valid := func() Base {
return Base{ASN: 64600, RouterID: net.ParseIP("192.168.14.13"), RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}}
}
if err := valid().validate(); err != nil {
t.Fatalf("base valide refusée : %v", err)
}
cases := map[string]struct {
mutate func(*Base)
msg string
}{
"asn nul": {func(b *Base) { b.ASN = 0 }, "asn must not be 0"},
"router-id absent": {func(b *Base) { b.RouterID = nil }, "router-id <nil> is not an IPv4 address"},
"router-id IPv6": {func(b *Base) { b.RouterID = net.ParseIP("fd00::1") }, "router-id fd00::1 is not an IPv4 address"},
"aucun RR": {func(b *Base) { b.RouteReflectors = nil }, "at least one route reflector is required"},
"RR IPv6": {func(b *Base) { b.RouteReflectors = append(b.RouteReflectors, net.ParseIP("fd00::2")) }, "route reflector fd00::2 is not an IPv4 address"},
}
for name, c := range cases {
b := valid()
c.mutate(&b)
if err := b.validate(); err == nil || !strings.Contains(err.Error(), c.msg) {
t.Errorf("%s : erreur %v, « %s » attendu", name, err, c.msg)
}
}
}
func TestVRFValidateRejects(t *testing.T) {
if err := labVRF(t).validate(); err != nil {
t.Fatalf("VRF valide refusée : %v", err)
}
cases := map[string]struct {
mutate func(*VRF)
msg string
}{
"asn nul": {func(v *VRF) { v.ASN = 0 }, "asn must not be 0"},
"id vide": {func(v *VRF) { v.ID = "" }, "vrf id must not be empty"},
"l3vni nul": {func(v *VRF) { v.L3VNI = 0 }, "l3vni 0 does not fit in 24 bits"},
"l3vni sur 25 bits": {func(v *VRF) { v.L3VNI = 16777216 }, "l3vni 16777216 does not fit in 24 bits"},
"vtep absent": {func(v *VRF) { v.VTEP = nil }, "vtep <nil> is not an IPv4 address"},
"vtep IPv6": {func(v *VRF) { v.VTEP = net.ParseIP("fd00::1") }, "vtep fd00::1 is not an IPv4 address"},
"supernet absent": {func(v *VRF) { v.Supernet = nil }, "supernet <nil> is not an IPv4 prefix"},
"supernet IPv6": {func(v *VRF) { v.Supernet = mustCIDR(t, "fd00::/48") }, "supernet fd00::/48 is not an IPv4 prefix"},
}
for name, c := range cases {
v := labVRF(t)
c.mutate(&v)
if err := v.validate(); err == nil || !strings.Contains(err.Error(), c.msg) {
t.Errorf("%s : erreur %v, « %s » attendu", name, err, c.msg)
}
}
}
func TestVRFAcceptsTheLargest24BitVNI(t *testing.T) {
v := labVRF(t)
v.L3VNI = 16777215
if err := v.validate(); err != nil {
t.Errorf("l3vni 16777215 refusée : %v", err)
}
}

49
internal/frr/status.go Normal file
View file

@ -0,0 +1,49 @@
package frr
import (
"fmt"
"net"
"strings"
)
type VNIState struct {
Type string
State string
RouterMAC string
}
func (c *Client) NeighborEstablished(peer net.IP) (bool, error) {
out, err := c.exec("show bgp neighbors " + peer.String())
if err != nil {
return false, err
}
return strings.Contains(out, "BGP state = Established"), nil
}
func (c *Client) VNI(vni uint32) (VNIState, error) {
out, err := c.exec(fmt.Sprintf("show evpn vni %d", vni))
if err != nil {
return VNIState{}, err
}
return parseVNI(out), nil
}
func parseVNI(out string) VNIState {
var s VNIState
for _, l := range strings.Split(out, "\n") {
key, value, ok := strings.Cut(strings.TrimSpace(l), ": ")
if !ok {
continue
}
value = strings.TrimSpace(value)
switch strings.ToLower(key) {
case "type":
s.Type = value
case "state":
s.State = value
case "router mac":
s.RouterMAC = strings.ToLower(value)
}
}
return s
}

View file

@ -0,0 +1,50 @@
package frr
import (
"net"
"strings"
"testing"
)
const vniOutput = `VNI: 2700
Type: L3
Tenant VRF: vrf-s7
Vxlan-Intf: vxl3-s7
SVI-If: brl3-s7
State: Up
VNI Filter: none
System MAC: 02:02:c0:a8:0e:aa
Router MAC: 02:02:C0:A8:0E:0B
L2 VNIs:
`
func TestVNIReadsTypeStateAndRouterMAC(t *testing.T) {
f := &fakeVtysh{fail: map[string]string{"show evpn vni 2700": vniOutput}}
s, err := f.client().VNI(2700)
if err != nil {
t.Fatal(err)
}
want := VNIState{Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}
if s != want {
t.Errorf("VNI = %+v, attendu %+v", s, want)
}
}
func TestVNIReportsAnUnknownVNI(t *testing.T) {
f := &fakeVtysh{fail: map[string]string{"show evpn vni 9": "% VNI 9 does not exist\n"}}
if _, err := f.client().VNI(9); err == nil || !strings.Contains(err.Error(), "does not exist") {
t.Errorf("erreur %v", err)
}
}
func TestNeighborEstablished(t *testing.T) {
rr := net.ParseIP("10.255.255.1")
f := &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1, remote AS 64600, local AS 64600, internal link\n BGP version 4, remote router ID 10.255.255.1\n BGP state = Established, up for 00:10:02\n"}}
if up, err := f.client().NeighborEstablished(rr); err != nil || !up {
t.Errorf("établie : %v, %v", up, err)
}
f = &fakeVtysh{fail: map[string]string{"show bgp neighbors 10.255.255.1": "BGP neighbor is 10.255.255.1\n BGP state = Active\n"}}
if up, err := f.client().NeighborEstablished(rr); err != nil || up {
t.Errorf("active : %v, %v", up, err)
}
}

84
internal/frr/teardown.go Normal file
View file

@ -0,0 +1,84 @@
package frr
import (
"fmt"
"strings"
)
type step []string
func routingTeardown(v VRF, running string) []step {
lines := strings.Split(running, "\n")
bgp := hasLine(lines, v.bgpInstance())
var steps []step
if bgp {
steps = append(steps, step{v.bgpInstance(), "address-family ipv4 unicast", "no redistribute kernel"})
}
if vni := vrfVNI(lines, v.Name()); vni != "" {
steps = append(steps, step{"vrf " + v.Name(), "no vni " + vni})
}
if bgp {
steps = append(steps, step{"no " + v.bgpInstance()})
}
if hasPrefix(lines, "route-map "+v.routeMap()+" ") {
steps = append(steps, step{"no route-map " + v.routeMap()})
}
if hasPrefix(lines, "ip prefix-list "+v.prefixList()+" ") {
steps = append(steps, step{"no ip prefix-list " + v.prefixList()})
}
return steps
}
func vrfTeardown(v VRF, running string) []step {
if hasLine(strings.Split(running, "\n"), "vrf "+v.Name()) {
return []step{{"no vrf " + v.Name()}}
}
return nil
}
func hasLine(lines []string, want string) bool {
for _, l := range lines {
if strings.TrimRight(l, " \r") == want {
return true
}
}
return false
}
func hasPrefix(lines []string, prefix string) bool {
for _, l := range lines {
if strings.HasPrefix(l, prefix) {
return true
}
}
return false
}
func vrfVNI(lines []string, name string) string {
in := false
for _, l := range lines {
l = strings.TrimRight(l, " \r")
switch {
case l == "vrf "+name:
in = true
case in && l == "exit-vrf":
return ""
case in:
if f := strings.Fields(l); len(f) >= 2 && f[0] == "vni" {
return f[1]
}
}
}
return ""
}
func remaining(steps []step) error {
if len(steps) == 0 {
return nil
}
var parts []string
for _, s := range steps {
parts = append(parts, strings.Join(s, " / "))
}
return fmt.Errorf("configuration still present after teardown: %s", strings.Join(parts, "; "))
}

View file

@ -0,0 +1,142 @@
package frr
import (
"reflect"
"testing"
)
const fullRunning = `frr version 10.7.1
frr defaults traditional
hostname hv1
!
vrf vrf-a1b2
vni 4001
exit-vrf
!
vrf vrf-a1b2c
vni 4002
exit-vrf
!
router bgp 64600
bgp router-id 192.168.14.11
no bgp default ipv4-unicast
neighbor fabric peer-group
neighbor fabric remote-as 64600
neighbor 10.255.255.1 peer-group fabric
!
address-family l2vpn evpn
neighbor fabric activate
advertise-all-vni
exit-address-family
exit
!
router bgp 64600 vrf vrf-a1b2
bgp router-id 192.168.14.11
no bgp default ipv4-unicast
!
address-family ipv4 unicast
redistribute kernel route-map EXPORT-a1b2
exit-address-family
!
address-family l2vpn evpn
advertise ipv4 unicast
rd 192.168.14.11:4001
route-target import 64600:4001
route-target export 64600:4001
exit-address-family
exit
!
router bgp 64600 vrf vrf-a1b2c
bgp router-id 192.168.14.11
exit
!
ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32
ip prefix-list PL-a1b2c seq 10 permit 10.20.0.0/16 le 32
!
route-map EXPORT-a1b2 permit 10
match ip address prefix-list PL-a1b2
exit
!
route-map EXPORT-a1b2 deny 20
exit
!
route-map EXPORT-a1b2c permit 10
exit
!
end
`
const otherOnlyRunning = `vrf vrf-a1b2
exit-vrf
!
vrf vrf-a1b2c
vni 4002
exit-vrf
!
router bgp 64600 vrf vrf-a1b2c
exit
!
ip prefix-list PL-a1b2c seq 10 permit 10.20.0.0/16 le 32
!
route-map EXPORT-a1b2c permit 10
exit
!
end
`
func TestRoutingTeardownFollowsTheOrderFRRRequires(t *testing.T) {
want := []step{
{"router bgp 64600 vrf vrf-a1b2", "address-family ipv4 unicast", "no redistribute kernel"},
{"vrf vrf-a1b2", "no vni 4001"},
{"no router bgp 64600 vrf vrf-a1b2"},
{"no route-map EXPORT-a1b2"},
{"no ip prefix-list PL-a1b2"},
}
if got := routingTeardown(labVRF(t), fullRunning); !reflect.DeepEqual(got, want) {
t.Errorf("démontage :\n%q\nattendu :\n%q", got, want)
}
}
func TestRoutingTeardownIgnoresAVRFWhoseNameExtendsOurs(t *testing.T) {
if got := routingTeardown(labVRF(t), otherOnlyRunning); got != nil {
t.Errorf("étapes visant vrf-a1b2c : %q", got)
}
}
func TestRoutingTeardownUsesTheVNIFoundInFRR(t *testing.T) {
v := labVRF(t)
v.L3VNI = 9999
running := "vrf vrf-a1b2\n vni 4001\nexit-vrf\n"
want := []step{{"vrf vrf-a1b2", "no vni 4001"}}
if got := routingTeardown(v, running); !reflect.DeepEqual(got, want) {
t.Errorf("démontage : %q, %q attendu", got, want)
}
}
func TestRoutingTeardownSkipsWhatIsAbsent(t *testing.T) {
running := "vrf vrf-a1b2\n vni 4001\nexit-vrf\n!\nip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32\n"
want := []step{
{"vrf vrf-a1b2", "no vni 4001"},
{"no ip prefix-list PL-a1b2"},
}
if got := routingTeardown(labVRF(t), running); !reflect.DeepEqual(got, want) {
t.Errorf("démontage :\n%q\nattendu :\n%q", got, want)
}
}
func TestRoutingTeardownDoesNotReadAVNIOutsideTheVRFBlock(t *testing.T) {
running := "vrf vrf-a1b2\nexit-vrf\n!\nvrf vrf-a1b2c\n vni 4002\nexit-vrf\n"
if got := routingTeardown(labVRF(t), running); got != nil {
t.Errorf("étapes : %q", got)
}
}
func TestVRFTeardown(t *testing.T) {
want := []step{{"no vrf vrf-a1b2"}}
if got := vrfTeardown(labVRF(t), otherOnlyRunning); !reflect.DeepEqual(got, want) {
t.Errorf("démontage : %q, %q attendu", got, want)
}
if got := vrfTeardown(labVRF(t), "vrf vrf-a1b2c\nexit-vrf\n"); got != nil {
t.Errorf("étapes visant vrf-a1b2c : %q", got)
}
}

View file

@ -19,3 +19,11 @@ func AddrAdd(iface string, ip net.IP) error {
}
return netlink.AddrAdd(link, addr)
}
func AddrAddPrefix(iface string, prefix *net.IPNet) error {
link, err := netlink.LinkByName(iface)
if err != nil {
return err
}
return netlink.AddrAdd(link, &netlink.Addr{IPNet: prefix})
}

View file

@ -1,6 +1,8 @@
package netif
import (
"errors"
"github.com/vishvananda/netlink"
)
@ -11,3 +13,15 @@ func DeleteLink(name string) error {
}
return netlink.LinkDel(link)
}
func DeleteLinkIfExists(name string) error {
link, err := netlink.LinkByName(name)
if err != nil {
var notFound netlink.LinkNotFoundError
if errors.As(err, &notFound) {
return nil
}
return err
}
return netlink.LinkDel(link)
}

36
internal/netif/route.go Normal file
View file

@ -0,0 +1,36 @@
package netif
import (
"errors"
"net"
"syscall"
"github.com/vishvananda/netlink"
)
func RouteReplaceVia(table int, dst *net.IPNet, gw net.IP, iface string) error {
link, err := netlink.LinkByName(iface)
if err != nil {
return err
}
return netlink.RouteReplace(&netlink.Route{
LinkIndex: link.Attrs().Index,
Dst: dst,
Gw: gw,
Table: table,
})
}
func RouteDel(table int, dst *net.IPNet) error {
return netlink.RouteDel(&netlink.Route{
Dst: dst,
Table: table,
})
}
func RouteDelIfExists(table int, dst *net.IPNet) error {
if err := RouteDel(table, dst); err != nil && !errors.Is(err, syscall.ESRCH) {
return err
}
return nil
}

View file

@ -19,3 +19,24 @@ func RouteAdd(iface string, subnet *net.IPNet) error {
Scope: netlink.SCOPE_LINK,
})
}
func TableRoutesVia(table int, iface string) ([]*net.IPNet, error) {
link, err := netlink.LinkByName(iface)
if err != nil {
return nil, err
}
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{
Table: table,
LinkIndex: link.Attrs().Index,
}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF)
if err != nil {
return nil, err
}
dsts := make([]*net.IPNet, 0, len(routes))
for _, r := range routes {
if r.Dst != nil && r.Gw != nil {
dsts = append(dsts, r.Dst)
}
}
return dsts, nil
}

View file

@ -2,8 +2,15 @@
package netif
import "net"
import (
"errors"
"net"
)
func RouteAdd(_ string, _ *net.IPNet) error {
return nil
}
func TableRoutesVia(_ int, _ string) ([]*net.IPNet, error) {
return nil, errors.New("route listing is only implemented on linux")
}

View file

@ -1,6 +1,8 @@
package netif
import (
"net"
"github.com/vishvananda/netlink"
)
@ -19,3 +21,11 @@ func LinkSetDown(name string) error {
}
return netlink.LinkSetDown(link)
}
func LinkSetHardwareAddr(name string, mac net.HardwareAddr) error {
link, err := netlink.LinkByName(name)
if err != nil {
return err
}
return netlink.LinkSetHardwareAddr(link, mac)
}

16
internal/netif/vrf.go Normal file
View file

@ -0,0 +1,16 @@
package netif
import (
"github.com/vishvananda/netlink"
)
func CreateVRF(name string, table uint32) error {
vrf := &netlink.Vrf{
LinkAttrs: netlink.LinkAttrs{Name: name},
Table: table,
}
if err := netlink.LinkAdd(vrf); err != nil {
return err
}
return netlink.LinkSetUp(vrf)
}

View file

@ -18,11 +18,7 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error {
if err != nil {
return err
}
addrs, err := netlink.AddrList(link, syscall.AF_INET)
if err != nil {
return err
}
local, err := vtepAddress(localIface, addrs)
local, err := linkVTEPAddress(link)
if err != nil {
return err
}
@ -40,6 +36,22 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error {
return netlink.LinkAdd(vxlan)
}
func VTEPAddress(iface string) (net.IP, error) {
link, err := netlink.LinkByName(iface)
if err != nil {
return nil, err
}
return linkVTEPAddress(link)
}
func linkVTEPAddress(link netlink.Link) (net.IP, error) {
addrs, err := netlink.AddrList(link, syscall.AF_INET)
if err != nil {
return nil, err
}
return vtepAddress(link.Attrs().Name, addrs)
}
func vtepAddress(iface string, addrs []netlink.Addr) (net.IP, error) {
for _, a := range addrs {
if a.IPNet == nil || a.IP.To4() == nil || a.Scope != scopeUniverse || a.Flags&addressSecondary != 0 {

View file

@ -10,6 +10,7 @@ import (
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"github.com/dgraph-io/badger/v4"
)
@ -37,6 +38,12 @@ func CreateSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e
return err
}
if d.mode == ModeVxlan {
if err := vpc.ExportRoute(db, d.vpc, d.cidr); err != nil {
return err
}
}
return state.Set(db, "subnet/"+subnetName, state.Running)
}
@ -70,6 +77,15 @@ func createSubnet(db *badger.DB, subnetName string, d subnetData, backend dhcpba
if err := netif.CreateBridge(d.bridge, 1500); err != nil {
return fmt.Errorf("create bridge: %w", err)
}
if d.mode == ModeVxlan {
mac, err := anycastGatewayMAC(d.vxlanID)
if err != nil {
return err
}
if err := netif.LinkSetHardwareAddr(d.bridge, mac); err != nil {
return fmt.Errorf("set anycast gateway mac %s: %w", mac, err)
}
}
return netif.BridgeSetMaster(vethI, d.bridge)
}); err != nil {
return fmt.Errorf("setup bridge in netns: %w", err)

View file

@ -9,6 +9,7 @@ import (
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
@ -33,6 +34,12 @@ func DeleteSubnet(db *badger.DB, subnetName string, cfg *configuration.Config) e
return err
}
if d.mode == ModeVxlan {
if err := vpc.WithdrawRoute(db, d.vpc, d.cidr); err != nil {
return err
}
}
if err := stopDHCP(db, subnetName, d, backend); err != nil {
return err
}

View file

@ -0,0 +1,19 @@
package subnet
import (
"fmt"
"net"
)
const (
maxVNI = 1<<24 - 1
roleAnycastGateway = 0x01
locallyAdministered = 0x02
)
func anycastGatewayMAC(vni int) (net.HardwareAddr, error) {
if vni < 0 || vni > maxVNI {
return nil, fmt.Errorf("vxlan_id %d does not fit in 24 bits", vni)
}
return net.HardwareAddr{locallyAdministered, roleAnycastGateway, 0x00, byte(vni >> 16), byte(vni >> 8), byte(vni)}, nil
}

View file

@ -0,0 +1,58 @@
package subnet
import "testing"
func TestAnycastGatewayMAC(t *testing.T) {
cases := []struct {
vni int
want string
}{
{2811, "02:01:00:00:0a:fb"},
{0, "02:01:00:00:00:00"},
{1, "02:01:00:00:00:01"},
{0x123456, "02:01:00:12:34:56"},
{maxVNI, "02:01:00:ff:ff:ff"},
}
for _, c := range cases {
got, err := anycastGatewayMAC(c.vni)
if err != nil {
t.Fatalf("VNI %d : erreur inattendue %v", c.vni, err)
}
if got.String() != c.want {
t.Errorf("VNI %d : %s, %s attendu", c.vni, got, c.want)
}
}
}
func TestAnycastGatewayMACIsLocalUnicast(t *testing.T) {
got, err := anycastGatewayMAC(2811)
if err != nil {
t.Fatal(err)
}
if got[0]&0x02 == 0 {
t.Errorf("%s : bit « administrée localement » absent", got)
}
if got[0]&0x01 != 0 {
t.Errorf("%s : MAC de groupe (multicast)", got)
}
}
func TestAnycastGatewayMACIsIdenticalForTheSameVNIAndDistinctOtherwise(t *testing.T) {
a, _ := anycastGatewayMAC(2811)
b, _ := anycastGatewayMAC(2811)
c, _ := anycastGatewayMAC(2812)
if a.String() != b.String() {
t.Errorf("même VNI, MAC différentes : %s et %s", a, b)
}
if a.String() == c.String() {
t.Errorf("VNI différentes, même MAC : %s", a)
}
}
func TestAnycastGatewayMACRejectsVNIOutside24Bits(t *testing.T) {
for _, vni := range []int{-1, maxVNI + 1} {
if mac, err := anycastGatewayMAC(vni); err == nil {
t.Errorf("VNI %d accepté : %s", vni, mac)
}
}
}

23
internal/sysctl/sysctl.go Normal file
View file

@ -0,0 +1,23 @@
package sysctl
import (
"fmt"
"os"
"path/filepath"
"strings"
)
var root = "/proc/sys"
func Set(key, value string) error {
path := filepath.Join(root, strings.ReplaceAll(key, ".", "/"))
f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0)
if err != nil {
return fmt.Errorf("sysctl %s: %w", key, err)
}
if _, err := f.WriteString(value); err != nil {
f.Close()
return fmt.Errorf("sysctl %s=%s: %w", key, value, err)
}
return f.Close()
}

View file

@ -0,0 +1,48 @@
package sysctl
import (
"os"
"path/filepath"
"strings"
"testing"
)
func fakeRoot(t *testing.T) string {
t.Helper()
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "net", "ipv4"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "net", "ipv4", "ip_forward"), []byte("0\n"), 0o644); err != nil {
t.Fatal(err)
}
old := root
root = dir
t.Cleanup(func() { root = old })
return dir
}
func TestSetWritesTheValueAtTheKeyPath(t *testing.T) {
dir := fakeRoot(t)
if err := Set("net.ipv4.ip_forward", "1"); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(filepath.Join(dir, "net", "ipv4", "ip_forward"))
if err != nil {
t.Fatal(err)
}
if string(got) != "1" {
t.Errorf("contenu %q, \"1\" attendu", got)
}
}
func TestSetRefusesAnUnknownKeyWithoutCreatingIt(t *testing.T) {
dir := fakeRoot(t)
err := Set("net.ipv4.ip_forwarding", "1")
if err == nil || !strings.Contains(err.Error(), "sysctl net.ipv4.ip_forwarding") {
t.Errorf("erreur %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "net", "ipv4", "ip_forwarding")); !os.IsNotExist(err) {
t.Errorf("clé inconnue créée : %v", err)
}
}

View file

@ -15,6 +15,7 @@ import (
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/qemu"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"github.com/dgraph-io/badger/v4"
)
@ -111,6 +112,10 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error {
return fmt.Errorf("start qemu: %w", err)
}
if err := applyHostRoutes(db, d, vpc.ExportRoute); err != nil {
return err
}
return state.Set(db, "vm/"+name, state.Running)
}

View file

@ -14,6 +14,7 @@ import (
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/qmp"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"github.com/dgraph-io/badger/v4"
)
@ -38,6 +39,10 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error {
return err
}
if err := applyHostRoutes(db, d, vpc.WithdrawRoute); err != nil {
return err
}
socketPath := filepath.Join(cfg.QEMU.QMPDir, name+".sock")
if _, err := os.Stat(socketPath); err == nil {

43
internal/vm/routes.go Normal file
View file

@ -0,0 +1,43 @@
package vm
import (
"fmt"
"net"
"github.com/dgraph-io/badger/v4"
)
type hostRoute struct {
vpc string
dst *net.IPNet
}
type routeFunc func(db *badger.DB, vpcName string, dst *net.IPNet) error
func hostRoutes(d vmData) ([]hostRoute, error) {
var routes []hostRoute
for _, n := range d.nics {
if n.mode != "vxlan" {
continue
}
ip := net.ParseIP(n.ip).To4()
if ip == nil {
return nil, fmt.Errorf("interface %d: %q is not an IPv4 address", n.index, n.ip)
}
routes = append(routes, hostRoute{vpc: n.vpcName, dst: &net.IPNet{IP: ip, Mask: net.CIDRMask(32, 32)}})
}
return routes, nil
}
func applyHostRoutes(db *badger.DB, d vmData, apply routeFunc) error {
routes, err := hostRoutes(d)
if err != nil {
return err
}
for _, r := range routes {
if err := apply(db, r.vpc, r.dst); err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,66 @@
package vm
import (
"errors"
"net"
"reflect"
"strings"
"testing"
"github.com/dgraph-io/badger/v4"
)
func routedVM() vmData {
return vmData{nics: []nicData{
{index: 0, vpcName: "vp-s7", mode: "vxlan", ip: "10.241.1.10"},
{index: 1, vpcName: "vp-s7", mode: "bridge", ip: "192.168.50.10"},
{index: 2, vpcName: "vp-s7", mode: "vxlan", ip: "10.241.2.10"},
}}
}
type recorder struct {
calls []string
fail string
}
func (r *recorder) apply(_ *badger.DB, vpcName string, dst *net.IPNet) error {
r.calls = append(r.calls, vpcName+" "+dst.String())
if dst.String() == r.fail {
return errors.New("refused")
}
return nil
}
func TestApplyHostRoutesCoversEveryVxlanInterfaceAsA32(t *testing.T) {
r := &recorder{}
if err := applyHostRoutes(nil, routedVM(), r.apply); err != nil {
t.Fatal(err)
}
want := []string{"vp-s7 10.241.1.10/32", "vp-s7 10.241.2.10/32"}
if !reflect.DeepEqual(r.calls, want) {
t.Errorf("appels %q, attendu %q", r.calls, want)
}
}
func TestApplyHostRoutesStopsAtTheFirstError(t *testing.T) {
r := &recorder{fail: "10.241.1.10/32"}
if err := applyHostRoutes(nil, routedVM(), r.apply); err == nil || err.Error() != "refused" {
t.Errorf("erreur %v", err)
}
if len(r.calls) != 1 {
t.Errorf("appels après l'échec : %q", r.calls)
}
}
func TestApplyHostRoutesRefusesANonIPv4AddressBeforeAnyCall(t *testing.T) {
d := routedVM()
d.nics[2].ip = "fd00::10"
r := &recorder{}
err := applyHostRoutes(nil, d, r.apply)
if err == nil || !strings.Contains(err.Error(), `interface 2: "fd00::10" is not an IPv4 address`) {
t.Errorf("erreur %v", err)
}
if len(r.calls) != 0 {
t.Errorf("appels : %q", r.calls)
}
}

View file

@ -3,6 +3,7 @@ package vpc
import (
"strings"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
@ -10,7 +11,7 @@ import (
"github.com/dgraph-io/badger/v4"
)
func CreateVPC(db *badger.DB, name string) error {
func CreateVPC(db *badger.DB, name string, cfg *configuration.Config) error {
if current, err := state.Get(db, "vpc/"+name); err != nil {
return err
} else if current == state.Creating {
@ -48,6 +49,15 @@ func CreateVPC(db *badger.DB, name string) error {
}); err != nil {
return err
}
l3, ok, err := loadL3(db, name, vpcID)
if err != nil {
return err
}
if ok {
if err := createL3(newSystem(), cfg, l3); err != nil {
return err
}
}
return state.Set(db, "vpc/"+name, state.Running)
}
return nil

View file

@ -3,6 +3,7 @@ package vpc
import (
"strings"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
@ -10,12 +11,22 @@ import (
"github.com/dgraph-io/badger/v4"
)
func DeleteVPC(db *badger.DB, name string) error {
func DeleteVPC(db *badger.DB, name string, cfg *configuration.Config) error {
if current, err := state.Get(db, "vpc/"+name); err != nil {
return err
} else if current == state.Deleting {
vpcID := strings.SplitN(name, "-", 2)[1]
l3, ok, err := loadL3(db, name, vpcID)
if err != nil {
return err
}
if ok {
if err := deleteL3(newSystem(), cfg, l3); err != nil {
return err
}
}
if err := netif.DeleteLink("vp-" + vpcID + "-e"); err != nil {
return err
}

97
internal/vpc/host.go Normal file
View file

@ -0,0 +1,97 @@
package vpc
import (
"net"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/sysctl"
)
type host interface {
VTEPAddress(iface string) (net.IP, error)
Sysctl(ns, key, value string) error
CreateVRF(name string, table uint32) error
CreateBridge(name string, mtu int) error
SetHardwareAddr(name string, mac net.HardwareAddr) error
SetMaster(iface, master string) error
CreateVxlan(name string, vni int, iface string, mtu int) error
CreateVeth(hostIf, nsIf, ns string, mtu int) error
AddrAdd(ns, iface string, prefix *net.IPNet) error
LinkUp(ns, iface string) error
RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error
DeleteLink(name string) error
TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error
TableRouteDel(table uint32, dst *net.IPNet) error
ApplyVRF(v frr.VRF) error
WithdrawVRF(v frr.VRF) error
RemoveVRF(v frr.VRF) error
}
type system struct {
frr *frr.Client
}
func newSystem() system {
return system{frr: frr.NewClient()}
}
func in(ns string, fn func() error) error {
if ns == "" {
return fn()
}
return netns.Call(ns, fn)
}
func (system) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) }
func (system) Sysctl(ns, key, value string) error {
return in(ns, func() error { return sysctl.Set(key, value) })
}
func (system) CreateVRF(name string, table uint32) error { return netif.CreateVRF(name, table) }
func (system) CreateBridge(name string, mtu int) error { return netif.CreateBridge(name, mtu) }
func (system) SetHardwareAddr(name string, mac net.HardwareAddr) error {
return netif.LinkSetHardwareAddr(name, mac)
}
func (system) SetMaster(iface, master string) error { return netif.BridgeSetMaster(iface, master) }
func (system) CreateVxlan(name string, vni int, iface string, mtu int) error {
return netif.CreateVxlan(name, vni, iface, mtu)
}
func (system) CreateVeth(hostIf, nsIf, ns string, mtu int) error {
return netif.CreateVethToNetns(hostIf, nsIf, "/var/run/netns/"+ns, mtu)
}
func (system) AddrAdd(ns, iface string, prefix *net.IPNet) error {
return in(ns, func() error { return netif.AddrAddPrefix(iface, prefix) })
}
func (system) LinkUp(ns, iface string) error {
return in(ns, func() error { return netif.LinkSetUp(iface) })
}
func (system) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error {
return in(ns, func() error { return netif.RouteReplaceVia(0, dst, gw, iface) })
}
func (system) DeleteLink(name string) error { return netif.DeleteLinkIfExists(name) }
func (system) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error {
return netif.RouteReplaceVia(int(table), dst, gw, iface)
}
func (system) TableRouteDel(table uint32, dst *net.IPNet) error {
return netif.RouteDelIfExists(int(table), dst)
}
func (s system) ApplyVRF(v frr.VRF) error { return s.frr.ApplyVRF(v) }
func (s system) WithdrawVRF(v frr.VRF) error { return s.frr.WithdrawVRF(v) }
func (s system) RemoveVRF(v frr.VRF) error { return s.frr.RemoveVRF(v) }

169
internal/vpc/l3.go Normal file
View file

@ -0,0 +1,169 @@
package vpc
import (
"fmt"
"net"
"strconv"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
const l3MTU = 1500
var (
transitHost = &net.IPNet{IP: net.IPv4(169, 254, 0, 1).To4(), Mask: net.CIDRMask(30, 32)}
transitNetns = &net.IPNet{IP: net.IPv4(169, 254, 0, 2).To4(), Mask: net.CIDRMask(30, 32)}
)
type l3VPC struct {
netns string
id string
vni uint32
supernet *net.IPNet
}
func loadL3(db *badger.DB, name, id string) (l3VPC, bool, error) {
raw, err := kv.GetFromDB(db, "vpc/"+name+"/l3vni")
if err != nil {
return l3VPC{}, false, nil
}
vni, err := strconv.ParseUint(raw, 10, 32)
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: invalid l3vni %q", name, raw)
}
if err := ValidL3VNI(uint32(vni)); err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: %w", name, err)
}
cidr, err := kv.GetFromDB(db, "vpc/"+name+"/cidr")
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: no cidr: %w", name, err)
}
_, supernet, err := net.ParseCIDR(cidr)
if err != nil {
return l3VPC{}, false, fmt.Errorf("vpc %q: invalid cidr %q: %w", name, cidr, err)
}
return l3VPC{netns: name, id: id, vni: uint32(vni), supernet: supernet}, true, nil
}
func (v l3VPC) vrf() string { return "vrf-" + v.id }
func (v l3VPC) bridge() string { return "brl3-" + v.id }
func (v l3VPC) vxlan() string { return "vxl3-" + v.id }
func (v l3VPC) vethHost() string { return "l3-" + v.id + "-o" }
func (v l3VPC) vethNetns() string { return "l3-" + v.id + "-i" }
func routerMAC(vtep net.IP) (net.HardwareAddr, error) {
ip := vtep.To4()
if ip == nil {
return nil, fmt.Errorf("vtep %v is not an IPv4 address", vtep)
}
return net.HardwareAddr{0x02, 0x02, ip[0], ip[1], ip[2], ip[3]}, nil
}
func bgpASN(cfg *configuration.Config) (uint32, error) {
if cfg == nil || !cfg.BGP.Enabled() {
return 0, fmt.Errorf("the vpc has an l3vni but the agent configuration has no bgp section")
}
return cfg.BGP.ASN, nil
}
func createL3(h host, cfg *configuration.Config, v l3VPC) error {
asn, err := bgpASN(cfg)
if err != nil {
return err
}
vtep, err := h.VTEPAddress(cfg.DefaultInterface)
if err != nil {
return fmt.Errorf("vtep: %w", err)
}
rmac, err := routerMAC(vtep)
if err != nil {
return err
}
steps := []struct {
what string
run func() error
}{
{"create " + v.vrf(), func() error { return h.CreateVRF(v.vrf(), v.vni) }},
{"create " + v.bridge(), func() error { return h.CreateBridge(v.bridge(), l3MTU) }},
{"set router mac on " + v.bridge(), func() error { return h.SetHardwareAddr(v.bridge(), rmac) }},
{"attach " + v.bridge(), func() error { return h.SetMaster(v.bridge(), v.vrf()) }},
{"create " + v.vxlan(), func() error { return h.CreateVxlan(v.vxlan(), int(v.vni), cfg.DefaultInterface, l3MTU) }},
{"attach " + v.vxlan(), func() error { return h.SetMaster(v.vxlan(), v.bridge()) }},
{"set up " + v.vxlan(), func() error { return h.LinkUp("", v.vxlan()) }},
{"create " + v.vethHost(), func() error { return h.CreateVeth(v.vethHost(), v.vethNetns(), v.netns, l3MTU) }},
{"attach " + v.vethHost(), func() error { return h.SetMaster(v.vethHost(), v.vrf()) }},
{"address " + v.vethHost(), func() error { return h.AddrAdd("", v.vethHost(), transitHost) }},
{"set up " + v.vethHost(), func() error { return h.LinkUp("", v.vethHost()) }},
{"forwarding on " + v.bridge(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.bridge()+".forwarding", "1") }},
{"forwarding on " + v.vethHost(), func() error { return h.Sysctl("", "net.ipv4.conf."+v.vethHost()+".forwarding", "1") }},
{"address " + v.vethNetns(), func() error { return h.AddrAdd(v.netns, v.vethNetns(), transitNetns) }},
{"set up " + v.vethNetns(), func() error { return h.LinkUp(v.netns, v.vethNetns()) }},
{"forwarding in " + v.netns, func() error { return h.Sysctl(v.netns, "net.ipv4.ip_forward", "1") }},
{"route " + v.supernet.String() + " in " + v.netns, func() error {
return h.RouteVia(v.netns, v.supernet, transitHost.IP, v.vethNetns())
}},
{"frr " + v.vrf(), func() error {
return h.ApplyVRF(frr.VRF{ASN: asn, ID: v.id, L3VNI: v.vni, VTEP: vtep, Supernet: v.supernet})
}},
}
for _, s := range steps {
if err := s.run(); err != nil {
return fmt.Errorf("l3vni %d: %s: %w", v.vni, s.what, err)
}
}
return nil
}
func deleteL3(h host, cfg *configuration.Config, v l3VPC) error {
asn, err := bgpASN(cfg)
if err != nil {
return err
}
frrVRF := frr.VRF{ASN: asn, ID: v.id}
if err := h.WithdrawVRF(frrVRF); err != nil {
return fmt.Errorf("l3vni %d: withdraw frr: %w", v.vni, err)
}
for _, link := range []string{v.vethHost(), v.vxlan(), v.bridge(), v.vrf()} {
if err := h.DeleteLink(link); err != nil {
return fmt.Errorf("l3vni %d: delete %s: %w", v.vni, link, err)
}
}
if err := h.RemoveVRF(frrVRF); err != nil {
return fmt.Errorf("l3vni %d: remove frr vrf: %w", v.vni, err)
}
return nil
}
type Routing struct {
VNI uint32
Supernet *net.IPNet
VRF string
Bridge string
Vxlan string
VethHost string
VethNetns string
}
func LoadRouting(db *badger.DB, vpcName string) (Routing, bool, error) {
v, ok, err := loadRouted(db, vpcName)
if err != nil || !ok {
return Routing{}, ok, err
}
return Routing{
VNI: v.vni,
Supernet: v.supernet,
VRF: v.vrf(),
Bridge: v.bridge(),
Vxlan: v.vxlan(),
VethHost: v.vethHost(),
VethNetns: v.vethNetns(),
}, true, nil
}
func RouterMAC(vtep net.IP) (net.HardwareAddr, error) {
return routerMAC(vtep)
}

254
internal/vpc/l3_test.go Normal file
View file

@ -0,0 +1,254 @@
package vpc
import (
"errors"
"fmt"
"net"
"reflect"
"strings"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/pkg/db/kv"
)
type fakeHost struct {
calls []string
fail map[string]error
}
func (f *fakeHost) record(call string) error {
f.calls = append(f.calls, call)
for prefix, err := range f.fail {
if strings.HasPrefix(call, prefix) {
return err
}
}
return nil
}
func (f *fakeHost) VTEPAddress(iface string) (net.IP, error) {
if err := f.record("vtep " + iface); err != nil {
return nil, err
}
return net.ParseIP("192.168.14.11"), nil
}
func (f *fakeHost) Sysctl(ns, key, value string) error {
return f.record(fmt.Sprintf("sysctl [%s] %s=%s", ns, key, value))
}
func (f *fakeHost) CreateVRF(name string, table uint32) error {
return f.record(fmt.Sprintf("vrf %s table %d", name, table))
}
func (f *fakeHost) CreateBridge(name string, mtu int) error {
return f.record(fmt.Sprintf("bridge %s mtu %d", name, mtu))
}
func (f *fakeHost) SetHardwareAddr(name string, mac net.HardwareAddr) error {
return f.record(fmt.Sprintf("mac %s %s", name, mac))
}
func (f *fakeHost) SetMaster(iface, master string) error {
return f.record(fmt.Sprintf("master %s %s", iface, master))
}
func (f *fakeHost) CreateVxlan(name string, vni int, iface string, mtu int) error {
return f.record(fmt.Sprintf("vxlan %s id %d dev %s mtu %d", name, vni, iface, mtu))
}
func (f *fakeHost) CreateVeth(hostIf, nsIf, ns string, mtu int) error {
return f.record(fmt.Sprintf("veth %s peer %s netns %s mtu %d", hostIf, nsIf, ns, mtu))
}
func (f *fakeHost) AddrAdd(ns, iface string, prefix *net.IPNet) error {
return f.record(fmt.Sprintf("addr [%s] %s %s", ns, iface, prefix))
}
func (f *fakeHost) LinkUp(ns, iface string) error {
return f.record(fmt.Sprintf("up [%s] %s", ns, iface))
}
func (f *fakeHost) RouteVia(ns string, dst *net.IPNet, gw net.IP, iface string) error {
return f.record(fmt.Sprintf("route [%s] %s via %s dev %s", ns, dst, gw, iface))
}
func (f *fakeHost) DeleteLink(name string) error {
return f.record("del " + name)
}
func (f *fakeHost) TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error {
return f.record(fmt.Sprintf("table %d route %s via %s dev %s", table, dst, gw, iface))
}
func (f *fakeHost) TableRouteDel(table uint32, dst *net.IPNet) error {
return f.record(fmt.Sprintf("table %d del %s", table, dst))
}
func (f *fakeHost) ApplyVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet))
}
func (f *fakeHost) WithdrawVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr withdraw asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI))
}
func (f *fakeHost) RemoveVRF(v frr.VRF) error {
return f.record(fmt.Sprintf("frr remove asn=%d id=%s vni=%d", v.ASN, v.ID, v.L3VNI))
}
func bgpConfig() *configuration.Config {
cfg := &configuration.Config{DefaultInterface: "br-000000"}
cfg.BGP.ASN = 64600
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
return cfg
}
func labL3(t *testing.T) l3VPC {
t.Helper()
_, supernet, err := net.ParseCIDR("10.241.0.0/16")
if err != nil {
t.Fatal(err)
}
return l3VPC{netns: "vp-s7", id: "s7", vni: 2700, supernet: supernet}
}
func TestCreateL3FollowsTheLabRecipe(t *testing.T) {
f := &fakeHost{}
if err := createL3(f, bgpConfig(), labL3(t)); err != nil {
t.Fatal(err)
}
want := []string{
"vtep br-000000",
"vrf vrf-s7 table 2700",
"bridge brl3-s7 mtu 1500",
"mac brl3-s7 02:02:c0:a8:0e:0b",
"master brl3-s7 vrf-s7",
"vxlan vxl3-s7 id 2700 dev br-000000 mtu 1500",
"master vxl3-s7 brl3-s7",
"up [] vxl3-s7",
"veth l3-s7-o peer l3-s7-i netns vp-s7 mtu 1500",
"master l3-s7-o vrf-s7",
"addr [] l3-s7-o 169.254.0.1/30",
"up [] l3-s7-o",
"sysctl [] net.ipv4.conf.brl3-s7.forwarding=1",
"sysctl [] net.ipv4.conf.l3-s7-o.forwarding=1",
"addr [vp-s7] l3-s7-i 169.254.0.2/30",
"up [vp-s7] l3-s7-i",
"sysctl [vp-s7] net.ipv4.ip_forward=1",
"route [vp-s7] 10.241.0.0/16 via 169.254.0.1 dev l3-s7-i",
"frr apply asn=64600 id=s7 vni=2700 vtep=192.168.14.11 supernet=10.241.0.0/16",
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
}
}
func TestCreateL3RequiresTheBGPSectionBeforeTouchingTheHost(t *testing.T) {
f := &fakeHost{}
cfg := &configuration.Config{DefaultInterface: "br-000000"}
if err := createL3(f, cfg, labL3(t)); err == nil || !strings.Contains(err.Error(), "no bgp section") {
t.Errorf("erreur %v", err)
}
if err := createL3(f, nil, labL3(t)); err == nil {
t.Error("config absente acceptée")
}
if len(f.calls) != 0 {
t.Errorf("appels : %q", f.calls)
}
}
func TestCreateL3StopsAtTheFirstFailure(t *testing.T) {
f := &fakeHost{fail: map[string]error{"vxlan ": errors.New("file exists")}}
err := createL3(f, bgpConfig(), labL3(t))
if err == nil || !strings.Contains(err.Error(), "l3vni 2700: create vxl3-s7: file exists") {
t.Errorf("erreur %v", err)
}
if last := f.calls[len(f.calls)-1]; !strings.HasPrefix(last, "vxlan ") {
t.Errorf("appels après l'échec : %q", f.calls)
}
}
func TestDeleteL3WithdrawsFRRThenTheKernelThenTheVRF(t *testing.T) {
f := &fakeHost{}
if err := deleteL3(f, bgpConfig(), labL3(t)); err != nil {
t.Fatal(err)
}
want := []string{
"frr withdraw asn=64600 id=s7 vni=0",
"del l3-s7-o",
"del vxl3-s7",
"del brl3-s7",
"del vrf-s7",
"frr remove asn=64600 id=s7 vni=0",
}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
}
}
func TestDeleteL3KeepsTheKernelWhenFRRRefusesTheWithdrawal(t *testing.T) {
f := &fakeHost{fail: map[string]error{"frr withdraw": errors.New("% Please unconfigure l3vni 2700")}}
err := deleteL3(f, bgpConfig(), labL3(t))
if err == nil || !strings.Contains(err.Error(), "withdraw frr") {
t.Errorf("erreur %v", err)
}
if len(f.calls) != 1 {
t.Errorf("le noyau a été touché malgré le refus de FRR : %q", f.calls)
}
}
func TestDeleteL3KeepsTheFRRVRFWhenAKernelLinkCannotBeDeleted(t *testing.T) {
f := &fakeHost{fail: map[string]error{"del brl3-s7": errors.New("device busy")}}
if err := deleteL3(f, bgpConfig(), labL3(t)); err == nil || !strings.Contains(err.Error(), "delete brl3-s7") {
t.Errorf("erreur %v", err)
}
for _, c := range f.calls {
if strings.HasPrefix(c, "frr remove") || c == "del vrf-s7" {
t.Errorf("étape exécutée après l'échec : %q", f.calls)
}
}
}
func TestDeleteL3RequiresTheBGPSection(t *testing.T) {
f := &fakeHost{}
if err := deleteL3(f, &configuration.Config{}, labL3(t)); err == nil || len(f.calls) != 0 {
t.Errorf("erreur %v, appels %q", err, f.calls)
}
}
func TestRouterMAC(t *testing.T) {
mac, err := routerMAC(net.ParseIP("192.168.14.11"))
if err != nil || mac.String() != "02:02:c0:a8:0e:0b" {
t.Errorf("routerMAC = %v, %v", mac, err)
}
if _, err := routerMAC(net.ParseIP("fd00::1")); err == nil {
t.Error("VTEP IPv6 accepté")
}
}
func TestLoadL3(t *testing.T) {
db := kv.InitDB(kv.Config{Path: t.TempDir()}, false)
t.Cleanup(func() { db.Close() })
kv.AddInDB(db, "vpc/vp-a/cidr", "10.241.0.0/16")
if _, ok, err := loadL3(db, "vp-a", "a"); ok || err != nil {
t.Errorf("VPC sans l3vni : ok=%v err=%v", ok, err)
}
kv.AddInDB(db, "vpc/vp-a/l3vni", "2700")
got, ok, err := loadL3(db, "vp-a", "a")
if !ok || err != nil || got.vni != 2700 || got.id != "a" || got.netns != "vp-a" || got.supernet.String() != "10.241.0.0/16" {
t.Errorf("loadL3 = %+v, %v, %v", got, ok, err)
}
for _, bad := range []string{"abc", "254", "0"} {
kv.AddInDB(db, "vpc/vp-a/l3vni", bad)
if _, _, err := loadL3(db, "vp-a", "a"); err == nil {
t.Errorf("l3vni %q acceptée", bad)
}
}
kv.AddInDB(db, "vpc/vp-b/l3vni", "2700")
if _, _, err := loadL3(db, "vp-b", "b"); err == nil || !strings.Contains(err.Error(), "no cidr") {
t.Errorf("VPC sans cidr : %v", err)
}
}

15
internal/vpc/l3vni.go Normal file
View file

@ -0,0 +1,15 @@
package vpc
import "fmt"
const maxL3VNI = 1<<24 - 1
func ValidL3VNI(id uint32) error {
if id == 0 || id > maxL3VNI {
return fmt.Errorf("l3vni %d does not fit in 24 bits (1-%d)", id, maxL3VNI)
}
if id >= 253 && id <= 255 {
return fmt.Errorf("l3vni %d is reserved: it is also the routing table id, and tables 253-255 belong to the kernel", id)
}
return nil
}

View file

@ -0,0 +1,29 @@
package vpc
import (
"strings"
"testing"
)
func TestValidL3VNI_AcceptsEvery24BitValueOutsideTheKernelTables(t *testing.T) {
for _, id := range []uint32{1, 252, 256, 4001, 16777215} {
if err := ValidL3VNI(id); err != nil {
t.Errorf("ValidL3VNI(%d) = %v", id, err)
}
}
}
func TestValidL3VNI_RejectsOutOfRangeAndReservedTables(t *testing.T) {
cases := map[uint32]string{
0: "does not fit in 24 bits",
16777216: "does not fit in 24 bits",
253: "is reserved",
254: "is reserved",
255: "is reserved",
}
for id, want := range cases {
if err := ValidL3VNI(id); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("ValidL3VNI(%d) = %v, attendu %q", id, err, want)
}
}
}

50
internal/vpc/route.go Normal file
View file

@ -0,0 +1,50 @@
package vpc
import (
"fmt"
"net"
"strings"
"github.com/dgraph-io/badger/v4"
)
func ExportRoute(db *badger.DB, vpcName string, dst *net.IPNet) error {
return exportRoute(newSystem(), db, vpcName, dst)
}
func WithdrawRoute(db *badger.DB, vpcName string, dst *net.IPNet) error {
return withdrawRoute(newSystem(), db, vpcName, dst)
}
func loadRouted(db *badger.DB, vpcName string) (l3VPC, bool, error) {
parts := strings.SplitN(vpcName, "-", 2)
if len(parts) != 2 {
return l3VPC{}, false, fmt.Errorf("invalid vpc name %q", vpcName)
}
return loadL3(db, vpcName, parts[1])
}
func exportRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error {
v, ok, err := loadRouted(db, vpcName)
if err != nil || !ok {
return err
}
if !v.supernet.Contains(dst.IP) {
return fmt.Errorf("vpc %q: %s is outside %s", vpcName, dst, v.supernet)
}
if err := h.TableRouteVia(v.vni, dst, transitNetns.IP, v.vethHost()); err != nil {
return fmt.Errorf("export %s in %s: %w", dst, v.vrf(), err)
}
return nil
}
func withdrawRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error {
v, ok, err := loadRouted(db, vpcName)
if err != nil || !ok {
return err
}
if err := h.TableRouteDel(v.vni, dst); err != nil {
return fmt.Errorf("withdraw %s from %s: %w", dst, v.vrf(), err)
}
return nil
}

View file

@ -0,0 +1,89 @@
package vpc
import (
"errors"
"net"
"reflect"
"strings"
"testing"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
func routedDB(t *testing.T) *badger.DB {
t.Helper()
db := kv.InitDB(kv.Config{Path: t.TempDir()}, false)
t.Cleanup(func() { db.Close() })
kv.AddInDB(db, "vpc/vp-s7/cidr", "10.241.0.0/16")
kv.AddInDB(db, "vpc/vp-s7/l3vni", "2700")
kv.AddInDB(db, "vpc/vp-flat/cidr", "10.242.0.0/16")
return db
}
func cidr(t *testing.T, s string) *net.IPNet {
t.Helper()
_, n, err := net.ParseCIDR(s)
if err != nil {
t.Fatal(err)
}
return n
}
func TestExportRouteInstallsTheRouteInTheVRFTowardsTheNetns(t *testing.T) {
f := &fakeHost{}
if err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.0/24")); err != nil {
t.Fatal(err)
}
want := []string{"table 2700 route 10.241.1.0/24 via 169.254.0.2 dev l3-s7-o"}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels %q, attendu %q", f.calls, want)
}
}
func TestExportRouteRefusesAPrefixOutsideTheVPC(t *testing.T) {
f := &fakeHost{}
err := exportRoute(f, routedDB(t), "vp-s7", cidr(t, "10.99.1.0/24"))
if err == nil || !strings.Contains(err.Error(), "outside 10.241.0.0/16") || len(f.calls) != 0 {
t.Errorf("erreur %v, appels %q", err, f.calls)
}
}
func TestWithdrawRouteDeletesFromTheVRFTable(t *testing.T) {
f := &fakeHost{}
if err := withdrawRoute(f, routedDB(t), "vp-s7", cidr(t, "10.241.1.10/32")); err != nil {
t.Fatal(err)
}
want := []string{"table 2700 del 10.241.1.10/32"}
if !reflect.DeepEqual(f.calls, want) {
t.Errorf("appels %q, attendu %q", f.calls, want)
}
}
func TestRoutesAreIgnoredForAVPCWithoutL3VNI(t *testing.T) {
f := &fakeHost{}
db := routedDB(t)
if err := exportRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil {
t.Error(err)
}
if err := withdrawRoute(f, db, "vp-flat", cidr(t, "10.242.1.0/24")); err != nil {
t.Error(err)
}
if len(f.calls) != 0 {
t.Errorf("appels %q", f.calls)
}
}
func TestRouteErrorsArePropagated(t *testing.T) {
db := routedDB(t)
f := &fakeHost{fail: map[string]error{"table ": errors.New("network unreachable")}}
if err := exportRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "export 10.241.1.0/24 in vrf-s7") {
t.Errorf("export : %v", err)
}
if err := withdrawRoute(f, db, "vp-s7", cidr(t, "10.241.1.0/24")); err == nil || !strings.Contains(err.Error(), "withdraw 10.241.1.0/24 from vrf-s7") {
t.Errorf("withdraw : %v", err)
}
if err := exportRoute(f, db, "novpc", cidr(t, "10.241.1.0/24")); err == nil {
t.Error("nom de VPC sans tiret accepté")
}
}

View file

@ -0,0 +1,232 @@
package watchdog
import (
"fmt"
"net"
"sort"
"strings"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"git.g3e.fr/syonad/two/internal/watchdog/notify"
"git.g3e.fr/syonad/two/pkg/db/kv"
"github.com/dgraph-io/badger/v4"
)
const kindBGP = "bgp"
type l3Probe interface {
VTEPAddress(iface string) (net.IP, error)
NeighborEstablished(peer net.IP) (bool, error)
VNI(vni uint32) (frr.VNIState, error)
ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error)
LinkProblem(ns, iface string) string
}
type systemProbe struct {
frr *frr.Client
}
func (systemProbe) VTEPAddress(iface string) (net.IP, error) { return netif.VTEPAddress(iface) }
func (p systemProbe) NeighborEstablished(peer net.IP) (bool, error) {
return p.frr.NeighborEstablished(peer)
}
func (p systemProbe) VNI(vni uint32) (frr.VNIState, error) { return p.frr.VNI(vni) }
func (systemProbe) ExportedRoutes(table uint32, iface string) ([]*net.IPNet, error) {
return netif.TableRoutesVia(int(table), iface)
}
func (systemProbe) LinkProblem(ns, iface string) string {
if ns == "" {
return linkProblem(iface)
}
var p string
if err := netns.Call(ns, func() error {
p = linkProblem(iface)
return nil
}); err != nil {
return fmt.Sprintf("cannot enter netns %s: %v", ns, err)
}
if p != "" {
return p + " (in netns)"
}
return ""
}
func CheckBGP(cfg *configuration.Config, p l3Probe, n notify.Notifier) {
if !cfg.BGP.Enabled() {
return
}
reflectors, err := cfg.BGP.Reflectors()
if err != nil {
n.Notify(kindBGP, "config", err.Error())
return
}
for _, rr := range reflectors {
up, err := p.NeighborEstablished(rr)
switch {
case err != nil:
n.Notify(kindBGP, rr.String(), fmt.Sprintf("session unreadable: %v", err))
case !up:
n.Notify(kindBGP, rr.String(), "session to the route reflector is not established")
}
}
}
func CheckL3VPCs(db *badger.DB, cfg *configuration.Config, p l3Probe, n notify.Notifier) error {
pairs, err := kv.ListByPrefix(db, prefixVPC)
if err != nil {
return fmt.Errorf("watchdog: listing vpcs: %w", err)
}
var rmac string
for _, name := range resourceNames(pairs, prefixVPC) {
if st, err := state.Get(db, prefixVPC+name); err != nil || st != state.Running {
continue
}
r, ok, err := vpc.LoadRouting(db, name)
if err != nil {
n.Notify(kindVPC, name, err.Error())
continue
}
if !ok {
continue
}
if !cfg.BGP.Enabled() {
n.Notify(kindVPC, name, "vpc has an l3vni but the agent configuration has no bgp section")
continue
}
if rmac == "" {
rmac = expectedRouterMAC(cfg, p, n)
}
checkL3VPC(db, name, r, rmac, p, n)
}
return nil
}
func expectedRouterMAC(cfg *configuration.Config, p l3Probe, n notify.Notifier) string {
vtep, err := p.VTEPAddress(cfg.DefaultInterface)
if err != nil {
n.Notify(kindBGP, "vtep", fmt.Sprintf("vtep address unreadable on %s: %v", cfg.DefaultInterface, err))
return "unknown"
}
mac, err := vpc.RouterMAC(vtep)
if err != nil {
n.Notify(kindBGP, "vtep", err.Error())
return "unknown"
}
return mac.String()
}
func checkL3VPC(db *badger.DB, name string, r vpc.Routing, rmac string, p l3Probe, n notify.Notifier) {
for _, iface := range []string{r.VRF, r.Bridge, r.Vxlan, r.VethHost} {
if problem := p.LinkProblem("", iface); problem != "" {
n.Notify(kindVPC, name, problem)
}
}
if problem := p.LinkProblem(name, r.VethNetns); problem != "" {
n.Notify(kindVPC, name, problem)
}
s, err := p.VNI(r.VNI)
switch {
case err != nil:
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d unreadable in frr: %v", r.VNI, err))
case s.Type != "L3" || s.State != "Up":
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d is type %q state %q in frr, expected L3 Up", r.VNI, s.Type, s.State))
case rmac != "unknown" && s.RouterMAC != rmac:
n.Notify(kindVPC, name, fmt.Sprintf("l3vni %d has router mac %q in frr, expected %s", r.VNI, s.RouterMAC, rmac))
}
actual, err := p.ExportedRoutes(r.VNI, r.VethHost)
if err != nil {
n.Notify(kindVPC, name, fmt.Sprintf("routes of %s unreadable: %v", r.VRF, err))
return
}
expected, tolerated, err := expectedExports(db, name)
if err != nil {
n.Notify(kindVPC, name, err.Error())
return
}
exported := make(map[string]bool, len(actual))
for _, dst := range actual {
exported[dst.String()] = true
}
for _, dst := range sortedKeys(exported) {
if !expected[dst] && !tolerated[dst] {
n.Notify(kindVPC, name, fmt.Sprintf("route %s exported in %s without a running owner: its traffic is black-holed", dst, r.VRF))
}
}
for _, dst := range sortedKeys(expected) {
if !exported[dst] {
n.Notify(kindVPC, name, fmt.Sprintf("route %s not exported in %s", dst, r.VRF))
}
}
}
func expectedExports(db *badger.DB, vpcName string) (expected, tolerated map[string]bool, err error) {
expected, tolerated = map[string]bool{}, map[string]bool{}
classify := func(st state.State, dst string) {
switch st {
case state.Running:
expected[dst] = true
case state.Creating, state.Deleting, state.Error:
tolerated[dst] = true
}
}
subnets, err := kv.ListByPrefix(db, prefixSubnet)
if err != nil {
return nil, nil, fmt.Errorf("watchdog: listing subnets: %w", err)
}
routed := map[string]bool{}
for key, value := range subnets {
subnet, ok := strings.CutSuffix(strings.TrimPrefix(key, prefixSubnet), "/vpc")
if !ok || strings.Contains(subnet, "/") || value != vpcName || subnets[prefixSubnet+subnet+"/mode"] != "vxlan" {
continue
}
routed[subnet] = true
_, cidr, err := net.ParseCIDR(subnets[prefixSubnet+subnet+"/cidr"])
if err != nil {
continue
}
if st, err := state.Get(db, prefixSubnet+subnet); err == nil {
classify(st, cidr.String())
}
}
vms, err := kv.ListByPrefix(db, prefixVM)
if err != nil {
return nil, nil, fmt.Errorf("watchdog: listing vms: %w", err)
}
for key, value := range vms {
parts := strings.Split(key, "/")
if len(parts) != 5 || parts[2] != "nic" || parts[4] != "subnet" || !routed[value] {
continue
}
ip := net.ParseIP(vms[strings.TrimSuffix(key, "subnet")+"ip"]).To4()
if ip == nil {
continue
}
if st, err := state.Get(db, prefixVM+parts[1]); err == nil {
classify(st, ip.String()+"/32")
}
}
return expected, tolerated, nil
}
func sortedKeys(m map[string]bool) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}

View file

@ -0,0 +1,242 @@
package watchdog
import (
"errors"
"net"
"reflect"
"testing"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/state"
)
type fakeProbe struct {
established map[string]bool
neighborErr error
vni map[uint32]frr.VNIState
routes map[uint32][]string
routesErr error
links map[string]string
vtepErr error
}
func (f *fakeProbe) VTEPAddress(string) (net.IP, error) {
if f.vtepErr != nil {
return nil, f.vtepErr
}
return net.ParseIP("192.168.14.11"), nil
}
func (f *fakeProbe) NeighborEstablished(peer net.IP) (bool, error) {
return f.established[peer.String()], f.neighborErr
}
func (f *fakeProbe) VNI(vni uint32) (frr.VNIState, error) {
s, ok := f.vni[vni]
if !ok {
return s, errors.New("% VNI does not exist")
}
return s, nil
}
func (f *fakeProbe) ExportedRoutes(table uint32, _ string) ([]*net.IPNet, error) {
var out []*net.IPNet
for _, r := range f.routes[table] {
_, n, _ := net.ParseCIDR(r)
out = append(out, n)
}
return out, f.routesErr
}
func (f *fakeProbe) LinkProblem(ns, iface string) string {
return f.links[ns+"/"+iface]
}
func bgpCfg() *configuration.Config {
cfg := &configuration.Config{DefaultInterface: "br-000000"}
cfg.BGP.ASN = 64600
cfg.BGP.RouteReflectors = []string{"10.255.255.1", "10.255.255.2"}
return cfg
}
func notified(r *recorder) []string {
var out []string
for _, c := range r.calls {
out = append(out, c.kind+" "+c.name+": "+c.problem)
}
return out
}
func TestCheckBGPReportsEachReflectorWithoutSession(t *testing.T) {
r := &recorder{}
CheckBGP(bgpCfg(), &fakeProbe{established: map[string]bool{"10.255.255.1": true}}, r)
want := []string{"bgp 10.255.255.2: session to the route reflector is not established"}
if got := notified(r); !reflect.DeepEqual(got, want) {
t.Errorf("notifications %q, attendu %q", got, want)
}
}
func TestCheckBGPReportsAnUnreadableSession(t *testing.T) {
r := &recorder{}
CheckBGP(bgpCfg(), &fakeProbe{neighborErr: errors.New("vtysh absent")}, r)
if len(r.calls) != 2 || !r.hasProblemContaining("session unreadable: vtysh absent") {
t.Errorf("notifications %q", notified(r))
}
}
func TestCheckBGPIsSilentWithoutBGPSection(t *testing.T) {
r := &recorder{}
CheckBGP(&configuration.Config{}, &fakeProbe{}, r)
if len(r.calls) != 0 {
t.Errorf("notifications %q", notified(r))
}
}
func seedRoutedVPC(t *testing.T) *fakeProbe {
return &fakeProbe{
vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:0b"}},
}
}
func TestCheckL3VPCsReconcilesExportedRoutesWithTheirOwners(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
seedResource(t, db, prefixVPC, "vp-flat", state.Running)
seedKV(t, db, "vpc/vp-flat/cidr", "10.242.0.0/16")
seedResource(t, db, prefixSubnet, "sn-a", state.Running)
seedKV(t, db, "subnet/sn-a/vpc", "vp-s7")
seedKV(t, db, "subnet/sn-a/mode", "vxlan")
seedKV(t, db, "subnet/sn-a/cidr", "10.241.1.0/24")
seedResource(t, db, prefixSubnet, "sn-b", state.Creating)
seedKV(t, db, "subnet/sn-b/vpc", "vp-s7")
seedKV(t, db, "subnet/sn-b/mode", "vxlan")
seedKV(t, db, "subnet/sn-b/cidr", "10.241.2.0/24")
seedResource(t, db, prefixSubnet, "sn-br", state.Running)
seedKV(t, db, "subnet/sn-br/vpc", "vp-s7")
seedKV(t, db, "subnet/sn-br/mode", "bridge")
seedKV(t, db, "subnet/sn-br/cidr", "192.168.50.0/24")
vm := func(name string, st state.State, subnet, ip string) {
seedResource(t, db, prefixVM, name, st)
seedKV(t, db, "vm/"+name+"/nic/0/subnet", subnet)
seedKV(t, db, "vm/"+name+"/nic/0/ip", ip)
}
vm("i-run", state.Running, "sn-a", "10.241.1.10")
vm("i-missing", state.Running, "sn-a", "10.241.1.11")
vm("i-starting", state.Creating, "sn-a", "10.241.1.12")
vm("i-gone", state.Deleted, "sn-a", "10.241.1.13")
vm("i-bridge", state.Running, "sn-br", "192.168.50.10")
seedResource(t, db, prefixSubnet, "sn-flat", state.Running)
seedKV(t, db, "subnet/sn-flat/vpc", "vp-flat")
seedKV(t, db, "subnet/sn-flat/mode", "vxlan")
seedKV(t, db, "subnet/sn-flat/cidr", "10.242.1.0/24")
vm("i-flat", state.Running, "sn-flat", "10.242.1.10")
p := seedRoutedVPC(t)
p.routes = map[uint32][]string{2700: {
"10.241.1.0/24", "10.241.2.0/24",
"10.241.1.10/32", "10.241.1.12/32", "10.241.1.13/32", "10.241.1.50/32",
}}
r := &recorder{}
if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil {
t.Fatal(err)
}
want := []string{
"vpc vp-s7: route 10.241.1.13/32 exported in vrf-s7 without a running owner: its traffic is black-holed",
"vpc vp-s7: route 10.241.1.50/32 exported in vrf-s7 without a running owner: its traffic is black-holed",
"vpc vp-s7: route 10.241.1.11/32 not exported in vrf-s7",
}
if got := notified(r); !reflect.DeepEqual(got, want) {
t.Errorf("notifications :\n%q\nattendu :\n%q", got, want)
}
}
func TestCheckL3VPCsChecksInterfacesAndTheL3VNI(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
seedResource(t, db, prefixVPC, "vp-down", state.Error)
seedKV(t, db, "vpc/vp-down/cidr", "10.243.0.0/16")
seedKV(t, db, "vpc/vp-down/l3vni", "2800")
p := &fakeProbe{
vni: map[uint32]frr.VNIState{2700: {Type: "L3", State: "Up", RouterMAC: "02:02:c0:a8:0e:99"}},
links: map[string]string{"/vxl3-s7": "interface vxl3-s7 down", "vp-s7/l3-s7-i": "interface l3-s7-i not found (in netns)"},
}
r := &recorder{}
if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil {
t.Fatal(err)
}
want := []string{
"vpc vp-s7: interface vxl3-s7 down",
"vpc vp-s7: interface l3-s7-i not found (in netns)",
`vpc vp-s7: l3vni 2700 has router mac "02:02:c0:a8:0e:99" in frr, expected 02:02:c0:a8:0e:0b`,
}
if got := notified(r); !reflect.DeepEqual(got, want) {
t.Errorf("notifications :\n%q\nattendu :\n%q", got, want)
}
p.vni[2700] = frr.VNIState{Type: "L2", State: "Up"}
p.links = nil
r = &recorder{}
CheckL3VPCs(db, bgpCfg(), p, r)
if !r.hasProblemContaining(`l3vni 2700 is type "L2" state "Up" in frr, expected L3 Up`) {
t.Errorf("notifications %q", notified(r))
}
p.vni[2700] = frr.VNIState{Type: "L3", State: "Down", RouterMAC: "02:02:c0:a8:0e:0b"}
r = &recorder{}
CheckL3VPCs(db, bgpCfg(), p, r)
if !r.hasProblemContaining(`l3vni 2700 is type "L3" state "Down" in frr, expected L3 Up`) {
t.Errorf("notifications %q", notified(r))
}
delete(p.vni, 2700)
r = &recorder{}
CheckL3VPCs(db, bgpCfg(), p, r)
if !r.hasProblemContaining("l3vni 2700 unreadable in frr") {
t.Errorf("notifications %q", notified(r))
}
}
func TestCheckL3VPCsReportsAnL3VNIWithoutBGP(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
r := &recorder{}
CheckL3VPCs(db, &configuration.Config{}, &fakeProbe{}, r)
want := []string{"vpc vp-s7: vpc has an l3vni but the agent configuration has no bgp section"}
if got := notified(r); !reflect.DeepEqual(got, want) {
t.Errorf("notifications %q, attendu %q", got, want)
}
}
func TestCheckL3VPCsReportsUnreadableRoutesAndVTEP(t *testing.T) {
db := newTestDB(t)
seedResource(t, db, prefixVPC, "vp-s7", state.Running)
seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16")
seedKV(t, db, "vpc/vp-s7/l3vni", "2700")
seedResource(t, db, prefixVPC, "vp-s8", state.Running)
seedKV(t, db, "vpc/vp-s8/cidr", "10.242.0.0/16")
seedKV(t, db, "vpc/vp-s8/l3vni", "2800")
p := seedRoutedVPC(t)
p.routesErr = errors.New("netlink refused")
p.vtepErr = errors.New("no address")
r := &recorder{}
CheckL3VPCs(db, bgpCfg(), p, r)
if n := len(r.forName("vtep")); n != 1 {
t.Errorf("VTEP illisible signalé %d fois, attendu une fois par passage : %q", n, notified(r))
}
if !r.hasProblemContaining("routes of vrf-s7 unreadable: netlink refused") || !r.hasProblemContaining("vtep address unreadable on br-000000") {
t.Errorf("notifications %q", notified(r))
}
if r.hasProblemContaining("router mac") {
t.Errorf("RMAC comparée sans VTEP : %q", notified(r))
}
}

View file

@ -6,6 +6,7 @@ import (
"time"
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/frr"
"git.g3e.fr/syonad/two/internal/watchdog/notify"
"git.g3e.fr/syonad/two/pkg/systemd"
@ -21,6 +22,7 @@ type Watchdog struct {
logger *slog.Logger
interval time.Duration
dbusDown bool
l3 l3Probe
}
func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *slog.Logger, interval time.Duration) *Watchdog {
@ -38,6 +40,7 @@ func New(db *badger.DB, cfg *configuration.Config, n notify.Notifier, logger *sl
notifier: n,
logger: logger,
interval: interval,
l3: systemProbe{frr: frr.NewClient()},
}
}
@ -71,6 +74,10 @@ func (w *Watchdog) tick() {
if err := CheckVMs(w.db, w.cfg, u, w.notifier); err != nil {
w.logger.Error("watchdog: vm check failed", "err", err)
}
CheckBGP(w.cfg, w.l3, w.notifier)
if err := CheckL3VPCs(w.db, w.cfg, w.l3, w.notifier); err != nil {
w.logger.Error("watchdog: l3 vpc check failed", "err", err)
}
}
func (w *Watchdog) units() (unitChecker, func()) {