diff --git a/api/agent.yaml b/api/agent.yaml index c7a546f..b28f183 100644 --- a/api/agent.yaml +++ b/api/agent.yaml @@ -91,7 +91,7 @@ paths: "404": $ref: "#/components/responses/NotFound" "409": - description: VPC not deletable — only running or error states can be deleted, and all its subnets must be deleted or deleting; subnets still deleting are awaited by the asynchronous deletion + description: VPC not deletable — only running or error states can be deleted, all its subnets must be deleted or deleting (subnets still deleting are awaited by the asynchronous deletion), and its NAT gateway, if any, must be deleted first content: application/json: schema: @@ -299,6 +299,101 @@ paths: # ── Components ────────────────────────────────────────────────────────────── + # ── NAT gateway ──────────────────────────────────────────────────────────── + + /nat-gateways: + get: + summary: List all NAT gateways of this agent + operationId: listNATGateways + responses: + "200": + description: List of NAT gateways + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/NATGateway" + "500": + $ref: "#/components/responses/InternalError" + + post: + summary: Create the NAT gateway of a VPC on this agent + operationId: createNATGateway + description: >- + Gives the VMs of a VPC without a public IP an Internet exit through this host. The + gateway owns the given public IP on the public bridge (interfaces.internet) and announces + a type-5 default route in the VRF of the VPC. Only VPCs created by this version or later + route their default traffic to the VRF, and can use it. To move the gateway, delete it on + the old host if it answers, then create it on the new one. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NATGatewayCreateRequest" + responses: + "202": + description: NAT gateway creation accepted + content: + application/json: + schema: + $ref: "#/components/schemas/NATGateway" + "400": + description: Missing required field, invalid request body, ip without prefix length or not IPv4, or gateway not IPv4 + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "409": + description: NAT gateway already exists, VPC unknown on this agent or not running, VPC without l3vni, or VPC that already has a NAT gateway on this agent + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + $ref: "#/components/responses/InternalError" + + /nat-gateways/{name}: + parameters: + - $ref: "#/components/parameters/ResourceName" + + get: + summary: Get NAT gateway status and info + operationId: getNATGateway + responses: + "200": + description: NAT gateway found + content: + application/json: + schema: + $ref: "#/components/schemas/NATGateway" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalError" + + delete: + summary: Delete a NAT gateway + operationId: deleteNATGateway + responses: + "202": + description: NAT gateway deletion accepted + content: + application/json: + schema: + $ref: "#/components/schemas/NATGateway" + "404": + $ref: "#/components/responses/NotFound" + "409": + description: NAT gateway not deletable — only running or error states can be deleted + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + $ref: "#/components/responses/InternalError" + components: parameters: @@ -626,6 +721,46 @@ components: type: boolean example: false + NATGatewayCreateRequest: + type: object + required: [name, vpc, ip, gateway] + properties: + name: + type: string + example: ng-42 + vpc: + type: string + description: VPC of the gateway; must be running on this agent and have an l3vni + example: vp-42 + ip: + type: string + description: Public exit IP of the VPC with its prefix length, chosen by the caller and not checked by the agent + example: "198.51.100.50/24" + gateway: + type: string + description: Gateway of the border routers on the public segment, not checked by the agent + example: "198.51.100.1" + + NATGateway: + type: object + properties: + name: + type: string + example: ng-42 + state: + type: string + enum: [creating, running, error, deleting, deleted] + example: running + vpc: + type: string + example: vp-42 + ip: + type: string + example: "198.51.100.50/24" + gateway: + type: string + example: "198.51.100.1" + Error: type: object properties: diff --git a/docs/architecture/stockage.rst b/docs/architecture/stockage.rst index 4931e13..c587c68 100644 --- a/docs/architecture/stockage.rst +++ b/docs/architecture/stockage.rst @@ -37,6 +37,11 @@ de sérialisation structurée. vm//sshkey → (optionnel) vm//metadata/ → (optionnel : user-data, vendor-data, …) + nat-gateway//state → creating | running | error | deleting | deleted + nat-gateway//vpc → (une passerelle par VPC) + nat-gateway//ip → / (IP de sortie, telle que fournie) + nat-gateway//gateway → (gateway des bordures) + Règles ------ diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index 90a05db..ae9a014 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -13,3 +13,4 @@ fonctionnent entre eux. Le contrat HTTP correspondant est dans cycle-de-vie metadata-cloud-init dns-ntp + passerelle-nat diff --git a/docs/concepts/passerelle-nat.rst b/docs/concepts/passerelle-nat.rst new file mode 100644 index 0000000..72df641 --- /dev/null +++ b/docs/concepts/passerelle-nat.rst @@ -0,0 +1,100 @@ +Passerelle NAT +============== + +Une VM sans IP publique sort vers Internet par la **passerelle NAT de son VPC**. C'est une +ressource de l'API de l'agent (``/nat-gateways``), créée sur un seul hyperviseur, choisi par +l'appelant. Elle porte l'IP de sortie du VPC sur le segment public de l'hôte (``br-public``, +``vxlan-public``) et annonce une route par défaut de type 5 dans la VRF du VPC : toutes les VM du +VPC, sur tous les hyperviseurs, sortent par elle, avec cette IP pour source. + +**L'autorisation Internet d'un VPC, c'est l'existence de sa passerelle** : le VPC ne porte aucun +champ pour cela. Le VPC doit avoir un ``l3vni``, et une seule passerelle existe par VPC. L'IP de +sortie, son préfixe et la gateway des bordures sont fournis par l'appelant ; l'agent ne les +vérifie pas. + +Chemin d'un paquet +------------------ + +.. mermaid:: + + graph LR + V["VM sur l'hôte B"] --> NB["netns du VPC, hôte B
route par défaut vers la VRF"] + NB --> RB["vrf-<id>, hôte B
route par défaut de type 5"] + RB -->|"L3VNI"| RA["vrf-<id>, hôte A
route par défaut vers 169.254.0.2"] + RA --> NA["netns du VPC, hôte A
SNAT vers l'IP de sortie"] + NA -->|"ng-<id>, br-public, vxlan-public"| B["bordures"] + +Le netns de chaque VPC routé envoie sa route par défaut vers la VRF. Sans passerelle, la route +``unreachable default`` de la VRF rejette ce trafic (voir :doc:`vpc-subnet-vm`) ; avec une +passerelle, la route par défaut de type 5 l'emporte, l'``unreachable`` ayant la métrique maximale. +Créer ou déplacer une passerelle ne demande donc rien aux autres hyperviseurs du VPC. Sur l'hôte +de la passerelle, les VM sortent directement par le netns, sans passer par la VRF. + +Les réponses reviennent à l'IP de sortie : les bordures la résolvent vers la MAC de la +passerelle, le netns défait la traduction et renvoie le paquet vers la VM par la VRF. + +Ce que l'agent pose +------------------- + +Dans le netns du VPC ``vp-``, sur l'hôte de la passerelle : + +* un ``macvlan`` ``ng-`` sur le bridge ``br-public`` du netns, déjà relié au bridge public de + l'hôte par ``vp--i`` / ``vp--e``. Sa MAC est **calculée** à partir du ``l3vni`` : + ``02:04:00`` suivi du ``l3vni`` sur 24 bits (``4600`` → ``02:04:00:00:11:f8``). Elle est la + même quel que soit l'hôte : c'est ce qui permet de déplacer la passerelle ; +* une entrée FDB locale de cette MAC sur ``br-public`` : un ``macvlan`` n'en crée pas dans son + bridge parent, et sans elle les trames vers la passerelle seraient inondées au lieu de lui être + remises (même mécanisme que la MAC anycast des subnets ``vxlan``). Elle est posée par + remplacement : après une bascule, le bridge a appris cette MAC sur le segment, vers l'ancien + hôte ; +* ``arp_ignore=1``, ``arp_announce=2`` et ``arp_notify=1`` sur ``ng-`` : la passerelle ne + répond à l'ARP que pour son IP de sortie, et annonce celle-ci par un ARP gratuit à sa mise en + route ; +* l'IP de sortie sur ``ng-``, puis une route par défaut vers la gateway des bordures par + ``ng-``, à la place de celle vers la VRF ; le ``cidr`` du VPC reste routé vers la VRF ; +* trois chaînes dans la table nftables ``ip two`` du netns : + + * ``snat-ng-`` : ``snat`` vers l'IP de sortie de tout ce qui sort par ``ng-`` ; + * ``fwd-ng-`` et ``in-ng-`` : rien n'entre par ``ng-`` — ni vers les VM, ni vers le + netns lui-même (DNS et NTP, métadonnées) — hors réponse à une connexion établie + (``ct state established,related``). L'ARP n'est pas filtré ; + +* en dernier, sur l'hôte, une route par défaut vers ``169.254.0.2`` dans la table de la VRF : + l'annonce n'est jamais faite avant que la traduction soit en place. + +La suppression retire d'abord la route par défaut de la VRF, puis les règles nftables, remet la +route par défaut du netns vers la VRF, retire l'entrée FDB et le ``macvlan``. Chaque étape tolère +ce qui est déjà absent : une suppression interrompue se relance depuis ``error``. Le retrait de la +route de la VRF désigne sa gateway et son interface, de sorte que l'``unreachable default`` reste +en place. + +Annonce de la route par défaut +------------------------------ + +Le route-map ``EXPORT-`` de la VRF porte une entrée ``permit 15`` : ``0.0.0.0/0``, à +condition que son next-hop soit ``169.254.0.2`` (prefix-lists ``PL--default`` et +``PL--transit``). Sans cette condition, ``redistribute kernel`` exporterait aussi +l'``unreachable default`` de chaque hôte (constaté sur FRR 8.4, hors VRF), et tous les +hyperviseurs annonceraient une sortie qu'ils n'ont pas. + +Ces entrées sont posées à la création du VPC et retirées à sa suppression ; seule la route du +noyau dépend de la passerelle. + +Déplacer la passerelle +---------------------- + +L'agent ne détecte aucune panne et ne coordonne rien avec les autres hyperviseurs. Pour déplacer +la passerelle, l'appelant la supprime sur l'ancien hôte s'il répond, puis la crée sur le nouveau +avec la même IP. La MAC étant la même, la bascule se réduit à un ARP gratuit et à la mobilité MAC +d'EVPN. Les connexions en cours sont perdues. + +Si l'hôte de la passerelle tombe, ses routes restent annoncées jusqu'à l'expiration des sessions +BGP vers les route reflectors (180 s), et le VPC n'a plus de sortie jusqu'à la création d'une +nouvelle passerelle. + +Limites +------- + +* Seuls les VPC créés par la 0.3.0 ou après envoient leur route par défaut vers la VRF et + exportent la route de la passerelle : un VPC plus ancien doit être recréé pour s'en servir. +* IPv4 seulement. diff --git a/docs/concepts/vpc-subnet-vm.rst b/docs/concepts/vpc-subnet-vm.rst index ba697b4..3d27bf9 100644 --- a/docs/concepts/vpc-subnet-vm.rst +++ b/docs/concepts/vpc-subnet-vm.rst @@ -25,6 +25,14 @@ public est celui de ``interfaces.internet``, ``br-public`` si la clé est absent accroche ``vxlan-public``, le VXLAN du segment public partagé avec les bordures (:doc:`/demarrage/installation`) ; sans ce bridge, toute création de VPC échoue. +Le ``br-public`` du netns n'est qu'un point de raccordement : rien n'entre dans le VPC par lui. +L'agent y pose ``arp_ignore=1`` (il ne répond pas à l'ARP pour les adresses du netns) et, dans la +table nftables ``ip two``, rejette tout IP arrivé par ``br-public`` (chaînes ``in-br-public`` et +``fwd-br-public``). Seules les interfaces publiques prévues — passerelle NAT, IP flottante — y +reçoivent du trafic. Il désactive aussi ``br_netfilter`` dans le netns +(``net.bridge.bridge-nf-call-{iptables,ip6tables,arptables}=0``, quand le module est chargé) : sinon +conntrack voit chaque trame pontée et le SNAT de la passerelle NAT ne s'applique plus. + Dans le netns, l'agent pose aussi la ``dummy`` ``hostsvc`` (``169.254.169.253/32``) et la chaîne iptables ``TWO-HOSTSVC``, puis demande à ``two-hostsvc`` de servir le VPC : c'est le chemin du DNS et du NTP des VM (voir :doc:`dns-ntp`). Si ``two-hostsvc`` ne répond pas, la création du VPC @@ -58,12 +66,15 @@ passe en ``error`` : ce VNI est déjà porté par ``vxlan-public``. │ └── vxl3- VXLAN du l3vni, VTEP = IPv4 primaire de default_interface └── l3--o 169.254.0.1/30 ── l3--i 169.254.0.2/30, dans le netns du VPC -Dans le netns, le ``cidr`` du VPC est routé vers ``169.254.0.1`` ; le transit ``/30`` est le même +Dans le netns, le ``cidr`` du VPC et la route par défaut sont routés vers ``169.254.0.1`` : sans +passerelle NAT, l'``unreachable default`` de la VRF rejette ce qui n'est pas dans le VPC (voir +:doc:`passerelle-nat`) ; le transit ``/30`` est le même sur tous les hyperviseurs et pour tous les VPC. L'agent active le forwarding IPv4 dans le netns, et sur l'hôte sur ``brl3-`` et ``l3--o``, les deux interfaces d'entrée de la VRF ; il ne touche pas au réglage global, que ``bootstrap_kvm.sh`` met déjà à 1. FRR reçoit la VRF, son ``vni``, une instance ``router bgp vrf vrf-`` avec RD ``:`` et RT ``:``, et -l'export des routes du ``cidr`` présentes dans la VRF. +l'export des routes du ``cidr`` présentes dans la VRF, et de la route par défaut de la passerelle +NAT. La table de la VRF reçoit, juste après sa création, une route ``unreachable default`` de métrique maximale (``4278198272``, valeur recommandée par la documentation VRF du noyau). Sans elle, une @@ -71,7 +82,8 @@ recherche qui échoue dans la VRF continue dans les règles suivantes, donc dans de l'hôte : un paquet d'un locataire vers une adresse inconnue de la VRF sortirait par l'underlay, avec ``169.254.0.2`` pour source (mesuré, `#81 `_). La métrique maximale laisse passer avant elle toute route réelle de la VRF. Cette route n'est pas -annoncée en EVPN : le route-map de la VRF n'exporte que le ``cidr`` du VPC. Elle n'est liée à +annoncée en EVPN : le route-map de la VRF n'exporte que le ``cidr`` du VPC, et la route par +défaut seulement quand son next-hop est ``169.254.0.2``, celle d'une passerelle NAT. Elle n'est liée à aucune interface et ne disparaît pas avec la VRF : la suppression la retire explicitement. IPv4 seulement. diff --git a/docs/developpement/lab.rst b/docs/developpement/lab.rst index dfceacf..0264b31 100644 --- a/docs/developpement/lab.rst +++ b/docs/developpement/lab.rst @@ -396,6 +396,10 @@ Générée par ``test/e2e/run.sh --matrice`` à partir des en-têtes ; à régé - segment public de l'hôte — vxlan-public (VNI 1) posé par le bootstrap sur br-public, VNI annoncé en EVPN et vu par les bordures, gateway anycast des bordures installée vers une seule bordure, gateway et destination de test joignables sans doublon, L2 entre hyperviseurs, veth des VPC sur le bridge public, watchdog - hv1 hv2 hv3 - #87 + * - ``46-passerelle-nat`` + - passerelle NAT d'un VPC — macvlan ng- sur le br-public du netns, MAC 02:04:00 + l3vni, SNAT vers l'IP de sortie (source vue à la sortie du netns), route par défaut en type 5 importée par un autre hôte, unreachable default jamais annoncée, rien n'entre sans connexion établie, refus 409, suppression propre, bascule vers un autre hôte par mobilité MAC + - hv1 hv2 hv3 + - #42 * - ``51-l3vni`` - routage inter-subnet entre hôtes par L3VNI (IRB symétrique) — VRF, RMAC, exports /24 et /32, RT-5 réfléchies, TTL, isolation par RT, unreachable default dans la VRF, démontage d'un VPC routé - hv1 hv2 diff --git a/docs/exploitation/observabilite.rst b/docs/exploitation/observabilite.rst index b6b8ba9..c34d770 100644 --- a/docs/exploitation/observabilite.rst +++ b/docs/exploitation/observabilite.rst @@ -18,6 +18,8 @@ Exposées sur le port ``prometheus.port`` (9090 par défaut), alimentées par l' - nombre de subnets, par état * - ``syonad_vms_total`` - nombre de VM, par état + * - ``syonad_nat_gateways_total`` + - nombre de passerelles NAT, par état Les états sont ceux du :doc:`cycle de vie `. Une valeur non nulle et durable sur ``error`` est l'alerte la plus utile à poser ; une valeur durable sur ``creating`` @@ -63,7 +65,8 @@ Avec une section ``bgp`` dans la configuration, il contrôle aussi le plan de co par une version antérieure à la 0.3.0 ne l'a pas : il est signalé à chaque tick jusqu'à sa recréation, l'agent ne la pose pas au démarrage. * **Réconciliation des routes exportées** dans la VRF : chaque ``/24`` de subnet ``vxlan`` et chaque - ``/32`` d'interface de VM ``running`` doit être exporté ; à l'inverse, une route exportée dont + ``/32`` d'interface de VM ``running`` doit être exporté, ainsi que la route par défaut quand le + VPC a une passerelle NAT ``running`` sur l'hôte (voir :doc:`../concepts/passerelle-nat`) ; à l'inverse, une route exportée dont le propriétaire n'existe plus — ni ``running``, ni en ``creating``, ``deleting`` ou ``error`` — est signalée comme un **trou noir** : l'annonce attire du trafic que plus rien ne reçoit. diff --git a/internal/api/agent/models.go b/internal/api/agent/models.go index c033a16..bd86adc 100644 --- a/internal/api/agent/models.go +++ b/internal/api/agent/models.go @@ -82,3 +82,18 @@ type VM struct { type ErrorResponse struct { Error string `json:"error"` } + +type NATGatewayCreateRequest struct { + Name string `json:"name"` + VPC string `json:"vpc"` + IP string `json:"ip"` + Gateway string `json:"gateway"` +} + +type NATGateway struct { + Name string `json:"name"` + State string `json:"state"` + VPC string `json:"vpc,omitempty"` + IP string `json:"ip,omitempty"` + Gateway string `json:"gateway,omitempty"` +} diff --git a/internal/api/agent/natgw.go b/internal/api/agent/natgw.go new file mode 100644 index 0000000..13ea847 --- /dev/null +++ b/internal/api/agent/natgw.go @@ -0,0 +1,65 @@ +package agentapi + +import ( + "encoding/json" + "net/http" + "strings" + + dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent" + "git.g3e.fr/syonad/two/pkg/db/kv" +) + +func (s *Server) NATGatewayByNameHandler(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/nat-gateways/") + w.Header().Set("Content-Type", "application/json") + if name == "" { + w.WriteHeader(http.StatusNotFound) + json.NewEncoder(w).Encode(ErrorResponse{Error: "resource not found"}) + return + } + switch r.Method { + case http.MethodGet: + s.getNATGateway(w, r, name) + case http.MethodDelete: + s.deleteNATGateway(w, r, name) + default: + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + } +} + +func (s *Server) getNATGateway(w http.ResponseWriter, _ *http.Request, name string) { + prefix := "nat-gateway/" + name + "/" + state, err := kv.GetFromDB(s.db, prefix+"state") + if err != nil { + w.WriteHeader(http.StatusNotFound) + json.NewEncoder(w).Encode(ErrorResponse{Error: "nat gateway not found"}) + return + } + vpc, _ := kv.GetFromDB(s.db, prefix+"vpc") + ip, _ := kv.GetFromDB(s.db, prefix+"ip") + gateway, _ := kv.GetFromDB(s.db, prefix+"gateway") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(NATGateway{Name: name, State: state, VPC: vpc, IP: ip, Gateway: gateway}) +} + +func (s *Server) deleteNATGateway(w http.ResponseWriter, _ *http.Request, name string) { + cmd := dispatcher.DeleteNATGatewayCommand{Name: name} + if err := s.dispatcher.Prepare(cmd); err != nil { + if _, dbErr := kv.GetFromDB(s.db, "nat-gateway/"+name+"/state"); dbErr != nil { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusConflict) + } + json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()}) + return + } + s.dispatcher.Dispatch(cmd) + state, err := kv.GetFromDB(s.db, "nat-gateway/"+name+"/state") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(ErrorResponse{Error: "failed to read nat gateway state"}) + return + } + w.WriteHeader(http.StatusAccepted) + json.NewEncoder(w).Encode(NATGateway{Name: name, State: state}) +} diff --git a/internal/api/agent/natgw_test.go b/internal/api/agent/natgw_test.go new file mode 100644 index 0000000..0f7e80b --- /dev/null +++ b/internal/api/agent/natgw_test.go @@ -0,0 +1,168 @@ +package agentapi + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +func seedRoutedVPC(t *testing.T, db *badger.DB) { + t.Helper() + kv.AddInDB(db, "vpc/vp-s7/state", "running") + kv.AddInDB(db, "vpc/vp-s7/cidr", "10.241.0.0/16") + kv.AddInDB(db, "vpc/vp-s7/l3vni", "2700") +} + +func postNATGateway(t *testing.T, s *Server, req NATGatewayCreateRequest) *httptest.ResponseRecorder { + t.Helper() + body, _ := json.Marshal(req) + w := httptest.NewRecorder() + s.NATGatewaysHandler(w, httptest.NewRequest(http.MethodPost, "/nat-gateways", bytes.NewReader(body))) + return w +} + +func validNATGatewayRequest() NATGatewayCreateRequest { + return NATGatewayCreateRequest{Name: "ng-1", VPC: "vp-s7", IP: "198.51.100.50/24", Gateway: "198.51.100.1"} +} + +func TestPostNATGateway_Accepted(t *testing.T) { + s, db := newTestServer(t) + seedRoutedVPC(t, db) + w := postNATGateway(t, s, validNATGatewayRequest()) + if w.Code != http.StatusAccepted { + t.Fatalf("attendu 202, obtenu %d : %s", w.Code, w.Body.String()) + } + var got NATGateway + json.NewDecoder(w.Body).Decode(&got) + want := NATGateway{Name: "ng-1", State: "creating", VPC: "vp-s7", IP: "198.51.100.50/24", Gateway: "198.51.100.1"} + if got != want { + t.Errorf("réponse %+v, attendu %+v", got, want) + } +} + +func TestPostNATGateway_BadRequest(t *testing.T) { + cases := []struct { + label string + mutate func(*NATGatewayCreateRequest) + msg string + }{ + {"nom absent", func(r *NATGatewayCreateRequest) { r.Name = "" }, "name is required"}, + {"vpc absent", func(r *NATGatewayCreateRequest) { r.VPC = "" }, "vpc is required"}, + {"ip absente", func(r *NATGatewayCreateRequest) { r.IP = "" }, "ip is required"}, + {"gateway absente", func(r *NATGatewayCreateRequest) { r.Gateway = "" }, "gateway is required"}, + {"ip sans préfixe", func(r *NATGatewayCreateRequest) { r.IP = "198.51.100.50" }, "invalid ip"}, + {"ip IPv6", func(r *NATGatewayCreateRequest) { r.IP = "2001:db8::50/64" }, "invalid ip"}, + {"gateway invalide", func(r *NATGatewayCreateRequest) { r.Gateway = "bordure" }, "invalid gateway"}, + {"gateway IPv6", func(r *NATGatewayCreateRequest) { r.Gateway = "2001:db8::1" }, "invalid gateway"}, + } + for _, c := range cases { + s, db := newTestServer(t) + seedRoutedVPC(t, db) + req := validNATGatewayRequest() + c.mutate(&req) + w := postNATGateway(t, s, req) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), c.msg) { + t.Errorf("%s : %d %s, 400 « %s » attendu", c.label, w.Code, w.Body.String(), c.msg) + } + if _, err := kv.GetFromDB(db, "nat-gateway/ng-1/state"); err == nil { + t.Errorf("%s : passerelle écrite malgré le refus", c.label) + } + } +} + +func TestPostNATGateway_InvalidBody(t *testing.T) { + s, _ := newTestServer(t) + w := httptest.NewRecorder() + s.NATGatewaysHandler(w, httptest.NewRequest(http.MethodPost, "/nat-gateways", bytes.NewReader([]byte("not json")))) + if w.Code != http.StatusBadRequest { + t.Errorf("attendu 400, obtenu %d", w.Code) + } +} + +func TestPostNATGateway_ConflictWithoutRunningRoutedVPC(t *testing.T) { + s, _ := newTestServer(t) + w := postNATGateway(t, s, validNATGatewayRequest()) + if w.Code != http.StatusConflict || !strings.Contains(w.Body.String(), `vpc \"vp-s7\" not found`) { + t.Errorf("%d %s, 409 attendu", w.Code, w.Body.String()) + } +} + +func TestListNATGateways(t *testing.T) { + s, db := newTestServer(t) + kv.AddInDB(db, "nat-gateway/ng-1/state", "running") + kv.AddInDB(db, "nat-gateway/ng-1/vpc", "vp-s7") + kv.AddInDB(db, "nat-gateway/ng-1/ip", "198.51.100.50/24") + kv.AddInDB(db, "nat-gateway/ng-1/gateway", "198.51.100.1") + w := httptest.NewRecorder() + s.NATGatewaysHandler(w, httptest.NewRequest(http.MethodGet, "/nat-gateways", nil)) + if w.Code != http.StatusOK { + t.Fatalf("attendu 200, obtenu %d", w.Code) + } + var got []NATGateway + json.NewDecoder(w.Body).Decode(&got) + want := NATGateway{Name: "ng-1", State: "running", VPC: "vp-s7", IP: "198.51.100.50/24", Gateway: "198.51.100.1"} + if len(got) != 1 || got[0] != want { + t.Errorf("liste %+v, attendu [%+v]", got, want) + } +} + +func TestListNATGateways_Empty(t *testing.T) { + s, _ := newTestServer(t) + w := httptest.NewRecorder() + s.NATGatewaysHandler(w, httptest.NewRequest(http.MethodGet, "/nat-gateways", nil)) + if w.Code != http.StatusOK || strings.TrimSpace(w.Body.String()) != "[]" { + t.Errorf("%d %q, 200 [] attendu", w.Code, w.Body.String()) + } +} + +func TestGetNATGateway(t *testing.T) { + s, db := newTestServer(t) + kv.AddInDB(db, "nat-gateway/ng-1/state", "running") + kv.AddInDB(db, "nat-gateway/ng-1/vpc", "vp-s7") + w := httptest.NewRecorder() + s.NATGatewayByNameHandler(w, httptest.NewRequest(http.MethodGet, "/nat-gateways/ng-1", nil)) + var got NATGateway + json.NewDecoder(w.Body).Decode(&got) + if w.Code != http.StatusOK || got.Name != "ng-1" || got.State != "running" || got.VPC != "vp-s7" { + t.Errorf("%d %+v", w.Code, got) + } + w = httptest.NewRecorder() + s.NATGatewayByNameHandler(w, httptest.NewRequest(http.MethodGet, "/nat-gateways/ng-x", nil)) + if w.Code != http.StatusNotFound { + t.Errorf("passerelle absente : %d, 404 attendu", w.Code) + } +} + +func TestDeleteNATGateway(t *testing.T) { + cases := map[string]int{"running": http.StatusAccepted, "error": http.StatusAccepted, "creating": http.StatusConflict, "": http.StatusNotFound} + for st, want := range cases { + s, db := newTestServer(t) + if st != "" { + kv.AddInDB(db, "nat-gateway/ng-1/state", st) + kv.AddInDB(db, "nat-gateway/ng-1/vpc", "vp-s7") + } + w := httptest.NewRecorder() + s.NATGatewayByNameHandler(w, httptest.NewRequest(http.MethodDelete, "/nat-gateways/ng-1", nil)) + if w.Code != want { + t.Errorf("état %q : %d, %d attendu (%s)", st, w.Code, want, w.Body.String()) + } + } +} + +func TestDeleteVpc_BlockedByNATGateway(t *testing.T) { + s, db := newTestServer(t) + seedRoutedVPC(t, db) + kv.AddInDB(db, "nat-gateway/ng-1/state", "running") + kv.AddInDB(db, "nat-gateway/ng-1/vpc", "vp-s7") + w := httptest.NewRecorder() + s.VpcByNameHandler(w, httptest.NewRequest(http.MethodDelete, "/vpcs/vp-s7", nil)) + if w.Code != http.StatusConflict || !strings.Contains(w.Body.String(), `nat gateway \"ng-1\" must be deleted`) { + t.Errorf("%d %s, 409 attendu", w.Code, w.Body.String()) + } +} diff --git a/internal/api/agent/natgws.go b/internal/api/agent/natgws.go new file mode 100644 index 0000000..2756412 --- /dev/null +++ b/internal/api/agent/natgws.go @@ -0,0 +1,100 @@ +package agentapi + +import ( + "encoding/json" + "net" + "net/http" + "strings" + + dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent" + "git.g3e.fr/syonad/two/pkg/db/kv" +) + +func (s *Server) NATGatewaysHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method { + case http.MethodGet: + s.listNATGateways(w, r) + case http.MethodPost: + s.postNATGateway(w, r) + default: + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + } +} + +func (s *Server) listNATGateways(w http.ResponseWriter, _ *http.Request) { + entries, err := kv.ListByPrefix(s.db, "nat-gateway/") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(ErrorResponse{Error: "failed to list nat gateways"}) + return + } + gateways := make(map[string]*NATGateway) + for key, value := range entries { + parts := strings.Split(key, "/") + if len(parts) != 3 { + continue + } + name := parts[1] + if _, ok := gateways[name]; !ok { + gateways[name] = &NATGateway{Name: name} + } + switch parts[2] { + case "state": + gateways[name].State = value + case "vpc": + gateways[name].VPC = value + case "ip": + gateways[name].IP = value + case "gateway": + gateways[name].Gateway = value + } + } + result := make([]NATGateway, 0, len(gateways)) + for _, g := range gateways { + result = append(result, *g) + } + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(result) +} + +func (s *Server) postNATGateway(w http.ResponseWriter, r *http.Request) { + var req NATGatewayCreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(ErrorResponse{Error: "invalid request body"}) + return + } + for _, f := range []struct{ name, value string }{{"name", req.Name}, {"vpc", req.VPC}, {"ip", req.IP}, {"gateway", req.Gateway}} { + if f.value == "" { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(ErrorResponse{Error: f.name + " is required"}) + return + } + } + if ip, _, err := net.ParseCIDR(req.IP); err != nil || ip.To4() == nil { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(ErrorResponse{Error: "invalid ip: an IPv4 address with its prefix length is expected"}) + return + } + if gw := net.ParseIP(req.Gateway); gw == nil || gw.To4() == nil { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(ErrorResponse{Error: "invalid gateway: an IPv4 address is expected"}) + return + } + cmd := dispatcher.CreateNATGatewayCommand{Name: req.Name, VPC: req.VPC, IP: req.IP, Gateway: req.Gateway} + if err := s.dispatcher.Prepare(cmd); err != nil { + w.WriteHeader(http.StatusConflict) + json.NewEncoder(w).Encode(ErrorResponse{Error: err.Error()}) + return + } + s.dispatcher.Dispatch(cmd) + state, err := kv.GetFromDB(s.db, "nat-gateway/"+req.Name+"/state") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(ErrorResponse{Error: "failed to read nat gateway state"}) + return + } + w.WriteHeader(http.StatusAccepted) + json.NewEncoder(w).Encode(NATGateway{Name: req.Name, State: state, VPC: req.VPC, IP: req.IP, Gateway: req.Gateway}) +} diff --git a/internal/api/agent/server.go b/internal/api/agent/server.go index 2b892bd..f5b2273 100644 --- a/internal/api/agent/server.go +++ b/internal/api/agent/server.go @@ -29,6 +29,8 @@ func New(d *dispatcher.Dispatcher, db *badger.DB, logger *slog.Logger, address s mux.HandleFunc("/subnets/", s.SubnetByNameHandler) mux.HandleFunc("/vms", s.VmsHandler) mux.HandleFunc("/vms/", s.VmByNameHandler) + mux.HandleFunc("/nat-gateways", s.NATGatewaysHandler) + mux.HandleFunc("/nat-gateways/", s.NATGatewayByNameHandler) s.srv = &http.Server{Addr: address, Handler: s.logMiddleware(mux)} return s } diff --git a/internal/dispatcher/agent/natgw_commands.go b/internal/dispatcher/agent/natgw_commands.go new file mode 100644 index 0000000..cabea56 --- /dev/null +++ b/internal/dispatcher/agent/natgw_commands.go @@ -0,0 +1,125 @@ +package dispatcher + +import ( + "fmt" + "net" + "strings" + + configuration "git.g3e.fr/syonad/two/internal/config/agent" + "git.g3e.fr/syonad/two/internal/natgw" + "git.g3e.fr/syonad/two/internal/state" + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +const natGatewayPrefix = "nat-gateway/" + +type CreateNATGatewayCommand struct { + Name string + VPC string + IP string + Gateway string +} + +func (c CreateNATGatewayCommand) Key() string { return natGatewayPrefix + c.Name } + +func (c CreateNATGatewayCommand) Prepare(db *badger.DB, _ *configuration.Config) error { + if _, err := kv.GetFromDB(db, c.Key()+"/state"); err == nil { + return fmt.Errorf("nat gateway %q already exists", c.Name) + } + if _, _, err := parseNATGateway(c.IP, c.Gateway); err != nil { + return err + } + vpcState, err := state.Get(db, "vpc/"+c.VPC) + if err != nil { + return fmt.Errorf("vpc %q not found", c.VPC) + } + if vpcState != state.Running { + return fmt.Errorf("vpc %q is %s", c.VPC, vpcState) + } + if _, err := kv.GetFromDB(db, "vpc/"+c.VPC+"/l3vni"); err != nil { + return fmt.Errorf("vpc %q has no l3vni", c.VPC) + } + other, err := natGatewayOf(db, c.VPC) + if err != nil { + return err + } + if other != "" { + return fmt.Errorf("vpc %q already has nat gateway %q", c.VPC, other) + } + for key, value := range map[string]string{"vpc": c.VPC, "ip": c.IP, "gateway": c.Gateway} { + if err := kv.AddInDB(db, c.Key()+"/"+key, value); err != nil { + return err + } + } + return state.Set(db, c.Key(), state.Creating) +} + +func (c CreateNATGatewayCommand) Execute(db *badger.DB, _ *configuration.Config) error { + ip, gw, err := parseNATGateway(c.IP, c.Gateway) + if err != nil { + return err + } + if err := natgw.Create(db, c.VPC, ip, gw); err != nil { + return err + } + return state.Set(db, c.Key(), state.Running) +} + +type DeleteNATGatewayCommand struct { + Name string +} + +func (c DeleteNATGatewayCommand) Key() string { return natGatewayPrefix + c.Name } + +func (c DeleteNATGatewayCommand) Prepare(db *badger.DB, _ *configuration.Config) error { + current, err := state.Get(db, c.Key()) + if err != nil { + return fmt.Errorf("nat gateway %q not found", c.Name) + } + if !state.CanDelete(current) { + return fmt.Errorf("nat gateway %q cannot be deleted while %s", c.Name, current) + } + return state.Set(db, c.Key(), state.Deleting) +} + +func (c DeleteNATGatewayCommand) Execute(db *badger.DB, _ *configuration.Config) error { + vpcName, err := kv.GetFromDB(db, c.Key()+"/vpc") + if err != nil { + return fmt.Errorf("nat gateway %q: reading vpc: %w", c.Name, err) + } + if err := natgw.Delete(db, vpcName); err != nil { + return err + } + if err := state.Set(db, c.Key(), state.Deleted); err != nil { + return err + } + return kv.DeleteInDB(db, c.Key()) +} + +func parseNATGateway(ip, gateway string) (*net.IPNet, net.IP, error) { + addr, prefix, err := net.ParseCIDR(ip) + if err != nil || addr.To4() == nil { + return nil, nil, fmt.Errorf("invalid ip %q: an IPv4 address with its prefix length is expected", ip) + } + prefix.IP = addr.To4() + gw := net.ParseIP(gateway).To4() + if gw == nil { + return nil, nil, fmt.Errorf("invalid gateway %q: an IPv4 address is expected", gateway) + } + return prefix, gw, nil +} + +func natGatewayOf(db *badger.DB, vpcName string) (string, error) { + entries, err := kv.ListByPrefix(db, natGatewayPrefix) + if err != nil { + return "", fmt.Errorf("failed to list nat gateways: %w", err) + } + for key, value := range entries { + name, ok := strings.CutSuffix(strings.TrimPrefix(key, natGatewayPrefix), "/vpc") + if ok && !strings.Contains(name, "/") && value == vpcName { + return name, nil + } + } + return "", nil +} diff --git a/internal/dispatcher/agent/natgw_commands_test.go b/internal/dispatcher/agent/natgw_commands_test.go new file mode 100644 index 0000000..2151610 --- /dev/null +++ b/internal/dispatcher/agent/natgw_commands_test.go @@ -0,0 +1,143 @@ +package dispatcher + +import ( + "strings" + "testing" + + "git.g3e.fr/syonad/two/pkg/db/kv" + "github.com/dgraph-io/badger/v4" +) + +func seedRoutedVPC(t *testing.T, db *badger.DB, name, st string) { + t.Helper() + kv.AddInDB(db, "vpc/"+name+"/state", st) + kv.AddInDB(db, "vpc/"+name+"/cidr", "10.241.0.0/16") + kv.AddInDB(db, "vpc/"+name+"/l3vni", "2700") +} + +func validNATGateway() CreateNATGatewayCommand { + return CreateNATGatewayCommand{Name: "ng-1", VPC: "vp-s7", IP: "198.51.100.50/24", Gateway: "198.51.100.1"} +} + +func TestCreateNATGatewayCommand_Prepare_WritesTheGateway(t *testing.T) { + _, db := newTestDispatcher(t) + seedRoutedVPC(t, db, "vp-s7", "running") + if err := validNATGateway().Prepare(db, nil); err != nil { + t.Fatalf("Prepare a échoué : %v", err) + } + for key, want := range map[string]string{ + "nat-gateway/ng-1/state": "creating", + "nat-gateway/ng-1/vpc": "vp-s7", + "nat-gateway/ng-1/ip": "198.51.100.50/24", + "nat-gateway/ng-1/gateway": "198.51.100.1", + } { + got, err := kv.GetFromDB(db, key) + if err != nil || got != want { + t.Errorf("%s : %q (%v), %q attendu", key, got, err, want) + } + } +} + +func TestCreateNATGatewayCommand_Prepare_Refuses(t *testing.T) { + cases := []struct { + label string + seed func(*badger.DB) + mutate func(*CreateNATGatewayCommand) + msg string + }{ + {"passerelle existante", func(db *badger.DB) { kv.AddInDB(db, "nat-gateway/ng-1/state", "running") }, nil, `nat gateway "ng-1" already exists`}, + {"VPC absent", nil, func(c *CreateNATGatewayCommand) { c.VPC = "vp-absent" }, `vpc "vp-absent" not found`}, + {"VPC en création", func(db *badger.DB) { seedRoutedVPC(t, db, "vp-new", "creating") }, func(c *CreateNATGatewayCommand) { c.VPC = "vp-new" }, `vpc "vp-new" is creating`}, + {"VPC sans l3vni", func(db *badger.DB) { + kv.AddInDB(db, "vpc/vp-flat/state", "running") + kv.AddInDB(db, "vpc/vp-flat/cidr", "10.242.0.0/16") + }, func(c *CreateNATGatewayCommand) { c.VPC = "vp-flat" }, `vpc "vp-flat" has no l3vni`}, + {"VPC qui a déjà une passerelle", func(db *badger.DB) { + kv.AddInDB(db, "nat-gateway/ng-0/state", "error") + kv.AddInDB(db, "nat-gateway/ng-0/vpc", "vp-s7") + }, nil, `vpc "vp-s7" already has nat gateway "ng-0"`}, + {"ip sans longueur de préfixe", nil, func(c *CreateNATGatewayCommand) { c.IP = "198.51.100.50" }, `invalid ip "198.51.100.50"`}, + {"ip IPv6", nil, func(c *CreateNATGatewayCommand) { c.IP = "2001:db8::50/64" }, `invalid ip "2001:db8::50/64"`}, + {"gateway invalide", nil, func(c *CreateNATGatewayCommand) { c.Gateway = "bordure" }, `invalid gateway "bordure"`}, + } + for _, c := range cases { + _, db := newTestDispatcher(t) + seedRoutedVPC(t, db, "vp-s7", "running") + if c.seed != nil { + c.seed(db) + } + cmd := validNATGateway() + if c.mutate != nil { + c.mutate(&cmd) + } + err := cmd.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), c.msg) { + t.Errorf("%s : erreur %v, « %s » attendu", c.label, err, c.msg) + continue + } + if c.label != "passerelle existante" { + if _, err := kv.GetFromDB(db, "nat-gateway/ng-1/state"); err == nil { + t.Errorf("%s : état écrit malgré le refus", c.label) + } + } + } +} + +func TestCreateNATGatewayCommand_Prepare_IgnoresTheGatewayOfAnotherVPC(t *testing.T) { + _, db := newTestDispatcher(t) + seedRoutedVPC(t, db, "vp-s7", "running") + kv.AddInDB(db, "nat-gateway/ng-0/state", "running") + kv.AddInDB(db, "nat-gateway/ng-0/vpc", "vp-s70") + if err := validNATGateway().Prepare(db, nil); err != nil { + t.Errorf("Prepare a échoué : %v", err) + } +} + +func TestDeleteNATGatewayCommand_Prepare(t *testing.T) { + for st, ok := range map[string]bool{"running": true, "error": true, "creating": false, "deleting": false} { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "nat-gateway/ng-1/state", st) + err := DeleteNATGatewayCommand{Name: "ng-1"}.Prepare(db, nil) + if ok != (err == nil) { + t.Errorf("état %s : erreur %v", st, err) + } + got, _ := kv.GetFromDB(db, "nat-gateway/ng-1/state") + if ok && got != "deleting" { + t.Errorf("état %s : %q après Prepare, deleting attendu", st, got) + } + } +} + +func TestDeleteNATGatewayCommand_Prepare_Unknown(t *testing.T) { + _, db := newTestDispatcher(t) + err := DeleteNATGatewayCommand{Name: "ng-x"}.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), `nat gateway "ng-x" not found`) { + t.Errorf("erreur %v", err) + } +} + +func TestDeleteVPCCommand_Prepare_RefusesWhileANATGatewayExists(t *testing.T) { + for _, st := range []string{"running", "error", "creating", "deleting"} { + _, db := newTestDispatcher(t) + kv.AddInDB(db, "vpc/vp-s7/state", "running") + kv.AddInDB(db, "nat-gateway/ng-1/state", st) + kv.AddInDB(db, "nat-gateway/ng-1/vpc", "vp-s7") + err := DeleteVPCCommand{Name: "vp-s7"}.Prepare(db, nil) + if err == nil || !strings.Contains(err.Error(), `nat gateway "ng-1" must be deleted before deleting vpc "vp-s7"`) { + t.Errorf("passerelle %s : erreur %v", st, err) + } + if got, _ := kv.GetFromDB(db, "vpc/vp-s7/state"); got != "running" { + t.Errorf("passerelle %s : VPC passé en %q malgré le refus", st, got) + } + } +} + +func TestParseNATGatewayKeepsTheHostAddress(t *testing.T) { + ip, gw, err := parseNATGateway("198.51.100.50/24", "198.51.100.1") + if err != nil { + t.Fatal(err) + } + if ip.String() != "198.51.100.50/24" || gw.String() != "198.51.100.1" { + t.Errorf("ip %s, gateway %s", ip, gw) + } +} diff --git a/internal/dispatcher/agent/vpc_commands.go b/internal/dispatcher/agent/vpc_commands.go index 7e73164..ade4d13 100644 --- a/internal/dispatcher/agent/vpc_commands.go +++ b/internal/dispatcher/agent/vpc_commands.go @@ -126,6 +126,13 @@ func (c DeleteVPCCommand) Prepare(db *badger.DB, _ *configuration.Config) error return fmt.Errorf("subnet %q must be deleted before deleting vpc %q", subnetName, c.Name) } } + gateway, err := natGatewayOf(db, c.Name) + if err != nil { + return err + } + if gateway != "" { + return fmt.Errorf("nat gateway %q must be deleted before deleting vpc %q", gateway, c.Name) + } return state.Set(db, c.Key(), state.Deleting) } diff --git a/internal/frr/client_test.go b/internal/frr/client_test.go index 0f9ec9e..13dc3a6 100644 --- a/internal/frr/client_test.go +++ b/internal/frr/client_test.go @@ -124,6 +124,8 @@ func TestWithdrawVRFRunsEachStepInOrderThenSaves(t *testing.T) { {"configure terminal", "no router bgp 64600 vrf vrf-a1b2"}, {"configure terminal", "no route-map EXPORT-a1b2"}, {"configure terminal", "no ip prefix-list PL-a1b2"}, + {"configure terminal", "no ip prefix-list PL-a1b2-default"}, + {"configure terminal", "no ip prefix-list PL-a1b2-transit"}, {"write memory"}, {"show running-config"}, } diff --git a/internal/frr/config.go b/internal/frr/config.go index 34e3210..9d1bd1d 100644 --- a/internal/frr/config.go +++ b/internal/frr/config.go @@ -20,6 +20,7 @@ type VRF struct { L3VNI uint32 VTEP net.IP Supernet *net.IPNet + Transit net.IP } func (b Base) validate() error { @@ -94,6 +95,14 @@ func (v VRF) prefixList() string { return "PL-" + v.ID } +func (v VRF) defaultPrefixList() string { + return "PL-" + v.ID + "-default" +} + +func (v VRF) transitPrefixList() string { + return "PL-" + v.ID + "-transit" +} + func (v VRF) routeMap() string { return "EXPORT-" + v.ID } @@ -125,15 +134,24 @@ func (v VRF) validate() error { if v.Supernet == nil || v.Supernet.IP.To4() == nil { return fmt.Errorf("frr: supernet %v is not an IPv4 prefix", v.Supernet) } + if v.Transit.To4() == nil { + return fmt.Errorf("frr: transit %v is not an IPv4 address", v.Transit) + } return nil } func (v VRF) commands() []string { return []string{ fmt.Sprintf("ip prefix-list %s seq 10 permit %s le 32", v.prefixList(), v.Supernet), + fmt.Sprintf("ip prefix-list %s seq 10 permit 0.0.0.0/0", v.defaultPrefixList()), + fmt.Sprintf("ip prefix-list %s seq 10 permit %s/32", v.transitPrefixList(), v.Transit.To4()), fmt.Sprintf("route-map %s permit 10", v.routeMap()), fmt.Sprintf("match ip address prefix-list %s", v.prefixList()), "exit", + fmt.Sprintf("route-map %s permit 15", v.routeMap()), + fmt.Sprintf("match ip address prefix-list %s", v.defaultPrefixList()), + fmt.Sprintf("match ip next-hop prefix-list %s", v.transitPrefixList()), + "exit", fmt.Sprintf("route-map %s deny 20", v.routeMap()), "exit", fmt.Sprintf("vrf %s", v.Name()), diff --git a/internal/frr/config_test.go b/internal/frr/config_test.go index 766554a..9ce27d9 100644 --- a/internal/frr/config_test.go +++ b/internal/frr/config_test.go @@ -23,6 +23,7 @@ func labVRF(t *testing.T) VRF { L3VNI: 4001, VTEP: net.ParseIP("192.168.14.11"), Supernet: mustCIDR(t, "10.10.0.0/16"), + Transit: net.ParseIP("169.254.0.2"), } } @@ -55,9 +56,15 @@ func TestBaseCommands(t *testing.T) { func TestVRFCommands(t *testing.T) { want := []string{ "ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32", + "ip prefix-list PL-a1b2-default seq 10 permit 0.0.0.0/0", + "ip prefix-list PL-a1b2-transit seq 10 permit 169.254.0.2/32", "route-map EXPORT-a1b2 permit 10", "match ip address prefix-list PL-a1b2", "exit", + "route-map EXPORT-a1b2 permit 15", + "match ip address prefix-list PL-a1b2-default", + "match ip next-hop prefix-list PL-a1b2-transit", + "exit", "route-map EXPORT-a1b2 deny 20", "exit", "vrf vrf-a1b2", @@ -124,6 +131,8 @@ func TestVRFValidateRejects(t *testing.T) { "vtep IPv6": {func(v *VRF) { v.VTEP = net.ParseIP("fd00::1") }, "vtep fd00::1 is not an IPv4 address"}, "supernet absent": {func(v *VRF) { v.Supernet = nil }, "supernet is not an IPv4 prefix"}, "supernet IPv6": {func(v *VRF) { v.Supernet = mustCIDR(t, "fd00::/48") }, "supernet fd00::/48 is not an IPv4 prefix"}, + "transit absent": {func(v *VRF) { v.Transit = nil }, "transit is not an IPv4 address"}, + "transit IPv6": {func(v *VRF) { v.Transit = net.ParseIP("fd00::2") }, "transit fd00::2 is not an IPv4 address"}, } for name, c := range cases { v := labVRF(t) diff --git a/internal/frr/teardown.go b/internal/frr/teardown.go index fcbc407..dcb43c8 100644 --- a/internal/frr/teardown.go +++ b/internal/frr/teardown.go @@ -25,8 +25,10 @@ func routingTeardown(v VRF, running string) []step { if hasPrefix(lines, "route-map "+v.routeMap()+" ") { steps = append(steps, step{"no route-map " + v.routeMap()}) } - if hasPrefix(lines, "ip prefix-list "+v.prefixList()+" ") { - steps = append(steps, step{"no ip prefix-list " + v.prefixList()}) + for _, list := range []string{v.prefixList(), v.defaultPrefixList(), v.transitPrefixList()} { + if hasPrefix(lines, "ip prefix-list "+list+" ") { + steps = append(steps, step{"no ip prefix-list " + list}) + } } return steps } diff --git a/internal/frr/teardown_test.go b/internal/frr/teardown_test.go index 210c943..91ca84b 100644 --- a/internal/frr/teardown_test.go +++ b/internal/frr/teardown_test.go @@ -51,12 +51,19 @@ router bgp 64600 vrf vrf-a1b2c exit ! ip prefix-list PL-a1b2 seq 10 permit 10.10.0.0/16 le 32 +ip prefix-list PL-a1b2-default seq 10 permit 0.0.0.0/0 +ip prefix-list PL-a1b2-transit seq 10 permit 169.254.0.2/32 ip prefix-list PL-a1b2c seq 10 permit 10.20.0.0/16 le 32 ! route-map EXPORT-a1b2 permit 10 match ip address prefix-list PL-a1b2 exit ! +route-map EXPORT-a1b2 permit 15 + match ip address prefix-list PL-a1b2-default + match ip next-hop prefix-list PL-a1b2-transit +exit +! route-map EXPORT-a1b2 deny 20 exit ! @@ -91,6 +98,8 @@ func TestRoutingTeardownFollowsTheOrderFRRRequires(t *testing.T) { {"no router bgp 64600 vrf vrf-a1b2"}, {"no route-map EXPORT-a1b2"}, {"no ip prefix-list PL-a1b2"}, + {"no ip prefix-list PL-a1b2-default"}, + {"no ip prefix-list PL-a1b2-transit"}, } if got := routingTeardown(labVRF(t), fullRunning); !reflect.DeepEqual(got, want) { t.Errorf("démontage :\n%q\nattendu :\n%q", got, want) @@ -104,6 +113,8 @@ func TestRoutingTeardownTakesTheASNOfTheInstanceFromFRR(t *testing.T) { {"no router bgp 64600 vrf vrf-a1b2"}, {"no route-map EXPORT-a1b2"}, {"no ip prefix-list PL-a1b2"}, + {"no ip prefix-list PL-a1b2-default"}, + {"no ip prefix-list PL-a1b2-transit"}, } for _, asn := range []uint32{0, 64601} { v := labVRF(t) diff --git a/internal/l3vni/host.go b/internal/l3vni/host.go index 24a6c98..a97c547 100644 --- a/internal/l3vni/host.go +++ b/internal/l3vni/host.go @@ -23,6 +23,7 @@ type host interface { DeleteLink(name string) error TableRouteVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error TableRouteDel(table uint32, dst *net.IPNet) error + TableRouteDelVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error TableUnreachableDefault(table uint32) error TableUnreachableDefaultDel(table uint32) error ApplyVRF(v frr.VRF) error @@ -89,6 +90,10 @@ func (system) TableRouteDel(table uint32, dst *net.IPNet) error { return netif.RouteDelIfExists(int(table), dst) } +func (system) TableRouteDelVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return netif.RouteDelViaIfExists(int(table), dst, gw, iface) +} + func (system) TableUnreachableDefault(table uint32) error { return netif.RouteReplaceUnreachableDefault(int(table), UnreachableMetric) } diff --git a/internal/l3vni/l3.go b/internal/l3vni/l3.go index ad22ac2..03b5658 100644 --- a/internal/l3vni/l3.go +++ b/internal/l3vni/l3.go @@ -19,6 +19,7 @@ const UnreachableMetric = 4278198272 var ( transitHost = &net.IPNet{IP: net.IPv4(169, 254, 0, 1).To4(), Mask: net.CIDRMask(30, 32)} transitNetns = &net.IPNet{IP: net.IPv4(169, 254, 0, 2).To4(), Mask: net.CIDRMask(30, 32)} + defaultRoute = &net.IPNet{IP: net.IPv4zero.To4(), Mask: net.CIDRMask(0, 32)} ) type l3VPC struct { @@ -104,8 +105,11 @@ func createL3(h host, cfg *configuration.Config, v l3VPC) error { {"route " + v.supernet.String() + " in " + v.netns, func() error { return h.RouteVia(v.netns, v.supernet, transitHost.IP, v.vethNetns()) }}, + {"default route in " + v.netns, func() error { + return h.RouteVia(v.netns, defaultRoute, transitHost.IP, v.vethNetns()) + }}, {"frr " + v.vrf(), func() error { - return h.ApplyVRF(frr.VRF{ASN: asn, ID: v.id, L3VNI: v.vni, VTEP: vtep, Supernet: v.supernet}) + return h.ApplyVRF(frr.VRF{ASN: asn, ID: v.id, L3VNI: v.vni, VTEP: vtep, Supernet: v.supernet, Transit: transitNetns.IP}) }}, } for _, s := range steps { diff --git a/internal/l3vni/l3_test.go b/internal/l3vni/l3_test.go index 7855f27..75cb17a 100644 --- a/internal/l3vni/l3_test.go +++ b/internal/l3vni/l3_test.go @@ -83,6 +83,10 @@ func (f *fakeHost) TableRouteDel(table uint32, dst *net.IPNet) error { return f.record(fmt.Sprintf("table %d del %s", table, dst)) } +func (f *fakeHost) TableRouteDelVia(table uint32, dst *net.IPNet, gw net.IP, iface string) error { + return f.record(fmt.Sprintf("table %d del %s via %s dev %s", table, dst, gw, iface)) +} + func (f *fakeHost) TableUnreachableDefault(table uint32) error { return f.record(fmt.Sprintf("table %d unreachable default", table)) } @@ -92,7 +96,7 @@ func (f *fakeHost) TableUnreachableDefaultDel(table uint32) error { } func (f *fakeHost) ApplyVRF(v frr.VRF) error { - return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet)) + return f.record(fmt.Sprintf("frr apply asn=%d id=%s vni=%d vtep=%s supernet=%s transit=%s", v.ASN, v.ID, v.L3VNI, v.VTEP, v.Supernet, v.Transit)) } func (f *fakeHost) WithdrawVRF(v frr.VRF) error { @@ -145,7 +149,8 @@ func TestCreateL3FollowsTheLabRecipe(t *testing.T) { "up [vp-s7] l3-s7-i", "sysctl [vp-s7] net.ipv4.ip_forward=1", "route [vp-s7] 10.241.0.0/16 via 169.254.0.1 dev l3-s7-i", - "frr apply asn=64600 id=s7 vni=2700 vtep=192.168.14.11 supernet=10.241.0.0/16", + "route [vp-s7] 0.0.0.0/0 via 169.254.0.1 dev l3-s7-i", + "frr apply asn=64600 id=s7 vni=2700 vtep=192.168.14.11 supernet=10.241.0.0/16 transit=169.254.0.2", } if !reflect.DeepEqual(f.calls, want) { t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) diff --git a/internal/l3vni/route.go b/internal/l3vni/route.go index 333abf6..d0ba3cf 100644 --- a/internal/l3vni/route.go +++ b/internal/l3vni/route.go @@ -16,6 +16,18 @@ func WithdrawRoute(db *badger.DB, vpcName string, dst *net.IPNet) error { return withdrawRoute(newSystem(), db, vpcName, dst) } +func ExportDefault(db *badger.DB, vpcName string) error { + return exportDefault(newSystem(), db, vpcName) +} + +func WithdrawDefault(db *badger.DB, vpcName string) error { + return withdrawDefault(newSystem(), db, vpcName) +} + +func NetnsDefaultToVRF(db *badger.DB, vpcName string) error { + return netnsDefaultToVRF(newSystem(), db, vpcName) +} + func loadRouted(db *badger.DB, vpcName string) (l3VPC, bool, error) { parts := strings.SplitN(vpcName, "-", 2) if len(parts) != 2 { @@ -48,3 +60,47 @@ func withdrawRoute(h host, db *badger.DB, vpcName string, dst *net.IPNet) error } return nil } + +func loadRequired(db *badger.DB, vpcName string) (l3VPC, error) { + v, ok, err := loadRouted(db, vpcName) + if err != nil { + return l3VPC{}, err + } + if !ok { + return l3VPC{}, fmt.Errorf("vpc %q has no l3vni", vpcName) + } + return v, nil +} + +func exportDefault(h host, db *badger.DB, vpcName string) error { + v, err := loadRequired(db, vpcName) + if err != nil { + return err + } + if err := h.TableRouteVia(v.vni, defaultRoute, transitNetns.IP, v.vethHost()); err != nil { + return fmt.Errorf("export default in %s: %w", v.vrf(), err) + } + return nil +} + +func withdrawDefault(h host, db *badger.DB, vpcName string) error { + v, err := loadRequired(db, vpcName) + if err != nil { + return err + } + if err := h.TableRouteDelVia(v.vni, defaultRoute, transitNetns.IP, v.vethHost()); err != nil { + return fmt.Errorf("withdraw default from %s: %w", v.vrf(), err) + } + return nil +} + +func netnsDefaultToVRF(h host, db *badger.DB, vpcName string) error { + v, err := loadRequired(db, vpcName) + if err != nil { + return err + } + if err := h.RouteVia(v.netns, defaultRoute, transitHost.IP, v.vethNetns()); err != nil { + return fmt.Errorf("default route of %s towards %s: %w", v.netns, v.vrf(), err) + } + return nil +} diff --git a/internal/l3vni/route_test.go b/internal/l3vni/route_test.go index 0377f68..0264692 100644 --- a/internal/l3vni/route_test.go +++ b/internal/l3vni/route_test.go @@ -87,3 +87,69 @@ func TestRouteErrorsArePropagated(t *testing.T) { t.Error("nom de VPC sans tiret accepté") } } + +func TestExportDefaultInstallsTheDefaultRouteInTheVRFTowardsTheNetns(t *testing.T) { + f := &fakeHost{} + if err := exportDefault(f, routedDB(t), "vp-s7"); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 route 0.0.0.0/0 via 169.254.0.2 dev l3-s7-o"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestWithdrawDefaultNamesTheGatewaySoTheUnreachableDefaultStays(t *testing.T) { + f := &fakeHost{} + if err := withdrawDefault(f, routedDB(t), "vp-s7"); err != nil { + t.Fatal(err) + } + want := []string{"table 2700 del 0.0.0.0/0 via 169.254.0.2 dev l3-s7-o"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestDefaultRouteRefusesAVPCWithoutL3VNI(t *testing.T) { + db := routedDB(t) + for label, run := range map[string]func(host, *badger.DB, string) error{ + "export": exportDefault, + "withdraw": withdrawDefault, + } { + f := &fakeHost{} + err := run(f, db, "vp-flat") + if err == nil || !strings.Contains(err.Error(), `vpc "vp-flat" has no l3vni`) || len(f.calls) != 0 { + t.Errorf("%s : erreur %v, appels %q", label, err, f.calls) + } + } +} + +func TestDefaultRouteErrorsArePropagated(t *testing.T) { + db := routedDB(t) + f := &fakeHost{fail: map[string]error{"table ": errors.New("network unreachable")}} + if err := exportDefault(f, db, "vp-s7"); err == nil || !strings.Contains(err.Error(), "export default in vrf-s7: network unreachable") { + t.Errorf("export : %v", err) + } + if err := withdrawDefault(f, db, "vp-s7"); err == nil || !strings.Contains(err.Error(), "withdraw default from vrf-s7: network unreachable") { + t.Errorf("withdraw : %v", err) + } +} + +func TestNetnsDefaultToVRFPointsTheNetnsDefaultAtTheVRF(t *testing.T) { + f := &fakeHost{} + if err := netnsDefaultToVRF(f, routedDB(t), "vp-s7"); err != nil { + t.Fatal(err) + } + want := []string{"route [vp-s7] 0.0.0.0/0 via 169.254.0.1 dev l3-s7-i"} + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels %q, attendu %q", f.calls, want) + } +} + +func TestNetnsDefaultToVRFRefusesAVPCWithoutL3VNI(t *testing.T) { + f := &fakeHost{} + err := netnsDefaultToVRF(f, routedDB(t), "vp-flat") + if err == nil || !strings.Contains(err.Error(), `vpc "vp-flat" has no l3vni`) || len(f.calls) != 0 { + t.Errorf("erreur %v, appels %q", err, f.calls) + } +} diff --git a/internal/migration/state.go b/internal/migration/state.go index 99b5517..27fe805 100644 --- a/internal/migration/state.go +++ b/internal/migration/state.go @@ -14,7 +14,7 @@ import ( ) // prefixes énumère les familles de ressources portant une clé /state. -var prefixes = []string{"vpc/", "subnet/", "vm/"} +var prefixes = []string{"vpc/", "subnet/", "vm/", "nat-gateway/"} // legacyStates traduit l'ancien vocabulaire (états VPC/subnet d'avant // l'unification, et états VM start/stop) vers l'enum courant. diff --git a/internal/migration/state_test.go b/internal/migration/state_test.go index 85137a4..6896489 100644 --- a/internal/migration/state_test.go +++ b/internal/migration/state_test.go @@ -38,6 +38,9 @@ func TestMigrateStates_LegacyMapping(t *testing.T) { "vm stopping": {"vm/vm-4", "stopping", state.Error}, "vpc creating": {"vpc/vpc-2", "creating", state.Error}, "subnet deleting": {"subnet/sn-2", "deleting", state.Error}, + "nat creating": {"nat-gateway/ng-1", "creating", state.Error}, + "nat deleting": {"nat-gateway/ng-2", "deleting", state.Error}, + "nat running": {"nat-gateway/ng-3", "running", state.Running}, } for _, c := range cases { kv.AddInDB(db, c.resource+"/state", c.from) diff --git a/internal/natgw/host.go b/internal/natgw/host.go new file mode 100644 index 0000000..5f25a7f --- /dev/null +++ b/internal/natgw/host.go @@ -0,0 +1,85 @@ +package natgw + +import ( + "net" + + "git.g3e.fr/syonad/two/internal/l3vni" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/netns" + "git.g3e.fr/syonad/two/internal/nft" + "git.g3e.fr/syonad/two/internal/sysctl" + + "github.com/dgraph-io/badger/v4" +) + +type host interface { + CreateMacvlan(ns, name, parent string, mac net.HardwareAddr) error + BridgeSetLocalMAC(ns, bridge string, mac net.HardwareAddr) error + BridgeDelLocalMAC(ns, bridge string, mac net.HardwareAddr) error + Sysctl(ns, key, value string) error + AddrAdd(ns, iface string, prefix *net.IPNet) error + LinkUp(ns, iface string) error + DefaultVia(ns string, gw net.IP, iface string) error + SetNAT(ns string, g nft.NATGateway) error + DeleteNAT(ns, iface string) error + DeleteLink(ns, name string) error + ExportDefault(vpc string) error + WithdrawDefault(vpc string) error + NetnsDefaultToVRF(vpc string) error +} + +var defaultRoute = &net.IPNet{IP: net.IPv4zero.To4(), Mask: net.CIDRMask(0, 32)} + +type system struct { + db *badger.DB +} + +func newSystem(db *badger.DB) system { + return system{db: db} +} + +func (system) CreateMacvlan(ns, name, parent string, mac net.HardwareAddr) error { + return netns.Call(ns, func() error { return netif.CreateMacvlan(name, parent, mac) }) +} + +func (system) BridgeSetLocalMAC(ns, bridge string, mac net.HardwareAddr) error { + return netns.Call(ns, func() error { return netif.BridgeSetLocalMAC(bridge, mac) }) +} + +func (system) BridgeDelLocalMAC(ns, bridge string, mac net.HardwareAddr) error { + return netns.Call(ns, func() error { return netif.BridgeDelLocalMACIfExists(bridge, mac) }) +} + +func (system) Sysctl(ns, key, value string) error { + return netns.Call(ns, func() error { return sysctl.Set(key, value) }) +} + +func (system) AddrAdd(ns, iface string, prefix *net.IPNet) error { + return netns.Call(ns, func() error { return netif.AddrAddPrefix(iface, prefix) }) +} + +func (system) LinkUp(ns, iface string) error { + return netns.Call(ns, func() error { return netif.LinkSetUp(iface) }) +} + +func (system) DefaultVia(ns string, gw net.IP, iface string) error { + return netns.Call(ns, func() error { return netif.RouteReplaceVia(0, defaultRoute, gw, iface) }) +} + +func (system) SetNAT(ns string, g nft.NATGateway) error { + return netns.Call(ns, func() error { return nft.SetNATGateway(g) }) +} + +func (system) DeleteNAT(ns, iface string) error { + return netns.Call(ns, func() error { return nft.DeleteNATGateway(iface) }) +} + +func (system) DeleteLink(ns, name string) error { + return netns.Call(ns, func() error { return netif.DeleteLinkIfExists(name) }) +} + +func (s system) ExportDefault(vpc string) error { return l3vni.ExportDefault(s.db, vpc) } + +func (s system) WithdrawDefault(vpc string) error { return l3vni.WithdrawDefault(s.db, vpc) } + +func (s system) NetnsDefaultToVRF(vpc string) error { return l3vni.NetnsDefaultToVRF(s.db, vpc) } diff --git a/internal/natgw/natgw.go b/internal/natgw/natgw.go new file mode 100644 index 0000000..52de536 --- /dev/null +++ b/internal/natgw/natgw.go @@ -0,0 +1,119 @@ +package natgw + +import ( + "fmt" + "net" + "strings" + + "git.g3e.fr/syonad/two/internal/l3vni" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/nft" + + "github.com/dgraph-io/badger/v4" +) + +const publicBridge = "br-public" + +type gateway struct { + vpc string + iface string + mac net.HardwareAddr + ip *net.IPNet + gateway net.IP +} + +func Iface(vpcName string) (string, error) { + parts := strings.SplitN(vpcName, "-", 2) + if len(parts) != 2 { + return "", fmt.Errorf("invalid vpc name %q", vpcName) + } + return "ng-" + parts[1], nil +} + +func load(db *badger.DB, vpcName string) (gateway, error) { + r, ok, err := l3vni.LoadRouting(db, vpcName) + if err != nil { + return gateway{}, err + } + if !ok { + return gateway{}, fmt.Errorf("vpc %q has no l3vni", vpcName) + } + iface, err := Iface(vpcName) + if err != nil { + return gateway{}, err + } + mac, err := netif.NATGatewayMAC(int(r.VNI)) + if err != nil { + return gateway{}, err + } + return gateway{vpc: vpcName, iface: iface, mac: mac}, nil +} + +func Create(db *badger.DB, vpcName string, ip *net.IPNet, gw net.IP) error { + g, err := load(db, vpcName) + if err != nil { + return err + } + g.ip, g.gateway = ip, gw + return create(newSystem(db), g) +} + +func Delete(db *badger.DB, vpcName string) error { + g, err := load(db, vpcName) + if err != nil { + return err + } + return remove(newSystem(db), g) +} + +type step struct { + what string + run func() error +} + +func runSteps(g gateway, steps []step) error { + for _, s := range steps { + if err := s.run(); err != nil { + return fmt.Errorf("nat gateway of %s: %s: %w", g.vpc, s.what, err) + } + } + return nil +} + +func sysctls(iface string) [][2]string { + return [][2]string{ + {"net.ipv4.conf." + iface + ".arp_ignore", "1"}, + {"net.ipv4.conf." + iface + ".arp_announce", "2"}, + {"net.ipv4.conf." + iface + ".arp_notify", "1"}, + } +} + +func create(h host, g gateway) error { + ns := g.vpc + steps := []step{ + {"create " + g.iface, func() error { return h.CreateMacvlan(ns, g.iface, publicBridge, g.mac) }}, + {"local fdb entry " + g.mac.String(), func() error { return h.BridgeSetLocalMAC(ns, publicBridge, g.mac) }}, + } + for _, kv := range sysctls(g.iface) { + steps = append(steps, step{kv[0], func() error { return h.Sysctl(ns, kv[0], kv[1]) }}) + } + steps = append(steps, + step{"address " + g.ip.String(), func() error { return h.AddrAdd(ns, g.iface, g.ip) }}, + step{"set up " + g.iface, func() error { return h.LinkUp(ns, g.iface) }}, + step{"default route via " + g.gateway.String(), func() error { return h.DefaultVia(ns, g.gateway, g.iface) }}, + step{"nat rules", func() error { return h.SetNAT(ns, nft.NATGateway{Iface: g.iface, IP: g.ip.IP.String()}) }}, + step{"export default", func() error { return h.ExportDefault(g.vpc) }}, + ) + return runSteps(g, steps) +} + +func remove(h host, g gateway) error { + ns := g.vpc + return runSteps(g, []step{ + {"withdraw default", func() error { return h.WithdrawDefault(g.vpc) }}, + {"nat rules", func() error { return h.DeleteNAT(ns, g.iface) }}, + {"default route towards the vrf", func() error { return h.NetnsDefaultToVRF(g.vpc) }}, + {"local fdb entry " + g.mac.String(), func() error { return h.BridgeDelLocalMAC(ns, publicBridge, g.mac) }}, + {"delete " + g.iface, func() error { return h.DeleteLink(ns, g.iface) }}, + }) +} diff --git a/internal/natgw/natgw_test.go b/internal/natgw/natgw_test.go new file mode 100644 index 0000000..14019c4 --- /dev/null +++ b/internal/natgw/natgw_test.go @@ -0,0 +1,202 @@ +package natgw + +import ( + "errors" + "fmt" + "net" + "reflect" + "strings" + "testing" + + "git.g3e.fr/syonad/two/internal/nft" + "git.g3e.fr/syonad/two/pkg/db/kv" +) + +type fakeHost struct { + calls []string + fail map[string]error +} + +func (f *fakeHost) record(call string) error { + f.calls = append(f.calls, call) + for prefix, err := range f.fail { + if strings.HasPrefix(call, prefix) { + return err + } + } + return nil +} + +func (f *fakeHost) CreateMacvlan(ns, name, parent string, mac net.HardwareAddr) error { + return f.record(fmt.Sprintf("macvlan [%s] %s parent %s mac %s", ns, name, parent, mac)) +} + +func (f *fakeHost) BridgeSetLocalMAC(ns, bridge string, mac net.HardwareAddr) error { + return f.record(fmt.Sprintf("fdb add [%s] %s %s", ns, bridge, mac)) +} + +func (f *fakeHost) BridgeDelLocalMAC(ns, bridge string, mac net.HardwareAddr) error { + return f.record(fmt.Sprintf("fdb del [%s] %s %s", ns, bridge, mac)) +} + +func (f *fakeHost) Sysctl(ns, key, value string) error { + return f.record(fmt.Sprintf("sysctl [%s] %s=%s", ns, key, value)) +} + +func (f *fakeHost) AddrAdd(ns, iface string, prefix *net.IPNet) error { + return f.record(fmt.Sprintf("addr [%s] %s %s", ns, iface, prefix)) +} + +func (f *fakeHost) LinkUp(ns, iface string) error { + return f.record(fmt.Sprintf("up [%s] %s", ns, iface)) +} + +func (f *fakeHost) DefaultVia(ns string, gw net.IP, iface string) error { + return f.record(fmt.Sprintf("default [%s] via %s dev %s", ns, gw, iface)) +} + +func (f *fakeHost) SetNAT(ns string, g nft.NATGateway) error { + return f.record(fmt.Sprintf("nat set [%s] %s %s", ns, g.Iface, g.IP)) +} + +func (f *fakeHost) DeleteNAT(ns, iface string) error { + return f.record(fmt.Sprintf("nat del [%s] %s", ns, iface)) +} + +func (f *fakeHost) DeleteLink(ns, name string) error { + return f.record(fmt.Sprintf("del [%s] %s", ns, name)) +} + +func (f *fakeHost) ExportDefault(vpc string) error { + return f.record("export default " + vpc) +} + +func (f *fakeHost) WithdrawDefault(vpc string) error { + return f.record("withdraw default " + vpc) +} + +func (f *fakeHost) NetnsDefaultToVRF(vpc string) error { + return f.record("netns default to vrf " + vpc) +} + +func labGateway(t *testing.T) gateway { + t.Helper() + ip, prefix, err := net.ParseCIDR("198.51.100.50/24") + if err != nil { + t.Fatal(err) + } + prefix.IP = ip + mac, err := net.ParseMAC("02:04:00:00:0a:8c") + if err != nil { + t.Fatal(err) + } + return gateway{vpc: "vp-s7", iface: "ng-s7", mac: mac, ip: prefix, gateway: net.ParseIP("198.51.100.1")} +} + +func TestCreateBuildsThePublicFaceThenTheRoutesThenTheNAT(t *testing.T) { + f := &fakeHost{} + if err := create(f, labGateway(t)); err != nil { + t.Fatal(err) + } + want := []string{ + "macvlan [vp-s7] ng-s7 parent br-public mac 02:04:00:00:0a:8c", + "fdb add [vp-s7] br-public 02:04:00:00:0a:8c", + "sysctl [vp-s7] net.ipv4.conf.ng-s7.arp_ignore=1", + "sysctl [vp-s7] net.ipv4.conf.ng-s7.arp_announce=2", + "sysctl [vp-s7] net.ipv4.conf.ng-s7.arp_notify=1", + "addr [vp-s7] ng-s7 198.51.100.50/24", + "up [vp-s7] ng-s7", + "default [vp-s7] via 198.51.100.1 dev ng-s7", + "nat set [vp-s7] ng-s7 198.51.100.50", + "export default vp-s7", + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) + } +} + +func TestCreateStopsAtTheFirstFailure(t *testing.T) { + f := &fakeHost{fail: map[string]error{"default ": errors.New("network unreachable")}} + err := create(f, labGateway(t)) + if err == nil || !strings.Contains(err.Error(), "nat gateway of vp-s7: default route via 198.51.100.1: network unreachable") { + t.Fatalf("erreur %v", err) + } + for _, c := range f.calls { + if strings.HasPrefix(c, "nat ") || strings.HasPrefix(c, "export ") { + t.Errorf("étape exécutée après l'échec : %q", c) + } + } +} + +func TestCreateExportsTheDefaultOnlyOnceTheNATIsInPlace(t *testing.T) { + f := &fakeHost{fail: map[string]error{"nat set ": errors.New("nft absent")}} + if err := create(f, labGateway(t)); err == nil { + t.Fatal("erreur attendue") + } + for _, c := range f.calls { + if strings.HasPrefix(c, "export ") { + t.Errorf("route par défaut annoncée sans NAT : %q", c) + } + } +} + +func TestRemoveWithdrawsTheDefaultFirstThenUndoesTheHost(t *testing.T) { + f := &fakeHost{} + if err := remove(f, labGateway(t)); err != nil { + t.Fatal(err) + } + want := []string{ + "withdraw default vp-s7", + "nat del [vp-s7] ng-s7", + "netns default to vrf vp-s7", + "fdb del [vp-s7] br-public 02:04:00:00:0a:8c", + "del [vp-s7] ng-s7", + } + if !reflect.DeepEqual(f.calls, want) { + t.Errorf("appels :\n%s\nattendu :\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) + } +} + +func TestRemoveKeepsTheHostWhenTheDefaultCannotBeWithdrawn(t *testing.T) { + f := &fakeHost{fail: map[string]error{"withdraw ": errors.New("no such table")}} + err := remove(f, labGateway(t)) + if err == nil || !strings.Contains(err.Error(), "withdraw default: no such table") { + t.Fatalf("erreur %v", err) + } + if len(f.calls) != 1 { + t.Errorf("appels après l'échec : %q", f.calls) + } +} + +func TestIface(t *testing.T) { + got, err := Iface("vp-s7") + if err != nil || got != "ng-s7" { + t.Errorf("%q, %v ; ng-s7 attendu", got, err) + } + if _, err := Iface("nodash"); err == nil { + t.Error("nom de VPC sans tiret accepté") + } +} + +func TestLoadDerivesTheMACFromTheL3VNI(t *testing.T) { + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vpc/vp-s7/cidr", "10.241.0.0/16") + kv.AddInDB(db, "vpc/vp-s7/l3vni", "2700") + g, err := load(db, "vp-s7") + if err != nil { + t.Fatal(err) + } + if g.vpc != "vp-s7" || g.iface != "ng-s7" || g.mac.String() != "02:04:00:00:0a:8c" { + t.Errorf("passerelle %+v", g) + } +} + +func TestLoadRefusesAVPCWithoutL3VNI(t *testing.T) { + db := kv.InitDB(kv.Config{Path: t.TempDir()}, false) + t.Cleanup(func() { db.Close() }) + kv.AddInDB(db, "vpc/vp-flat/cidr", "10.242.0.0/16") + if _, err := load(db, "vp-flat"); err == nil || !strings.Contains(err.Error(), `vpc "vp-flat" has no l3vni`) { + t.Errorf("erreur %v", err) + } +} diff --git a/internal/netif/fdb_linux.go b/internal/netif/fdb_linux.go index 462cab2..939be88 100644 --- a/internal/netif/fdb_linux.go +++ b/internal/netif/fdb_linux.go @@ -4,6 +4,7 @@ package netif import ( "bytes" + "errors" "net" "syscall" @@ -38,6 +39,14 @@ func BridgeAddLocalMAC(bridge string, mac net.HardwareAddr) error { return netlink.NeighAdd(bridgeLocalEntry(link.Attrs().Index, mac)) } +func BridgeSetLocalMAC(bridge string, mac net.HardwareAddr) error { + link, err := netlink.LinkByName(bridge) + if err != nil { + return err + } + return netlink.NeighSet(bridgeLocalEntry(link.Attrs().Index, mac)) +} + func BridgeHasLocalMAC(bridge string, mac net.HardwareAddr) (bool, error) { link, err := netlink.LinkByName(bridge) if err != nil { @@ -49,3 +58,14 @@ func BridgeHasLocalMAC(bridge string, mac net.HardwareAddr) (bool, error) { } return hasBridgeLocalMAC(entries, link.Attrs().Index, mac), nil } + +func BridgeDelLocalMACIfExists(bridge string, mac net.HardwareAddr) error { + link, err := netlink.LinkByName(bridge) + if err != nil { + return err + } + if err := netlink.NeighDel(bridgeLocalEntry(link.Attrs().Index, mac)); err != nil && !errors.Is(err, syscall.ENOENT) { + return err + } + return nil +} diff --git a/internal/netif/fdb_other.go b/internal/netif/fdb_other.go index 8c861ee..4669b01 100644 --- a/internal/netif/fdb_other.go +++ b/internal/netif/fdb_other.go @@ -14,3 +14,11 @@ func BridgeAddLocalMAC(_ string, _ net.HardwareAddr) error { func BridgeHasLocalMAC(_ string, _ net.HardwareAddr) (bool, error) { return false, errors.New("not supported on this platform") } + +func BridgeDelLocalMACIfExists(_ string, _ net.HardwareAddr) error { + return errors.New("not supported on this platform") +} + +func BridgeSetLocalMAC(_ string, _ net.HardwareAddr) error { + return errors.New("not supported on this platform") +} diff --git a/internal/netif/mac.go b/internal/netif/mac.go index 3b1ae23..7dd4559 100644 --- a/internal/netif/mac.go +++ b/internal/netif/mac.go @@ -11,6 +11,7 @@ const ( RoleAnycastGateway byte = 0x01 RoleRouter byte = 0x02 RoleHostGateway byte = 0x03 + RoleNATGateway byte = 0x04 MaxVNI = 1<<24 - 1 ) @@ -24,8 +25,16 @@ func HostMAC(role byte, vtep net.IP) (net.HardwareAddr, error) { } func AnycastMAC(vni int) (net.HardwareAddr, error) { + return vniMAC(RoleAnycastGateway, vni) +} + +func NATGatewayMAC(l3vni int) (net.HardwareAddr, error) { + return vniMAC(RoleNATGateway, l3vni) +} + +func vniMAC(role byte, vni int) (net.HardwareAddr, error) { if vni < 1 || vni > MaxVNI { return nil, fmt.Errorf("vni %d out of range 1-%d", vni, MaxVNI) } - return net.HardwareAddr{locallyAdministered, RoleAnycastGateway, 0x00, byte(vni >> 16), byte(vni >> 8), byte(vni)}, nil + return net.HardwareAddr{locallyAdministered, role, 0x00, byte(vni >> 16), byte(vni >> 8), byte(vni)}, nil } diff --git a/internal/netif/mac_test.go b/internal/netif/mac_test.go index 55f6350..b302bff 100644 --- a/internal/netif/mac_test.go +++ b/internal/netif/mac_test.go @@ -98,3 +98,46 @@ func TestAnycastMACDiffersFromHostMACs(t *testing.T) { } } } + +func TestNATGatewayMAC(t *testing.T) { + cases := map[int]string{ + 1: "02:04:00:00:00:01", + 2911: "02:04:00:00:0b:5f", + 65535: "02:04:00:00:ff:ff", + MaxVNI: "02:04:00:ff:ff:ff", + } + for l3vni, want := range cases { + got, err := NATGatewayMAC(l3vni) + if err != nil { + t.Fatalf("l3vni %d : erreur inattendue %v", l3vni, err) + } + if got.String() != want { + t.Errorf("l3vni %d : %s, %s attendu", l3vni, got, want) + } + if got[0]&0x02 == 0 || got[0]&0x01 != 0 { + t.Errorf("l3vni %d : %s n'est pas une MAC unicast administrée localement", l3vni, got) + } + } +} + +func TestNATGatewayMACRefusesOutOfRangeVNI(t *testing.T) { + for _, l3vni := range []int{0, -1, MaxVNI + 1} { + if _, err := NATGatewayMAC(l3vni); err == nil { + t.Errorf("l3vni %d : une erreur était attendue", l3vni) + } + } +} + +func TestNATGatewayMACDiffersFromOtherRoles(t *testing.T) { + nat, _ := NATGatewayMAC(2911) + anycast, _ := AnycastMAC(2911) + if nat.String() == anycast.String() { + t.Errorf("même MAC que la gateway anycast de même VNI : %s", nat) + } + for _, role := range []byte{RoleRouter, RoleHostGateway} { + host, _ := HostMAC(role, net.ParseIP("192.168.14.11")) + if host[1] == nat[1] { + t.Errorf("rôle %#x : même octet de rôle que la passerelle NAT (%s, %s)", role, host, nat) + } + } +} diff --git a/internal/netif/route_linux.go b/internal/netif/route_linux.go index 1001a53..75ef3df 100644 --- a/internal/netif/route_linux.go +++ b/internal/netif/route_linux.go @@ -51,6 +51,24 @@ func TableRoutesVia(table int, iface string) ([]*net.IPNet, error) { return dsts, nil } +func RouteDelViaIfExists(table int, dst *net.IPNet, gw net.IP, iface string) error { + link, err := netlink.LinkByName(iface) + if err != nil { + return err + } + err = netlink.RouteDel(&netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: dst, + Gw: gw, + Table: table, + Type: unix.RTN_UNICAST, + }) + if err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} + func unreachableDefault(table, metric int) *netlink.Route { return &netlink.Route{ Dst: defaultIPv4, diff --git a/internal/netif/route_other.go b/internal/netif/route_other.go index 9b7a12b..2ea9883 100644 --- a/internal/netif/route_other.go +++ b/internal/netif/route_other.go @@ -30,3 +30,7 @@ func RouteDelUnreachableDefaultIfExists(_, _ int) error { func HasUnreachableDefault(_, _ int) (bool, error) { return false, errors.New("route listing is only implemented on linux") } + +func RouteDelViaIfExists(_ int, _ *net.IPNet, _ net.IP, _ string) error { + return errors.New("not supported on this platform") +} diff --git a/internal/nft/guard.go b/internal/nft/guard.go new file mode 100644 index 0000000..9bfa39f --- /dev/null +++ b/internal/nft/guard.go @@ -0,0 +1,28 @@ +package nft + +import ( + "fmt" + "strings" +) + +var guardChains = []natChain{ + {"in-", "filter", "input", "filter"}, + {"fwd-", "filter", "forward", "filter"}, +} + +func guardScript(bridge string) string { + var b strings.Builder + fmt.Fprintf(&b, "add table ip %s\n", Table) + for _, c := range guardChains { + name := c.declare(&b, bridge) + fmt.Fprintf(&b, "add rule ip %s %s iifname %q drop\n", Table, name, bridge) + } + return b.String() +} + +func SetBridgeGuard(bridge string) error { + if err := apply(guardScript(bridge)); err != nil { + return fmt.Errorf("guard %s: %w", bridge, err) + } + return nil +} diff --git a/internal/nft/guard_test.go b/internal/nft/guard_test.go new file mode 100644 index 0000000..95e1b46 --- /dev/null +++ b/internal/nft/guard_test.go @@ -0,0 +1,17 @@ +package nft + +import "testing" + +func TestGuardScript(t *testing.T) { + want := `add table ip two +add chain ip two in-br-public { type filter hook input priority filter; policy accept; } +flush chain ip two in-br-public +add rule ip two in-br-public iifname "br-public" drop +add chain ip two fwd-br-public { type filter hook forward priority filter; policy accept; } +flush chain ip two fwd-br-public +add rule ip two fwd-br-public iifname "br-public" drop +` + if got := guardScript("br-public"); got != want { + t.Errorf("script :\n%s\nattendu :\n%s", got, want) + } +} diff --git a/internal/nft/nat.go b/internal/nft/nat.go new file mode 100644 index 0000000..308b804 --- /dev/null +++ b/internal/nft/nat.go @@ -0,0 +1,77 @@ +package nft + +import ( + "fmt" + "strings" +) + +type NATGateway struct { + Iface string + IP string +} + +type natChain struct { + prefix string + kind string + hook string + priority string +} + +var natChains = []natChain{ + {"snat-", "nat", "postrouting", "srcnat"}, + {"fwd-", "filter", "forward", "filter"}, + {"in-", "filter", "input", "filter"}, +} + +func (c natChain) declare(b *strings.Builder, iface string) string { + name := c.prefix + iface + fmt.Fprintf(b, "add chain ip %s %s { type %s hook %s priority %s; policy accept; }\n", Table, name, c.kind, c.hook, c.priority) + fmt.Fprintf(b, "flush chain ip %s %s\n", Table, name) + return name +} + +func (g NATGateway) rules(c natChain) []string { + if c.kind == "nat" { + return []string{fmt.Sprintf("oifname %q snat to %s", g.Iface, g.IP)} + } + return []string{ + fmt.Sprintf("iifname %q ct state established,related accept", g.Iface), + fmt.Sprintf("iifname %q drop", g.Iface), + } +} + +func natSetScript(g NATGateway) string { + var b strings.Builder + fmt.Fprintf(&b, "add table ip %s\n", Table) + for _, c := range natChains { + name := c.declare(&b, g.Iface) + for _, r := range g.rules(c) { + fmt.Fprintf(&b, "add rule ip %s %s %s\n", Table, name, r) + } + } + return b.String() +} + +func natDeleteScript(iface string) string { + var b strings.Builder + fmt.Fprintf(&b, "add table ip %s\n", Table) + for _, c := range natChains { + name := c.declare(&b, iface) + fmt.Fprintf(&b, "delete chain ip %s %s\n", Table, name) + } + return b.String() +} + +func SetNATGateway(g NATGateway) error { + if err := apply(natSetScript(g)); err != nil { + return fmt.Errorf("set nat gateway %s: %w", g.Iface, err) + } + return nil +} + +func DeleteNATGateway(iface string) error { + if err := apply(natDeleteScript(iface)); err != nil { + return fmt.Errorf("delete nat gateway %s: %w", iface, err) + } + return nil +} diff --git a/internal/nft/nat_test.go b/internal/nft/nat_test.go new file mode 100644 index 0000000..43a3b0e --- /dev/null +++ b/internal/nft/nat_test.go @@ -0,0 +1,64 @@ +package nft + +import ( + "strings" + "testing" +) + +func TestNATSetScript(t *testing.T) { + want := `add table ip two +add chain ip two snat-ng-42 { type nat hook postrouting priority srcnat; policy accept; } +flush chain ip two snat-ng-42 +add rule ip two snat-ng-42 oifname "ng-42" snat to 198.51.100.50 +add chain ip two fwd-ng-42 { type filter hook forward priority filter; policy accept; } +flush chain ip two fwd-ng-42 +add rule ip two fwd-ng-42 iifname "ng-42" ct state established,related accept +add rule ip two fwd-ng-42 iifname "ng-42" drop +add chain ip two in-ng-42 { type filter hook input priority filter; policy accept; } +flush chain ip two in-ng-42 +add rule ip two in-ng-42 iifname "ng-42" ct state established,related accept +add rule ip two in-ng-42 iifname "ng-42" drop +` + if got := natSetScript(NATGateway{Iface: "ng-42", IP: "198.51.100.50"}); got != want { + t.Errorf("script :\n%s\nattendu :\n%s", got, want) + } +} + +func TestNATDeleteScript(t *testing.T) { + want := `add table ip two +add chain ip two snat-ng-42 { type nat hook postrouting priority srcnat; policy accept; } +flush chain ip two snat-ng-42 +delete chain ip two snat-ng-42 +add chain ip two fwd-ng-42 { type filter hook forward priority filter; policy accept; } +flush chain ip two fwd-ng-42 +delete chain ip two fwd-ng-42 +add chain ip two in-ng-42 { type filter hook input priority filter; policy accept; } +flush chain ip two in-ng-42 +delete chain ip two in-ng-42 +` + if got := natDeleteScript("ng-42"); got != want { + t.Errorf("script :\n%s\nattendu :\n%s", got, want) + } +} + +func TestNATScriptsLeaveTheBridgeTableAlone(t *testing.T) { + for label, script := range map[string]string{ + "set": natSetScript(NATGateway{Iface: "ng-42", IP: "198.51.100.50"}), + "delete": natDeleteScript("ng-42"), + } { + for _, line := range strings.Split(strings.TrimSpace(script), "\n") { + if !strings.Contains(line, " ip two") { + t.Errorf("%s : ligne hors de la table ip two : %q", label, line) + } + } + } +} + +func TestNATScriptsOnlyTouchTheirInterface(t *testing.T) { + script := natSetScript(NATGateway{Iface: "ng-42", IP: "198.51.100.50"}) + natDeleteScript("ng-42") + for _, line := range strings.Split(strings.TrimSpace(script), "\n") { + if strings.HasPrefix(line, "add rule") && !strings.Contains(line, `ifname "ng-42"`) { + t.Errorf("règle qui ne vise pas ng-42 : %q", line) + } + } +} diff --git a/internal/prometheus/agent/collector.go b/internal/prometheus/agent/collector.go index 073dd4b..6e1be01 100644 --- a/internal/prometheus/agent/collector.go +++ b/internal/prometheus/agent/collector.go @@ -18,6 +18,7 @@ type AgentCollector struct { vpcsTotal *prometheus.Desc subnetsTotal *prometheus.Desc vmsTotal *prometheus.Desc + natGateways *prometheus.Desc } func NewAgentCollector(db *badger.DB) *AgentCollector { @@ -38,6 +39,11 @@ func NewAgentCollector(db *badger.DB) *AgentCollector { "Number of VMs by state.", []string{"state"}, nil, ), + natGateways: prometheus.NewDesc( + "syonad_nat_gateways_total", + "Number of NAT gateways by state.", + []string{"state"}, nil, + ), } } @@ -45,12 +51,14 @@ func (c *AgentCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.vpcsTotal ch <- c.subnetsTotal ch <- c.vmsTotal + ch <- c.natGateways } func (c *AgentCollector) Collect(ch chan<- prometheus.Metric) { c.collectStates(ch, "vpc/", c.vpcsTotal) c.collectStates(ch, "subnet/", c.subnetsTotal) c.collectStates(ch, "vm/", c.vmsTotal) + c.collectStates(ch, "nat-gateway/", c.natGateways) } // collectStates counts resources under the given DB prefix by their state value diff --git a/internal/prometheus/agent/collector_test.go b/internal/prometheus/agent/collector_test.go index 49272f9..e425f62 100644 --- a/internal/prometheus/agent/collector_test.go +++ b/internal/prometheus/agent/collector_test.go @@ -44,6 +44,8 @@ func collect(t *testing.T, c *AgentCollector) map[string]map[string]float64 { name = "subnets" case strings.Contains(desc, "syonad_vms_total"): name = "vms" + case strings.Contains(desc, "syonad_nat_gateways_total"): + name = "nat_gateways" default: t.Fatalf("métrique inattendue : %s", desc) } @@ -62,12 +64,13 @@ func collect(t *testing.T, c *AgentCollector) map[string]map[string]float64 { func TestCollector_CountsByState(t *testing.T) { db := newTestDB(t) for resource, s := range map[string]state.State{ - "vpc/vpc-1": state.Running, - "vpc/vpc-2": state.Running, - "vpc/vpc-3": state.Error, - "subnet/sn-1": state.Creating, - "vm/vm-1": state.Running, - "vm/vm-2": state.Error, + "vpc/vpc-1": state.Running, + "vpc/vpc-2": state.Running, + "vpc/vpc-3": state.Error, + "subnet/sn-1": state.Creating, + "vm/vm-1": state.Running, + "vm/vm-2": state.Error, + "nat-gateway/ng-1": state.Running, } { if err := state.Set(db, resource, s); err != nil { t.Fatalf("préparation du test : %v", err) @@ -89,6 +92,9 @@ func TestCollector_CountsByState(t *testing.T) { t.Errorf("vms{state=%q} = %v, attendu %v", s, got["vms"][s], want) } } + if got["nat_gateways"]["running"] != 1 || got["vpcs"]["running"] != 2 { + t.Errorf("nat_gateways{state=running} = %v, vpcs{state=running} = %v, attendu 1 et 2", got["nat_gateways"]["running"], got["vpcs"]["running"]) + } } // Les clés vm//disk/ ne portent pas de suffixe /state : elles ne @@ -111,7 +117,7 @@ func TestCollector_IgnoresVMDiskKeys(t *testing.T) { // côté Prometheus casse les alertes qui s'en servent. func TestCollector_EmitsEveryState(t *testing.T) { got := collect(t, NewAgentCollector(newTestDB(t))) - for _, family := range []string{"vpcs", "subnets", "vms"} { + for _, family := range []string{"vpcs", "subnets", "vms", "nat_gateways"} { if len(got[family]) != len(state.All()) { t.Errorf("%s : %d séries, attendu %d", family, len(got[family]), len(state.All())) } @@ -145,7 +151,7 @@ func TestCollector_Describe(t *testing.T) { for range ch { n++ } - if n != 3 { - t.Errorf("%d descripteurs, attendu 3", n) + if n != 4 { + t.Errorf("%d descripteurs, attendu 4", n) } } diff --git a/internal/sysctl/sysctl.go b/internal/sysctl/sysctl.go index 50f0a1a..9b7c599 100644 --- a/internal/sysctl/sysctl.go +++ b/internal/sysctl/sysctl.go @@ -1,7 +1,9 @@ package sysctl import ( + "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -21,3 +23,13 @@ func Set(key, value string) error { } return f.Close() } + +func SetIfPresent(key, value string) (bool, error) { + path := filepath.Join(root, strings.ReplaceAll(key, ".", "/")) + if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("sysctl %s: %w", key, err) + } + return true, Set(key, value) +} diff --git a/internal/sysctl/sysctl_test.go b/internal/sysctl/sysctl_test.go index e07674f..6566f47 100644 --- a/internal/sysctl/sysctl_test.go +++ b/internal/sysctl/sysctl_test.go @@ -46,3 +46,25 @@ func TestSetRefusesAnUnknownKeyWithoutCreatingIt(t *testing.T) { t.Errorf("clé inconnue créée : %v", err) } } + +func TestSetIfPresentSkipsAnAbsentKey(t *testing.T) { + dir := fakeRoot(t) + set, err := SetIfPresent("net.bridge.bridge-nf-call-iptables", "0") + if err != nil || set { + t.Errorf("clé absente : %v, %v ; false, nil attendus", set, err) + } + if _, err := os.Stat(filepath.Join(dir, "net", "bridge")); err == nil { + t.Error("répertoire créé pour une clé absente") + } +} + +func TestSetIfPresentWritesAPresentKey(t *testing.T) { + dir := fakeRoot(t) + set, err := SetIfPresent("net.ipv4.ip_forward", "1") + if err != nil || !set { + t.Fatalf("clé présente : %v, %v", set, err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "net", "ipv4", "ip_forward")); string(got) != "1" { + t.Errorf("contenu %q", got) + } +} diff --git a/internal/vpc/create.go b/internal/vpc/create.go index 8020a79..6d18437 100644 --- a/internal/vpc/create.go +++ b/internal/vpc/create.go @@ -32,6 +32,10 @@ func CreateVPC(db *badger.DB, name string, cfg *configuration.Config) error { return err } + if err := netns.Call(name, guardPublicBridge); err != nil { + return err + } + if err := netif.BridgeSetMaster("vp-"+vpcID+"-e", cfg.PublicBridge()); err != nil { return err } diff --git a/internal/vpc/guard.go b/internal/vpc/guard.go new file mode 100644 index 0000000..d6f9cbd --- /dev/null +++ b/internal/vpc/guard.go @@ -0,0 +1,26 @@ +package vpc + +import ( + "git.g3e.fr/syonad/two/internal/nft" + "git.g3e.fr/syonad/two/internal/sysctl" +) + +const netnsPublicBridge = "br-public" + +var bridgeNetfilterKeys = []string{ + "net.bridge.bridge-nf-call-iptables", + "net.bridge.bridge-nf-call-ip6tables", + "net.bridge.bridge-nf-call-arptables", +} + +func guardPublicBridge() error { + for _, key := range bridgeNetfilterKeys { + if _, err := sysctl.SetIfPresent(key, "0"); err != nil { + return err + } + } + if err := sysctl.Set("net.ipv4.conf."+netnsPublicBridge+".arp_ignore", "1"); err != nil { + return err + } + return nft.SetBridgeGuard(netnsPublicBridge) +} diff --git a/internal/watchdog/check_l3.go b/internal/watchdog/check_l3.go index fae7759..ae66f76 100644 --- a/internal/watchdog/check_l3.go +++ b/internal/watchdog/check_l3.go @@ -232,6 +232,20 @@ func expectedExports(db *badger.DB, vpcName string) (expected, tolerated map[str classify(st, ip.String()+"/32") } } + + gateways, err := kv.ListByPrefix(db, prefixNATGW) + if err != nil { + return nil, nil, fmt.Errorf("watchdog: listing nat gateways: %w", err) + } + for key, value := range gateways { + name, ok := strings.CutSuffix(strings.TrimPrefix(key, prefixNATGW), "/vpc") + if !ok || strings.Contains(name, "/") || value != vpcName { + continue + } + if st, err := state.Get(db, prefixNATGW+name); err == nil { + classify(st, "0.0.0.0/0") + } + } return expected, tolerated, nil } diff --git a/internal/watchdog/check_l3_test.go b/internal/watchdog/check_l3_test.go index f84a6a3..23fc1db 100644 --- a/internal/watchdog/check_l3_test.go +++ b/internal/watchdog/check_l3_test.go @@ -278,3 +278,41 @@ func TestCheckL3VPCsReportsAVRFWithoutUnreachableDefault(t *testing.T) { t.Errorf("notifications %q", notified(r)) } } + +func TestCheckL3VPCsExpectsTheDefaultRouteOfARunningNATGateway(t *testing.T) { + cases := []struct { + label string + gateway state.State + vpc string + exported bool + want []string + }{ + {"passerelle running, route exportée", state.Running, "vp-s7", true, nil}, + {"passerelle running, route absente", state.Running, "vp-s7", false, + []string{"vpc vp-s7: route 0.0.0.0/0 not exported in vrf-s7"}}, + {"passerelle en création, route exportée", state.Creating, "vp-s7", true, nil}, + {"passerelle en erreur, route absente", state.Error, "vp-s7", false, nil}, + {"passerelle d'un autre VPC, route exportée", state.Running, "vp-s70", true, + []string{"vpc vp-s7: route 0.0.0.0/0 exported in vrf-s7 without a running owner: its traffic is black-holed"}}, + } + for _, c := range cases { + db := newTestDB(t) + seedResource(t, db, prefixVPC, "vp-s7", state.Running) + seedKV(t, db, "vpc/vp-s7/cidr", "10.241.0.0/16") + seedKV(t, db, "vpc/vp-s7/l3vni", "2700") + seedResource(t, db, prefixNATGW, "ng-1", c.gateway) + seedKV(t, db, "nat-gateway/ng-1/vpc", c.vpc) + p := seedRoutedVPC(t) + p.routes = map[uint32][]string{2700: nil} + if c.exported { + p.routes[2700] = []string{"0.0.0.0/0"} + } + r := &recorder{} + if err := CheckL3VPCs(db, bgpCfg(), p, r); err != nil { + t.Fatal(err) + } + if got := notified(r); !reflect.DeepEqual(got, c.want) { + t.Errorf("%s :\n%q\nattendu :\n%q", c.label, got, c.want) + } + } +} diff --git a/internal/watchdog/resources.go b/internal/watchdog/resources.go index 37219a5..015791e 100644 --- a/internal/watchdog/resources.go +++ b/internal/watchdog/resources.go @@ -14,6 +14,7 @@ const ( prefixVPC = "vpc/" prefixSubnet = "subnet/" prefixVM = "vm/" + prefixNATGW = "nat-gateway/" kindVPC = "vpc" kindSubnet = "subnet" diff --git a/release_notes/0.3.0.md b/release_notes/0.3.0.md index 41743d7..dc46da6 100644 --- a/release_notes/0.3.0.md +++ b/release_notes/0.3.0.md @@ -57,6 +57,26 @@ (`br-public` par défaut, jamais `default_interface`), au lieu d'un nom en dur - Le watchdog signale un bridge public absent, éteint ou sans port VXLAN +**Passerelle NAT des VPC (#42)** + +- Nouvelle ressource `/nat-gateways` (`POST`, `GET`, `DELETE`) : `name`, `vpc`, `ip` (IP de + sortie avec son préfixe) et `gateway` (gateway des bordures), fournis par l'appelant et non + vérifiés. Créée sur un seul hyperviseur, elle donne une sortie Internet à toutes les VM du VPC, + sur tous les hyperviseurs, avec l'IP de sortie pour source. Une passerelle par VPC ; le VPC + doit être `running` sur l'hôte et avoir un `l3vni` (409 sinon) +- Dans le netns du VPC : `macvlan` `ng-` sur le bridge `br-public` du netns, de MAC + `02:04:00` + `l3vni` (la même sur tout hôte), SNAT en nftables (table `ip two`) ; rien n'entre + par la passerelle hors réponse à une connexion établie. La route par défaut est annoncée en + type 5 dans la VRF du VPC +- Le netns de chaque VPC routé envoie désormais sa route par défaut vers la VRF ; sans passerelle, + l'`unreachable default` de #81 la rejette. Le route-map de la VRF exporte `0.0.0.0/0` seulement + si son next-hop est `169.254.0.2`, ce qui écarte l'`unreachable default` +- Déplacer la passerelle : la supprimer sur l'ancien hôte s'il répond, la créer sur le nouveau + avec la même IP. Les connexions en cours sont perdues +- La suppression d'un VPC est refusée (409) tant qu'il a une passerelle. Nouvelle métrique + `syonad_nat_gateways_total` ; le watchdog attend la route par défaut d'une passerelle `running` + dans la VRF + ## Correctifs - **Chaque VRF de VPC porte une route `unreachable default` (#81).** Une recherche qui échouait @@ -87,6 +107,14 @@ `delete vxlan: Link not found` et le watchdog a signalé `br-000000` et des VRF absents. L'agent revient désormais au netns du thread courant, et un thread qui ne peut pas y revenir est détruit au lieu d'être réutilisé. La création d'un netns est corrigée de la même façon +- **Rien n'entre plus dans un VPC par le `br-public` de son netns (#42).** Ce bridge, qui relie le + netns au segment public de l'hôte, répondait à l'ARP pour toutes les adresses du netns + (`arp_ignore=0`) et laissait entrer l'IP : un équipement du segment public pouvait router vers + les VM du VPC (mesuré en lab). L'agent pose désormais `arp_ignore=1` sur ce bridge et rejette en + nftables tout IP arrivé par lui +- **`br_netfilter` est désactivé dans le netns de chaque VPC (#42)** quand le module est chargé + (`bridge-nf-call-iptables`, `-ip6tables`, `-arptables` à 0) : conntrack voyait chaque trame + pontée, et le SNAT de la passerelle NAT ne s'appliquait pas (mesuré en lab) - **Le serveur de métadonnées crée sa socket dans le netns du VPC (#92).** Il faisait entrer un seul thread dans le VPC, puis créait sa socket sur le thread où il s'exécutait à cet instant : sur un autre thread, la socket était créée dans le netns de l'hôte, l'écoute échouait et le @@ -119,6 +147,13 @@ la fuite vers l'underlay décrite plus haut reste possible. Pour la fermer sans recréer le VPC, la poser à la main sur chaque hyperviseur, la table étant le `l3vni` du VPC : `ip route replace unreachable default metric 4278198272 table ` +- **`br-public` du netns (#42) : seuls les VPC créés par la 0.3.0 sont protégés.** L'agent ne + reprend pas les VPC existants au démarrage : recréer chaque VPC pour fermer l'entrée par le + segment public +- **Passerelle NAT (#42) : seuls les VPC créés par la 0.3.0 peuvent s'en servir.** Les VPC + routés existants n'envoient pas leur route par défaut vers la VRF et leur route-map n'exporte + pas celle de la passerelle ; l'agent ne les reprend pas au démarrage. Recréer le VPC avant de + lui donner une passerelle - **La section `dispatcher` n'est plus lue** : restée dans `/etc/two/agent.yml`, elle est ignorée, et les valeurs par défaut de `times` s'appliquent. Une valeur personnalisée de `dispatcher.timeout_seconds` est à reporter dans `times.timeout_vms_stop` (et dans les délais diff --git a/test/e2e/lib/fixture.sh b/test/e2e/lib/fixture.sh index 248d804..63fbbe9 100644 --- a/test/e2e/lib/fixture.sh +++ b/test/e2e/lib/fixture.sh @@ -2,7 +2,7 @@ STATE_DIR=/var/tmp/e2e SCRATCH_DIR=/tmp owned () { - grep -E "^(vp-|sn-)?t${T}" + grep -E "^(vp-|sn-|ng-)?t${T}" } api_names () { @@ -39,11 +39,11 @@ clean_scratch () { } teardown_owned () { - delete_owned vms && delete_owned subnets && delete_owned vpcs || return 1 + delete_owned vms && delete_owned subnets && delete_owned nat-gateways && delete_owned vpcs || return 1 rm -f "${VOLUMES}/t${T}"-* clean_scratch || return 1 local left="" kind names - for kind in vms subnets vpcs; do + for kind in vms subnets nat-gateways vpcs; do names=$(api_names "${kind}") || return 1 left="${left}$(owned <<<"${names}" | sed "s|^|${kind}/|" | tr '\n' ' ')" done @@ -67,7 +67,7 @@ host_state () { prepare_test () { mkdir -p "${STATE_DIR}" curl -fsS -o /dev/null "${API}/vpcs" || echo "ÉCHOUÉ: ${HOST} : API de l'agent injoignable avant le test" - if [ -n "$(for kind in vms subnets vpcs; do api_names "${kind}" | owned; done)$(scratch_netns)$(scratch_files)" ]; then + if [ -n "$(for kind in vms subnets nat-gateways vpcs; do api_names "${kind}" | owned; done)$(scratch_netns)$(scratch_files)" ]; then info "${HOST} : ressources, netns ou fichiers t${T} d'un passage précédent, suppression" check "${HOST} : nettoyage des ressources t${T} restantes" teardown_owned fi diff --git a/test/e2e/lib/net.sh b/test/e2e/lib/net.sh index 55bb2a3..681dced 100644 --- a/test/e2e/lib/net.sh +++ b/test/e2e/lib/net.sh @@ -126,6 +126,8 @@ def match(kind, a, f): return t == b'\x08\x06' and f[20:22] == b'\x00\x01' and f[38:42] == ipb(a[0]) if kind == 'arp-from': return t == b'\x08\x06' and f[28:32] == ipb(a[0]) + if kind == 'icmp-from': + return t == b'\x08\x00' and f[23] == 1 and f[26:30] == ipb(a[0]) if kind == 'udp-from': return t == b'\x08\x00' and f[23] == 17 and f[26:30] == ipb(a[0]) and struct.unpack('!H', f[34:36])[0] == int(a[1]) if kind == 'marker': diff --git a/test/e2e/run_test.sh b/test/e2e/run_test.sh index f90f285..10f219d 100755 --- a/test/e2e/run_test.sh +++ b/test/e2e/run_test.sh @@ -399,19 +399,19 @@ test_matrix_lists_every_test_from_its_header () { } test_owned_keeps_only_the_resources_of_the_test () { - setup "owned : vp-tNN, sn-tNN, tNN- seulement" - [[ $(node 'printf "%s\n" vp-t41a sn-t41b t41-vm vp-t42a t4-x vp-s4 sn-s41 | T=41 owned | tr "\n" " "') == "vp-t41a sn-t41b t41-vm " ]] \ - || fail "$(node 'printf "%s\n" vp-t41a sn-t41b t41-vm vp-t42a t4-x vp-s4 sn-s41 | T=41 owned | tr "\n" " "')" + setup "owned : vp-tNN, sn-tNN, ng-tNN, tNN- seulement" + [[ $(node 'printf "%s\n" vp-t41a sn-t41b ng-t41c t41-vm vp-t42a ng-t42 t4-x vp-s4 sn-s41 ng-s41 | T=41 owned | tr "\n" " "') == "vp-t41a sn-t41b ng-t41c t41-vm " ]] \ + || fail "$(node 'printf "%s\n" vp-t41a sn-t41b ng-t41c t41-vm vp-t42a ng-t42 t4-x vp-s4 sn-s41 ng-s41 | T=41 owned | tr "\n" " "')" teardown } test_teardown_deletes_vms_then_subnets_then_vpcs () { - setup "teardown_owned : VM, puis subnets, puis VPC, et rien d'autre" - local stub='api_names () { case $1 in vms) [ -f /tmp/.t70.$$ ] || echo t41-a; echo t42-z ;; subnets) echo sn-t41a ;; vpcs) echo vp-t41a ;; esac; }; api_delete () { echo "DEL $1"; [ "$1" = vms/t41-a ] && touch /tmp/.t70.$$; :; }; wait_state () { :; }; rm () { :; }; scratch_netns () { :; }; scratch_files () { :; }; ' + setup "teardown_owned : VM, puis subnets, puis passerelles NAT, puis VPC, et rien d'autre" + local stub='api_names () { case $1 in vms) [ -f /tmp/.t70.$$ ] || echo t41-a; echo t42-z ;; subnets) echo sn-t41a ;; nat-gateways) echo ng-t41a; echo ng-t42 ;; vpcs) echo vp-t41a ;; esac; }; api_delete () { echo "DEL $1"; [ "$1" = vms/t41-a ] && touch /tmp/.t70.$$; :; }; wait_state () { :; }; rm () { :; }; scratch_netns () { :; }; scratch_files () { :; }; ' local out out=$(node "T=41; ${stub}teardown_owned; command rm -f /tmp/.t70.\$\$") - [[ $(grep '^DEL' <<< "${out}" | tr '\n' ' ') == "DEL vms/t41-a DEL subnets/sn-t41a DEL vpcs/vp-t41a " ]] || fail "ordre : ${out}" - grep -q 'encore présents : subnets/sn-t41a vpcs/vp-t41a' <<< "${out}" || fail "restes non signalés : ${out}" + [[ $(grep '^DEL' <<< "${out}" | tr '\n' ' ') == "DEL vms/t41-a DEL subnets/sn-t41a DEL nat-gateways/ng-t41a DEL vpcs/vp-t41a " ]] || fail "ordre : ${out}" + grep -q 'encore présents : subnets/sn-t41a nat-gateways/ng-t41a vpcs/vp-t41a' <<< "${out}" || fail "restes non signalés : ${out}" teardown } @@ -805,6 +805,27 @@ PY teardown } +test_frames_match_selects_icmp_by_source () { + setup "frames : icmp-from retient l'ICMP émis depuis une adresse, et rien d'autre" + local src + src=$(node 'frames_py') + python3 - "${src}" <<'PY' || fail "icmp-from" +import sys +exec(sys.argv[1].split('cmd, a = ')[0]) +def ip4(proto, s, d): + h = struct.pack('!BBHHHBBH4s4s', 0x45, 0, 28, 0, 0, 64, proto, 0, ipb(s), ipb(d)) + return eth('ff:ff:ff:ff:ff:ff', '02:00:00:00:00:0a', 0x0800, h + bytes(8)) +echo = ip4(1, '240.0.0.46', '240.0.0.146') +private = ip4(1, '10.46.1.10', '240.0.0.146') +udp = ip4(17, '240.0.0.46', '240.0.0.146') +assert match('icmp-from', ['240.0.0.46'], echo), 'ICMP depuis 240.0.0.46' +assert not match('icmp-from', ['240.0.0.46'], private), 'ICMP depuis une autre adresse' +assert not match('icmp-from', ['240.0.0.46'], udp), 'UDP depuis 240.0.0.46' +assert match('icmp-from', ['10.46.1.10'], private), 'témoin' +PY + teardown +} + for T in $(declare -F | awk '{print $3}' | grep '^test_'); do "${T}" done diff --git a/test/e2e/suite/46-passerelle-nat.sh b/test/e2e/suite/46-passerelle-nat.sh new file mode 100644 index 0000000..af85235 --- /dev/null +++ b/test/e2e/suite/46-passerelle-nat.sh @@ -0,0 +1,136 @@ +# fonctionnalité: passerelle NAT d'un VPC — macvlan ng- sur le br-public du netns, MAC 02:04:00 + l3vni, SNAT vers l'IP de sortie, route par défaut en type 5 importée par les autres hôtes, l'unreachable default jamais annoncée, rien n'entre sans connexion établie, bascule vers un autre hôte par mobilité MAC, refus 409, suppression propre, watchdog +# hôtes: hv1 hv2 hv3 +# tickets: #42 + +on hv1 <<'NODE' +KEY=$(vm_key) +check "hv1 : VPC vp-t46, l3vni 4600" vpc_create vp-t46 10.46.0.0/16 4600 +check "hv1 : subnet sn-t46a (VNI 4611)" subnet_create sn-t46a vp-t46 4611 10.46.1.0/24 +check "hv1 : VM t46-a" vm_create t46-a sn-t46a 10.46.1.10 "${KEY}" +check "t46-a joignable" vm_wait vp-t46 10.46.1.10 +check "hv1 : route par défaut du netns vp-t46 vers la VRF (169.254.0.1)" sh -c "ip -n vp-t46 -4 route show default | grep -q '^default via 169.254.0.1 dev l3-t46-i'" +check "sans passerelle, t46-a ne joint pas 240.255.0.1 : refusé par vrf-t46 (169.254.0.1)" vm_grep vp-t46 10.46.1.10 'ping -n -c 2 -W 2 240.255.0.1' 'From 169.254.0.1 .*Unreachable' +NODE + +on hv2 <<'NODE' +KEY=$(vm_key) +check "hv2 : VPC vp-t46, l3vni 4600" vpc_create vp-t46 10.46.0.0/16 4600 +check "hv2 : subnet sn-t46b (VNI 4612)" subnet_create sn-t46b vp-t46 4612 10.46.2.0/24 +check "hv2 : VM t46-b" vm_create t46-b sn-t46b 10.46.2.10 "${KEY}" +check "t46-b joignable" vm_wait vp-t46 10.46.2.10 +NODE + +on hv3 <<'NODE' +NS=t46-pub +ip netns add "${NS}" +ip link add t46-pub-h type veth peer name t46-pub-n netns "${NS}" +ip link set t46-pub-h master br-public up +ip -n "${NS}" link set t46-pub-n up +ip -n "${NS}" addr add 240.0.0.146/24 dev t46-pub-n +check "hv3 : netns de test ${NS} sur br-public, 240.0.0.146/24" sh -c "ip -n ${NS} -4 -o addr show dev t46-pub-n | grep -q ' 240.0.0.146/24 '" +check "hv3 : témoin, ${NS} joint la gateway des bordures 240.0.0.1" ip netns exec "${NS}" ping -n -c 3 -W 1 240.0.0.1 +NODE + +on hv1 <<'NODE' +MAC=02:04:00:00:11:f8 +check "hv1 : passerelle ng-t46 de vp-t46 (240.0.0.46/24, gateway 240.0.0.1)" api_post nat-gateways '{"name":"ng-t46","vpc":"vp-t46","ip":"240.0.0.46/24","gateway":"240.0.0.1"}' +check "hv1 : ng-t46 running" wait_state nat-gateways/ng-t46 running +check "hv1 : seconde passerelle pour vp-t46 refusée (409)" expect_code 409 api_post_code nat-gateways '{"name":"ng-t46x","vpc":"vp-t46","ip":"240.0.0.45/24","gateway":"240.0.0.1"}' +check "hv1 : macvlan ng-t46 sur le br-public du netns, MAC ${MAC} (l3vni 4600)" sh -c "ip -n vp-t46 -d -o link show ng-t46 | grep -q 'ng-t46@br-public:.*link/ether ${MAC} .*macvlan mode bridge'" +check "hv1 : entrée FDB locale ${MAC} sur le br-public du netns" sh -c "ip netns exec vp-t46 bridge fdb show br br-public | grep -q '^${MAC} dev br-public master br-public permanent'" +check "hv1 : ng-t46 porte 240.0.0.46/24" sh -c "ip -n vp-t46 -4 -o addr show dev ng-t46 | grep -q ' 240.0.0.46/24 '" +check "hv1 : arp_ignore=1, arp_announce=2, arp_notify=1 sur ng-t46" sh -c "[ \"\$(ip netns exec vp-t46 sysctl -n net.ipv4.conf.ng-t46.arp_ignore net.ipv4.conf.ng-t46.arp_announce net.ipv4.conf.ng-t46.arp_notify | tr '\n' ' ')\" = '1 2 1 ' ]" +check "hv1 : route par défaut du netns vers les bordures par ng-t46" sh -c "ip -n vp-t46 -4 route show default | grep -q '^default via 240.0.0.1 dev ng-t46'" +check "hv1 : route du VPC toujours vers la VRF" sh -c "ip -n vp-t46 -4 route show 10.46.0.0/16 | grep -q 'via 169.254.0.1 dev l3-t46-i'" +check "hv1 : table 4600 : route par défaut vers le netns (169.254.0.2)" sh -c "ip -4 route show table 4600 default | grep -q '^default via 169.254.0.2 dev l3-t46-o'" +check "hv1 : table 4600 : unreachable default toujours là" sh -c "ip -4 route show table 4600 default | grep -q '^unreachable default metric 4278198272'" +check "hv1 : SNAT vers 240.0.0.46 en sortie par ng-t46" sh -c "ip netns exec vp-t46 nft list chain ip two snat-ng-t46 | grep -q 'oifname \"ng-t46\" snat to 240.0.0.46'" +for chain in fwd in; do + check "hv1 : ${chain}-ng-t46 n'accepte par ng-t46 que les connexions établies" sh -c "ip netns exec vp-t46 nft list chain ip two ${chain}-ng-t46 | grep -q 'iifname \"ng-t46\" ct state established,related accept' && ip netns exec vp-t46 nft list chain ip two ${chain}-ng-t46 | grep -q 'iifname \"ng-t46\" drop'" +done +S=$(mktemp -d) +sniff_start "${S}/public" vp-t46-e 12 icmp-from 240.0.0.46 +sniff_start "${S}/private" vp-t46-e 12 icmp-from 10.46.1.10 +for i in $(seq 1 10); do vm_ssh vp-t46 10.46.1.10 'ping -n -c 1 -W 2 240.255.0.1' >/dev/null 2>&1 && break; sleep 2; done +check "t46-a (hôte de la passerelle) joint 240.255.0.1 derrière les bordures" vm_ssh vp-t46 10.46.1.10 'ping -n -c 3 -W 2 240.255.0.1' +wait +info "ICMP sortis du netns par vp-t46-e : depuis 240.0.0.46 $(cat "${S}/public"), depuis 10.46.1.10 $(cat "${S}/private")" +check "témoin : l'ICMP de t46-a sort du netns, source 240.0.0.46" test "$(cat "${S}/public")" -gt 0 +check "aucun ICMP ne sort du netns avec l'adresse privée 10.46.1.10" test "$(cat "${S}/private")" -eq 0 +rm -r "${S}" +NODE + +on rr1 <<'NODE' +check "rr1 : RT-5 0.0.0.0/0 de hv1 (RD 192.168.14.11:4600) reçue et réfléchie" sh -c "vtysh -c 'show bgp l2vpn evpn route rd 192.168.14.11:4600 type prefix' | grep -q '\[0.0.0.0\]'" +check "rr1 : aucune RT-5 0.0.0.0/0 de hv2 (RD 192.168.14.12:4600) : son unreachable default n'est pas annoncée" sh -c "! vtysh -c 'show bgp l2vpn evpn route rd 192.168.14.12:4600 type prefix' | grep -q '\[0.0.0.0\]'" +NODE + +on hv2 <<'NODE' +check "hv2 : vrf-t46 apprend la route par défaut via le VTEP de hv1" wait_vrf_route vp-t46 default 192.168.14.11 +check "hv2 : aucune interface ng-t46 sur un hôte sans passerelle" sh -c "! ip -n vp-t46 link show ng-t46 >/dev/null 2>&1" +check "hv2 : route par défaut du netns vers la VRF" sh -c "ip -n vp-t46 -4 route show default | grep -q '^default via 169.254.0.1 dev l3-t46-i'" +for i in $(seq 1 10); do vm_ssh vp-t46 10.46.2.10 'ping -n -c 1 -W 2 240.255.0.1' >/dev/null 2>&1 && break; sleep 2; done +check "t46-b (hv2) joint 240.255.0.1 par la passerelle de hv1" vm_ssh vp-t46 10.46.2.10 'ping -n -c 3 -W 2 240.255.0.1' +NODE + +on hv3 <<'NODE' +NS=t46-pub +MAC=02:04:00:00:11:f8 +check "hv3 : ${MAC} apprise en type 2 sur le VNI 1 via hv1" wait_evpn_mac 1 "${MAC}" 192.168.14.11 +check "hv3 : ${NS} ne joint pas 240.0.0.46 (entrée neuve refusée)" sh -c "! ip netns exec ${NS} ping -n -c 3 -W 1 240.0.0.46 >/dev/null 2>&1" +check "hv3 : 240.0.0.46 résolue vers ${MAC} (l'ARP passe, seul l'IP est filtré)" sh -c "ip -n ${NS} neigh show 240.0.0.46 | grep -q 'lladdr ${MAC}'" +ip -n "${NS}" route add 10.46.0.0/16 via 240.0.0.46 +check "hv3 : ${NS} ne joint pas t46-a à travers la passerelle (transit neuf refusé)" sh -c "! ip netns exec ${NS} ping -n -c 3 -W 1 10.46.1.10 >/dev/null 2>&1" +NODE + +on hv3 <<'NODE' +check "hv3 : VPC vp-t46c, l3vni 4601, sans subnet" vpc_create vp-t46c 10.47.0.0/16 4601 +check "hv3 : passerelle ng-t46c de vp-t46c" api_post nat-gateways '{"name":"ng-t46c","vpc":"vp-t46c","ip":"240.0.0.47/24","gateway":"240.0.0.1"}' +check "hv3 : ng-t46c running" wait_state nat-gateways/ng-t46c running +check "hv3 : suppression de vp-t46c refusée tant que sa passerelle existe (409)" expect_code 409 api_code DELETE vpcs/vp-t46c +check "hv3 : le refus nomme la passerelle" sh -c "curl -s -X DELETE ${API}/vpcs/vp-t46c | grep -q 'nat gateway .*ng-t46c.* must be deleted'" +check "hv3 : VPC vp-t46f sans l3vni" vpc_create vp-t46f 10.48.0.0/16 +check "hv3 : passerelle refusée pour un VPC sans l3vni (409)" expect_code 409 api_post_code nat-gateways '{"name":"ng-t46f","vpc":"vp-t46f","ip":"240.0.0.48/24","gateway":"240.0.0.1"}' +check "hv3 : passerelle refusée pour un VPC absent de l'hôte (409)" expect_code 409 api_post_code nat-gateways '{"name":"ng-t46z","vpc":"vp-t46z","ip":"240.0.0.49/24","gateway":"240.0.0.1"}' +check "hv3 : ip sans longueur de préfixe refusée (400)" expect_code 400 api_post_code nat-gateways '{"name":"ng-t46y","vpc":"vp-t46c","ip":"240.0.0.50","gateway":"240.0.0.1"}' +check "hv3 : suppression de ng-t46c" api_delete nat-gateways/ng-t46c +check "hv3 : ng-t46c supprimée" wait_state nat-gateways/ng-t46c deleted +check "hv3 : vp-t46c supprimable une fois la passerelle partie" api_delete vpcs/vp-t46c +check "hv3 : vp-t46c supprimé" wait_state vpcs/vp-t46c deleted +NODE + +on hv1 <<'NODE' +check "hv1 : watchdog muet sur la route par défaut de vrf-t46" watchdog_quiet_about 120 '0.0.0.0/0' +check "hv1 : bascule, suppression de ng-t46" api_delete nat-gateways/ng-t46 +check "hv1 : ng-t46 supprimée" wait_state nat-gateways/ng-t46 deleted +check "hv1 : plus d'interface ng-t46" sh -c "! ip -n vp-t46 link show ng-t46 >/dev/null 2>&1" +check "hv1 : plus d'entrée FDB 02:04:00:00:11:f8 sur le br-public du netns" sh -c "! ip netns exec vp-t46 bridge fdb show br br-public | grep -q '^02:04:00:00:11:f8 '" +check "hv1 : plus de chaîne de la passerelle dans la table ip two du netns" sh -c "! ip netns exec vp-t46 nft list table ip two | grep -q 'ng-t46'" +check "hv1 : route par défaut du netns revenue vers la VRF" sh -c "ip -n vp-t46 -4 route show default | grep -q '^default via 169.254.0.1 dev l3-t46-i'" +check "hv1 : table 4600 : plus de route par défaut vers le netns" sh -c "! ip -4 route show table 4600 default | grep -q 'via 169.254.0.2'" +check "hv1 : table 4600 : l'unreachable default a survécu au retrait" sh -c "ip -4 route show table 4600 default | grep -q '^unreachable default metric 4278198272'" +check "sans passerelle, t46-a ne sort plus" vm_fails vp-t46 10.46.1.10 'ping -n -c 2 -W 2 240.255.0.1' +NODE + +on hv2 <<'NODE' +check "hv2 : bascule, passerelle ng-t46 recréée sur hv2, même IP" api_post nat-gateways '{"name":"ng-t46","vpc":"vp-t46","ip":"240.0.0.46/24","gateway":"240.0.0.1"}' +check "hv2 : ng-t46 running" wait_state nat-gateways/ng-t46 running +check "hv2 : même MAC 02:04:00:00:11:f8 qu'avant la bascule" sh -c "ip -n vp-t46 -o link show ng-t46 | grep -q 'link/ether 02:04:00:00:11:f8 '" +NODE + +on hv3 <<'NODE' +check "hv3 : 02:04:00:00:11:f8 a suivi la passerelle, apprise via hv2" wait_evpn_mac 1 02:04:00:00:11:f8 192.168.14.12 +NODE + +on hv2 <<'NODE' +for i in $(seq 1 15); do vm_ssh vp-t46 10.46.2.10 'ping -n -c 1 -W 2 240.255.0.1' >/dev/null 2>&1 && break; sleep 2; done +check "t46-b (hôte de la passerelle désormais) joint 240.255.0.1" vm_ssh vp-t46 10.46.2.10 'ping -n -c 3 -W 2 240.255.0.1' +NODE + +on hv1 <<'NODE' +check "hv1 : vrf-t46 apprend la route par défaut via le VTEP de hv2" wait_vrf_route vp-t46 default 192.168.14.12 +START=$(date +%s) +for i in $(seq 1 30); do vm_ssh vp-t46 10.46.1.10 'ping -n -c 1 -W 2 240.255.0.1' >/dev/null 2>&1 && break; sleep 2; done +info "hv1 : sortie de t46-a rétablie $(( $(date +%s) - START )) s après l'import de la route par défaut de hv2" +check "t46-a (hv1) sort par la passerelle de hv2 après la bascule" vm_ssh vp-t46 10.46.1.10 'ping -n -c 3 -W 2 240.255.0.1' +NODE