f-41: config, frr: base BGP pilotée par l'agent au démarrage #41
Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
parent
94c2f82867
commit
bf0e83862f
12 changed files with 331 additions and 18 deletions
|
|
@ -13,7 +13,9 @@ import (
|
|||
agentapi "git.g3e.fr/syonad/two/internal/api/agent"
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
dispatcher "git.g3e.fr/syonad/two/internal/dispatcher/agent"
|
||||
"git.g3e.fr/syonad/two/internal/frr"
|
||||
"git.g3e.fr/syonad/two/internal/migration"
|
||||
"git.g3e.fr/syonad/two/internal/netif"
|
||||
agentmetrics "git.g3e.fr/syonad/two/internal/prometheus/agent"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog"
|
||||
"git.g3e.fr/syonad/two/internal/watchdog/notify"
|
||||
|
|
@ -58,6 +60,14 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
if cfg.BGP.Enabled() {
|
||||
if err := applyBGPBase(cfg, frr.NewClient()); err != nil {
|
||||
log.Error("failed to apply the bgp base configuration", "error", err)
|
||||
return
|
||||
}
|
||||
log.Info("bgp base configuration applied", "asn", cfg.BGP.ASN, "route_reflectors", cfg.BGP.RouteReflectors)
|
||||
}
|
||||
|
||||
q := worker.New(cfg.Worker.BufferSize)
|
||||
q.Start(cfg.Worker.Count)
|
||||
|
||||
|
|
@ -110,6 +120,22 @@ func main() {
|
|||
closeDB = shutdown(log, q, servers, shutdownTimeout)
|
||||
}
|
||||
|
||||
type baseApplier interface {
|
||||
ApplyBase(frr.Base) error
|
||||
}
|
||||
|
||||
func applyBGPBase(cfg *configuration.Config, frrClient baseApplier) error {
|
||||
reflectors, err := cfg.BGP.Reflectors()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vtep, err := netif.VTEPAddress(cfg.DefaultInterface)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return frrClient.ApplyBase(frr.Base{ASN: cfg.BGP.ASN, RouterID: vtep, RouteReflectors: reflectors})
|
||||
}
|
||||
|
||||
type httpShutdowner interface {
|
||||
Shutdown(context.Context) error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
configuration "git.g3e.fr/syonad/two/internal/config/agent"
|
||||
"git.g3e.fr/syonad/two/internal/frr"
|
||||
"git.g3e.fr/syonad/two/pkg/worker"
|
||||
)
|
||||
|
||||
|
|
@ -96,3 +98,22 @@ func TestShutdown_SansServeur(t *testing.T) {
|
|||
t.Fatal("l'absence de serveur ne doit pas empêcher un arrêt propre")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeBase struct{ called bool }
|
||||
|
||||
func (f *fakeBase) ApplyBase(frr.Base) error {
|
||||
f.called = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestApplyBGPBase_RefusesAnInvalidSectionBeforeTouchingFRR(t *testing.T) {
|
||||
cfg := &configuration.Config{}
|
||||
cfg.BGP.RouteReflectors = []string{"10.255.255.1"}
|
||||
f := &fakeBase{}
|
||||
if err := applyBGPBase(cfg, f); err == nil || !strings.Contains(err.Error(), "bgp.asn must be set") {
|
||||
t.Errorf("erreur %v", err)
|
||||
}
|
||||
if f.called {
|
||||
t.Error("ApplyBase appelé avec une section bgp invalide")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,15 @@ interfaces:
|
|||
internet: br-000000
|
||||
admin: br-000000
|
||||
|
||||
# BGP control plane of the hypervisor (optional).
|
||||
# Absent: the agent leaves FRR alone. Present: at every start, before serving its
|
||||
# API, the agent applies the base BGP configuration through vtysh (router-id = the
|
||||
# primary IPv4 of default_interface) and saves it with "write memory".
|
||||
# Changing the asn requires removing the previous instance by hand.
|
||||
# bgp:
|
||||
# asn: 64600
|
||||
# route_reflectors: [10.255.255.1]
|
||||
|
||||
# Metadata server runtime directory (cloud-init files per VM)
|
||||
metadata:
|
||||
run_dir: "/run/two/metadata"
|
||||
|
|
|
|||
|
|
@ -27,8 +27,18 @@ l'agent désactive l'apprentissage et ne configure aucun voisin — cf.
|
|||
Configuration de FRR
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. D'un hyperviseur à l'autre ne changent que
|
||||
``hostname`` et ``router-id``, l'adresse de l'hyperviseur sur son segment.
|
||||
Seul ``bgpd`` est activé dans ``/etc/frr/daemons``. Le reste est posé par l'agent, à chaque
|
||||
démarrage, à partir de la section ``bgp`` de ``agent.yml`` (:doc:`/exploitation/configuration`) :
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
bgp:
|
||||
asn: 64600
|
||||
route_reflectors: [10.255.255.1]
|
||||
|
||||
L'agent en déduit la configuration ci-dessous ; le ``router-id`` est l'adresse IPv4 primaire de
|
||||
``default_interface``, la seule valeur qui change d'un hyperviseur à l'autre. ``hostname`` et
|
||||
``log`` ne sont pas gérés par l'agent.
|
||||
|
||||
.. literalinclude:: ../../test/e2e/topologies/frr/hv1.conf
|
||||
:language: text
|
||||
|
|
|
|||
|
|
@ -104,6 +104,38 @@ Traduit les clés logiques ``iface_type`` de l'API vers les bridges physiques de
|
|||
inconnue ou omise retombe silencieusement sur ``default_interface`` — ce n'est pas une erreur,
|
||||
mais c'est une source de subnets branchés au mauvais endroit sans le dire.
|
||||
|
||||
Plan de contrôle BGP
|
||||
--------------------
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
bgp:
|
||||
asn: 64600
|
||||
route_reflectors: [10.255.255.1]
|
||||
|
||||
Section **facultative**. Absente, l'agent ne touche pas à FRR : sa configuration reste à poser
|
||||
à la main, comme avant. Présente, l'agent **pilote la configuration BGP de l'hyperviseur** :
|
||||
à chaque démarrage, avant d'ouvrir son API, il applique par ``vtysh`` la base décrite dans
|
||||
:doc:`/deploiement/premier-hyperviseur`, puis l'enregistre par ``write memory``.
|
||||
|
||||
* ``asn`` — l'AS de l'hyperviseur, commun à tous les hyperviseurs du cluster (iBGP vers le
|
||||
route reflector). Obligatoire dès que la section existe.
|
||||
* ``route_reflectors`` — au moins une adresse IPv4, sans doublon. Un route reflector retiré de
|
||||
la liste est retiré de FRR au démarrage suivant.
|
||||
|
||||
Le ``router-id`` n'est pas configurable : c'est l'adresse IPv4 primaire de
|
||||
``default_interface``, celle qui sert de VTEP.
|
||||
|
||||
Le démarrage **échoue** si la section est incomplète, si ``default_interface`` n'a pas
|
||||
d'adresse IPv4 primaire, si ``vtysh`` refuse une commande, ou si FRR tourne déjà sous un autre
|
||||
AS : changer d'``asn`` suppose de retirer l'ancienne instance à la main.
|
||||
|
||||
.. warning::
|
||||
|
||||
``/etc/frr/frr.conf`` devient un fichier **généré** : ``write memory`` le réécrit à partir de
|
||||
la configuration en mémoire de FRR. Une modification faite dans le fichier sans être
|
||||
rechargée est perdue au prochain démarrage de l'agent.
|
||||
|
||||
Metadata et QEMU
|
||||
----------------
|
||||
|
||||
|
|
|
|||
38
internal/config/agent/bgp.go
Normal file
38
internal/config/agent/bgp.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
type BGP struct {
|
||||
ASN uint32 `mapstructure:"asn"`
|
||||
RouteReflectors []string `mapstructure:"route_reflectors"`
|
||||
}
|
||||
|
||||
func (b BGP) Enabled() bool {
|
||||
return b.ASN != 0 || len(b.RouteReflectors) > 0
|
||||
}
|
||||
|
||||
func (b BGP) Reflectors() ([]net.IP, error) {
|
||||
if b.ASN == 0 {
|
||||
return nil, fmt.Errorf("bgp.asn must be set")
|
||||
}
|
||||
if len(b.RouteReflectors) == 0 {
|
||||
return nil, fmt.Errorf("bgp.route_reflectors must list at least one address")
|
||||
}
|
||||
seen := make(map[string]bool, len(b.RouteReflectors))
|
||||
ips := make([]net.IP, 0, len(b.RouteReflectors))
|
||||
for _, rr := range b.RouteReflectors {
|
||||
ip := net.ParseIP(rr).To4()
|
||||
if ip == nil {
|
||||
return nil, fmt.Errorf("bgp.route_reflectors: %q is not an IPv4 address", rr)
|
||||
}
|
||||
if seen[ip.String()] {
|
||||
return nil, fmt.Errorf("bgp.route_reflectors: %s is listed twice", ip)
|
||||
}
|
||||
seen[ip.String()] = true
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
return ips, nil
|
||||
}
|
||||
54
internal/config/agent/bgp_test.go
Normal file
54
internal/config/agent/bgp_test.go
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadConfig_BGPAbsentIsDisabled(t *testing.T) {
|
||||
cfg, err := LoadConfig(writeConfig(t, "database:\n path: /tmp/two\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if cfg.BGP.Enabled() {
|
||||
t.Errorf("bgp enabled without a bgp section: %+v", cfg.BGP)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_ReadsTheBGPSection(t *testing.T) {
|
||||
cfg, err := LoadConfig(writeConfig(t, "bgp:\n asn: 4200000000\n route_reflectors: [10.255.255.1, 10.255.255.2]\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if !cfg.BGP.Enabled() || cfg.BGP.ASN != 4200000000 {
|
||||
t.Fatalf("bgp = %+v", cfg.BGP)
|
||||
}
|
||||
rrs, err := cfg.BGP.Reflectors()
|
||||
if err != nil {
|
||||
t.Fatalf("Reflectors: %v", err)
|
||||
}
|
||||
if len(rrs) != 2 || rrs[0].String() != "10.255.255.1" || rrs[1].String() != "10.255.255.2" {
|
||||
t.Errorf("reflectors = %v", rrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBGPReflectors_RejectsAnIncompleteOrInvalidSection(t *testing.T) {
|
||||
cases := []struct {
|
||||
bgp BGP
|
||||
want string
|
||||
}{
|
||||
{BGP{RouteReflectors: []string{"10.255.255.1"}}, "bgp.asn must be set"},
|
||||
{BGP{ASN: 64600}, "at least one address"},
|
||||
{BGP{ASN: 64600, RouteReflectors: []string{"rr1"}}, `"rr1" is not an IPv4 address`},
|
||||
{BGP{ASN: 64600, RouteReflectors: []string{"fd00::1"}}, `"fd00::1" is not an IPv4 address`},
|
||||
{BGP{ASN: 64600, RouteReflectors: []string{"10.255.255.1", "10.255.255.1"}}, "10.255.255.1 is listed twice"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if !c.bgp.Enabled() {
|
||||
t.Errorf("%+v: Enabled() = false", c.bgp)
|
||||
}
|
||||
if _, err := c.bgp.Reflectors(); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%+v: erreur %v, attendu %q", c.bgp, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -55,6 +55,7 @@ type Config struct {
|
|||
Enabled bool `mapstructure:"enabled"`
|
||||
IntervalSeconds int `mapstructure:"interval_seconds"`
|
||||
} `mapstructure:"watchdog"`
|
||||
BGP BGP `mapstructure:"bgp"`
|
||||
DefaultInterface string `mapstructure:"default_interface"`
|
||||
Interfaces map[string]string `mapstructure:"interfaces"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,9 +22,26 @@ func (c *Client) ApplyBase(b Base) error {
|
|||
if err := b.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
running, err := c.runningConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stale, err := b.staleReflectors(running)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.configure(b.commands()...); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
cmds := []string{fmt.Sprintf("router bgp %d", b.ASN)}
|
||||
for _, rr := range stale {
|
||||
cmds = append(cmds, "no neighbor "+rr)
|
||||
}
|
||||
if err := c.configure(append(cmds, "exit")...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return c.save()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -79,17 +79,6 @@ func TestApplyVRFRefusesAnInvalidVRFWithoutCallingVtysh(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestApplyBaseConfiguresThenSaves(t *testing.T) {
|
||||
f := &fakeVtysh{}
|
||||
b := Base{ASN: 64600, RouterID: labVRF(t).VTEP, RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}}
|
||||
if err := f.client().ApplyBase(b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.calls) != 2 || f.calls[0][0] != "configure terminal" || f.calls[0][1] != "router bgp 64600" || f.calls[1][0] != "write memory" {
|
||||
t.Errorf("appels : %q", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecReportsAnErrorPrintedByVtysh(t *testing.T) {
|
||||
f := &fakeVtysh{fail: map[string]string{"vni 4001": "% Please unconfigure l3vni 4001\n"}}
|
||||
err := f.client().ApplyVRF(labVRF(t))
|
||||
|
|
@ -182,3 +171,81 @@ func TestTeardownNeedsOnlyTheASNAndTheID(t *testing.T) {
|
|||
t.Errorf("démontage sans id : erreur %v, appels %q", err, f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
const baseRunning = `frr version 10.7.1
|
||||
router bgp 64600
|
||||
bgp router-id 192.168.14.13
|
||||
neighbor fabric peer-group
|
||||
neighbor fabric remote-as 64600
|
||||
neighbor 10.255.255.1 peer-group fabric
|
||||
neighbor 10.255.255.9 peer-group fabric
|
||||
neighbor 10.0.0.5 peer-group monitoring
|
||||
!
|
||||
address-family l2vpn evpn
|
||||
neighbor fabric activate
|
||||
advertise-all-vni
|
||||
exit-address-family
|
||||
exit
|
||||
!
|
||||
router bgp 64600 vrf vrf-a1b2
|
||||
neighbor 10.255.255.7 peer-group fabric
|
||||
exit
|
||||
`
|
||||
|
||||
func labBase() Base {
|
||||
return Base{ASN: 64600, RouterID: net.ParseIP("192.168.14.13"), RouteReflectors: []net.IP{net.ParseIP("10.255.255.1")}}
|
||||
}
|
||||
|
||||
func TestApplyBaseOnAFreshFRRConfiguresThenSaves(t *testing.T) {
|
||||
f := &fakeVtysh{running: []string{""}}
|
||||
if err := f.client().ApplyBase(labBase()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [][]string{
|
||||
{"show running-config"},
|
||||
append([]string{"configure terminal"}, labBase().commands()...),
|
||||
{"write memory"},
|
||||
}
|
||||
if !reflect.DeepEqual(f.calls, want) {
|
||||
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBaseRemovesReflectorsNoLongerConfiguredAfterAddingTheNewOnes(t *testing.T) {
|
||||
f := &fakeVtysh{running: []string{baseRunning}}
|
||||
if err := f.client().ApplyBase(labBase()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [][]string{
|
||||
{"show running-config"},
|
||||
append([]string{"configure terminal"}, labBase().commands()...),
|
||||
{"configure terminal", "router bgp 64600", "no neighbor 10.255.255.9", "exit"},
|
||||
{"write memory"},
|
||||
}
|
||||
if !reflect.DeepEqual(f.calls, want) {
|
||||
t.Errorf("appels :\n%q\nattendu :\n%q", f.calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBaseRefusesToChangeTheASN(t *testing.T) {
|
||||
f := &fakeVtysh{running: []string{baseRunning}}
|
||||
b := labBase()
|
||||
b.ASN = 64601
|
||||
err := f.client().ApplyBase(b)
|
||||
if err == nil || !strings.Contains(err.Error(), "bgp already runs as AS 64600, not 64601") {
|
||||
t.Errorf("erreur %v", err)
|
||||
}
|
||||
if len(f.calls) != 1 {
|
||||
t.Errorf("appels après le refus : %q", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBaseStopsWhenTheRunningConfigIsUnreadable(t *testing.T) {
|
||||
f := &fakeVtysh{}
|
||||
if err := f.client().ApplyBase(labBase()); err == nil {
|
||||
t.Fatal("aucune erreur")
|
||||
}
|
||||
if f.sent("write memory") || f.sent("router bgp 64600") {
|
||||
t.Errorf("appels : %q", f.calls)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package frr
|
|||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxVNI = 1<<24 - 1
|
||||
|
|
@ -60,6 +61,31 @@ func (b Base) commands() []string {
|
|||
)
|
||||
}
|
||||
|
||||
func (b Base) staleReflectors(running string) ([]string, error) {
|
||||
want := make(map[string]bool, len(b.RouteReflectors))
|
||||
for _, rr := range b.RouteReflectors {
|
||||
want[rr.String()] = true
|
||||
}
|
||||
var stale []string
|
||||
in := false
|
||||
for _, l := range strings.Split(running, "\n") {
|
||||
l = strings.TrimRight(l, " \r")
|
||||
f := strings.Fields(l)
|
||||
switch {
|
||||
case len(f) == 3 && f[0] == "router" && f[1] == "bgp":
|
||||
if f[2] != fmt.Sprint(b.ASN) {
|
||||
return nil, fmt.Errorf("frr: bgp already runs as AS %s, not %d: changing the asn requires removing that instance by hand", f[2], b.ASN)
|
||||
}
|
||||
in = true
|
||||
case in && l == "exit":
|
||||
in = false
|
||||
case in && len(f) == 4 && f[0] == "neighbor" && f[2] == "peer-group" && f[3] == "fabric" && !want[f[1]]:
|
||||
stale = append(stale, f[1])
|
||||
}
|
||||
}
|
||||
return stale, nil
|
||||
}
|
||||
|
||||
func (v VRF) Name() string {
|
||||
return "vrf-" + v.ID
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,7 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
local, err := vtepAddress(localIface, addrs)
|
||||
local, err := linkVTEPAddress(link)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -40,6 +36,22 @@ func CreateVxlan(name string, vxlanID int, localIface string, mtu int) error {
|
|||
return netlink.LinkAdd(vxlan)
|
||||
}
|
||||
|
||||
func VTEPAddress(iface string) (net.IP, error) {
|
||||
link, err := netlink.LinkByName(iface)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return linkVTEPAddress(link)
|
||||
}
|
||||
|
||||
func linkVTEPAddress(link netlink.Link) (net.IP, error) {
|
||||
addrs, err := netlink.AddrList(link, syscall.AF_INET)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return vtepAddress(link.Attrs().Name, addrs)
|
||||
}
|
||||
|
||||
func vtepAddress(iface string, addrs []netlink.Addr) (net.IP, error) {
|
||||
for _, a := range addrs {
|
||||
if a.IPNet == nil || a.IP.To4() == nil || a.Scope != scopeUniverse || a.Flags&addressSecondary != 0 {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue