diff --git a/internal/ebtables/ebtables.go b/internal/ebtables/ebtables.go index 40ea0f0..f99c084 100644 --- a/internal/ebtables/ebtables.go +++ b/internal/ebtables/ebtables.go @@ -58,3 +58,34 @@ func DeleteDHCP(iface, ip string) error { "--ip-source", ip, "-j", "DROP") } + +func confineRules(iface, mac string) [][]string { + return [][]string{ + {"-t", "nat", "PREROUTING", "--in-interface", iface, "-s", mac, "-j", "DROP"}, + {"-t", "filter", "FORWARD", "--out-interface", iface, "-s", mac, "-j", "DROP"}, + {"-t", "filter", "OUTPUT", "--out-interface", iface, "-s", mac, "-j", "DROP"}, + } +} + +func withAction(action string, rule []string) []string { + return append([]string{rule[0], rule[1], action}, rule[2:]...) +} + +func ConfineMAC(iface, mac string) error { + for _, rule := range confineRules(iface, mac) { + if err := exec.Command("ebtables", withAction("-A", rule)...).Run(); err != nil { + return fmt.Errorf("ebtables confine %s on %s (%s %s): %w", mac, iface, rule[1], rule[2], err) + } + } + return nil +} + +func DeleteConfineMAC(iface, mac string) error { + var first error + for _, rule := range confineRules(iface, mac) { + if err := exec.Command("ebtables", withAction("-D", rule)...).Run(); err != nil && first == nil { + first = fmt.Errorf("ebtables unconfine %s on %s (%s %s): %w", mac, iface, rule[1], rule[2], err) + } + } + return first +} diff --git a/internal/ebtables/ebtables_test.go b/internal/ebtables/ebtables_test.go new file mode 100644 index 0000000..c64123f --- /dev/null +++ b/internal/ebtables/ebtables_test.go @@ -0,0 +1,43 @@ +package ebtables + +import ( + "strings" + "testing" +) + +func TestConfineRules(t *testing.T) { + want := []string{ + "-t nat -A PREROUTING --in-interface v-s9r1-i -s 02:01:00:00:0b:5f -j DROP", + "-t filter -A FORWARD --out-interface v-s9r1-i -s 02:01:00:00:0b:5f -j DROP", + "-t filter -A OUTPUT --out-interface v-s9r1-i -s 02:01:00:00:0b:5f -j DROP", + } + rules := confineRules("v-s9r1-i", "02:01:00:00:0b:5f") + if len(rules) != len(want) { + t.Fatalf("%d règles, %d attendues", len(rules), len(want)) + } + for i, rule := range rules { + if got := strings.Join(withAction("-A", rule), " "); got != want[i] { + t.Errorf("règle %d : %q, %q attendu", i, got, want[i]) + } + } +} + +func TestDeleteUsesTheSameRules(t *testing.T) { + for _, rule := range confineRules("v-x-i", "02:01:00:00:00:01") { + added := withAction("-A", rule) + deleted := withAction("-D", rule) + if strings.Replace(strings.Join(added, " "), " -A ", " -D ", 1) != strings.Join(deleted, " ") { + t.Errorf("ajout %v et suppression %v divergent", added, deleted) + } + } +} + +func TestWithActionDoesNotAlterTheRule(t *testing.T) { + rule := confineRules("v-x-i", "02:01:00:00:00:01")[0] + before := strings.Join(rule, " ") + withAction("-A", rule) + withAction("-D", rule) + if strings.Join(rule, " ") != before { + t.Errorf("la règle a été modifiée : %q devenu %q", before, strings.Join(rule, " ")) + } +} diff --git a/internal/subnet/anycast.go b/internal/subnet/anycast.go new file mode 100644 index 0000000..cabe22a --- /dev/null +++ b/internal/subnet/anycast.go @@ -0,0 +1,93 @@ +package subnet + +import ( + "errors" + "fmt" + "net" + + "git.g3e.fr/syonad/two/internal/ebtables" + "git.g3e.fr/syonad/two/internal/netif" + "git.g3e.fr/syonad/two/internal/netns" + "git.g3e.fr/syonad/two/internal/sysctl" +) + +func (d subnetData) anycast() bool { return d.mode == ModeVxlan && d.relayIP != nil } + +func (d subnetData) gatewayLink() string { return "gw-" + d.subnetID } + +func arpSysctls(iface string) [][2]string { + return [][2]string{ + {"net.ipv4.conf." + iface + ".arp_ignore", "1"}, + {"net.ipv4.conf." + iface + ".arp_announce", "2"}, + } +} + +func setupAnycastGateway(d subnetData, vethI string) error { + mac, err := netif.AnycastMAC(d.vxlanID) + if err != nil { + return err + } + gw := d.gatewayLink() + steps := []struct { + name string + fn func() error + }{ + {"confine " + mac.String(), func() error { return ebtables.ConfineMAC(vethI, mac.String()) }}, + {"arp sysctls on " + d.bridge, func() error { return setSysctls(arpSysctls(d.bridge)) }}, + {"relay address on " + d.bridge, func() error { return netif.AddrAdd(d.bridge, d.relayIP) }}, + {"route " + d.cidr.String(), func() error { return netif.RouteAdd(d.bridge, d.cidr) }}, + {"create " + gw, func() error { return netif.CreateMacvlan(gw, d.bridge, mac) }}, + {"arp sysctls on " + gw, func() error { return setSysctls(arpSysctls(gw)) }}, + {"gateway address on " + gw, func() error { return netif.AddrAdd(gw, d.interfaceIP) }}, + {"set up " + gw, func() error { return netif.LinkSetUp(gw) }}, + {"drop arp to gateway", func() error { return ebtables.DropARPToGateway(vethI, d.interfaceIP.String()) }}, + {"drop arp to relay", func() error { return ebtables.DropARPToGateway(vethI, d.relayIP.String()) }}, + {"drop dhcp from gateway", func() error { return ebtables.DropDHCP(vethI, d.interfaceIP.String()) }}, + {"drop dhcp from relay", func() error { return ebtables.DropDHCP(vethI, d.relayIP.String()) }}, + } + return netns.Call(d.vpc, func() error { + for _, step := range steps { + if err := step.fn(); err != nil { + return fmt.Errorf("%s: %w", step.name, err) + } + } + return nil + }) +} + +func deleteAnycastGateway(d subnetData, vethI string) error { + mac, err := netif.AnycastMAC(d.vxlanID) + if err != nil { + return err + } + return netns.Call(d.vpc, func() error { + var errs []error + for _, ip := range []net.IP{d.interfaceIP, d.relayIP} { + if err := ebtables.DeleteARPToGateway(vethI, ip.String()); err != nil { + errs = append(errs, fmt.Errorf("delete ebtables arp rule %s: %w", ip, err)) + } + if err := ebtables.DeleteDHCP(vethI, ip.String()); err != nil { + errs = append(errs, fmt.Errorf("delete ebtables dhcp rule %s: %w", ip, err)) + } + } + if err := ebtables.DeleteConfineMAC(vethI, mac.String()); err != nil { + errs = append(errs, err) + } + if err := netif.DeleteLink(d.gatewayLink()); err != nil { + errs = append(errs, fmt.Errorf("delete %s: %w", d.gatewayLink(), err)) + } + if err := netif.DeleteLink(d.bridge); err != nil { + errs = append(errs, fmt.Errorf("delete bridge: %w", err)) + } + return errors.Join(errs...) + }) +} + +func setSysctls(values [][2]string) error { + for _, kv := range values { + if err := sysctl.Set(kv[0], kv[1]); err != nil { + return err + } + } + return nil +} diff --git a/internal/subnet/anycast_test.go b/internal/subnet/anycast_test.go new file mode 100644 index 0000000..63d6ebc --- /dev/null +++ b/internal/subnet/anycast_test.go @@ -0,0 +1,51 @@ +package subnet + +import ( + "net" + "testing" +) + +func TestAnycastOnlyForVxlanWithRelay(t *testing.T) { + relay := net.ParseIP("10.0.0.253") + cases := []struct { + label string + d subnetData + want bool + }{ + {"vxlan avec relay_ip", subnetData{mode: ModeVxlan, relayIP: relay}, true}, + {"vxlan existant sans relay_ip", subnetData{mode: ModeVxlan}, false}, + {"bridge", subnetData{mode: ModeBridge, relayIP: relay}, false}, + {"public_ip", subnetData{mode: ModePublicIP, relayIP: relay}, false}, + } + for _, c := range cases { + if got := c.d.anycast(); got != c.want { + t.Errorf("%s : %v, %v attendu", c.label, got, c.want) + } + } +} + +func TestGatewayLinkFitsInterfaceNameLimit(t *testing.T) { + d := subnetData{subnetID: "s9r1", bridge: "br-s9r1"} + if d.gatewayLink() != "gw-s9r1" { + t.Errorf("%s, gw-s9r1 attendu", d.gatewayLink()) + } + if len(d.gatewayLink()) != len(d.bridge) { + t.Errorf("%s et %s doivent avoir la même longueur", d.gatewayLink(), d.bridge) + } +} + +func TestArpSysctls(t *testing.T) { + want := [][2]string{ + {"net.ipv4.conf.br-s9r1.arp_ignore", "1"}, + {"net.ipv4.conf.br-s9r1.arp_announce", "2"}, + } + got := arpSysctls("br-s9r1") + if len(got) != len(want) { + t.Fatalf("%d sysctls, %d attendus", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("sysctl %d : %v, %v attendu", i, got[i], want[i]) + } + } +} diff --git a/internal/subnet/create.go b/internal/subnet/create.go index 12d345e..6857678 100644 --- a/internal/subnet/create.go +++ b/internal/subnet/create.go @@ -105,8 +105,12 @@ func createSubnet(db *badger.DB, subnetName string, d subnetData, backend dhcpba return fmt.Errorf("set up interfaces in netns: %w", err) } - switch d.mode { - case "vxlan": + switch { + case d.anycast(): + if err := setupAnycastGateway(d, vethI); err != nil { + return fmt.Errorf("configure netns: %w", err) + } + case d.mode == ModeVxlan: if err := netns.Call(d.vpc, func() error { if err := netif.AddrAdd(d.bridge, d.interfaceIP); err != nil { return fmt.Errorf("add addr: %w", err) @@ -121,7 +125,7 @@ func createSubnet(db *badger.DB, subnetName string, d subnetData, backend dhcpba }); err != nil { return fmt.Errorf("configure netns: %w", err) } - case "bridge": + case d.mode == ModeBridge: if err := netns.Call(d.vpc, func() error { if err := netif.AddrAdd(d.bridge, d.interfaceIP); err != nil { return fmt.Errorf("add addr: %w", err) diff --git a/internal/subnet/data.go b/internal/subnet/data.go index 6f2d302..43a5cf0 100644 --- a/internal/subnet/data.go +++ b/internal/subnet/data.go @@ -18,6 +18,7 @@ type subnetData struct { vxlanID int localIface string interfaceIP net.IP + relayIP net.IP cidr *net.IPNet vpcCIDR *net.IPNet defaultRoute bool @@ -70,6 +71,14 @@ func loadSubnet(db *badger.DB, name string) (subnetData, error) { } d.interfaceIP = interfaceIP + if relayStr, err := kv.GetFromDB(db, "subnet/"+name+"/relay_ip"); err == nil && relayStr != "" { + relayIP := net.ParseIP(relayStr) + if relayIP == nil { + return d, fmt.Errorf("invalid relay_ip: %s", relayStr) + } + d.relayIP = relayIP + } + cidrStr, err := kv.GetFromDB(db, "subnet/"+name+"/cidr") if err != nil { return d, fmt.Errorf("get cidr: %w", err) diff --git a/internal/subnet/delete.go b/internal/subnet/delete.go index 8ee4d78..f528eb5 100644 --- a/internal/subnet/delete.go +++ b/internal/subnet/delete.go @@ -79,7 +79,11 @@ func deleteSubnetVxlan(d subnetData) error { vxlanIface := fmt.Sprintf("vxlan-%d", d.vxlanID) vethI := "v-" + d.subnetID + "-i" - if err := netns.Call(d.vpc, func() error { + if d.anycast() { + if err := deleteAnycastGateway(d, vethI); err != nil { + return fmt.Errorf("delete netns resources: %w", err) + } + } else if err := netns.Call(d.vpc, func() error { if err := ebtables.DeleteARPToGateway(vethI, d.interfaceIP.String()); err != nil { return fmt.Errorf("delete ebtables arp rule: %w", err) }