f-90: vm, vpc, watchdog: redirections en nft, retrait d'iptables #90

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-31 14:15:07 +01:00
commit 682696214a
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
10 changed files with 31 additions and 173 deletions

View file

@ -1,60 +0,0 @@
package iptables
import (
"errors"
"fmt"
"os/exec"
)
const HostSvcChain = "TWO-HOSTSVC"
var hostSvcExcluded = []string{"br-public", "l3-+"}
func hostSvcJump() []string {
return []string{"PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", HostSvcChain}
}
func hostSvcRules(target string) [][]string {
var rules [][]string
for _, iface := range hostSvcExcluded {
rules = append(rules, []string{HostSvcChain, "-i", iface, "-j", "RETURN"})
}
for _, p := range []struct{ proto, port string }{{"udp", "53"}, {"tcp", "53"}, {"udp", "123"}} {
rules = append(rules, []string{HostSvcChain,
"-p", p.proto, "-m", p.proto, "--dport", p.port,
"-j", "DNAT", "--to-destination", target,
})
}
return rules
}
func nat(args ...string) *exec.Cmd {
return exec.Command("iptables", append([]string{"-t", "nat"}, args...)...)
}
func AddHostSvcRedirect(target string) error {
if out, err := nat("-N", HostSvcChain).CombinedOutput(); err != nil {
return fmt.Errorf("iptables create chain %s: %w: %s", HostSvcChain, err, out)
}
for _, rule := range append(hostSvcRules(target), hostSvcJump()) {
if out, err := nat(append([]string{"-A"}, rule...)...).CombinedOutput(); err != nil {
return fmt.Errorf("iptables hostsvc redirect %v: %w: %s", rule, err, out)
}
}
return nil
}
func HasHostSvcRedirect(target string) (bool, error) {
for _, rule := range append(hostSvcRules(target), hostSvcJump()) {
err := nat(append([]string{"-C"}, rule...)...).Run()
var exit *exec.ExitError
switch {
case err == nil:
case errors.As(err, &exit) && exit.ExitCode() == 1:
return false, nil
default:
return false, fmt.Errorf("iptables check %v: %w", rule, err)
}
}
return true, nil
}

View file

@ -1,48 +0,0 @@
package iptables
import (
"reflect"
"strings"
"testing"
)
func TestHostSvcJump_OnlyLocalDestinations(t *testing.T) {
want := []string{"PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "TWO-HOSTSVC"}
if got := hostSvcJump(); !reflect.DeepEqual(got, want) {
t.Errorf("jump = %v, want %v: traffic to an outside resolver must not be hijacked", got, want)
}
}
func TestHostSvcRules_ExclusionsComeFirst(t *testing.T) {
rules := hostSvcRules("169.254.169.253")
if len(rules) != 5 {
t.Fatalf("%d rules, want 2 exclusions and 3 redirections: %v", len(rules), rules)
}
for i, iface := range []string{"br-public", "l3-+"} {
want := []string{"TWO-HOSTSVC", "-i", iface, "-j", "RETURN"}
if !reflect.DeepEqual(rules[i], want) {
t.Errorf("rule %d = %v, want %v: the public segment and the vrf link must never reach the relay", i, rules[i], want)
}
}
}
func TestHostSvcRules_RedirectDNSAndNTPOnly(t *testing.T) {
var got []string
for _, rule := range hostSvcRules("169.254.169.253") {
if rule[len(rule)-3] != "DNAT" {
continue
}
if rule[len(rule)-1] != "169.254.169.253" {
t.Errorf("rule %v does not target 169.254.169.253", rule)
}
got = append(got, strings.Join(rule[1:7], " "))
}
want := []string{
"-p udp -m udp --dport 53",
"-p tcp -m tcp --dport 53",
"-p udp -m udp --dport 123",
}
if !reflect.DeepEqual(got, want) {
t.Errorf("redirections = %v, want %v", got, want)
}
}

View file

@ -1,44 +0,0 @@
package iptables
import (
"fmt"
"os/exec"
"git.g3e.fr/syonad/two/internal/metadata"
)
func addRule(args ...string) error {
return exec.Command("iptables", append([]string{"-t", "nat", "-A"}, args...)...).Run()
}
func deleteRule(args ...string) error {
return exec.Command("iptables", append([]string{"-t", "nat", "-D"}, args...)...).Run()
}
func AddMetadataRedirect(vmIP, gatewayIP, metadataPort string) error {
if err := addRule("PREROUTING",
"-s", vmIP+"/32",
"-d", metadata.ServiceIP+"/32",
"-p", "tcp", "-m", "tcp",
"--dport", "80",
"-j", "DNAT",
"--to-destination", gatewayIP+":"+metadataPort,
); err != nil {
return fmt.Errorf("iptables metadata redirect: %w", err)
}
return nil
}
func DeleteMetadataRedirect(vmIP, gatewayIP, metadataPort string) error {
if err := deleteRule("PREROUTING",
"-s", vmIP+"/32",
"-d", metadata.ServiceIP+"/32",
"-p", "tcp", "-m", "tcp",
"--dport", "80",
"-j", "DNAT",
"--to-destination", gatewayIP+":"+metadataPort,
); err != nil {
return fmt.Errorf("iptables delete metadata redirect: %w", err)
}
return nil
}

View file

@ -1,7 +1,7 @@
package metadata
// ServiceIP is the well-known address guests use to reach the metadata server.
// Traffic to it is DNATed to the subnet gateway by internal/iptables, and the
// Traffic to it is DNATed to the subnet gateway by internal/nft, and the
// route to it is advertised by internal/dhcp.
const ServiceIP = "169.254.169.254"

View file

@ -10,7 +10,6 @@ import (
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/dhcpbackend"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/l3vni"
"git.g3e.fr/syonad/two/internal/metadata"
"git.g3e.fr/syonad/two/internal/netif"
@ -63,12 +62,7 @@ func StartVM(db *badger.DB, name string, cfg *configuration.Config) error {
// métadonnées de l'interface primaire. Toutes les interfaces étant dans le
// même VPC, donc le même netns, il est joignable depuis n'importe laquelle.
if err := netns.Call(nic.vpcName, func() error {
for _, n := range d.nics {
if err := iptables.AddMetadataRedirect(n.ip, nic.interfaceIP, d.metadataPort); err != nil {
return fmt.Errorf("interface %d: %w", n.index, err)
}
}
return nil
return nft.SetMetadataRedirect(name, nicIPs(d.nics), nic.interfaceIP+":"+d.metadataPort)
}); err != nil {
return fmt.Errorf("add metadata redirect: %w", err)
}
@ -186,6 +180,14 @@ func copyFile(src, dst string) error {
return out.Sync()
}
func nicIPs(nics []nicData) []string {
ips := make([]string, len(nics))
for i, n := range nics {
ips[i] = n.ip
}
return ips
}
func nicMACs(nics []nicData) []string {
macs := make([]string, len(nics))
for i, n := range nics {

View file

@ -18,3 +18,17 @@ func TestNicMACs_KeepsTheOrderAndLowersTheCase(t *testing.T) {
t.Errorf("nicMACs = %v, attendu %v", got, want)
}
}
func TestNicIPs_EveryInterfaceGetsTheRedirect(t *testing.T) {
nics := []nicData{
{index: 0, ip: "10.49.1.10"},
{index: 1, ip: "10.49.2.10", primary: true},
}
got := nicIPs(nics)
want := []string{"10.49.1.10", "10.49.2.10"}
if !reflect.DeepEqual(got, want) {
t.Errorf("nicIPs = %v, attendu %v", got, want)
}
}

View file

@ -11,7 +11,6 @@ import (
configuration "git.g3e.fr/syonad/two/internal/config/agent"
"git.g3e.fr/syonad/two/internal/dhcpbackend"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/l3vni"
"git.g3e.fr/syonad/two/internal/metadata"
"git.g3e.fr/syonad/two/internal/netif"
@ -60,12 +59,7 @@ func StopVM(db *badger.DB, name string, cfg *configuration.Config) error {
}
if err := netns.Call(nic.vpcName, func() error {
for _, n := range d.nics {
if err := iptables.DeleteMetadataRedirect(n.ip, nic.interfaceIP, d.metadataPort); err != nil {
return fmt.Errorf("interface %d: %w", n.index, err)
}
}
return nil
return nft.DeleteMetadataRedirect(name)
}); err != nil {
return fmt.Errorf("delete metadata redirect: %w", err)
}

View file

@ -7,9 +7,9 @@ import (
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
hostsvcclient "git.g3e.fr/syonad/two/internal/client/hostsvc"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/nft"
)
const HostSvcLink = "hostsvc"
@ -34,7 +34,7 @@ func newSystemHostSvc() systemHostSvc {
func (systemHostSvc) InNetns(vpc string, fn func() error) error { return netns.Call(vpc, fn) }
func (systemHostSvc) CreateDummy(name string) error { return netif.CreateDummy(name) }
func (systemHostSvc) AddrAdd(iface string, ip net.IP) error { return netif.AddrAdd(iface, ip) }
func (systemHostSvc) AddRedirect(target string) error { return iptables.AddHostSvcRedirect(target) }
func (systemHostSvc) AddRedirect(target string) error { return nft.SetHostSvcRedirect(target) }
func (s systemHostSvc) Open(vpc string) error { return s.client.Open(vpc) }
func (s systemHostSvc) Close(vpc string) error { return s.client.Close(vpc) }

View file

@ -7,9 +7,9 @@ import (
hostsvcapi "git.g3e.fr/syonad/two/internal/api/hostsvc"
hostsvcclient "git.g3e.fr/syonad/two/internal/client/hostsvc"
"git.g3e.fr/syonad/two/internal/hostsvc"
"git.g3e.fr/syonad/two/internal/iptables"
"git.g3e.fr/syonad/two/internal/netif"
"git.g3e.fr/syonad/two/internal/netns"
"git.g3e.fr/syonad/two/internal/nft"
"git.g3e.fr/syonad/two/internal/state"
"git.g3e.fr/syonad/two/internal/vpc"
"git.g3e.fr/syonad/two/internal/watchdog/notify"
@ -58,7 +58,7 @@ func (systemHostSvcProbe) Redirect(name string) (bool, error) {
var ok bool
err := netns.Call(name, func() error {
var err error
ok, err = iptables.HasHostSvcRedirect(hostsvc.ServiceIP)
ok, err = nft.HasHostSvcRedirect(hostsvc.ServiceIP)
return err
})
return ok, err
@ -103,7 +103,7 @@ func CheckHostSvc(db *badger.DB, p hostSvcProbe, n notify.Notifier) error {
if ok, err := p.Redirect(name); err != nil {
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection unreadable: %v", err))
} else if !ok {
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection to %s missing or incomplete (chain %s)", hostsvc.ServiceIP, iptables.HostSvcChain))
n.Notify(kindVPC, name, fmt.Sprintf("dns and ntp redirection to %s missing or incomplete (nft chain ip %s %s)", hostsvc.ServiceIP, nft.Table, nft.HostSvcChain))
}
}
return nil

View file

@ -97,7 +97,7 @@ func TestCheckHostSvc_ReportsEachMissingPiece(t *testing.T) {
for name, want := range map[string]string{
"vp-unserved": "not served by two-hostsvc: its vms get no dns nor ntp",
"vp-noaddr": "169.254.169.253 missing on hostsvc (in netns)",
"vp-noredirect": "dns and ntp redirection to 169.254.169.253 missing or incomplete (chain TWO-HOSTSVC)",
"vp-noredirect": "dns and ntp redirection to 169.254.169.253 missing or incomplete (nft chain ip two hostsvc)",
} {
got := r.forName(name)
if len(got) != 1 || got[0].kind != "vpc" || got[0].problem != want {