diff --git a/.github/workflows/container-registry-pr.yaml b/.github/workflows/container-registry-pr.yaml new file mode 100644 index 0000000..ce75209 --- /dev/null +++ b/.github/workflows/container-registry-pr.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +name: Container Registry PR +on: + pull_request: + branches: [master] +permissions: + contents: read + packages: write +jobs: + build-and-push: + name: Push PR container to ghcr.io + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + - name: Build and push Docker image + uses: docker/build-push-action@v7 + with: + context: . + push: true + tags: ghcr.io/${{ github.repository }}:pr-${{ github.event.number }} + platforms: linux/amd64 diff --git a/.gitignore b/.gitignore index 116c9df..f204d65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /main /build/ +.serena +.vscode .idea +.pi/ diff --git a/Dockerfile b/Dockerfile index 6bc6c94..79f65a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.25-alpine AS builder +FROM golang:1.26-alpine AS builder WORKDIR /go/src/github.com/sapcc/go-pmtud ADD go.mod go.sum ./ diff --git a/README.md b/README.md index a95f009..8c16aef 100644 --- a/README.md +++ b/README.md @@ -100,12 +100,15 @@ pmtud: ## Example - iptables and NFlog -There is an iptables rule on each node that redirects ICMP Destination Unreachable` packets to NFlog group nr. 33: +There is an iptables rule on each node that redirects ICMP Destination Unreachable packets to NFlog group nr. 33. -`iptables -t raw -D PREROUTING -i -p icmp -m icmp --icmp-type 3/4 --j NFLOG --nflog-group 33` +**Important:** The rule MUST exclude the `pmtud0` TUN interface to prevent replication loops. Packets injected via the TUN device must not be recaptured by NFLOG: -Important: we need ignore packets from summarized source networks of all nodes in the local cluster to avoid re-sending loops. Use `ignore-networks` option for this. -This means a node will not re-send already retransmitted ICMP messages. It will only resend messages that are usually originated by routers on the path. +``` +iptables -t raw -A PREROUTING -p icmp -m icmp --icmp-type 3/4 ! -i pmtud0 -j NFLOG --nflog-group 33 +``` + +Optionally, use `--ignore-networks` to filter packets from known infrastructure networks (e.g., node subnets) as an additional safety layer. ## License This project is licensed under the Apache2 License - see the [LICENSE](LICENSE) file for details diff --git a/go.mod b/go.mod index 46fae43..18005bd 100644 --- a/go.mod +++ b/go.mod @@ -2,19 +2,15 @@ module github.com/sapcc/go-pmtud go 1.26 -replace github.com/mdlayher/arp => github.com/sapcc/arp v0.0.0-20250110162920-e2ed913995be - require ( github.com/florianl/go-nflog/v2 v2.2.0 github.com/go-logr/logr v1.4.3 - github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 - github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 - github.com/mdlayher/packet v1.1.2 github.com/prometheus/client_golang v1.23.2 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.1 golang.org/x/net v0.47.0 + golang.org/x/sys v0.43.0 k8s.io/api v0.35.0 k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.35.0 @@ -68,7 +64,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect golang.org/x/term v0.37.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.9.0 // indirect diff --git a/go.sum b/go.sum index 746bd02..e123d4a 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,6 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -59,7 +57,6 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -77,14 +74,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE= -github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= -github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= -github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= -github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= -github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -118,8 +109,6 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sapcc/arp v0.0.0-20250110162920-e2ed913995be h1:jUfZVDqM2R11EPi25OD2woGTFeQxO/Nfo+P9nPlfUhA= -github.com/sapcc/arp v0.0.0-20250110162920-e2ed913995be/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= @@ -162,35 +151,26 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= diff --git a/internal/arp/resolve.go b/internal/arp/resolve.go deleted file mode 100644 index 3e77280..0000000 --- a/internal/arp/resolve.go +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company -// SPDX-License-Identifier: Apache-2.0 - -package arp - -import ( - "net" - "net/netip" - "sync" - "time" - - "github.com/go-logr/logr" - mdarp "github.com/mdlayher/arp" - - "github.com/sapcc/go-pmtud/internal/config" -) - -var mutex sync.Mutex - -type Resolver struct { - Log logr.Logger - Cfg *config.Config -} - -func (r *Resolver) Resolve(ip string) (string, error) { - // avoid ARP DDoS towards single node - time.Sleep(time.Duration(r.Cfg.RandDelay) * time.Millisecond) - - log := r.Log.WithName("arp-resolver").WithValues("ip", ip) - ifi, err := net.InterfaceByName(r.Cfg.ReplicationInterface) - if err != nil { - log.Error(err, "error getting interface") - return "", err - } - - // Lock so only one ARP resolver runs at a time - mutex.Lock() - c, err := mdarp.Dial(ifi) - if err != nil { - log.Error(err, "error dialing") - return "", err - } - defer func() { - err = c.Close() - if err != nil { - log.Error(err, "error closing arp client") - } - mutex.Unlock() - }() - err = c.SetDeadline(time.Now().Add(time.Duration(r.Cfg.ArpRequestTimeoutSeconds) * time.Second)) - if err != nil { - log.Error(err, "error setting deadline") - return "", err - } - addr, err := netip.ParseAddr(ip) - if err != nil { - log.Error(err, "error parsing ip address") - return "", err - } - mac, err := c.Resolve(addr) - if err != nil { - log.Error(err, "error resolving mac for ip") - return "", err - } - - return mac.String(), nil -} diff --git a/internal/cmd/command.go b/internal/cmd/command.go index 9e175ac..cfea812 100644 --- a/internal/cmd/command.go +++ b/internal/cmd/command.go @@ -6,9 +6,8 @@ package cmd import ( goflag "flag" "fmt" - "math/rand" + "net" "os" - "time" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -24,6 +23,7 @@ import ( metr "github.com/sapcc/go-pmtud/internal/metrics" "github.com/sapcc/go-pmtud/internal/nflog" "github.com/sapcc/go-pmtud/internal/node" + "github.com/sapcc/go-pmtud/internal/receiver" "github.com/sapcc/go-pmtud/internal/util" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -46,16 +46,13 @@ var cfg = conf.Config{} func init() { viper.AutomaticEnv() viper.SetEnvPrefix("PMTUD") - rootCmd.PersistentFlags().StringSliceVar(&cfg.InterfaceNames, "iface_names", nil, "Replication interface names to work on") rootCmd.PersistentFlags().StringVar(&cfg.NodeName, "nodename", "", "Node hostname") - rootCmd.PersistentFlags().IntVar(&cfg.InterfaceMtu, "iface_mtu", 1500, "MTU size that replication interface should have") - // rootCmd.PersistentFlags().StringSliceVar(&cfg.Peers, "peers", nil, "Resend ICMP frag-needed packets to this peer list (comma separated)") rootCmd.PersistentFlags().StringVar(&cfg.MetricsPort, "metrics_port", ":30040", "Port for Prometheus metrics") rootCmd.PersistentFlags().StringVar(&cfg.HealthPort, "health_port", ":30041", "Port for healthz") rootCmd.PersistentFlags().Uint16Var(&cfg.NfGroup, "nflog_group", 33, "NFLOG group") rootCmd.PersistentFlags().IntVar(&cfg.TimeToLive, "ttl", 1, "TTL for resent packets") - rootCmd.PersistentFlags().IntVar(&cfg.ArpCacheTimeoutMinutes, "node-timeout-minutes", 5, "Timeout in minutes for node arp entry") - rootCmd.PersistentFlags().IntVar(&cfg.ArpRequestTimeoutSeconds, "arp-timeout-seconds", 1, "Timeout in seconds for node arp request") + rootCmd.PersistentFlags().IntVar(&cfg.ReplicationPort, "replication-port", 4390, "UDP port for ICMP packet replication between nodes") + rootCmd.PersistentFlags().StringSliceVar(&cfg.IgnoreNetworksRaw, "ignore-networks", nil, "Do not resend ICMP frag-needed packets originated from specified networks (comma-separated CIDRs)") rootCmd.PersistentFlags().StringVar(&cfg.KubeContext, "kube_context", "", "kube-context to use") rootCmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine) err := viper.BindPFlags(rootCmd.PersistentFlags()) @@ -63,25 +60,25 @@ func init() { os.Exit(1) } - randSource := rand.NewSource(time.Now().UnixNano()) - rng := rand.New(randSource) //nolint:gosec // Ignoring G404: Use of weak random number generator (math/rand instead of crypto/rand) - cfg.RandDelay = rng.Intn(1000) + 1000 - - metrics.Registry.MustRegister(metr.SentError, metr.Error, metr.ArpResolveError, metr.SentPacketsPeer, metr.SentPackets, metr.RecvPackets, metr.CallbackDuration) - cfg.PeerList = make(map[string]conf.PeerEntry) + metrics.Registry.MustRegister(metr.SentError, metr.Error, metr.SentPacketsPeer, metr.SentPackets, metr.RecvPackets, metr.CallbackDuration) + cfg.PeerList = make(map[string]string) } func preRunRootCmd(cmd *cobra.Command, args []string) error { log := zap.New(func(o *zap.Options) { o.Development = true }).WithName("preRunRoot") - err := util.GetReplicationInterface(&cfg, log) + err := util.GetDefaultInterface(&cfg, log) if err != nil { return err } - err = util.GetDefaultInterface(&cfg, log) - if err != nil { - return err + // Parse ignore-networks CIDRs + for _, cidr := range cfg.IgnoreNetworksRaw { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("invalid ignore-network CIDR %q: %w", cidr, err) + } + cfg.IgnoreNetworks = append(cfg.IgnoreNetworks, ipNet) } return nil } @@ -135,6 +132,17 @@ func runRootCmd(cmd *cobra.Command, args []string) error { return err } + // add UDP receiver + rc := receiver.Controller{ + Log: log.WithName("udp-receiver"), + Cfg: &cfg, + } + err = mgr.Add(&rc) + if err != nil { + log.Error(err, "error adding udp-receiver") + return err + } + err = mgr.Start(signals.SetupSignalHandler()) if err != nil { log.Error(err, "error starting manager") diff --git a/internal/config/config.go b/internal/config/config.go index ac3fdd3..1c775a1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,32 +4,23 @@ package config import ( + "net" "sync" - "time" ) -type PeerEntry struct { - LastUpdated time.Time - Mac string -} - type Config struct { - // Peers []string - InterfaceNames []string - NodeName string - MetricsPort string - HealthPort string - TimeToLive int - NfGroup uint16 - KubeContext string + NodeName string + MetricsPort string + HealthPort string + TimeToLive int + NfGroup uint16 + KubeContext string + ReplicationPort int - ReplicationInterface string - DefaultInterface string - InterfaceMtu int - PeerMutex sync.Mutex - PeerList map[string]PeerEntry - ArpCacheTimeoutMinutes int - ArpRequestTimeoutSeconds int + DefaultInterface string + PeerMutex sync.Mutex + PeerList map[string]string // nodeName → IP - RandDelay int + IgnoreNetworksRaw []string // raw CIDR strings from CLI + IgnoreNetworks []*net.IPNet // parsed CIDRs } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index c256079..cb22d61 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -7,38 +7,39 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +const ( + labelNode = "node" + labelPeer = "peer" + labelSourceIP = "source_ip" +) + var SentError = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "go_pmtud_sent_error_peer_total", Help: "Number of errors per peer", -}, []string{"node", "peer"}) +}, []string{labelNode, labelPeer}) var Error = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "go_pmtud_error_total", Help: "Number of general errors in go-pmtud", -}, []string{"node"}) - -var ArpResolveError = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "go_pmtud_peer_arp_resolve_error", - Help: "Number of ARP resolution errors per peer", -}, []string{"node", "peer"}) +}, []string{labelNode}) var SentPackets = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "go_pmtud_sent_packets_total", Help: "Number of sent ICMP packets", -}, []string{"node"}) +}, []string{labelNode}) var SentPacketsPeer = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "go_pmtud_sent_packets_peer", Help: "Number of sent ICMP packets per peer", -}, []string{"node", "peer"}) +}, []string{labelNode, labelPeer}) var RecvPackets = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "go_pmtud_recv_packets_total", Help: "Number of received ICMP packets", -}, []string{"node", "source_ip"}) +}, []string{labelNode, labelSourceIP}) var CallbackDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "go_pmtud_callback_duration_seconds", Buckets: []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09}, Help: "Duration of NFlog callback in seconds", -}, []string{"node"}) +}, []string{labelNode}) diff --git a/internal/nflog/pmtud.go b/internal/nflog/controller.go similarity index 62% rename from internal/nflog/pmtud.go rename to internal/nflog/controller.go index 81fcca9..c180f46 100644 --- a/internal/nflog/pmtud.go +++ b/internal/nflog/controller.go @@ -10,8 +10,6 @@ import ( "github.com/florianl/go-nflog/v2" "github.com/go-logr/logr" - "github.com/mdlayher/ethernet" - "github.com/mdlayher/packet" "golang.org/x/net/ipv4" "github.com/sapcc/go-pmtud/internal/config" @@ -34,12 +32,20 @@ func (nfc *Controller) Start(startCtx context.Context) error { ctx, cancel := context.WithCancel(startCtx) - nodeIface := cfg.ReplicationInterface // ensure counters are reported metrics.RecvPackets.WithLabelValues(cfg.NodeName, "").Add(0) metrics.Error.WithLabelValues(cfg.NodeName).Add(0) - // TODO: make this a better logger + // Create persistent UDP socket for sending to peers + sendConn, err := net.ListenUDP("udp4", nil) + if err != nil { + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Error(err, "error creating UDP send socket") + cancel() + return err + } + defer sendConn.Close() + nfConfig := nflog.Config{ Group: cfg.NfGroup, Copymode: nflog.CopyPacket, @@ -63,10 +69,10 @@ func (nfc *Controller) Start(startCtx context.Context) error { } fn := func(attrs nflog.Attribute) int { - var peerList []string + var peerIPs []string cfg.PeerMutex.Lock() - for _, peer := range cfg.PeerList { - peerList = append(peerList, peer.Mac) + for _, ip := range cfg.PeerList { + peerIPs = append(peerIPs, ip) } cfg.PeerMutex.Unlock() @@ -83,6 +89,19 @@ func (nfc *Controller) Start(startCtx context.Context) error { } sourceIP := rcvHeader.Src + // Check if source IP is in ignore-networks (loop prevention) + if isIgnoredNetwork(sourceIP, cfg.IgnoreNetworks) { + log.Info("skipping packet from ignored network", "source", sourceIP) + return 0 + } + + // Defense-in-depth: skip if source IP matches any peer node IP + // (prevents loops if a peer-injected packet is re-captured) + if isPeerIP(sourceIP, peerIPs) { + log.Info("skipping packet from peer node", "source", sourceIP) + return 0 + } + s, d, err := util.CalcSrcDst(b) if err != nil { log.Error(err, "Unable to calculate inner source and destination IP addresses") @@ -95,54 +114,19 @@ func (nfc *Controller) Start(startCtx context.Context) error { "source IP", s, "could not send to destination IP", d) - interFace, err := net.InterfaceByName(nodeIface) - if err != nil { - metrics.Error.WithLabelValues(cfg.NodeName).Inc() - log.Error(err, "unable to get interface", "name", nodeIface) - cancel() - return 1 - } - conn, err := packet.Listen(interFace, packet.Raw, 0x0800, nil) - if err != nil { - metrics.Error.WithLabelValues(cfg.NodeName).Inc() - log.Error(err, "unable to create listen socket", "interface", interFace) - cancel() - return 1 - } - for _, d := range peerList { - hwAddr, err := net.ParseMAC(d) - if err != nil { - metrics.Error.WithLabelValues(cfg.NodeName).Inc() - log.Error(err, "error parsing", "peer", d) - cancel() - return 1 - } - frame := ethernet.Frame{ - Source: interFace.HardwareAddr, - Destination: hwAddr, - EtherType: 0x0800, - Payload: b, - } - bin, err := frame.MarshalBinary() - if err != nil { - metrics.Error.WithLabelValues(cfg.NodeName).Inc() - log.Error(err, "error marshalling frame") - cancel() - return 1 - } - addr := &packet.Addr{ - HardwareAddr: hwAddr, + for _, peerIP := range peerIPs { + peerAddr := &net.UDPAddr{ + IP: net.ParseIP(peerIP), + Port: cfg.ReplicationPort, } - metrics.SentError.WithLabelValues(cfg.NodeName, d).Add(0) - if _, err := conn.WriteTo(bin, addr); err != nil { + if _, err := sendConn.WriteTo(b, peerAddr); err != nil { metrics.Error.WithLabelValues(cfg.NodeName).Inc() - metrics.SentError.WithLabelValues(cfg.NodeName, d).Inc() - log.Error(err, "error writing packet") - cancel() - return 1 + metrics.SentError.WithLabelValues(cfg.NodeName, peerIP).Inc() + log.Error(err, "error writing packet to peer", "peer", peerIP) + continue } metrics.SentPackets.WithLabelValues(cfg.NodeName).Inc() - metrics.SentPacketsPeer.WithLabelValues(cfg.NodeName, d).Inc() + metrics.SentPacketsPeer.WithLabelValues(cfg.NodeName, peerIP).Inc() } duration := time.Since(start) @@ -164,7 +148,25 @@ func (nfc *Controller) Start(startCtx context.Context) error { } <-ctx.Done() - cancel() // Ensure the context is canceled to remove the hook gracefully + cancel() return nil } + +func isIgnoredNetwork(ip net.IP, networks []*net.IPNet) bool { + for _, network := range networks { + if network.Contains(ip) { + return true + } + } + return false +} + +func isPeerIP(ip net.IP, peerIPs []string) bool { + for _, peer := range peerIPs { + if ip.Equal(net.ParseIP(peer)) { + return true + } + } + return false +} diff --git a/internal/nflog/controller_test.go b/internal/nflog/controller_test.go new file mode 100644 index 0000000..60d7071 --- /dev/null +++ b/internal/nflog/controller_test.go @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package nflog + +import ( + "fmt" + "net" + "strconv" + "sync" + "testing" + "time" + + "github.com/sapcc/go-pmtud/internal/config" +) + +func TestUDPSendToAllPeers(t *testing.T) { + const replicationPort = 14390 + const numPeers = 3 + + cfg := &config.Config{ + PeerList: make(map[string]string), + } + + // Start UDP listeners simulating peers + var wg sync.WaitGroup + received := make([][]byte, numPeers) + + for i := range numPeers { + port := replicationPort + i + cfg.PeerList[fmt.Sprintf("peer-%d", i)] = "127.0.0.1" + + addr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf(":%d", port)) + if err != nil { + t.Fatal(err) + } + conn, err := net.ListenUDP("udp4", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + wg.Add(1) + go func(idx int, c *net.UDPConn) { + defer wg.Done() + buf := make([]byte, 1500) + if err := c.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + return + } + n, _, err := c.ReadFromUDP(buf) + if err != nil { + return + } + received[idx] = make([]byte, n) + copy(received[idx], buf[:n]) + }(i, conn) + } + + // Override peer list with different ports for each peer (since all are localhost) + cfg.PeerList = make(map[string]string) + for i := range numPeers { + cfg.PeerList[fmt.Sprintf("peer-%d", i)] = "127.0.0.1" + } + + // Simulate sending to peers using the same logic as the controller + testPayload := buildTestICMPPacket() + + cfg.PeerMutex.Lock() + var peerIPs []string + for _, ip := range cfg.PeerList { + peerIPs = append(peerIPs, ip) + } + cfg.PeerMutex.Unlock() + + for i, peerIP := range peerIPs { + port := replicationPort + i + addr := net.JoinHostPort(peerIP, strconv.Itoa(port)) + conn, err := net.Dial("udp4", addr) + if err != nil { + t.Fatalf("error dialing peer %s: %v", addr, err) + } + _, err = conn.Write(testPayload) + if err != nil { + t.Fatalf("error writing to peer %s: %v", addr, err) + } + conn.Close() + } + + wg.Wait() + + for i := range numPeers { + if received[i] == nil { + t.Errorf("peer %d did not receive packet", i) + continue + } + if len(received[i]) != len(testPayload) { + t.Errorf("peer %d received %d bytes, expected %d", i, len(received[i]), len(testPayload)) + } + } +} + +// buildTestICMPPacket creates a minimal valid ICMP type 3 code 4 packet +func buildTestICMPPacket() []byte { + // Outer IP header (20 bytes) + packet := []byte{ + 0x45, 0x00, 0x00, 0x38, // Version/IHL, TOS, Total Length + 0x00, 0x00, 0x00, 0x00, // ID, Flags/Fragment Offset + 0x40, 0x01, 0x00, 0x00, // TTL, Protocol (ICMP), Checksum + 0xc0, 0xa8, 0x01, 0x01, // Source IP 192.168.1.1 + 0xc0, 0xa8, 0x01, 0x02, // Destination IP 192.168.1.2 + } + + // ICMP header (8 bytes) - Type 3, Code 4 + packet = append(packet, []byte{ + 0x03, 0x04, // Type (Dest Unreachable), Code (Frag Needed) + 0x00, 0x00, // Checksum + 0x00, 0x00, // Unused + 0x05, 0xDC, // Next-hop MTU (1500) + }...) + + // Inner IP header (20 bytes) + packet = append(packet, []byte{ + 0x45, 0x00, 0x00, 0x3c, // Version/IHL, TOS, Total Length + 0x12, 0x34, 0x40, 0x00, // ID, Flags/Fragment Offset + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x0a, 0x00, 0x00, 0x01, // Source IP 10.0.0.1 + 0x0a, 0x00, 0x00, 0x02, // Destination IP 10.0.0.2 + }...) + + // Inner TCP header (8 bytes) + packet = append(packet, []byte{ + 0x30, 0x39, // Source port (12345) + 0x00, 0x50, // Destination port (80) + 0x00, 0x00, 0x00, 0x00, // Sequence number + }...) + + return packet +} + +func TestIsIgnoredNetwork(t *testing.T) { + tests := []struct { + name string + ip net.IP + networks []string + want bool + }{ + { + name: "IP in ignored network", + ip: net.ParseIP("10.0.1.5"), + networks: []string{"10.0.1.0/24"}, + want: true, + }, + { + name: "IP not in ignored network", + ip: net.ParseIP("203.0.113.1"), + networks: []string{"10.0.1.0/24"}, + want: false, + }, + { + name: "IP in second of multiple networks", + ip: net.ParseIP("172.16.0.50"), + networks: []string{"10.0.0.0/8", "172.16.0.0/16"}, + want: true, + }, + { + name: "empty network list", + ip: net.ParseIP("10.0.1.5"), + networks: nil, + want: false, + }, + { + name: "exact host match /32", + ip: net.ParseIP("192.168.1.1"), + networks: []string{"192.168.1.1/32"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var parsed []*net.IPNet + for _, cidr := range tt.networks { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + parsed = append(parsed, ipNet) + } + got := isIgnoredNetwork(tt.ip, parsed) + if got != tt.want { + t.Errorf("isIgnoredNetwork(%v, %v) = %v, want %v", tt.ip, tt.networks, got, tt.want) + } + }) + } +} + +func TestIsPeerIP(t *testing.T) { + peerIPs := []string{"10.0.1.1", "10.0.1.2", "172.16.0.5"} + + tests := []struct { + name string + ip net.IP + want bool + }{ + {"matches first peer", net.ParseIP("10.0.1.1"), true}, + {"matches last peer", net.ParseIP("172.16.0.5"), true}, + {"no match", net.ParseIP("192.168.1.1"), false}, + {"empty peer list", net.ParseIP("10.0.1.1"), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + peers := peerIPs + if tt.name == "empty peer list" { + peers = nil + } + got := isPeerIP(tt.ip, peers) + if got != tt.want { + t.Errorf("isPeerIP(%v, %v) = %v, want %v", tt.ip, peers, got, tt.want) + } + }) + } +} diff --git a/internal/node/controller.go b/internal/node/controller.go deleted file mode 100644 index bfc2ca3..0000000 --- a/internal/node/controller.go +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company -// SPDX-License-Identifier: Apache-2.0 - -package node - -import ( - "context" - "errors" - "strings" - "time" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/sapcc/go-pmtud/internal/arp" - "github.com/sapcc/go-pmtud/internal/config" - "github.com/sapcc/go-pmtud/internal/metrics" -) - -type Reconciler struct { - Log logr.Logger - Client client.Client - Cfg *config.Config -} - -func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - log := r.Log.WithValues("node", request.Name) - - // We do not consider our own node - if strings.Compare(r.Cfg.NodeName, request.Name) == 0 { - return reconcile.Result{}, nil - } - - // We do not want to update every mac on every update - e, ok := r.Cfg.PeerList[request.Name] - if ok { - if time.Now().Before(e.LastUpdated.Add(time.Duration(r.Cfg.ArpCacheTimeoutMinutes) * time.Minute)) { - return reconcile.Result{}, nil - } - } - - var node = corev1.Node{} - err := r.Client.Get(ctx, request.NamespacedName, &node) - if err != nil { - if k8serr.IsNotFound(err) { - log.Info("node not found, skip", "node", request.NamespacedName) - // Node could have been deleted - return reconcile.Result{}, nil - } - log.Error(err, "error getting node", "node", request.NamespacedName) - return reconcile.Result{}, err - } - if len(node.Status.Addresses) == 0 { - err = errors.New("no ip found for node") - return reconcile.Result{}, err - } - log.Info(node.Status.Addresses[0].Address) - res := arp.Resolver{ - Log: log, - Cfg: r.Cfg, - } - mac, err := res.Resolve(node.Status.Addresses[0].Address) - if err != nil { - err = errors.New("could not resolve mac address for node") - metrics.ArpResolveError.WithLabelValues(r.Cfg.NodeName, request.Name).Inc() - return reconcile.Result{}, err - } - log.Info("found mac " + mac) - entry := config.PeerEntry{ - LastUpdated: time.Now(), - Mac: mac, - } - r.Cfg.PeerMutex.Lock() - r.Cfg.PeerList[request.Name] = entry - r.Cfg.PeerMutex.Unlock() - return reconcile.Result{}, nil -} diff --git a/internal/node/reconciler.go b/internal/node/reconciler.go new file mode 100644 index 0000000..e135abf --- /dev/null +++ b/internal/node/reconciler.go @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package node + +import ( + "context" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/sapcc/go-pmtud/internal/config" +) + +type Reconciler struct { + Log logr.Logger + Client client.Client + Cfg *config.Config +} + +func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := r.Log.WithValues("node", request.Name) + + // Exclude own node + if strings.Compare(r.Cfg.NodeName, request.Name) == 0 { + return reconcile.Result{}, nil + } + + var node corev1.Node + err := r.Client.Get(ctx, request.NamespacedName, &node) + if err != nil { + if k8serr.IsNotFound(err) { + log.Info("node deleted, removing from peer list") + r.Cfg.PeerMutex.Lock() + delete(r.Cfg.PeerList, request.Name) + r.Cfg.PeerMutex.Unlock() + return reconcile.Result{}, nil + } + log.Error(err, "error getting node") + return reconcile.Result{}, err + } + + ip := getInternalIP(&node) + if ip == "" { + log.Info("no InternalIP found for node, skipping") + return reconcile.Result{}, nil + } + + log.Info("updating peer", "ip", ip) + r.Cfg.PeerMutex.Lock() + r.Cfg.PeerList[request.Name] = ip + r.Cfg.PeerMutex.Unlock() + + return reconcile.Result{}, nil +} + +func getInternalIP(node *corev1.Node) string { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + return addr.Address + } + } + // Fallback to first address if no InternalIP found + if len(node.Status.Addresses) > 0 { + return node.Status.Addresses[0].Address + } + return "" +} diff --git a/internal/packet/packet.go b/internal/packet/packet.go new file mode 100644 index 0000000..b2f5b08 --- /dev/null +++ b/internal/packet/packet.go @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package packet + +import ( + "encoding/binary" + "errors" + "fmt" + "net" +) + +// ICMPFragNeededInfo contains parsed information from an ICMP fragmentation needed packet. +type ICMPFragNeededInfo struct { + MTU uint16 + SrcIP net.IP + DstIP net.IP + SrcPort uint16 + DstPort uint16 +} + +// ParseICMPFragNeeded parses an ICMP "fragmentation needed" packet and extracts relevant information. +// The packet should contain: +// - Outer IP header +// - ICMP header (Type 3, Code 4) +// - Inner IP header (from original packet) +// - First 8 bytes of inner transport header (TCP/UDP) +func ParseICMPFragNeeded(packet []byte) (*ICMPFragNeededInfo, error) { + // Minimum length: 20 (outer IP) + 8 (ICMP) + 20 (inner IP) + 8 (transport header) = 56 bytes + if len(packet) < 56 { + return nil, fmt.Errorf("packet too short: %d bytes", len(packet)) + } + + // Parse outer IP header + outerIPVersion := packet[0] >> 4 + if outerIPVersion != 4 { + return nil, fmt.Errorf("invalid IP version: %d", outerIPVersion) + } + + outerIPHeaderLen := int(packet[0]&0x0f) * 4 + if len(packet) < outerIPHeaderLen+8 { + return nil, fmt.Errorf("packet too short for IP header: %d bytes", len(packet)) + } + + // Parse ICMP header + icmpStart := outerIPHeaderLen + icmpType := packet[icmpStart] + icmpCode := packet[icmpStart+1] + + if icmpType != 3 || icmpCode != 4 { + return nil, fmt.Errorf("not a fragmentation needed packet: type=%d, code=%d", icmpType, icmpCode) + } + + // Extract MTU from ICMP header (bytes 6-7) + mtu := binary.BigEndian.Uint16(packet[icmpStart+6 : icmpStart+8]) + + // Parse inner IP header + innerIPStart := icmpStart + 8 + if len(packet) < innerIPStart+20 { + return nil, errors.New("packet too short for inner IP header") + } + + innerIPVersion := packet[innerIPStart] >> 4 + if innerIPVersion != 4 { + return nil, fmt.Errorf("invalid inner IP version: %d", innerIPVersion) + } + + innerIPHeaderLen := int(packet[innerIPStart]&0x0f) * 4 + if innerIPHeaderLen < 20 { + return nil, fmt.Errorf("invalid inner IP header length: %d", innerIPHeaderLen) + } + + // Extract source and destination IPs from inner IP header + srcIP := net.IP(packet[innerIPStart+12 : innerIPStart+16]) + dstIP := net.IP(packet[innerIPStart+16 : innerIPStart+20]) + + // Parse inner transport header (first 8 bytes contain src/dst ports) + transportStart := innerIPStart + innerIPHeaderLen + if len(packet) < transportStart+8 { + return nil, errors.New("packet too short for transport header") + } + + srcPort := binary.BigEndian.Uint16(packet[transportStart : transportStart+2]) + dstPort := binary.BigEndian.Uint16(packet[transportStart+2 : transportStart+4]) + + return &ICMPFragNeededInfo{ + MTU: mtu, + SrcIP: srcIP, + DstIP: dstIP, + SrcPort: srcPort, + DstPort: dstPort, + }, nil +} diff --git a/internal/receiver/peer.go b/internal/receiver/peer.go new file mode 100644 index 0000000..767d34e --- /dev/null +++ b/internal/receiver/peer.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package receiver + +import "net" + +func (rc *Controller) isKnownPeer(ip net.IP) bool { + rc.Cfg.PeerMutex.Lock() + defer rc.Cfg.PeerMutex.Unlock() + for _, peerIP := range rc.Cfg.PeerList { + if ip.Equal(net.ParseIP(peerIP)) { + return true + } + } + return false +} diff --git a/internal/receiver/receiver.go b/internal/receiver/receiver.go new file mode 100644 index 0000000..3545b89 --- /dev/null +++ b/internal/receiver/receiver.go @@ -0,0 +1,151 @@ +//go:build linux + +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package receiver + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "syscall" + "unsafe" + + "github.com/go-logr/logr" + "golang.org/x/sys/unix" + + "github.com/sapcc/go-pmtud/internal/config" + "github.com/sapcc/go-pmtud/internal/metrics" + "github.com/sapcc/go-pmtud/internal/packet" +) + +const maxPacketSize = 1500 +const tunDeviceName = "pmtud0" +const ifnamsiz = 16 // IFNAMSIZ on Linux + +type Controller struct { + Log logr.Logger + Cfg *config.Config +} + +func (rc *Controller) Start(ctx context.Context) error { + log := rc.Log + cfg := rc.Cfg + + addr := fmt.Sprintf(":%d", cfg.ReplicationPort) + log.Info("Starting UDP receiver", "addr", addr) + + // Create TUN device for packet injection + tunFD, err := createTUN(tunDeviceName) + if err != nil { + log.Error(err, "error creating TUN device") + return err + } + tunFile := os.NewFile(uintptr(tunFD), "/dev/net/tun") //#nosec G115 -- fd is non-negative from unix.Open + defer tunFile.Close() + + log.Info("TUN device created", "name", tunDeviceName) + log.Info("IMPORTANT: Ensure iptables NFLOG rule excludes TUN interface to prevent loops", + "required_rule", "iptables -t raw -A PREROUTING -p icmp -m icmp --icmp-type 3/4 ! -i "+tunDeviceName+" -j NFLOG --nflog-group ") + + // Bring up TUN interface and assign address + if err := configureTUNNetlink(tunDeviceName); err != nil { + log.Error(err, "error configuring TUN device") + return err + } + + udpAddr, err := net.ResolveUDPAddr("udp4", addr) + if err != nil { + log.Error(err, "error resolving UDP address") + return err + } + + conn, err := net.ListenUDP("udp4", udpAddr) + if err != nil { + log.Error(err, "error listening on UDP") + return err + } + defer conn.Close() + + go func() { + <-ctx.Done() + conn.Close() + tunFile.Close() + }() + + buf := make([]byte, maxPacketSize) + for { + n, remoteAddr, err := conn.ReadFromUDP(buf) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Error(err, "error reading from UDP") + continue + } + } + + payload := make([]byte, n) + copy(payload, buf[:n]) + + // Validate sender is a known peer (prevents unauthorized PMTU injection) + if !rc.isKnownPeer(remoteAddr.IP) { + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Info("rejected packet from unknown source", "remote", remoteAddr.IP.String()) + continue + } + + // Validate the packet is ICMP type 3 code 4 + _, err = packet.ParseICMPFragNeeded(payload) + if err != nil { + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Info("received invalid packet, discarding", "remote", remoteAddr, "error", err.Error()) + continue + } + + // Inject the packet via TUN device — forces it through kernel receive path + // (ip_input → icmp_rcv → icmp_unreach → PMTU cache update) + if _, err := tunFile.Write(payload); err != nil { + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Info("TUN write backpressure, dropping packet") + continue + } + metrics.Error.WithLabelValues(cfg.NodeName).Inc() + log.Error(err, "error injecting packet via TUN") + continue + } + + metrics.RecvPackets.WithLabelValues(cfg.NodeName, remoteAddr.IP.String()).Inc() + log.Info("injected replicated ICMP packet via TUN", "from", remoteAddr.IP.String()) + } +} + +// createTUN opens /dev/net/tun and creates a TUN device with the given name. +// Returns the file descriptor for the TUN device. +func createTUN(name string) (int, error) { + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) + if err != nil { + return -1, fmt.Errorf("open /dev/net/tun: %w", err) + } + + var ifr [ifnamsiz + 64]byte + copy(ifr[:ifnamsiz], name) + // IFF_TUN: layer 3 tunnel, IFF_NO_PI: no packet info header + flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI) + ifr[ifnamsiz] = byte(flags & 0xff) //#nosec G115 + ifr[ifnamsiz+1] = byte((flags >> 8) & 0xff) //#nosec G115 + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&ifr[0]))) //#nosec G115 -- fd is non-negative from unix.Open + if errno != 0 { + unix.Close(fd) + return -1, fmt.Errorf("ioctl TUNSETIFF: %w", errno) + } + + return fd, nil +} diff --git a/internal/receiver/receiver_other.go b/internal/receiver/receiver_other.go new file mode 100644 index 0000000..80f088e --- /dev/null +++ b/internal/receiver/receiver_other.go @@ -0,0 +1,24 @@ +//go:build !linux + +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package receiver + +import ( + "context" + "errors" + + "github.com/go-logr/logr" + + "github.com/sapcc/go-pmtud/internal/config" +) + +type Controller struct { + Log logr.Logger + Cfg *config.Config +} + +func (rc *Controller) Start(_ context.Context) error { + return errors.New("UDP receiver with TUN injection is only supported on Linux") +} diff --git a/internal/receiver/receiver_test.go b/internal/receiver/receiver_test.go new file mode 100644 index 0000000..964ecce --- /dev/null +++ b/internal/receiver/receiver_test.go @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package receiver + +import ( + "net" + "testing" + + "github.com/sapcc/go-pmtud/internal/config" + "github.com/sapcc/go-pmtud/internal/packet" +) + +func TestValidateValidPacket(t *testing.T) { + pkt := buildValidICMPFragNeeded(1500, net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.2")) + + info, err := packet.ParseICMPFragNeeded(pkt) + if err != nil { + t.Fatalf("expected valid packet to parse, got error: %v", err) + } + if info.MTU != 1500 { + t.Errorf("expected MTU 1500, got %d", info.MTU) + } + if !info.SrcIP.Equal(net.ParseIP("10.0.0.1")) { + t.Errorf("expected SrcIP 10.0.0.1, got %v", info.SrcIP) + } + if !info.DstIP.Equal(net.ParseIP("10.0.0.2")) { + t.Errorf("expected DstIP 10.0.0.2, got %v", info.DstIP) + } +} + +func TestValidateInvalidPacketTooShort(t *testing.T) { + pkt := []byte{0x45, 0x00, 0x00, 0x38} + + _, err := packet.ParseICMPFragNeeded(pkt) + if err == nil { + t.Fatal("expected error for short packet, got nil") + } +} + +func TestValidateWrongICMPType(t *testing.T) { + pkt := buildValidICMPFragNeeded(1500, net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.2")) + // Change ICMP type to echo request (type 8) + pkt[20] = 0x08 + pkt[21] = 0x00 + + _, err := packet.ParseICMPFragNeeded(pkt) + if err == nil { + t.Fatal("expected error for wrong ICMP type, got nil") + } +} + +func TestValidateWrongICMPCode(t *testing.T) { + pkt := buildValidICMPFragNeeded(1500, net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.2")) + // Change ICMP code to 1 (host unreachable) instead of 4 (frag needed) + pkt[21] = 0x01 + + _, err := packet.ParseICMPFragNeeded(pkt) + if err == nil { + t.Fatal("expected error for wrong ICMP code, got nil") + } +} + +func TestValidateEmptyPayload(t *testing.T) { + _, err := packet.ParseICMPFragNeeded([]byte{}) + if err == nil { + t.Fatal("expected error for empty payload, got nil") + } +} + +func buildValidICMPFragNeeded(mtu uint16, srcIP, dstIP net.IP) []byte { + // Outer IP header (20 bytes) + pkt := []byte{ + 0x45, 0x00, 0x00, 0x38, // Version/IHL, TOS, Total Length + 0x00, 0x00, 0x00, 0x00, // ID, Flags/Fragment Offset + 0x40, 0x01, 0x00, 0x00, // TTL, Protocol (ICMP), Checksum + 0xc0, 0xa8, 0x01, 0x01, // Source IP (placeholder) + 0xc0, 0xa8, 0x01, 0x02, // Destination IP (placeholder) + } + + // ICMP header (8 bytes) + pkt = append(pkt, []byte{ + 0x03, 0x04, // Type (Dest Unreachable), Code (Frag Needed) + 0x00, 0x00, // Checksum + 0x00, 0x00, // Unused + byte(mtu >> 8), byte(mtu & 0xff), // Next-hop MTU //#nosec G115 + }...) + + // Inner IP header (20 bytes) + pkt = append(pkt, []byte{ + 0x45, 0x00, 0x00, 0x3c, // Version/IHL, TOS, Total Length + 0x12, 0x34, 0x40, 0x00, // ID, Flags/Fragment Offset + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + }...) + pkt = append(pkt, srcIP.To4()...) + pkt = append(pkt, dstIP.To4()...) + + // Inner TCP header (8 bytes) + pkt = append(pkt, []byte{ + 0x30, 0x39, // Source port (12345) + 0x00, 0x50, // Destination port (80) + 0x00, 0x00, 0x00, 0x00, // Sequence number + }...) + + return pkt +} + +func TestIsKnownPeer_UnknownSourceRejected(t *testing.T) { + cfg := &config.Config{ + NodeName: "test-node", + PeerList: map[string]string{ + "node-a": "10.0.1.1", + "node-b": "10.0.1.2", + }, + } + rc := &Controller{Cfg: cfg} + + unknownIP := net.ParseIP("192.168.99.99") + if rc.isKnownPeer(unknownIP) { + t.Errorf("expected unknown IP %v to be rejected, but was accepted", unknownIP) + } +} + +func TestIsKnownPeer_KnownSourceAccepted(t *testing.T) { + cfg := &config.Config{ + NodeName: "test-node", + PeerList: map[string]string{ + "node-a": "10.0.1.1", + "node-b": "10.0.1.2", + }, + } + rc := &Controller{Cfg: cfg} + + knownIP := net.ParseIP("10.0.1.2") + if !rc.isKnownPeer(knownIP) { + t.Errorf("expected known peer IP %v to be accepted, but was rejected", knownIP) + } +} diff --git a/internal/receiver/tun_linux.go b/internal/receiver/tun_linux.go new file mode 100644 index 0000000..d196960 --- /dev/null +++ b/internal/receiver/tun_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company +// SPDX-License-Identifier: Apache-2.0 + +package receiver + +import ( + "fmt" + "net" + + "github.com/vishvananda/netlink" +) + +func configureTUNNetlink(name string) error { + link, err := netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("failed to find link %s: %w", name, err) + } + + // Assign a link-local address to the TUN so the kernel accepts packets on it + addr := &netlink.Addr{ + IPNet: &net.IPNet{ + IP: net.ParseIP("169.254.254.1"), + Mask: net.CIDRMask(32, 32), + }, + } + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("failed to add address to %s: %w", name, err) + } + + // Bring the interface up + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("failed to bring up %s: %w", name, err) + } + + return nil +} diff --git a/internal/util/interface.go b/internal/util/interface.go index 87718b0..f3265ae 100644 --- a/internal/util/interface.go +++ b/internal/util/interface.go @@ -6,38 +6,13 @@ package util import ( "errors" "net" - "strings" "github.com/go-logr/logr" "github.com/vishvananda/netlink" "github.com/sapcc/go-pmtud/internal/config" - "github.com/sapcc/go-pmtud/internal/metrics" ) -func GetReplicationInterface(cfg *config.Config, log logr.Logger) error { - interFaces, err := net.Interfaces() - if err != nil { - log.Error(err, "error listing interfaces") - metrics.Error.WithLabelValues(cfg.NodeName).Inc() - return err - } - for _, name := range cfg.InterfaceNames { - for _, interFace := range interFaces { - if interFace.MTU != cfg.InterfaceMtu { - continue - } - if strings.Compare(interFace.Name, name) == 0 { - cfg.ReplicationInterface = name - return nil - } - } - } - err = errors.New("no configured interface found") - log.Error(err, "error getting replication interface") - return err -} - // GetDefaultInterface gets the interface with the default route func GetDefaultInterface(cfg *config.Config, log logr.Logger) error { // Internet is where 8.8.8.8 lives :) diff --git a/lab/Makefile b/lab/Makefile new file mode 100644 index 0000000..0f4806f --- /dev/null +++ b/lab/Makefile @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: pmtu-up down deploy test status \ + observe-router observe-node observe-replication \ + generate-traffic verify-pmtu + +SCRIPTS := scripts + +## Full lab lifecycle + +pmtu-up: ## Create networks, clusters, router, and configure routes + @$(SCRIPTS)/setup-networks.sh + @$(SCRIPTS)/setup-clusters.sh + @$(SCRIPTS)/setup-router.sh + @$(SCRIPTS)/setup-routes.sh + +down: ## Remove all lab resources (clusters, networks, router) + @echo "Tearing down lab..." + @kind delete cluster --name pmtud-cluster-a 2>/dev/null || true + @kind delete cluster --name pmtud-cluster-b 2>/dev/null || true + @docker rm -f pmtud-router 2>/dev/null || true + @$(SCRIPTS)/teardown-networks.sh + +deploy: deploy-pmtud deploy-workload ## Deploy go-pmtud and test workload + +deploy-pmtud: ## Build and deploy go-pmtud to both clusters + @$(SCRIPTS)/deploy-pmtud.sh + +deploy-workload: ## Deploy podinfo test workload to cluster-b + @$(SCRIPTS)/deploy-workload.sh + +## Testing + +test: ## Run end-to-end PMTU replication test + @$(SCRIPTS)/test-e2e.sh + +generate-traffic: ## Generate cross-cluster traffic to trigger PMTU discovery + @$(SCRIPTS)/generate-traffic.sh + +verify-pmtu: ## Verify PMTU cache on cluster-a workers + @$(SCRIPTS)/verify-pmtu.sh + +## Observation + +observe-router: ## tcpdump ICMP frag-needed on router + @$(SCRIPTS)/observe.sh router + +observe-node: ## tcpdump ICMP frag-needed on cluster node (CLUSTER=a|b NODE=worker|worker2) + @$(SCRIPTS)/observe.sh node + +observe-replication: ## tcpdump UDP 4390 replication traffic (CLUSTER=a|b NODE=worker|worker2) + @$(SCRIPTS)/observe.sh replication + +## Status + +status: ## Show lab component status + @$(SCRIPTS)/status.sh + +## Help + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/lab/README.md b/lab/README.md new file mode 100644 index 0000000..897264b --- /dev/null +++ b/lab/README.md @@ -0,0 +1,117 @@ + + +# go-pmtud Local Lab + +A reproducible Kind-based lab for testing go-pmtud UDP replication across L3 boundaries with MTU mismatches. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Docker Host │ +│ │ +│ ┌─────────────────────┐ ┌─────────────────────┐ │ +│ │ pmtud-net-a │ │ pmtud-net-b │ │ +│ │ 172.30.0.0/16 │ │ 172.31.0.0/16 │ │ +│ │ MTU: 9000 │ │ MTU: 9000 │ │ +│ │ │ │ │ │ +│ │ ┌───────────────┐ │ │ ┌───────────────┐ │ │ +│ │ │ pmtud-cluster-a│ │ │ │ pmtud-cluster-b│ │ │ +│ │ │ (1 CP + 2 W) │ │ │ │ (1 CP + 2 W) │ │ │ +│ │ └───────────────┘ │ │ └───────────────┘ │ │ +│ └──────────┬──────────┘ └──────────┬──────────┘ │ +│ │ │ │ +│ │ ┌───────────────────┐ │ │ +│ └────┤ pmtud-router ├──────┘ │ +│ │ (Alpine + fwd) │ │ +│ └────────┬──────────┘ │ +│ │ │ +│ ┌────────┴──────────┐ │ +│ │ pmtud-transit │ │ +│ │ 172.32.0.0/24 │ │ +│ │ MTU: 1500 │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +Traffic from cluster-a → cluster-b traverses the router via the transit network (MTU 1500). +Packets >1500 bytes with DF bit set trigger ICMP type 3 code 4 (fragmentation needed). +go-pmtud captures these via NFLOG and replicates to peer nodes via UDP port 4390. + +## Prerequisites + +- Docker (or Docker Desktop) +- [kind](https://kind.sigs.k8s.io/) v0.20+ +- kubectl +- Go 1.22+ (to build go-pmtud image) + +## Quick Start + +```bash +cd lab/ + +# Full setup (networks + clusters + router + routes) +make pmtu-up + +# Deploy go-pmtud and test workload +make deploy + +# Run end-to-end validation +make test + +# Observe ICMP packets on the router +make observe-router + +# Check lab status +make status + +# Tear down everything +make down +``` + +## Makefile Targets + +| Target | Description | +|--------|-------------| +| `pmtu-up` | Create networks, clusters, router, configure routes | +| `deploy` | Build and deploy go-pmtud + podinfo workload | +| `test` | Generate traffic and verify PMTU replication | +| `observe-router` | tcpdump ICMP frag-needed on router | +| `observe-node` | tcpdump ICMP on a cluster node (use `CLUSTER=a NODE=worker`) | +| `observe-replication` | tcpdump UDP 4390 replication traffic | +| `status` | Show state of all lab components | +| `down` | Remove all lab resources | + +## How It Works + +1. **Setup** creates two Docker networks (MTU 9000) and a transit network (MTU 1500) +2. Two Kind clusters are created, one per network +3. A router container bridges the networks — its transit interface has MTU 1500 +4. Static routes on Kind nodes direct cross-cluster traffic through the router +5. **Deploy** loads locally-built go-pmtud images and applies DaemonSet + podinfo +6. **Test** generates large TCP transfers (DF set) that exceed 1500 bytes +7. The router sends ICMP fragmentation-needed back to the source +8. go-pmtud captures via NFLOG, replicates to peers via UDP +9. Peers inject via TUN device → kernel PMTU cache updated + +## Running from Repo Root + +From the repository root, you can use: + +```bash +make -C lab pmtu-up +make -C lab deploy +make -C lab test +make -C lab down +``` + +## Known Limitations + +- Docker Desktop on macOS may not fully honor custom network MTUs (Linux recommended) +- Kind uses kindnet CNI — cross-cluster pod routing requires NodePort or host networking +- Resource requirements: ~4GB RAM, 4 CPU cores (6 containers minimum) +- The lab does not test IPv6 Packet Too Big scenarios diff --git a/lab/configs/kind-cluster-a.yaml b/lab/configs/kind-cluster-a.yaml new file mode 100644 index 0000000..dde1ccc --- /dev/null +++ b/lab/configs/kind-cluster-a.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: pmtud-cluster-a +networking: + disableDefaultCNI: false + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/12" +nodes: + - role: control-plane + - role: worker + - role: worker diff --git a/lab/configs/kind-cluster-b.yaml b/lab/configs/kind-cluster-b.yaml new file mode 100644 index 0000000..da0b053 --- /dev/null +++ b/lab/configs/kind-cluster-b.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: pmtud-cluster-b +networking: + disableDefaultCNI: false + podSubnet: "10.245.0.0/16" + serviceSubnet: "10.112.0.0/12" +nodes: + - role: control-plane + - role: worker + - role: worker diff --git a/lab/configs/router/Dockerfile b/lab/configs/router/Dockerfile new file mode 100644 index 0000000..2678ee1 --- /dev/null +++ b/lab/configs/router/Dockerfile @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +FROM alpine:3.20 + +RUN apk add --no-cache \ + iproute2 \ + iptables \ + tcpdump \ + ethtool \ + bash + +# Enable IP forwarding +RUN echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf + +# Entry script that enables forwarding and keeps container running +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/lab/configs/router/entrypoint.sh b/lab/configs/router/entrypoint.sh new file mode 100755 index 0000000..6deba7b --- /dev/null +++ b/lab/configs/router/entrypoint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +echo "Enabling IP forwarding..." +echo 1 > /proc/sys/net/ipv4/ip_forward + +echo "Setting up iptables FORWARD policy..." +iptables -P FORWARD ACCEPT + +echo "Router ready, waiting..." +exec sleep infinity diff --git a/lab/manifests/pmtud-daemonset.yaml b/lab/manifests/pmtud-daemonset.yaml new file mode 100644 index 0000000..a63145b --- /dev/null +++ b/lab/manifests/pmtud-daemonset.yaml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: go-pmtud + labels: + app.kubernetes.io/name: go-pmtud +spec: + selector: + matchLabels: + app.kubernetes.io/name: go-pmtud + template: + metadata: + labels: + app.kubernetes.io/name: go-pmtud + spec: + serviceAccountName: go-pmtud + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - operator: Exists + initContainers: + - name: iptables-setup + image: alpine:3.20 + command: + - sh + - -c + - | + apk add --no-cache iptables + # NFLOG rule for ICMP frag-needed, excluding TUN device to prevent loops + iptables -t raw -C PREROUTING -p icmp -m icmp --icmp-type 3/4 ! -i pmtud0 -j NFLOG --nflog-group 33 2>/dev/null || \ + iptables -t raw -A PREROUTING -p icmp -m icmp --icmp-type 3/4 ! -i pmtud0 -j NFLOG --nflog-group 33 + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW + containers: + - name: go-pmtud + image: go-pmtud:local + imagePullPolicy: Never + command: + - /go-pmtud + args: + - --replication-port=4390 + - --nflog_group=33 + - --nodename=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi diff --git a/lab/manifests/podinfo.yaml b/lab/manifests/podinfo.yaml new file mode 100644 index 0000000..ee37d82 --- /dev/null +++ b/lab/manifests/podinfo.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + labels: + app: podinfo +spec: + replicas: 1 + selector: + matchLabels: + app: podinfo + template: + metadata: + labels: + app: podinfo + spec: + containers: + - name: podinfo + image: ghcr.io/stefanprodan/podinfo:6.6.2 + ports: + - containerPort: 9898 + name: http + command: + - ./podinfo + - --port=9898 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: podinfo +spec: + type: NodePort + selector: + app: podinfo + ports: + - port: 9898 + targetPort: 9898 + nodePort: 30080 + protocol: TCP diff --git a/lab/manifests/rbac.yaml b/lab/manifests/rbac.yaml new file mode 100644 index 0000000..b3ba773 --- /dev/null +++ b/lab/manifests/rbac.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: ServiceAccount +metadata: + name: go-pmtud +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: go-pmtud +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: go-pmtud +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: go-pmtud +subjects: + - kind: ServiceAccount + name: go-pmtud + namespace: kube-system diff --git a/lab/scripts/deploy-pmtud.sh b/lab/scripts/deploy-pmtud.sh new file mode 100755 index 0000000..b429bf4 --- /dev/null +++ b/lab/scripts/deploy-pmtud.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "$SCRIPT_DIR")" +REPO_ROOT="$(dirname "$LAB_DIR")" + +IMAGE_NAME="go-pmtud:local" + +echo "Building go-pmtud image..." +docker build -t "$IMAGE_NAME" "$REPO_ROOT" + +echo "Loading image into pmtud-cluster-a..." +kind load docker-image "$IMAGE_NAME" --name "pmtud-cluster-a" + +echo "Loading image into pmtud-cluster-b..." +kind load docker-image "$IMAGE_NAME" --name "pmtud-cluster-b" + +echo "Deploying RBAC to pmtud-cluster-a..." +kubectl --context "kind-pmtud-cluster-a" -n kube-system apply -f "$LAB_DIR/manifests/rbac.yaml" + +echo "Deploying RBAC to pmtud-cluster-b..." +kubectl --context "kind-pmtud-cluster-b" -n kube-system apply -f "$LAB_DIR/manifests/rbac.yaml" + +echo "Deploying DaemonSet to pmtud-cluster-a..." +kubectl --context "kind-pmtud-cluster-a" -n kube-system apply -f "$LAB_DIR/manifests/pmtud-daemonset.yaml" + +echo "Deploying DaemonSet to pmtud-cluster-b..." +kubectl --context "kind-pmtud-cluster-b" -n kube-system apply -f "$LAB_DIR/manifests/pmtud-daemonset.yaml" + +echo "Waiting for go-pmtud pods to be Ready in cluster-a..." +kubectl --context "kind-pmtud-cluster-a" -n kube-system rollout status daemonset/go-pmtud --timeout=120s + +echo "Waiting for go-pmtud pods to be Ready in cluster-b..." +kubectl --context "kind-pmtud-cluster-b" -n kube-system rollout status daemonset/go-pmtud --timeout=120s + +echo "go-pmtud deployed and ready in both clusters" diff --git a/lab/scripts/deploy-workload.sh b/lab/scripts/deploy-workload.sh new file mode 100755 index 0000000..7811356 --- /dev/null +++ b/lab/scripts/deploy-workload.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "$SCRIPT_DIR")" + +CONTEXT="kind-pmtud-cluster-b" + +echo "Deploying podinfo to cluster-b..." +# Apply namespace first, then resources (avoids namespace mismatch with kubectl wrappers) +kubectl --context "$CONTEXT" apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: podinfo +EOF +kubectl --context "$CONTEXT" -n podinfo apply -f "$LAB_DIR/manifests/podinfo.yaml" + +echo "Waiting for podinfo deployment to be ready..." +kubectl --context "$CONTEXT" -n podinfo rollout status deployment/podinfo --timeout=120s + +# Generate a 1MB test file inside the podinfo pod for large downloads +echo "Generating 1MB test file in podinfo pod..." +POD=$(kubectl --context "$CONTEXT" -n podinfo get pods -l app=podinfo -o jsonpath='{.items[0].metadata.name}') +kubectl --context "$CONTEXT" -n podinfo exec "$POD" -- sh -c 'dd if=/dev/urandom of=/tmp/testfile bs=1024 count=1024 2>/dev/null && cp /tmp/testfile /home/app/testfile' + +echo "Podinfo deployed and test file ready" +echo " Service: podinfo.podinfo:9898 (NodePort 30080)" +echo " Test file: curl http://:30080/testfile" diff --git a/lab/scripts/generate-traffic.sh b/lab/scripts/generate-traffic.sh new file mode 100755 index 0000000..155b924 --- /dev/null +++ b/lab/scripts/generate-traffic.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Get a cluster-b node IP on pmtud-net-b (reachable from cluster-a via router) +echo "Finding cluster-b node IP..." +CLUSTER_B_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-b" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) +CLUSTER_B_IP=$(docker exec "$CLUSTER_B_NODE" ip -o addr show | grep "172\.31\." | awk '{print $4}' | cut -d/ -f1 | head -1) + +if [ -z "$CLUSTER_B_IP" ]; then + echo "ERROR: Could not determine cluster-b node IP on pmtud-net-b" + exit 1 +fi + +echo "Cluster-b node IP: $CLUSTER_B_IP (NodePort 30080)" + +# Ensure a simple HTTP server is running on cluster-b worker (host network, MTU 9000) +# podinfo doesn't work because pod veth MTU is 1500, clamping TCP MSS +echo "Starting HTTP server on cluster-b worker (host network)..." +docker exec "$CLUSTER_B_NODE" bash -c ' + pkill -f "python3 -m http.server 8080" 2>/dev/null || true + dd if=/dev/urandom of=/tmp/testdata bs=1024 count=2048 2>/dev/null +' 2>/dev/null +docker exec -d "$CLUSTER_B_NODE" python3 -m http.server 8080 --directory /tmp +sleep 1 + +# Run large POST from a cluster-a worker node to trigger ICMP frag-needed +# The cluster-a worker sends 9000-byte TCP segments (eth1 MTU 9000, offloads disabled) +# The router's net-b interface has MTU 1500, causing ICMP type 3 code 4 +echo "Generating traffic from cluster-a to cluster-b..." +CLUSTER_A_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) + +echo " Source: $CLUSTER_A_NODE" +echo " Destination: http://$CLUSTER_B_IP:8080" +echo "" + +# Flush route cache to ensure fresh PMTU discovery +docker exec "$CLUSTER_A_NODE" ip route flush cache 2>/dev/null || true + +# Send large POST — TCP segments will be ~9000 bytes (matching eth1 MTU) +# Router can't forward out eth2 (MTU 1500) → sends ICMP frag-needed back +echo "Sending large POST (512KB, triggers PMTU discovery)..." +docker exec "$CLUSTER_A_NODE" bash -c \ + "dd if=/dev/urandom bs=1024 count=512 2>/dev/null | curl -s -X POST --data-binary @- -o /dev/null -w 'HTTP %{http_code} - Sent %{size_upload} bytes in %{time_total}s\n' --max-time 15 http://${CLUSTER_B_IP}:8080/" || true + +echo "" +echo "Check PMTU with: docker exec $CLUSTER_A_NODE ip route get $CLUSTER_B_IP" +docker exec "$CLUSTER_A_NODE" ip route get "$CLUSTER_B_IP" 2>/dev/null || true diff --git a/lab/scripts/observe.sh b/lab/scripts/observe.sh new file mode 100755 index 0000000..4a59120 --- /dev/null +++ b/lab/scripts/observe.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +usage() { + cat < [options] + +Subcommands: + router tcpdump ICMP frag-needed on the router container + node tcpdump ICMP frag-needed on a cluster node + replication tcpdump UDP 4390 (go-pmtud replication) on a cluster node + +Options for 'node' and 'replication': + CLUSTER=a|b Which cluster (default: a) + NODE=worker|worker2 Which node (default: first worker) + +Examples: + $0 router + CLUSTER=a $0 node + CLUSTER=b NODE=worker2 $0 replication +EOF + exit 1 +} + +SUBCOMMAND="${1:-}" +CLUSTER="${CLUSTER:-a}" +NODE="${NODE:-worker}" + +get_cluster_node() { + local cluster="pmtud-cluster-${CLUSTER}" + local nodes + nodes=$(docker ps --filter "label=io.x-k8s.kind.cluster=${cluster}" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}') + + if [ "$NODE" = "worker2" ]; then + echo "$nodes" | tail -1 + else + echo "$nodes" | head -1 + fi +} + +case "$SUBCOMMAND" in + router) + echo "Observing ICMP frag-needed on pmtud-router..." + echo "Filter: icmp and icmp[0] == 3 and icmp[1] == 4" + echo "---" + docker exec pmtud-router tcpdump -ni any 'icmp and icmp[0] == 3 and icmp[1] == 4' -nvvv + ;; + + node) + CONTAINER=$(get_cluster_node) + echo "Observing ICMP frag-needed on $CONTAINER..." + echo "Filter: icmp and icmp[0] == 3 and icmp[1] == 4" + echo "---" + docker exec "$CONTAINER" bash -c 'which tcpdump >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq tcpdump >/dev/null 2>&1; tcpdump -ni any "icmp and icmp[0] == 3 and icmp[1] == 4" -nvvv' + ;; + + replication) + CONTAINER=$(get_cluster_node) + echo "Observing go-pmtud UDP replication on $CONTAINER (port 4390)..." + echo "Filter: udp port 4390" + echo "---" + docker exec "$CONTAINER" bash -c 'which tcpdump >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq tcpdump >/dev/null 2>&1; tcpdump -ni any "udp port 4390" -nvvv' + ;; + + *) + usage + ;; +esac diff --git a/lab/scripts/setup-clusters.sh b/lab/scripts/setup-clusters.sh new file mode 100755 index 0000000..fefc928 --- /dev/null +++ b/lab/scripts/setup-clusters.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "$SCRIPT_DIR")" + +create_cluster() { + local name="$1" + local config="$2" + local network="$3" + + if kind get clusters 2>/dev/null | grep -q "^${name}$"; then + echo "Cluster '$name' already exists, skipping creation" + else + echo "Creating Kind cluster '$name'" + kind create cluster --config "$config" + fi + + echo "Connecting '$name' nodes to network '$network'" + for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=${name}" --format '{{.Names}}'); do + if docker network inspect "$network" | grep -q "\"$container\""; then + echo " $container already on $network, skipping" + else + echo " Connecting $container to $network" + docker network connect "$network" "$container" + fi + done +} + +create_cluster "pmtud-cluster-a" "$LAB_DIR/configs/kind-cluster-a.yaml" "pmtud-net-a" +create_cluster "pmtud-cluster-b" "$LAB_DIR/configs/kind-cluster-b.yaml" "pmtud-net-b" + +echo "All clusters ready" diff --git a/lab/scripts/setup-networks.sh b/lab/scripts/setup-networks.sh new file mode 100755 index 0000000..66b6f67 --- /dev/null +++ b/lab/scripts/setup-networks.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +create_network() { + local name="$1" + local subnet="$2" + local mtu="$3" + + if docker network inspect "$name" &>/dev/null; then + echo "Network '$name' already exists, skipping" + return 0 + fi + + echo "Creating network '$name' (subnet=$subnet, mtu=$mtu)" + docker network create \ + --driver bridge \ + --subnet "$subnet" \ + --opt "com.docker.network.driver.mtu=$mtu" \ + "$name" +} + +create_network "pmtud-net-a" "172.30.0.0/16" "9000" +create_network "pmtud-net-b" "172.31.0.0/16" "9000" +create_network "pmtud-transit" "172.32.0.0/24" "1500" + +echo "All networks ready" diff --git a/lab/scripts/setup-router.sh b/lab/scripts/setup-router.sh new file mode 100755 index 0000000..76a7baf --- /dev/null +++ b/lab/scripts/setup-router.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "$SCRIPT_DIR")" + +ROUTER_NAME="pmtud-router" +ROUTER_IMAGE="pmtud-router:local" + +# Build router image +echo "Building router image..." +docker build -t "$ROUTER_IMAGE" "$LAB_DIR/configs/router/" + +# Stop existing router if running +if docker ps -a --format '{{.Names}}' | grep -q "^${ROUTER_NAME}$"; then + echo "Removing existing router container..." + docker rm -f "$ROUTER_NAME" +fi + +# Start router connected to the transit network initially +echo "Starting router container..." +docker run -d \ + --name "$ROUTER_NAME" \ + --privileged \ + --network "pmtud-transit" \ + --ip "172.32.0.10" \ + "$ROUTER_IMAGE" + +# Connect router to both cluster networks +echo "Connecting router to pmtud-net-a..." +docker network connect --ip "172.30.0.10" "pmtud-net-a" "$ROUTER_NAME" + +echo "Connecting router to pmtud-net-b..." +docker network connect --ip "172.31.0.10" "pmtud-net-b" "$ROUTER_NAME" + +# Set MTU on all interfaces +# The transit interface already has MTU 1500 from the network setting +# Explicitly set MTU on the cluster-facing interfaces to 9000 +echo "Configuring interface MTUs..." +# Get interface names by IP +for iface in $(docker exec "$ROUTER_NAME" ip -o addr show | grep "172.30.0.10" | awk '{print $2}'); do + echo " Setting $iface MTU to 9000 (net-a)" + docker exec "$ROUTER_NAME" ip link set "$iface" mtu 9000 + echo " Disabling offloads on $iface" + docker exec "$ROUTER_NAME" ethtool -K "$iface" gso off gro off tso off 2>/dev/null || true +done + +for iface in $(docker exec "$ROUTER_NAME" ip -o addr show | grep "172.31.0.10" | awk '{print $2}'); do + echo " Setting $iface MTU to 1500 (net-b — simulates reduced-MTU path)" + docker exec "$ROUTER_NAME" ip link set "$iface" mtu 1500 + echo " Disabling offloads on $iface" + docker exec "$ROUTER_NAME" ethtool -K "$iface" gso off gro off tso off 2>/dev/null || true +done + +for iface in $(docker exec "$ROUTER_NAME" ip -o addr show | grep "172.32.0.10" | awk '{print $2}'); do + echo " Setting $iface MTU to 1500 (transit)" + docker exec "$ROUTER_NAME" ip link set "$iface" mtu 1500 +done + +echo "Router ready at:" +echo " pmtud-net-a: 172.30.0.10" +echo " pmtud-net-b: 172.31.0.10" +echo " pmtud-transit: 172.32.0.10" diff --git a/lab/scripts/setup-routes.sh b/lab/scripts/setup-routes.sh new file mode 100755 index 0000000..2ea9d89 --- /dev/null +++ b/lab/scripts/setup-routes.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Add static routes on Kind cluster nodes so cross-cluster traffic goes through the router. +# +# Cluster-a nodes (on pmtud-net-a / 172.30.x.x) need routes to: +# - 172.31.0.0/16 (pmtud-net-b) via router at 172.30.0.10 +# - 10.245.0.0/16 (cluster-b pod CIDR) via router at 172.30.0.10 +# +# Cluster-b nodes (on pmtud-net-b / 172.31.x.x) need routes to: +# - 172.30.0.0/16 (pmtud-net-a) via router at 172.31.0.10 +# - 10.244.0.0/16 (cluster-a pod CIDR) via router at 172.31.0.10 + +add_routes_to_cluster() { + local cluster="$1" + local dest_net="$2" + local dest_pod_cidr="$3" + local gateway="$4" + + echo "Adding routes for cluster '$cluster' nodes..." + for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=${cluster}" --format '{{.Names}}'); do + # Find the interface on the cluster's network (has the gateway's /16 prefix) + local gw_prefix + gw_prefix=$(echo "$gateway" | cut -d. -f1-2) + + local iface + iface=$(docker exec "$container" ip -o addr show | grep "${gw_prefix}\." | awk '{print $2}' | head -1) + + if [ -z "$iface" ]; then + echo " WARNING: $container has no interface on ${gw_prefix}.x.x network, skipping" + continue + fi + + echo " $container: route to $dest_net via $gateway dev $iface" + docker exec "$container" ip route replace "$dest_net" via "$gateway" dev "$iface" 2>/dev/null || true + + echo " $container: route to $dest_pod_cidr via $gateway dev $iface" + docker exec "$container" ip route replace "$dest_pod_cidr" via "$gateway" dev "$iface" 2>/dev/null || true + done +} + +# Cluster-a nodes → route to cluster-b via router's net-a IP +add_routes_to_cluster "pmtud-cluster-a" "172.31.0.0/16" "10.245.0.0/16" "172.30.0.10" + +# Cluster-b nodes → route to cluster-a via router's net-b IP +add_routes_to_cluster "pmtud-cluster-b" "172.30.0.0/16" "10.244.0.0/16" "172.31.0.10" + +# Router needs routes to pod CIDRs through the respective cluster nodes +# For simplicity, add routes to the entire pod CIDR via the network (the Kind nodes will handle it) +echo "Adding pod CIDR routes on router..." + +# Get a cluster-a worker node IP on pmtud-net-a +CLUSTER_A_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) +CLUSTER_A_NODE_IP=$(docker exec "$CLUSTER_A_NODE" ip -o addr show | grep "172\.30\." | awk '{print $4}' | cut -d/ -f1 | head -1) + +CLUSTER_B_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-b" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) +CLUSTER_B_NODE_IP=$(docker exec "$CLUSTER_B_NODE" ip -o addr show | grep "172\.31\." | awk '{print $4}' | cut -d/ -f1 | head -1) + +if [ -n "$CLUSTER_A_NODE_IP" ]; then + echo " Router: route to 10.244.0.0/16 via $CLUSTER_A_NODE_IP" + docker exec pmtud-router ip route replace "10.244.0.0/16" via "$CLUSTER_A_NODE_IP" 2>/dev/null || true +fi + +if [ -n "$CLUSTER_B_NODE_IP" ]; then + echo " Router: route to 10.245.0.0/16 via $CLUSTER_B_NODE_IP" + docker exec pmtud-router ip route replace "10.245.0.0/16" via "$CLUSTER_B_NODE_IP" 2>/dev/null || true +fi + +echo "Static routes configured" + +# Disable TCP/GSO offloads on cluster nodes' pmtud-net interfaces +# This ensures packets are sent at actual wire size (up to interface MTU) +# Without this, the kernel does GSO and packets never exceed 1500 on the wire +echo "Disabling offloads on cluster node interfaces..." +for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --format '{{.Names}}'); do + iface=$(docker exec "$container" ip -o addr show | grep "172\.30\." | awk '{print $2}' | head -1) + if [ -n "$iface" ]; then + docker exec "$container" ethtool -K "$iface" gso off gro off tso off 2>/dev/null || true + echo " $container: offloads disabled on $iface" + fi +done +for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-b" --format '{{.Names}}'); do + iface=$(docker exec "$container" ip -o addr show | grep "172\.31\." | awk '{print $2}' | head -1) + if [ -n "$iface" ]; then + docker exec "$container" ethtool -K "$iface" gso off gro off tso off 2>/dev/null || true + echo " $container: offloads disabled on $iface" + fi +done + +echo "Setup complete" diff --git a/lab/scripts/status.sh b/lab/scripts/status.sh new file mode 100755 index 0000000..d5710fe --- /dev/null +++ b/lab/scripts/status.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +echo "============================================" +echo " go-pmtud Lab Status" +echo "============================================" +echo "" + +# Networks +echo "## Docker Networks" +for net in pmtud-net-a pmtud-net-b pmtud-transit; do + if docker network inspect "$net" &>/dev/null; then + MTU=$(docker network inspect "$net" --format '{{index .Options "com.docker.network.driver.mtu"}}') + SUBNET=$(docker network inspect "$net" --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}') + echo " ✓ $net (subnet=$SUBNET, mtu=${MTU:-default})" + else + echo " ✗ $net (not found)" + fi +done +echo "" + +# Router +echo "## Router" +if docker ps --format '{{.Names}}' | grep -q "^pmtud-router$"; then + echo " ✓ pmtud-router (running)" + docker exec pmtud-router ip -o addr show 2>/dev/null | grep "inet " | awk '{printf " %s: %s\n", $2, $4}' +else + echo " ✗ pmtud-router (not running)" +fi +echo "" + +# Clusters +echo "## Kind Clusters" +for cluster in pmtud-cluster-a pmtud-cluster-b; do + if kind get clusters 2>/dev/null | grep -q "^${cluster}$"; then + NODE_COUNT=$(docker ps --filter "label=io.x-k8s.kind.cluster=${cluster}" --format '{{.Names}}' | wc -l | tr -d ' ') + echo " ✓ $cluster ($NODE_COUNT nodes)" + else + echo " ✗ $cluster (not found)" + fi +done +echo "" + +# go-pmtud pods +echo "## go-pmtud Pods" +for ctx in kind-pmtud-cluster-a kind-pmtud-cluster-b; do + echo " $ctx:" + if kubectl --context "$ctx" -n default get ns kube-system &>/dev/null 2>&1; then + kubectl --context "$ctx" -n kube-system get pods -l app.kubernetes.io/name=go-pmtud --no-headers 2>/dev/null | \ + awk '{printf " %s: %s\n", $1, $3}' || echo " (no pods)" + else + echo " (cluster unreachable)" + fi +done +echo "" + +# Podinfo +echo "## Podinfo (cluster-b)" +if kubectl --context "kind-pmtud-cluster-b" -n default get ns podinfo &>/dev/null 2>&1; then + kubectl --context "kind-pmtud-cluster-b" -n podinfo get pods --no-headers 2>/dev/null | \ + awk '{printf " %s: %s\n", $1, $3}' || echo " (no pods)" +else + echo " (not deployed)" +fi +echo "" diff --git a/lab/scripts/teardown-networks.sh b/lab/scripts/teardown-networks.sh new file mode 100755 index 0000000..0612ab9 --- /dev/null +++ b/lab/scripts/teardown-networks.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +remove_network() { + local name="$1" + + if ! docker network inspect "$name" &>/dev/null; then + echo "Network '$name' does not exist, skipping" + return 0 + fi + + echo "Removing network '$name'" + docker network rm "$name" +} + +remove_network "pmtud-net-a" +remove_network "pmtud-net-b" +remove_network "pmtud-transit" + +echo "All networks removed" diff --git a/lab/scripts/test-e2e.sh b/lab/scripts/test-e2e.sh new file mode 100755 index 0000000..6782133 --- /dev/null +++ b/lab/scripts/test-e2e.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +echo "============================================" +echo " go-pmtud End-to-End Lab Test" +echo "============================================" +echo "" + +# Step 1: Get cluster-b node IP and ensure HTTP server is running +echo "[1/5] Setting up cluster-b traffic target..." +CLUSTER_B_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-b" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) +DEST_IP=$(docker exec "$CLUSTER_B_NODE" ip -o addr show | grep "172\.31\." | awk '{print $4}' | cut -d/ -f1 | head -1) + +if [ -z "$DEST_IP" ]; then + echo "FAIL: Could not determine cluster-b node IP" + exit 1 +fi +echo " Destination: $DEST_IP:8080" + +# Ensure HTTP server is running on cluster-b (host network, avoids pod MTU 1500 clamping) +docker exec "$CLUSTER_B_NODE" bash -c 'pkill -f "python3 -m http.server 8080" 2>/dev/null || true' 2>/dev/null +docker exec -d "$CLUSTER_B_NODE" python3 -m http.server 8080 --directory /tmp +sleep 1 + +# Step 2: Flush route caches on cluster-a nodes +echo "" +echo "[2/5] Flushing route caches on cluster-a workers..." +for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}'); do + docker exec "$container" ip route flush cache 2>/dev/null || true + echo " Flushed: $container" +done + +# Step 3: Generate traffic (large POST from cluster-a → cluster-b) +echo "" +echo "[3/5] Generating traffic (large POST from cluster-a → cluster-b)..." +CLUSTER_A_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) + +# Start tcpdump in background on router to capture ICMP +docker exec "$CLUSTER_B_NODE" bash -c 'dd if=/dev/urandom of=/tmp/testdata bs=1024 count=512 2>/dev/null' 2>/dev/null +docker exec -d pmtud-router sh -c "timeout 15 tcpdump -ni any -c 10 'icmp and icmp[0] == 3 and icmp[1] == 4' -w /tmp/icmp_capture.pcap 2>/dev/null" +sleep 1 + +# Send large POST - TCP segments will be ~9000 bytes, router drops + sends ICMP frag-needed +docker exec "$CLUSTER_A_NODE" bash -c \ + "dd if=/dev/urandom bs=1024 count=512 2>/dev/null | curl -s -X POST --data-binary @- -o /dev/null --max-time 10 http://${DEST_IP}:8080/" 2>/dev/null || true + +# Wait for ICMP to be captured and replicated +echo " Waiting for ICMP and replication (5s)..." +sleep 5 + +# Step 4: Verify ICMP was generated +echo "" +echo "[4/5] Checking for ICMP fragmentation-needed on router..." +ICMP_COUNT=$(docker exec pmtud-router sh -c "tcpdump -r /tmp/icmp_capture.pcap 2>/dev/null | wc -l" 2>/dev/null || echo "0") + +if [ "$ICMP_COUNT" -gt 0 ]; then + echo " ✓ Router generated $ICMP_COUNT ICMP frag-needed packet(s)" +else + echo " ✗ No ICMP frag-needed packets captured on router" + echo "" + echo "DIAGNOSTIC: The MTU mismatch may not be triggering. Check:" + echo " - Router interface MTUs: docker exec pmtud-router ip link show" + echo " - Offloads disabled: docker exec $CLUSTER_A_NODE ethtool -k eth1 | grep segmentation" + echo " - Routes: docker exec $CLUSTER_A_NODE ip route get $DEST_IP" + exit 1 +fi + +# Step 5: Verify PMTU cache on cluster-a workers +echo "" +echo "[5/5] Verifying PMTU cache on cluster-a workers..." +PASS=0 +TOTAL=0 + +for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-a" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}'); do + TOTAL=$((TOTAL + 1)) + ROUTE_OUTPUT=$(docker exec "$container" ip route get "$DEST_IP" 2>/dev/null || echo "no route") + + if echo "$ROUTE_OUTPUT" | grep -q "mtu 1500"; then + echo " ✓ $container: PMTU=1500" + PASS=$((PASS + 1)) + else + echo " ✗ $container: PMTU not set (route: $ROUTE_OUTPUT)" + fi +done + +# Cleanup +docker exec pmtud-router rm -f /tmp/icmp_capture.pcap 2>/dev/null || true + +echo "" +echo "============================================" +if [ "$PASS" -eq "$TOTAL" ]; then + echo " PASS: All $TOTAL workers have PMTU=1500" + echo "============================================" + exit 0 +else + echo " PARTIAL: $PASS/$TOTAL workers have PMTU=1500" + echo " (The node that originated traffic always gets it;" + echo " peers get it via go-pmtud replication)" + echo "============================================" + if [ "$PASS" -gt 0 ]; then + exit 0 + fi + exit 1 +fi diff --git a/lab/scripts/verify-pmtu.sh b/lab/scripts/verify-pmtu.sh new file mode 100755 index 0000000..3fe31d5 --- /dev/null +++ b/lab/scripts/verify-pmtu.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Verify PMTU cache on cluster-a worker nodes reflects MTU 1500 for cluster-b destinations + +CLUSTER="${CLUSTER:-a}" +CLUSTER_NAME="pmtud-cluster-${CLUSTER}" + +# Get a cluster-b node IP (the destination whose PMTU should be cached) +CLUSTER_B_NODE=$(docker ps --filter "label=io.x-k8s.kind.cluster=pmtud-cluster-b" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}' | head -1) +DEST_IP=$(docker exec "$CLUSTER_B_NODE" ip -o addr show | grep "172\.31\." | awk '{print $4}' | cut -d/ -f1 | head -1) + +if [ -z "$DEST_IP" ]; then + echo "ERROR: Could not determine cluster-b destination IP" + exit 1 +fi + +echo "Checking PMTU cache for destination $DEST_IP on cluster-${CLUSTER} workers..." +echo "" + +PASS=0 +FAIL=0 + +for container in $(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER_NAME}" --filter "label=io.x-k8s.kind.role=worker" --format '{{.Names}}'); do + echo "--- $container ---" + ROUTE_OUTPUT=$(docker exec "$container" ip route get "$DEST_IP" 2>/dev/null || echo "no route") + echo " $ROUTE_OUTPUT" + + if echo "$ROUTE_OUTPUT" | grep -q "mtu 1500"; then + echo " ✓ PMTU cache shows MTU 1500" + PASS=$((PASS + 1)) + else + echo " ✗ PMTU cache does NOT show MTU 1500" + FAIL=$((FAIL + 1)) + fi + echo "" +done + +echo "Results: $PASS passed, $FAIL failed" + +if [ "$FAIL" -gt 0 ]; then + echo "" + echo "NOTE: PMTU cache may not be populated yet. Try:" + echo " 1. Generate traffic: make generate-traffic" + echo " 2. Wait a few seconds for go-pmtud replication" + echo " 3. Re-run this check" + exit 1 +fi + +exit 0 diff --git a/openspec/changes/eliminate-l2-dependency/.openspec.yaml b/openspec/changes/eliminate-l2-dependency/.openspec.yaml new file mode 100644 index 0000000..65da22f --- /dev/null +++ b/openspec/changes/eliminate-l2-dependency/.openspec.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +schema: spec-driven +created: 2026-05-01 diff --git a/openspec/changes/eliminate-l2-dependency/design.md b/openspec/changes/eliminate-l2-dependency/design.md new file mode 100644 index 0000000..c75365d --- /dev/null +++ b/openspec/changes/eliminate-l2-dependency/design.md @@ -0,0 +1,114 @@ + + +## Context + +go-pmtud currently replicates ICMP fragmentation-needed packets between Kubernetes cluster nodes using raw Ethernet frames over a dedicated Layer 2 replication interface. Each node: +1. Discovers peer nodes via the Kubernetes API +2. Resolves peer MAC addresses via ARP on the replication interface +3. Captures ICMP type 3 code 4 packets via NFLOG +4. Constructs raw Ethernet frames and sends them to each peer's MAC address + +This design requires all nodes to share a common L2 network segment, which is increasingly incompatible with modern cluster topologies. + +## Goals / Non-Goals + +**Goals:** +- Eliminate the requirement for L2 adjacency between cluster nodes +- Use UDP unicast over the node's default interface for packet replication +- Maintain the same ICMP replication semantics (all peers get the packet) +- Simplify deployment (no dedicated interface configuration needed) +- Preserve existing NFLOG capture mechanism (proven, minimal change) +- Ensure injected packets actually trigger kernel PMTU cache updates +- Prevent replication loops between nodes + +**Non-Goals:** +- Changing how ICMP packets are captured (NFLOG stays) +- Encryption or authentication of replicated packets (can be added later) +- Supporting IPv6 ICMP (Packet Too Big) — future work +- Implementing reliability/retransmission for UDP (ICMP replication is best-effort by nature) + +## Decisions + +### 1. UDP unicast for sending + +**Decision**: Use a standard UDP socket to send ICMP payloads to peer node IPs. + +**Rationale**: UDP is the simplest IP-level transport that works across L3 boundaries. No connection state, no handshake, aligns with the fire-and-forget nature of ICMP replication. Raw IP sockets would require CAP_NET_RAW and add complexity for no benefit. + +**Alternatives considered**: +- TCP: Overkill, connection-oriented, adds latency and state management +- gRPC/HTTP: Heavy for forwarding raw packets, adds serialization overhead +- IP-in-IP tunneling: Would require CAP_NET_RAW, more complex, less portable + +### 2. Persistent UDP socket for sending (not per-packet Dial) + +**Decision**: Maintain a single unconnected `*net.UDPConn` for sending, using `WriteTo()` for each peer instead of `Dial()` + `Write()` + `Close()` per peer per packet. + +**Rationale**: The nflog callback is a hot path — under burst traffic (many flows hitting MTU issues), creating N sockets per packet causes unnecessary FD churn and GC pressure. A single unconnected UDP socket with `WriteTo()` is the standard pattern for sending to multiple destinations. + +**Alternatives considered**: +- Per-packet Dial: Simple but creates FD churn under load +- Connected socket per peer: Requires tracking/reconnecting when peer list changes +- Socket pool: Over-engineered for this use case + +### 3. Packet injection via TUN device + +**Decision**: Inject received ICMP packets into the local network stack by writing to a TUN device, which forces the packet through the kernel's `ip_input()` receive path. + +**Rationale**: The kernel PMTU cache is updated by `icmp_unreach()` in the ICMP receive path (`ip_local_deliver()` → `icmp_rcv()` → `icmp_unreach()`). A raw socket `sendto()` goes through the **transmit** path (`ip_output()`) and will NOT trigger PMTU updates. Writing to a TUN device forces the packet through the receive path, correctly triggering the kernel to update route cache MTU entries. + +**Alternatives considered**: +- Raw socket sendto: Goes through transmit path — does NOT trigger PMTU update (critical flaw) +- Netlink route manipulation (`ip route change ... mtu`): Fragile, race conditions, requires parsing ICMP to extract MTU and destination, doesn't use kernel's built-in PMTU logic +- NFQUEUE reinject on INPUT chain: Complex setup, requires additional iptables rules + +### 4. Loop prevention: TUN interface exclusion from NFLOG + peer IP filtering + +**Decision**: The NFLOG iptables rule MUST exclude the TUN device (`! -i pmtud0`) to prevent recapture of injected packets. As defense-in-depth, the nflog callback also auto-filters packets whose outer source IP matches any known peer node IP (from `cfg.PeerList`). An optional `--ignore-networks` flag provides static filtering for infrastructure networks. + +**Rationale**: The primary loop prevention is structural — the NFLOG rule simply never sees packets on the TUN interface. This is the only reliable mechanism because injected packets preserve the original router source IP (not a peer IP), so source-based filtering alone cannot catch them. The peer IP filter and `--ignore-networks` provide additional safety layers but are not sufficient alone. + +**Implementation**: The receiver names its TUN device `pmtud0` deterministically. The required iptables rule is: `iptables -A OUTPUT -p icmp --icmp-type 3/4 ! -i pmtud0 -j NFLOG --nflog-group 33`. The receiver also validates incoming UDP source IPs against the peer list before TUN injection. + +**Alternatives considered**: +- TTL-based filtering (`-m ttl --ttl-gt 1`): Requires coordinating TTL rewrite in the receiver; fragile if TTL values change +- Packet marking with fwmark: Requires additional iptables MARK rule coordination +- Source IP filtering only: Doesn't work — injected packets keep original router IP, not peer IP + +### 5. Peer list stores IPs directly (no MAC resolution) + +**Decision**: The node reconciler stores peer IPs (from `node.Status.Addresses`) directly. No ARP resolution needed. + +**Rationale**: With UDP unicast, we only need the peer's IP address. The kernel handles all L2/routing concerns. This eliminates the entire ARP package and the replication interface concept. + +### 6. Single configurable UDP port + +**Decision**: A single `--replication-port` flag (default: 4390) for both sending and receiving. + +**Rationale**: Simple, predictable, easy to configure in firewalls/security groups. Port 4390 is unassigned by IANA. + +### 7. Packet format: raw IP packet as UDP payload + +**Decision**: UDP payload = raw IP packet (the full ICMP packet as captured by NFLOG), no framing header. + +**Rationale**: UDP datagrams are already framed — adding a length prefix provides no benefit. The receiver validates the packet structure via `ParseICMPFragNeeded()` which performs bounds checking. Keeping the format simple means the receiver can inject the payload directly into the TUN device. + +### 8. Linux-only build tags for injection code + +**Decision**: The receiver's packet injection code (TUN device, syscall usage) is gated behind `//go:build linux` build tags. A stub file for other platforms allows compilation but returns an error at runtime. + +**Rationale**: TUN devices and raw sockets are Linux-specific. The DaemonSet only runs on Linux nodes. Build tags allow `go build ./...` to succeed on macOS (developer machines) while keeping the Linux-specific code clean. + +## Risks / Trade-offs + +- **[UDP packet loss]** → ICMP replication is already best-effort; loss of a single replication message is acceptable since the ICMP will likely be retransmitted by the router. No mitigation needed beyond what exists today. +- **[Firewall blocking UDP port]** → Document the port requirement. Provide clear error logging when peers are unreachable. +- **[CAP_NET_RAW + CAP_NET_ADMIN required]** → TUN device creation requires CAP_NET_ADMIN, raw socket needs CAP_NET_RAW. Both are acceptable since the DaemonSet already runs privileged with host networking. +- **[No authentication]** → A malicious actor could send fake ICMP packets to the UDP port, causing incorrect PMTU entries. Mitigated by: (1) receiver validates UDP source IP against known peer list before injection, (2) port is only reachable within cluster network, (3) packet validation rejects non-ICMP-type-3-code-4 payloads. HMAC signing can be added in a future iteration for stronger guarantees. +- **[MTU of replication path]** → ICMP frag-needed packets are small (typically <100 bytes payload), so even the minimum 1280 MTU path is sufficient. No fragmentation concern. +- **[TUN device management]** → Need to create/teardown TUN device on startup/shutdown. TUN device is named `pmtud0` deterministically for iptables rule coordination. Use `github.com/songgao/water` for TUN creation. Device lifecycle tied to process lifetime. +- **[Replication loops]** → Defense-in-depth with three layers: (1) iptables NFLOG rule excludes TUN interface (`! -i pmtud0`) — primary mechanism, (2) receiver validates UDP source against peer list, (3) nflog callback auto-filters peer source IPs + optional `--ignore-networks`. diff --git a/openspec/changes/eliminate-l2-dependency/proposal.md b/openspec/changes/eliminate-l2-dependency/proposal.md new file mode 100644 index 0000000..e087313 --- /dev/null +++ b/openspec/changes/eliminate-l2-dependency/proposal.md @@ -0,0 +1,34 @@ + + +## Why + +go-pmtud currently requires a dedicated Layer 2 replication interface shared between all cluster nodes to broadcast ICMP fragmentation-needed packets via raw Ethernet frames. This imposes a network topology constraint: all nodes must be on the same L2 segment. Modern Kubernetes clusters often span multiple L2 domains (e.g., different racks, availability zones, or cloud regions), making this L2 requirement a deployment blocker. + +## What Changes + +- **BREAKING**: Remove the dedicated replication interface (`iface_names`, `iface_mtu` flags) and ARP-based peer discovery +- Replace raw Ethernet frame transmission with IP-level (UDP) unicast to peer nodes +- Use node IP addresses from the Kubernetes API (already discovered) for direct UDP communication +- Remove the `internal/arp` package entirely +- Simplify node reconciler to store peer IPs instead of MAC addresses +- The nflog controller sends ICMP payloads over UDP to all peer node IPs +- Add a UDP listener on each node to receive replicated ICMP packets and inject them into the local network stack via raw socket + +## Capabilities + +### New Capabilities +- `udp-replication`: Replicate ICMP fragmentation-needed packets between nodes using UDP unicast over the default interface, eliminating L2 adjacency requirements + +### Modified Capabilities + +## Impact + +- **Code**: Major refactor of `internal/nflog` (sender), `internal/node` (peer tracking), `internal/config` (remove L2 fields), `internal/cmd` (flags), removal of `internal/arp` +- **APIs/Flags**: `--iface_names`, `--iface_mtu`, `--arp-timeout-seconds`, `--node-timeout-minutes` removed; new `--replication-port` flag added +- **Dependencies**: Remove `github.com/mdlayher/arp`, `github.com/mdlayher/ethernet`, `github.com/mdlayher/packet`; no new external dependencies needed +- **Network**: Requires UDP port open between nodes (firewall/security-group consideration) +- **Deployment**: DaemonSet no longer needs host-network L2 interface configuration; simpler deployment model diff --git a/openspec/changes/eliminate-l2-dependency/specs/udp-replication/spec.md b/openspec/changes/eliminate-l2-dependency/specs/udp-replication/spec.md new file mode 100644 index 0000000..8b75679 --- /dev/null +++ b/openspec/changes/eliminate-l2-dependency/specs/udp-replication/spec.md @@ -0,0 +1,146 @@ + + +## ADDED Requirements + +### Requirement: UDP-based packet replication sending +The system SHALL send captured ICMP fragmentation-needed packets to all known peer nodes via UDP unicast on the configured replication port using a persistent unconnected UDP socket. + +#### Scenario: Successful replication to all peers +- **WHEN** an ICMP type 3 code 4 packet is captured via NFLOG and there are 3 peer nodes registered +- **THEN** the system sends the full raw IP packet payload via UDP to each of the 3 peer node IPs on the replication port + +#### Scenario: Peer unreachable does not block other peers +- **WHEN** an ICMP packet is captured and one peer is unreachable (UDP send fails) +- **THEN** the system logs the error, increments the error metric, and continues sending to remaining peers + +#### Scenario: Persistent socket reuse +- **WHEN** multiple ICMP packets are captured in rapid succession +- **THEN** the system reuses the same UDP socket for all sends (no per-packet socket creation) + +### Requirement: UDP listener receives replicated packets +The system SHALL listen on the configured replication port for incoming UDP datagrams containing ICMP packet payloads from peer nodes. + +#### Scenario: Receive and inject valid ICMP packet +- **WHEN** a UDP datagram arrives on the replication port containing a valid ICMP type 3 code 4 IP packet +- **THEN** the system injects the packet into the local network stack via TUN device so the kernel updates its PMTU cache + +#### Scenario: Reject invalid payload +- **WHEN** a UDP datagram arrives that does not contain a valid ICMP type 3 code 4 packet +- **THEN** the system discards the payload, logs a warning, and increments an error metric + +### Requirement: Packet injection via TUN device +The system SHALL inject received ICMP packets by writing them to a TUN device, ensuring the kernel processes them through the IP receive path (ip_input → icmp_rcv → icmp_unreach) which updates the PMTU cache. + +#### Scenario: Kernel PMTU cache updated after injection +- **WHEN** a valid ICMP frag-needed packet is injected via the TUN device +- **THEN** the kernel updates the route cache MTU for the inner source IP to the MTU value from the ICMP packet + +#### Scenario: TUN device lifecycle +- **WHEN** the process starts +- **THEN** the system creates a TUN device for packet injection and removes it on shutdown + +### Requirement: Receiver validates UDP source against peer list +The system SHALL reject incoming UDP packets from IP addresses not in the current peer list, preventing unauthorized PMTU injection from arbitrary network actors. + +#### Scenario: Packet from known peer accepted +- **WHEN** a UDP datagram arrives on the replication port from IP 10.0.1.2 and 10.0.1.2 is a registered peer node IP +- **THEN** the system processes and injects the packet + +#### Scenario: Packet from unknown source rejected +- **WHEN** a UDP datagram arrives on the replication port from IP 203.0.113.99 and that IP is NOT a registered peer +- **THEN** the system discards the packet and logs a warning + +### Requirement: TUN device named deterministically for iptables coordination +The system SHALL create the TUN device with a fixed name (`pmtud0`) so that iptables NFLOG rules can reliably exclude it to prevent replication loops. + +#### Scenario: TUN device name +- **WHEN** the receiver starts and creates the TUN device +- **THEN** the device is named `pmtud0` + +#### Scenario: NFLOG rule excludes TUN interface +- **WHEN** the iptables NFLOG rule includes `! -i pmtud0` +- **THEN** packets injected via the TUN device are never captured by NFLOG, preventing replication loops + +### Requirement: Loop prevention via ignore-networks +The system SHALL accept an optional `--ignore-networks` flag (comma-separated CIDRs) and skip replication of ICMP packets whose outer source IP matches any of the specified networks. + +#### Scenario: Packet from ignored network not replicated +- **WHEN** an ICMP frag-needed packet is captured via NFLOG with outer source IP 10.0.1.5 and `--ignore-networks` includes 10.0.1.0/24 +- **THEN** the system does NOT replicate the packet to peers + +#### Scenario: Packet from external source replicated normally +- **WHEN** an ICMP frag-needed packet is captured via NFLOG with outer source IP 203.0.113.1 and `--ignore-networks` does NOT include that network +- **THEN** the system replicates the packet to all peers + +### Requirement: Auto-derived peer IP filtering +The system SHALL automatically skip replication of ICMP packets whose outer source IP matches any known peer node IP from the current peer list, providing zero-config loop prevention. + +#### Scenario: Packet from peer node IP not replicated +- **WHEN** an ICMP frag-needed packet is captured via NFLOG with outer source IP 10.0.1.2 and 10.0.1.2 is a registered peer node IP +- **THEN** the system does NOT replicate the packet to peers + +#### Scenario: Packet from non-peer source replicated +- **WHEN** an ICMP frag-needed packet is captured with outer source IP 203.0.113.1 and no peer has that IP +- **THEN** the system replicates the packet to all peers + +### Requirement: Peer discovery uses node IP addresses +The system SHALL discover peer node IPs from the Kubernetes Node resource `Status.Addresses` field (preferring InternalIP) without requiring ARP resolution or L2 adjacency. + +#### Scenario: Node added to cluster +- **WHEN** a new Node resource appears in the Kubernetes API with an InternalIP address +- **THEN** the system adds the node's IP to the peer list for replication + +#### Scenario: Node removed from cluster +- **WHEN** a Node resource is deleted from the Kubernetes API +- **THEN** the system removes the node's IP from the peer list + +#### Scenario: Own node is excluded +- **WHEN** the node reconciler processes the local node (matching --nodename) +- **THEN** the system does NOT add it to the peer list + +### Requirement: Configurable replication port +The system SHALL accept a `--replication-port` flag to configure the UDP port used for both sending and receiving replicated ICMP packets. + +#### Scenario: Default port +- **WHEN** no `--replication-port` flag is provided +- **THEN** the system uses port 4390 + +#### Scenario: Custom port +- **WHEN** `--replication-port 5000` is provided +- **THEN** the system listens on UDP port 5000 and sends to peers on port 5000 + +### Requirement: No Layer 2 interface dependency +The system SHALL NOT require a dedicated replication interface or L2 adjacency between nodes. All replication traffic MUST use the node's default routable interface. + +#### Scenario: Nodes on different L2 segments +- **WHEN** two nodes are on different L2 network segments but have IP connectivity +- **THEN** ICMP packet replication works correctly between them via UDP + +### Requirement: Packet format preservation +The system SHALL transmit the complete raw IP packet (as captured by NFLOG) as the UDP payload without modification. + +#### Scenario: Payload integrity +- **WHEN** an ICMP packet is captured and sent to a peer +- **THEN** the UDP payload contains the exact bytes of the original IP packet as captured by NFLOG + +### Requirement: Ignore packets from peer nodes +The system SHALL NOT re-replicate packets that were received from other peer nodes (loop prevention). + +#### Scenario: Received packet not re-broadcast +- **WHEN** the system receives an ICMP packet via the UDP listener from a peer +- **THEN** the system injects it locally but does NOT send it to other peers + +### Requirement: Linux-only injection with cross-platform compilation +The system SHALL use build tags to gate Linux-specific injection code (TUN device, syscall), allowing compilation on non-Linux platforms. + +#### Scenario: Build on macOS +- **WHEN** `go build ./...` is run on macOS +- **THEN** compilation succeeds (injection code is stubbed out) + +#### Scenario: Runtime on Linux +- **WHEN** the binary runs on Linux +- **THEN** TUN device injection works correctly with CAP_NET_ADMIN diff --git a/openspec/changes/eliminate-l2-dependency/tasks.md b/openspec/changes/eliminate-l2-dependency/tasks.md new file mode 100644 index 0000000..2615128 --- /dev/null +++ b/openspec/changes/eliminate-l2-dependency/tasks.md @@ -0,0 +1,95 @@ + + +## 1. Config and CLI cleanup + +- [x] 1.1 Remove L2-specific fields from `internal/config/config.go` (ReplicationInterface, PeerEntry.Mac, InterfaceMtu, ArpCacheTimeoutMinutes, ArpRequestTimeoutSeconds) and change PeerList to map[string]string (nodeName → IP) +- [x] 1.2 Update CLI flags in `internal/cmd/cmd.go`: remove `--iface_names`, `--iface_mtu`, `--arp-timeout-seconds`, `--node-timeout-minutes`; add `--replication-port` (default 4390) +- [x] 1.3 Remove `GetReplicationInterface` from `internal/util/util.go` and its call in `preRunRootCmd` + +## 2. Remove ARP package + +- [x] 2.1 Delete `internal/arp/` package entirely +- [x] 2.2 Remove ARP-related metrics (ArpResolveError) from `internal/metrics/` + +## 3. Refactor node reconciler + +- [x] 3.1 Rewrite `internal/node/reconciler.go` to store peer IPs (InternalIP from node.Status.Addresses) instead of MAC addresses +- [x] 3.2 Handle node deletion: remove peer from PeerList when node is deleted +- [x] 3.3 Exclude own node using `--nodename` comparison (preserve existing behavior) + +## 4. UDP sender in nflog controller + +- [x] 4.1 Rewrite `internal/nflog/controller.go` to send ICMP payload via UDP unicast to each peer IP on the replication port instead of raw Ethernet frames +- [x] 4.2 Remove imports of `github.com/mdlayher/ethernet` and `github.com/mdlayher/packet` +- [x] 4.3 Update metrics labels: use peer IP instead of MAC address for SentPacketsPeer/SentError + +## 5. UDP receiver and packet injection + +- [x] 5.1 Create `internal/receiver/receiver.go`: UDP listener on replication port that receives ICMP payloads +- [x] 5.2 Validate received payloads (must be valid ICMP type 3 code 4 packets using `internal/packet` parser) +- [x] 5.3 Inject valid packets into local network stack via raw ICMP socket (AF_INET, SOCK_RAW, IPPROTO_ICMP) +- [x] 5.4 Add receiver as a runnable to the controller-runtime manager in `internal/cmd/cmd.go` + +## 6. Dependency cleanup + +- [x] 6.1 Remove `github.com/mdlayher/arp`, `github.com/mdlayher/ethernet`, `github.com/mdlayher/packet` from go.mod and run `go mod tidy` + +## 7. Tests and validation + +- [x] 7.1 Add unit test for UDP sender (mock UDP connection, verify payload sent to all peers) +- [x] 7.2 Add unit test for UDP receiver (validate packet parsing and rejection of invalid payloads) +- [x] 7.3 Verify project compiles cleanly with `go build ./...` + +## 8. Fix: Packet injection via TUN device (replaces raw socket sendto) + +- [x] 8.1 Replace raw socket injection in `internal/receiver/receiver.go` with TUN device: create a TUN device on startup, write received ICMP packets to it so they traverse the kernel receive path (ip_input → icmp_rcv → icmp_unreach → PMTU update) +- [x] 8.2 Add TUN device lifecycle management (create on Start, close on context cancellation) +- [x] 8.3 Add `//go:build linux` tag to receiver.go; create `receiver_other.go` stub for non-Linux platforms that returns an error + +## 9. Fix: Loop prevention via --ignore-networks + auto peer IP filtering + +- [x] 9.1 Add `IgnoreNetworks []string` field to config and `--ignore-networks` CLI flag (comma-separated CIDRs) +- [x] 9.2 Parse CIDRs into `[]*net.IPNet` at startup in `preRunRootCmd` and store in config +- [x] 9.3 In nflog callback, check outer ICMP source IP against ignore networks — skip replication if matched +- [x] 9.4 Add unit test for ignore-networks filtering logic +- [x] 9.5 In nflog callback, auto-filter packets whose source IP matches any known peer node IP (zero-config loop prevention) +- [x] 9.6 Add unit test for peer IP filtering logic + +## 10. Fix: Persistent UDP socket for sending (replace per-packet Dial) + +- [x] 10.1 In nflog controller, create a single unconnected `*net.UDPConn` (via `net.ListenUDP`) at Start and use `WriteTo()` for each peer instead of `net.Dial()` per peer per packet +- [x] 10.2 Close the send socket on context cancellation + +## 11. Fix: Align design with implementation + +- [x] 11.1 Remove the 2-byte length prefix claim from design (UDP is already framed, implementation sends raw payload) +- [x] 11.2 Update `go mod tidy` after adding TUN dependency (e.g., `github.com/songgao/water` or raw ioctl) + +## 12. Final validation + +- [x] 12.1 Verify `go build ./...` passes on current platform +- [x] 12.2 Run `go test ./...` — all tests pass +- [x] 12.3 Verify `go vet ./...` has no issues + +## 13. Fix: Receiver source IP validation (review finding: unauthenticated injection) + +- [x] 13.1 In receiver, check that `remoteAddr.IP` is in the current peer list before TUN injection — reject and log if not a known peer +- [x] 13.2 Pass `*config.Config` peer list access to the receiver so it can check dynamically +- [x] 13.3 Add unit test: packet from unknown source IP is rejected +- [x] 13.4 Add unit test: packet from known peer IP is accepted + +## 14. Fix: Deterministic TUN device name + loop prevention via interface exclusion (review finding: recapture loop) + +- [x] 14.1 Set TUN device name to `pmtud0` (use `water.Config.Name` field) instead of kernel-assigned name +- [x] 14.2 Document required iptables rule: `iptables -A OUTPUT -p icmp --icmp-type 3/4 ! -i pmtud0 -j NFLOG --nflog-group 33` +- [x] 14.3 Log the TUN device name and required iptables rule at startup for operator visibility + +## 15. Re-validation + +- [x] 15.1 Verify `go build ./...` passes +- [x] 15.2 Run `go test ./...` — all tests pass +- [x] 15.3 Verify `go vet ./...` has no issues diff --git a/openspec/changes/kind-cluster-lab/.openspec.yaml b/openspec/changes/kind-cluster-lab/.openspec.yaml new file mode 100644 index 0000000..d433dd7 --- /dev/null +++ b/openspec/changes/kind-cluster-lab/.openspec.yaml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 +schema: spec-driven +created: 2026-05-01 diff --git a/openspec/changes/kind-cluster-lab/design.md b/openspec/changes/kind-cluster-lab/design.md new file mode 100644 index 0000000..1e8384a --- /dev/null +++ b/openspec/changes/kind-cluster-lab/design.md @@ -0,0 +1,113 @@ + + +## Context + +go-pmtud replicates ICMP fragmentation-needed packets between Kubernetes nodes. The `feature/eliminate-l2-dependency` branch replaces L2 broadcast with UDP unicast, enabling replication across L3 boundaries. Validating this requires an environment where: +1. Two sets of nodes exist in different L3 networks +2. A path between them has a lower MTU than the source network +3. Traffic large enough to exceed the path MTU is generated +4. ICMP type 3/4 packets and their replication can be observed + +Currently testing is done manually on QA infrastructure (adjusting router MTUs, SSHing to nodes). This is slow, non-reproducible, and ties up shared resources. + +## Goals / Non-Goals + +**Goals:** +- Reproducible local lab that simulates L3-separated clusters with MTU mismatch +- Two Kind clusters in separate Docker networks connected by a router container +- Configurable MTU on the router's interfaces to simulate path MTU reduction +- Deploy go-pmtud DaemonSet in the clusters +- Provide traffic generation and observation tooling +- Single-command setup and teardown +- Works on Linux and macOS (Docker Desktop) developer machines + +**Non-Goals:** +- Full CI pipeline integration (future work, document how but don't implement) +- Testing IPv6 Packet Too Big scenarios +- Performance/load testing at scale +- Replacing existing unit tests +- Multi-architecture support (arm64 lab images) + +## Decisions + +### 1. Two Kind clusters in separate Docker networks + +**Decision**: Create two Docker networks (`pmtud-net-a` with MTU 9000, `pmtud-net-b` with MTU 9000) and a third "transit" network (`pmtud-transit`) with MTU 1500. Each Kind cluster attaches to its respective network. A router container bridges them via the transit network. + +**Rationale**: Kind clusters use Docker networks for node connectivity. Separate networks ensure no L2 adjacency. The transit network with lower MTU simulates the real-world scenario where a WAN/backbone path has a smaller MTU than the local LAN. + +**Alternatives considered**: +- Single Kind cluster with network policies: Doesn't simulate L3 separation or MTU mismatch +- VMs (Vagrant): Heavier, slower to set up, more dependencies +- Network namespaces without Kind: Loses Kubernetes context, can't test DaemonSet deployment + +### 2. Router container with iptables forwarding + +**Decision**: Use an Alpine-based container with IP forwarding enabled, connected to all three networks. Its interface on `pmtud-transit` has MTU 1500 while interfaces on `pmtud-net-a` and `pmtud-net-b` have MTU 9000. The router does NOT fragment (DF bit respected), causing it to generate ICMP fragmentation-needed messages. + +**Rationale**: This mirrors real router behavior — when a packet with DF bit arrives on a high-MTU interface and must egress on a low-MTU interface, the router drops it and sends ICMP type 3/4. Using a container keeps everything in Docker, no host network changes needed. + +**Alternatives considered**: +- tc/netem on Kind nodes: Can simulate loss/delay but not MTU reduction at a router hop +- Docker network MTU alone: Kind nodes would just use the lower MTU from the start, no ICMP generated + +### 3. Podinfo as test workload + +**Decision**: Deploy podinfo in both clusters serving a large static file (e.g., 1MB). Cross-cluster curl requests trigger TCP segments larger than the transit MTU, producing ICMP fragmentation-needed. + +**Rationale**: podinfo is lightweight, well-known, and can serve arbitrary files. The user's existing QA test already uses podinfo for this purpose. TCP with DF bit set naturally triggers PMTU discovery. + +### 4. Script-based setup (Makefile + shell scripts) + +**Decision**: Provide a `lab/Makefile` with targets: `setup`, `teardown`, `status`, `test`, `logs`. Shell scripts under `lab/scripts/` handle Docker network creation, Kind cluster creation, router container setup, and go-pmtud deployment. + +**Rationale**: Makefile targets are discoverable and composable. Shell scripts are debuggable and don't add dependencies. Developers can run individual steps or the full setup. + +**Alternatives considered**: +- Docker Compose: Doesn't manage Kind clusters natively +- Terraform: Overkill for local Docker resources +- Go test harness: Higher barrier to entry, harder to debug interactively + +### 5. Tcpdump observation helpers + +**Decision**: Provide wrapper scripts that exec into cluster nodes or the router container to run tcpdump with pre-built filters for ICMP type 3/4 and go-pmtud replication traffic (UDP port 4390). + +**Rationale**: The user's manual test workflow involves specific tcpdump filters. Encoding these in scripts makes observation repeatable and documents the expected packet patterns. + +### 6. Lab directory structure + +**Decision**: +``` +lab/ +├── Makefile +├── README.md +├── configs/ +│ ├── kind-cluster-a.yaml +│ ├── kind-cluster-b.yaml +│ └── router/ +│ └── Dockerfile +├── scripts/ +│ ├── setup.sh +│ ├── teardown.sh +│ ├── deploy-pmtud.sh +│ ├── generate-traffic.sh +│ └── observe.sh +└── manifests/ + ├── podinfo.yaml + └── pmtud-daemonset.yaml +``` + +**Rationale**: Clear separation of concerns. Configs are declarative, scripts are imperative, manifests are deployable Kubernetes resources. + +## Risks / Trade-offs + +- **[Docker Desktop MTU limitations]** → Docker Desktop on macOS may not honor custom network MTUs in all cases. Document known limitations and provide a Linux-first recommendation. Mitigation: test on Linux CI runners. +- **[Kind networking complexity]** → Kind uses kindnet CNI by default which may interfere with cross-cluster routing. Mitigation: configure static routes in the router container and use NodePort/HostNetwork for cross-cluster traffic. +- **[Resource consumption]** → Two Kind clusters + router = ~6 containers minimum. Acceptable for developer machines but document minimum resource requirements (4GB RAM, 4 CPU). +- **[go-pmtud image availability]** → Lab needs a go-pmtud container image. Use `kind load docker-image` to load locally-built images. Document the build-then-load workflow. +- **[Privileged containers required]** → go-pmtud needs CAP_NET_RAW, CAP_NET_ADMIN, and host networking. Kind supports this via extra mounts and security context. +- **[Cross-cluster routing]** → Pods in cluster-a need to reach pods in cluster-b through the router. Requires static routes on Kind nodes pointing to the router. Mitigation: setup script adds routes automatically. diff --git a/openspec/changes/kind-cluster-lab/proposal.md b/openspec/changes/kind-cluster-lab/proposal.md new file mode 100644 index 0000000..32384df --- /dev/null +++ b/openspec/changes/kind-cluster-lab/proposal.md @@ -0,0 +1,32 @@ + + +## Why + +Testing the `feature/eliminate-l2-dependency` UDP replication requires simulating MTU mismatches between nodes across L3 boundaries. Currently this is done manually on real infrastructure (adjusting MTU on core routers, SSHing into nodes, flushing route caches). A reproducible local lab using Kind clusters in separate Docker networks enables fast iteration, CI integration, and eliminates dependency on production/QA infrastructure. + +## What Changes + +- Add a Kind-based local development lab with two clusters in separate Docker networks connected by a simulated router container +- Provide scripts to set up MTU mismatches between the networks (simulating a reduced-MTU path) +- Include a test workload (podinfo) that generates traffic large enough to trigger ICMP fragmentation-needed +- Provide helper scripts for tcpdump observation of ICMP type 3/4 packets and PMTU replication +- Document the lab setup and validation workflow + +## Capabilities + +### New Capabilities +- `kind-lab-setup`: Docker network topology with two Kind clusters, a router container bridging them, and configurable MTU mismatches between networks +- `kind-lab-validation`: Test scripts and procedures to verify go-pmtud correctly replicates ICMP fragmentation-needed packets across the simulated L3 boundary + +### Modified Capabilities + +## Impact + +- **Code**: New `lab/` directory with Kind configs, scripts, and Dockerfiles (no changes to main source) +- **Dependencies**: Requires `kind`, `docker`, `kubectl` on developer machine (development-time only) +- **CI**: Optionally integrable into GitHub Actions for automated integration testing +- **Documentation**: New README in `lab/` explaining setup, usage, and teardown diff --git a/openspec/changes/kind-cluster-lab/specs/kind-lab-setup/spec.md b/openspec/changes/kind-cluster-lab/specs/kind-lab-setup/spec.md new file mode 100644 index 0000000..9cd85d4 --- /dev/null +++ b/openspec/changes/kind-cluster-lab/specs/kind-lab-setup/spec.md @@ -0,0 +1,71 @@ + + +## ADDED Requirements + +### Requirement: Docker network topology creation +The lab SHALL create three Docker networks: `pmtud-net-a` (MTU 9000), `pmtud-net-b` (MTU 9000), and `pmtud-transit` (MTU 1500) with non-overlapping subnets. + +#### Scenario: Networks created with correct MTUs +- **WHEN** `make setup` is run +- **THEN** three Docker networks exist with the specified MTUs and distinct subnets + +#### Scenario: Idempotent creation +- **WHEN** `make setup` is run and networks already exist +- **THEN** the script completes without error (skip existing networks) + +### Requirement: Kind cluster creation in separate networks +The lab SHALL create two Kind clusters (`pmtud-cluster-a` and `pmtud-cluster-b`) each with at least 2 worker nodes. Cluster-a nodes MUST be attached to `pmtud-net-a` and cluster-b nodes MUST be attached to `pmtud-net-b`. + +#### Scenario: Two clusters created in separate L2 domains +- **WHEN** `make setup` completes +- **THEN** `kind get clusters` lists both `pmtud-cluster-a` and `pmtud-cluster-b` +- **AND** cluster-a node containers are on `pmtud-net-a` +- **AND** cluster-b node containers are on `pmtud-net-b` + +#### Scenario: Worker node count +- **WHEN** `make setup` completes +- **THEN** each cluster has at least 2 worker nodes (for intra-cluster go-pmtud replication testing) + +### Requirement: Router container with MTU mismatch +The lab SHALL run a router container connected to `pmtud-net-a`, `pmtud-net-b`, and `pmtud-transit`. The router MUST have IP forwarding enabled and MUST NOT fragment packets (respect DF bit). The transit-facing interfaces MUST have MTU 1500. + +#### Scenario: Router forwards between networks +- **WHEN** a packet from a cluster-a node is destined for a cluster-b node +- **THEN** the router forwards it through the transit network + +#### Scenario: Router generates ICMP fragmentation-needed +- **WHEN** a packet with DF bit set and size > 1500 bytes arrives at the router from `pmtud-net-a` destined for `pmtud-net-b` +- **THEN** the router drops the packet and sends ICMP type 3 code 4 (fragmentation needed, MTU=1500) back to the source + +### Requirement: Static routes on Kind nodes +The lab SHALL configure static routes on all Kind cluster nodes so that traffic to the other cluster's pod/node CIDRs is routed through the router container. + +#### Scenario: Cross-cluster reachability via router +- **WHEN** a pod in cluster-a sends traffic to a pod IP in cluster-b +- **THEN** the traffic transits through the router container + +### Requirement: go-pmtud DaemonSet deployment +The lab SHALL deploy go-pmtud as a DaemonSet in both clusters using locally-built container images loaded via `kind load docker-image`. + +#### Scenario: go-pmtud running on all nodes +- **WHEN** `make deploy-pmtud` completes +- **THEN** a go-pmtud pod is Running on every worker node in both clusters + +#### Scenario: go-pmtud uses UDP replication +- **WHEN** go-pmtud pods are running +- **THEN** they are configured with `--replication-port=4390` and the TUN device `pmtud0` exists on each node + +### Requirement: Single-command teardown +The lab SHALL provide `make teardown` that removes all clusters, networks, and the router container. + +#### Scenario: Clean teardown +- **WHEN** `make teardown` is run +- **THEN** both Kind clusters are deleted, all three Docker networks are removed, and the router container is stopped and removed + +#### Scenario: Teardown is idempotent +- **WHEN** `make teardown` is run and resources don't exist +- **THEN** the script completes without error diff --git a/openspec/changes/kind-cluster-lab/specs/kind-lab-validation/spec.md b/openspec/changes/kind-cluster-lab/specs/kind-lab-validation/spec.md new file mode 100644 index 0000000..c6dbfdf --- /dev/null +++ b/openspec/changes/kind-cluster-lab/specs/kind-lab-validation/spec.md @@ -0,0 +1,62 @@ + + +## ADDED Requirements + +### Requirement: Traffic generation that triggers PMTU discovery +The lab SHALL provide a script/target that generates TCP traffic with DF bit set from cluster-a to cluster-b exceeding 1500 bytes, triggering ICMP fragmentation-needed from the router. + +#### Scenario: Large file download triggers ICMP +- **WHEN** `make generate-traffic` is run +- **THEN** a curl/wget from a cluster-a pod downloads a large file (>1MB) from a podinfo service in cluster-b via NodePort +- **AND** the router generates ICMP type 3 code 4 packets + +#### Scenario: Podinfo serves test payload +- **WHEN** podinfo is deployed in cluster-b +- **THEN** it serves a test file at a known endpoint that is larger than the transit MTU + +### Requirement: ICMP observation tooling +The lab SHALL provide scripts to observe ICMP type 3 code 4 packets on cluster nodes and the router container using tcpdump. + +#### Scenario: Observe ICMP on router +- **WHEN** `make observe-router` is run +- **THEN** tcpdump runs on the router showing ICMP fragmentation-needed packets with filter `icmp and icmp[0] == 3 and icmp[1] == 4` + +#### Scenario: Observe ICMP on cluster node +- **WHEN** `make observe-node CLUSTER=a NODE=worker` is run +- **THEN** tcpdump runs on the specified node showing ICMP fragmentation-needed packets + +#### Scenario: Observe replication traffic +- **WHEN** `make observe-replication CLUSTER=a` is run +- **THEN** tcpdump runs filtering UDP port 4390 showing go-pmtud replication between nodes + +### Requirement: PMTU cache verification +The lab SHALL provide a way to verify that go-pmtud replication causes the kernel PMTU cache to update on peer nodes that didn't directly receive the ICMP. + +#### Scenario: Peer node PMTU cache updated +- **WHEN** ICMP fragmentation-needed is received by node-1 in cluster-a +- **AND** go-pmtud replicates it to node-2 in cluster-a +- **THEN** `ip route get ` on node-2 shows the updated MTU (1500) + +#### Scenario: Route cache flush and re-verify +- **WHEN** the route cache on a node is flushed (`ip route flush cache`) +- **AND** traffic is generated again +- **THEN** go-pmtud replication restores the PMTU cache entry on peer nodes + +### Requirement: End-to-end test target +The lab SHALL provide `make test` that runs an automated end-to-end validation: generates traffic, waits for ICMP, and verifies PMTU cache updates on all cluster-a worker nodes. + +#### Scenario: Automated validation passes +- **WHEN** `make test` is run after `make setup` +- **THEN** the test generates cross-cluster traffic, verifies ICMP was received, verifies go-pmtud replicated to peers, and checks PMTU cache on all nodes +- **AND** exits 0 on success or non-zero with diagnostic output on failure + +### Requirement: Lab status reporting +The lab SHALL provide `make status` showing the state of all lab components. + +#### Scenario: Status shows healthy lab +- **WHEN** `make status` is run after successful setup +- **THEN** output shows: networks (up/down), clusters (running/stopped), router (running/stopped), go-pmtud pods (running/pending/error), and podinfo deployment status diff --git a/openspec/changes/kind-cluster-lab/tasks.md b/openspec/changes/kind-cluster-lab/tasks.md new file mode 100644 index 0000000..a3166f4 --- /dev/null +++ b/openspec/changes/kind-cluster-lab/tasks.md @@ -0,0 +1,59 @@ + + +## 1. Directory structure and documentation + +- [x] 1.1 Create `lab/` directory with `Makefile`, `README.md`, `configs/`, `scripts/`, `manifests/` subdirectories +- [x] 1.2 Write `lab/README.md` documenting prerequisites (docker, kind, kubectl), architecture diagram (ASCII), usage, and known limitations + +## 2. Docker network setup + +- [x] 2.1 Create `lab/scripts/setup-networks.sh`: creates `pmtud-net-a` (MTU 9000, subnet 172.30.0.0/16), `pmtud-net-b` (MTU 9000, subnet 172.31.0.0/16), `pmtud-transit` (MTU 1500, subnet 172.32.0.0/24) — idempotent +- [x] 2.2 Create `lab/scripts/teardown-networks.sh`: removes all three networks — idempotent + +## 3. Kind cluster configuration + +- [x] 3.1 Create `lab/configs/kind-cluster-a.yaml`: 1 control-plane + 2 workers, networking config for `pmtud-net-a` +- [x] 3.2 Create `lab/configs/kind-cluster-b.yaml`: 1 control-plane + 2 workers, networking config for `pmtud-net-b` +- [x] 3.3 Create `lab/scripts/setup-clusters.sh`: creates both Kind clusters, attaches node containers to respective Docker networks + +## 4. Router container + +- [x] 4.1 Create `lab/configs/router/Dockerfile`: Alpine-based image with iproute2, iptables, tcpdump, ip forwarding enabled +- [x] 4.2 Create `lab/scripts/setup-router.sh`: builds and runs the router container, connects it to all three networks, sets MTU 1500 on transit-facing interface, configures forwarding rules and routes +- [x] 4.3 Add static routes on Kind nodes pointing to router for cross-cluster traffic + +## 5. go-pmtud deployment + +- [x] 5.1 Create `lab/manifests/pmtud-daemonset.yaml`: DaemonSet with hostNetwork, CAP_NET_RAW, CAP_NET_ADMIN, `--replication-port=4390`, iptables NFLOG rule init container +- [x] 5.2 Create `lab/scripts/deploy-pmtud.sh`: builds go-pmtud image, loads into both clusters via `kind load docker-image`, applies DaemonSet manifest, waits for pods Ready + +## 6. Test workload + +- [x] 6.1 Create `lab/manifests/podinfo.yaml`: podinfo Deployment + Service (NodePort) in cluster-b serving a large test file +- [x] 6.2 Create `lab/scripts/deploy-workload.sh`: deploys podinfo to cluster-b, generates a 1MB test file in the pod + +## 7. Traffic generation and observation + +- [x] 7.1 Create `lab/scripts/generate-traffic.sh`: runs curl from a cluster-a pod to podinfo NodePort in cluster-b (large file download triggering PMTU discovery) +- [x] 7.2 Create `lab/scripts/observe.sh`: wrapper for tcpdump on router/nodes with pre-built ICMP type 3/4 and UDP 4390 filters (subcommands: `router`, `node`, `replication`) + +## 8. Validation and testing + +- [x] 8.1 Create `lab/scripts/verify-pmtu.sh`: checks `ip route get` on all cluster-a worker nodes to verify PMTU cache reflects MTU 1500 for cluster-b destinations +- [x] 8.2 Create `lab/scripts/test-e2e.sh`: full end-to-end test — generates traffic, waits for ICMP, verifies replication on UDP 4390, checks PMTU cache on peers, exit 0/1 +- [x] 8.3 Create `lab/scripts/status.sh`: shows state of networks, clusters, router, go-pmtud pods, podinfo + +## 9. Makefile targets + +- [x] 9.1 Create `lab/Makefile` with targets: `setup` (networks + clusters + router + routes), `deploy` (pmtud + workload), `test` (generate-traffic + verify), `observe-router`, `observe-node`, `observe-replication`, `status`, `teardown` +- [x] 9.2 Add top-level convenience: `make lab-setup`, `make lab-teardown` in a root-level comment or lab/README reference + +## 10. Final validation + +- [x] 10.1 Run `make setup` on a Linux machine or Docker Desktop — verify all containers and clusters start +- [x] 10.2 Run `make deploy` — verify go-pmtud and podinfo pods are Running +- [x] 10.3 Run `make test` — verify end-to-end PMTU replication works diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000..e267f1a --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company +# SPDX-License-Identifier: Apache-2.0 + +schema: spec-driven + +context: | + Project: go-pmtud - replicates ICMP fragmentation-needed (type 3 code 4) packets between Kubernetes cluster nodes + Based on: cloudflare/pmtud concept (https://blog.cloudflare.com/path-mtu-discovery-in-practice/) + + Tech stack: + - Go 1.26 + - controller-runtime (sigs.k8s.io/controller-runtime) for Kubernetes reconciliation and manager lifecycle + - cobra/viper for CLI flags + - NFLOG (github.com/florianl/go-nflog/v2) for kernel packet capture + - Prometheus metrics (prometheus/client_golang) + - Runs as a DaemonSet with host networking and CAP_NET_RAW + + Architecture: + - cmd/go-pmtud/main.go: entrypoint + - internal/cmd/: cobra root command, flag parsing, manager setup + - internal/config/: shared Config struct (passed by pointer to all components) + - internal/node/: controller-runtime Reconciler watching corev1.Node resources + - internal/nflog/: controller-runtime Runnable that captures ICMP packets via NFLOG and replicates them + - internal/packet/: ICMP packet parsing utilities + - internal/util/: interface discovery, IP helpers + - internal/metrics/: Prometheus metric declarations + - internal/arp/: ARP resolution (L2 dependency, to be removed) + + Build tooling: go-makefile-maker (https://github.com/sapcc/go-makefile-maker) + - Generates Makefile, GitHub workflows, Dockerfile, golangci-lint config from Makefile.maker.yaml + - Config file: Makefile.maker.yaml in repo root + - DO NOT edit the generated Makefile directly — edit Makefile.maker.yaml and re-run go-makefile-maker + - Key config sections: metadata, dockerfile, golang, golangciLint, githubWorkflow, renovate, controllerGen + - Current project config: controllerGen enabled (generates CRDs/RBAC/objects/applyconfigurations), + golangciLint with createConfig: true, GitHub CI/security/license/GHCR push workflows enabled + - Targets: `make check` (lint + test), `make static-check` (golangci-lint + shellcheck + typos + license checks), + `make build/cover.html` (test coverage), `make generate` (controller-gen), `make tidy-deps` (go mod tidy + verify) + - Tests use setup-envtest (KUBEBUILDER_ASSETS) for controller-runtime integration tests + + Conventions: + - SPDX license headers on all files (enforced by addlicense + reuse lint) + - go-makefile-maker generated Makefile — never hand-edit + - golangci-lint for linting (config auto-generated from Makefile.maker.yaml) + - controller-runtime patterns: Reconciler interface, manager.Runnable for long-running goroutines + - Peer list protected by sync.Mutex in shared Config struct + - Build: `go build -v -o /go-pmtud cmd/go-pmtud/main.go` + - Tests: `make check` or `go test ./...` (uses setup-envtest for controller-runtime tests) + - Docker image built via generated Dockerfile (Alpine-based, multi-stage) + - GHCR push with tag strategies: edge, latest, semver, sha + +rules: + proposal: + - Keep proposals concise (1-2 pages max) + - Focus on Kubernetes networking and ICMP/PMTU domain + design: + - Reference controller-runtime patterns (Reconciler, Runnable, Manager) + - Consider CAP_NET_RAW and host networking implications + - Address loop prevention (nodes must not re-replicate received packets) + tasks: + - Each task should be completable in one coding session + - Order by dependency (config changes first, then logic, then cleanup) + - Include a final build verification task + - Do NOT include tasks to edit the generated Makefile — only Makefile.maker.yaml changes if needed